{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 10.0, "eval_steps": 500, "global_step": 490, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 3.0517578125e-05, "clip_ratio/low_min": 3.0517578125e-05, "clip_ratio/region_mean": 3.0517578125e-05, "completions/clipped_ratio": 0.8125, "completions/max_length": 4096.0, "completions/max_terminated_length": 3653.0, "completions/mean_length": 3795.8125, "completions/mean_terminated_length": 2495.0, "completions/min_length": 687.0, "completions/min_terminated_length": 687.0, "entropy": 0.048652240191586316, "epoch": 0.02040816326530612, "frac_reward_zero_std": 0.0, "grad_norm": 0.022123996168375015, "learning_rate": 1e-05, "loss": 0.2841, "num_tokens": 79469.0, "reward": 0.452539324760437, "reward_std": 0.21963119506835938, "rewards/reward_commands_completeness/mean": 0.07499999552965164, "rewards/reward_commands_completeness/std": 0.04926120862364769, "rewards/reward_commands_correctness/mean": 0.0468749962747097, "rewards/reward_commands_correctness/std": 0.03156343102455139, "rewards/reward_diversity/mean": 0.13128933310508728, "rewards/reward_diversity/std": 0.05721068009734154, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.054375000298023224, "rewards/reward_problem_validity/std": 0.03558440878987312, "rewards/reward_solution_effectiveness/mean": 0.08250000327825546, "rewards/reward_solution_effectiveness/std": 0.04837355017662048, "sampling/importance_sampling_ratio/max": 1.7174105644226074, "sampling/importance_sampling_ratio/mean": 0.7894679307937622, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6752933263778687, "sampling/sampling_logp_difference/mean": 0.002170279622077942, "step": 1, "step_time": 300.6198643948883 }, { "clip_ratio/high_max": 0.0007521576335420832, "clip_ratio/high_mean": 0.0007521576335420832, "clip_ratio/low_mean": 0.0002334105138288578, "clip_ratio/low_min": 0.0002334105138288578, "clip_ratio/region_mean": 0.000985568147370941, "completions/clipped_ratio": 0.625, "completions/max_length": 4096.0, "completions/max_terminated_length": 2535.0, "completions/mean_length": 2750.4375, "completions/mean_terminated_length": 1409.3333740234375, "completions/min_length": 738.0, "completions/min_terminated_length": 738.0, "entropy": 0.07360569387674332, "epoch": 0.04081632653061224, "frac_reward_zero_std": 0.0, "grad_norm": 0.026590902358293533, "learning_rate": 9.979591836734694e-06, "loss": 0.1413, "num_tokens": 167148.0, "reward": 0.40186646580696106, "reward_std": 0.24742040038108826, "rewards/reward_commands_completeness/mean": 0.06749999523162842, "rewards/reward_commands_completeness/std": 0.044944409281015396, "rewards/reward_commands_correctness/mean": 0.04124999791383743, "rewards/reward_commands_correctness/std": 0.032223179936409, "rewards/reward_diversity/mean": 0.1149914413690567, "rewards/reward_diversity/std": 0.07111777365207672, "rewards/reward_format/mean": 0.05624999850988388, "rewards/reward_format/std": 0.05123475566506386, "rewards/reward_problem_validity/mean": 0.05062500014901161, "rewards/reward_problem_validity/std": 0.03714274242520332, "rewards/reward_solution_effectiveness/mean": 0.07124999910593033, "rewards/reward_solution_effectiveness/std": 0.04500000178813934, "sampling/importance_sampling_ratio/max": 1.738100290298462, "sampling/importance_sampling_ratio/mean": 0.5859519839286804, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7011744976043701, "sampling/sampling_logp_difference/mean": 0.003054490080103278, "step": 2, "step_time": 386.80731361731887 }, { "clip_ratio/high_max": 0.0007145240233512595, "clip_ratio/high_mean": 0.0007145240233512595, "clip_ratio/low_mean": 0.00012740682905132417, "clip_ratio/low_min": 0.00012740682905132417, "clip_ratio/region_mean": 0.0008419308614975307, "completions/clipped_ratio": 0.5, "completions/max_length": 4096.0, "completions/max_terminated_length": 3345.0, "completions/mean_length": 2749.9375, "completions/mean_terminated_length": 1403.875, "completions/min_length": 861.0, "completions/min_terminated_length": 861.0, "entropy": 0.0793635519221425, "epoch": 0.061224489795918366, "frac_reward_zero_std": 0.0, "grad_norm": 0.031851451843976974, "learning_rate": 9.959183673469387e-06, "loss": -0.0155, "num_tokens": 238235.0, "reward": 0.40751713514328003, "reward_std": 0.23129980266094208, "rewards/reward_commands_completeness/mean": 0.07499999552965164, "rewards/reward_commands_completeness/std": 0.036147844046354294, "rewards/reward_commands_correctness/mean": 0.058125000447034836, "rewards/reward_commands_correctness/std": 0.0325000025331974, "rewards/reward_diversity/mean": 0.061892107129096985, "rewards/reward_diversity/std": 0.11516812443733215, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.0625, "rewards/reward_problem_validity/std": 0.03130495175719261, "rewards/reward_solution_effectiveness/mean": 0.07500000298023224, "rewards/reward_solution_effectiveness/std": 0.040987804532051086, "sampling/importance_sampling_ratio/max": 2.5373923778533936, "sampling/importance_sampling_ratio/mean": 0.8051360845565796, "sampling/importance_sampling_ratio/min": 0.12871091067790985, "sampling/sampling_logp_difference/max": 0.5645434856414795, "sampling/sampling_logp_difference/mean": 0.0027758816722780466, "step": 3, "step_time": 271.05148808285594 }, { "clip_ratio/high_max": 0.00038919919097679667, "clip_ratio/high_mean": 0.00038919919097679667, "clip_ratio/low_mean": 0.0002712102432269603, "clip_ratio/low_min": 0.0002712102432269603, "clip_ratio/region_mean": 0.0006604094378417358, "completions/clipped_ratio": 0.5625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1414.0, "completions/mean_length": 2786.0625, "completions/mean_terminated_length": 1101.857177734375, "completions/min_length": 857.0, "completions/min_terminated_length": 857.0, "entropy": 0.07299393322318792, "epoch": 0.08163265306122448, "frac_reward_zero_std": 0.0, "grad_norm": 0.02426210604608059, "learning_rate": 9.938775510204082e-06, "loss": 0.0262, "num_tokens": 304028.0, "reward": 0.565079391002655, "reward_std": 0.2090739756822586, "rewards/reward_commands_completeness/mean": 0.10625000298023224, "rewards/reward_commands_completeness/std": 0.055000003427267075, "rewards/reward_commands_correctness/mean": 0.05687499791383743, "rewards/reward_commands_correctness/std": 0.03048907034099102, "rewards/reward_diversity/mean": 0.1344543993473053, "rewards/reward_diversity/std": 0.0490889847278595, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06687499582767487, "rewards/reward_problem_validity/std": 0.034199174493551254, "rewards/reward_solution_effectiveness/mean": 0.12562499940395355, "rewards/reward_solution_effectiveness/std": 0.07201562076807022, "sampling/importance_sampling_ratio/max": 1.2906166315078735, "sampling/importance_sampling_ratio/mean": 0.7956347465515137, "sampling/importance_sampling_ratio/min": 0.27653831243515015, "sampling/sampling_logp_difference/max": 1.1023814678192139, "sampling/sampling_logp_difference/mean": 0.002275706036016345, "step": 4, "step_time": 268.81965524517 }, { "clip_ratio/high_max": 0.0003900919546140358, "clip_ratio/high_mean": 0.0003900919546140358, "clip_ratio/low_mean": 0.0001040825718519045, "clip_ratio/low_min": 0.0001040825718519045, "clip_ratio/region_mean": 0.0004941745246469509, "completions/clipped_ratio": 0.3125, "completions/max_length": 4096.0, "completions/max_terminated_length": 3813.0, "completions/mean_length": 2163.0625, "completions/mean_terminated_length": 1284.45458984375, "completions/min_length": 743.0, "completions/min_terminated_length": 743.0, "entropy": 0.08119554026052356, "epoch": 0.10204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.03950459510087967, "learning_rate": 9.918367346938776e-06, "loss": -0.0012, "num_tokens": 365717.0, "reward": 0.5430622100830078, "reward_std": 0.18637555837631226, "rewards/reward_commands_completeness/mean": 0.09999999403953552, "rewards/reward_commands_completeness/std": 0.052662450820207596, "rewards/reward_commands_correctness/mean": 0.061250001192092896, "rewards/reward_commands_correctness/std": 0.02895398810505867, "rewards/reward_diversity/mean": 0.1080622524023056, "rewards/reward_diversity/std": 0.04364072531461716, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07062499970197678, "rewards/reward_problem_validity/std": 0.03043435700237751, "rewards/reward_solution_effectiveness/mean": 0.12187500298023224, "rewards/reward_solution_effectiveness/std": 0.07138802856206894, "sampling/importance_sampling_ratio/max": 1.7584123611450195, "sampling/importance_sampling_ratio/mean": 0.5942248702049255, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.9435843229293823, "sampling/sampling_logp_difference/mean": 0.003044495126232505, "step": 5, "step_time": 286.22183033265173 }, { "clip_ratio/high_max": 0.0007542476159869693, "clip_ratio/high_mean": 0.0007542476159869693, "clip_ratio/low_mean": 0.000522652939253021, "clip_ratio/low_min": 0.000522652939253021, "clip_ratio/region_mean": 0.0012769005552399904, "completions/clipped_ratio": 0.375, "completions/max_length": 4096.0, "completions/max_terminated_length": 1304.0, "completions/mean_length": 2063.3125, "completions/mean_terminated_length": 843.7000122070312, "completions/min_length": 582.0, "completions/min_terminated_length": 582.0, "entropy": 0.11393151199445128, "epoch": 0.12244897959183673, "frac_reward_zero_std": 0.0, "grad_norm": 0.02719634212553501, "learning_rate": 9.89795918367347e-06, "loss": 0.0358, "num_tokens": 422826.0, "reward": 0.5994054675102234, "reward_std": 0.0989157184958458, "rewards/reward_commands_completeness/mean": 0.10624999552965164, "rewards/reward_commands_completeness/std": 0.03774917125701904, "rewards/reward_commands_correctness/mean": 0.05937500298023224, "rewards/reward_commands_correctness/std": 0.02568235620856285, "rewards/reward_diversity/mean": 0.1575305163860321, "rewards/reward_diversity/std": 0.027573535218834877, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07437500357627869, "rewards/reward_problem_validity/std": 0.025552237406373024, "rewards/reward_solution_effectiveness/mean": 0.11437499523162842, "rewards/reward_solution_effectiveness/std": 0.04802343621850014, "sampling/importance_sampling_ratio/max": 1.6615134477615356, "sampling/importance_sampling_ratio/mean": 0.5669702291488647, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0049055814743042, "sampling/sampling_logp_difference/mean": 0.003555847564712167, "step": 6, "step_time": 261.8866520822048 }, { "clip_ratio/high_max": 0.0005866574865649454, "clip_ratio/high_mean": 0.0005866574865649454, "clip_ratio/low_mean": 0.0007632654887856916, "clip_ratio/low_min": 0.0007632654887856916, "clip_ratio/region_mean": 0.0013499229717126582, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1774.0, "completions/mean_length": 1358.5, "completions/mean_terminated_length": 967.4285888671875, "completions/min_length": 707.0, "completions/min_terminated_length": 707.0, "entropy": 0.14017293881624937, "epoch": 0.14285714285714285, "frac_reward_zero_std": 0.0, "grad_norm": 0.06299114972352982, "learning_rate": 9.877551020408164e-06, "loss": -0.0518, "num_tokens": 472678.0, "reward": 0.5872423648834229, "reward_std": 0.1402631402015686, "rewards/reward_commands_completeness/mean": 0.10499999672174454, "rewards/reward_commands_completeness/std": 0.047046080231666565, "rewards/reward_commands_correctness/mean": 0.06749999523162842, "rewards/reward_commands_correctness/std": 0.025166116654872894, "rewards/reward_diversity/mean": 0.1347423493862152, "rewards/reward_diversity/std": 0.05085825175046921, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07437500357627869, "rewards/reward_problem_validity/std": 0.025552235543727875, "rewards/reward_solution_effectiveness/mean": 0.11812500655651093, "rewards/reward_solution_effectiveness/std": 0.060466933995485306, "sampling/importance_sampling_ratio/max": 1.5249344110488892, "sampling/importance_sampling_ratio/mean": 0.7105346918106079, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.374692678451538, "sampling/sampling_logp_difference/mean": 0.004656690172851086, "step": 7, "step_time": 307.54011243768036 }, { "clip_ratio/high_max": 0.0003985636467405129, "clip_ratio/high_mean": 0.0003985636467405129, "clip_ratio/low_mean": 0.00030071781657170504, "clip_ratio/low_min": 0.00030071781657170504, "clip_ratio/region_mean": 0.0006992814596742392, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1694.0, "completions/mean_length": 1674.375, "completions/mean_terminated_length": 1115.5384521484375, "completions/min_length": 707.0, "completions/min_terminated_length": 707.0, "entropy": 0.13556275889277458, "epoch": 0.16326530612244897, "frac_reward_zero_std": 0.0, "grad_norm": 0.0484846830368042, "learning_rate": 9.857142857142859e-06, "loss": 0.1313, "num_tokens": 523832.0, "reward": 0.6625246405601501, "reward_std": 0.13638421893119812, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.04226898401975632, "rewards/reward_commands_correctness/mean": 0.07062499970197678, "rewards/reward_commands_correctness/std": 0.018786076456308365, "rewards/reward_diversity/mean": 0.13689963519573212, "rewards/reward_diversity/std": 0.028630131855607033, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0806250050663948, "rewards/reward_problem_validity/std": 0.019482901319861412, "rewards/reward_solution_effectiveness/mean": 0.15562500059604645, "rewards/reward_solution_effectiveness/std": 0.06120661646127701, "sampling/importance_sampling_ratio/max": 2.752209186553955, "sampling/importance_sampling_ratio/mean": 0.6116431951522827, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9212453365325928, "sampling/sampling_logp_difference/mean": 0.004744302947074175, "step": 8, "step_time": 274.47765354439616 }, { "clip_ratio/high_max": 0.0004751539454446174, "clip_ratio/high_mean": 0.0004751539454446174, "clip_ratio/low_mean": 0.0003224953979952261, "clip_ratio/low_min": 0.0003224953979952261, "clip_ratio/region_mean": 0.0007976493507158011, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1100.0, "completions/mean_length": 1456.3125, "completions/mean_terminated_length": 847.1538696289062, "completions/min_length": 658.0, "completions/min_terminated_length": 658.0, "entropy": 0.12003156216815114, "epoch": 0.1836734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.05547052621841431, "learning_rate": 9.836734693877552e-06, "loss": -0.0125, "num_tokens": 565133.0, "reward": 0.6604683995246887, "reward_std": 0.055918339639902115, "rewards/reward_commands_completeness/mean": 0.11999999731779099, "rewards/reward_commands_completeness/std": 0.020655911415815353, "rewards/reward_commands_correctness/mean": 0.078125, "rewards/reward_commands_correctness/std": 0.020072786137461662, "rewards/reward_diversity/mean": 0.14171844720840454, "rewards/reward_diversity/std": 0.029847286641597748, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500774860382, "rewards/reward_problem_validity/std": 0.008139412850141525, "rewards/reward_solution_effectiveness/mean": 0.13499999046325684, "rewards/reward_solution_effectiveness/std": 0.04242640733718872, "sampling/importance_sampling_ratio/max": 1.367466926574707, "sampling/importance_sampling_ratio/mean": 0.8097851276397705, "sampling/importance_sampling_ratio/min": 0.014956453815102577, "sampling/sampling_logp_difference/max": 2.2255566120147705, "sampling/sampling_logp_difference/mean": 0.004242502152919769, "step": 9, "step_time": 265.35438827611506 }, { "clip_ratio/high_max": 0.0003859794742311351, "clip_ratio/high_mean": 0.0003859794742311351, "clip_ratio/low_mean": 0.0002052947565971408, "clip_ratio/low_min": 0.0002052947565971408, "clip_ratio/region_mean": 0.0005912742308282759, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 1310.9375, "completions/mean_terminated_length": 913.0714721679688, "completions/min_length": 755.0, "completions/min_terminated_length": 755.0, "entropy": 0.13812571065500379, "epoch": 0.20408163265306123, "frac_reward_zero_std": 0.25, "grad_norm": 0.08714073151350021, "learning_rate": 9.816326530612245e-06, "loss": -0.2983, "num_tokens": 616912.0, "reward": 0.44262954592704773, "reward_std": 0.1059701144695282, "rewards/reward_commands_completeness/mean": 0.08374999463558197, "rewards/reward_commands_completeness/std": 0.06031860038638115, "rewards/reward_commands_correctness/mean": 0.045625001192092896, "rewards/reward_commands_correctness/std": 0.03346017748117447, "rewards/reward_diversity/mean": 0.10200454294681549, "rewards/reward_diversity/std": 0.07611805945634842, "rewards/reward_format/mean": 0.05624999850988388, "rewards/reward_format/std": 0.05123475566506386, "rewards/reward_problem_validity/mean": 0.05375000089406967, "rewards/reward_problem_validity/std": 0.03997916728258133, "rewards/reward_solution_effectiveness/mean": 0.10125000029802322, "rewards/reward_solution_effectiveness/std": 0.07499999552965164, "sampling/importance_sampling_ratio/max": 2.75816011428833, "sampling/importance_sampling_ratio/mean": 0.9738132953643799, "sampling/importance_sampling_ratio/min": 0.01670793816447258, "sampling/sampling_logp_difference/max": 2.498704195022583, "sampling/sampling_logp_difference/mean": 0.005345502402633429, "step": 10, "step_time": 280.2753277607262 }, { "clip_ratio/high_max": 0.0006317410407064017, "clip_ratio/high_mean": 0.0006317410407064017, "clip_ratio/low_mean": 0.00032771911355666816, "clip_ratio/low_min": 0.00032771911355666816, "clip_ratio/region_mean": 0.0009594601688149851, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1269.0, "completions/mean_length": 1313.0, "completions/mean_terminated_length": 915.4285888671875, "completions/min_length": 721.0, "completions/min_terminated_length": 721.0, "entropy": 0.15296374401077628, "epoch": 0.22448979591836735, "frac_reward_zero_std": 0.0, "grad_norm": 0.0916745513677597, "learning_rate": 9.795918367346939e-06, "loss": -0.0966, "num_tokens": 666652.0, "reward": 0.4807194173336029, "reward_std": 0.223340705037117, "rewards/reward_commands_completeness/mean": 0.08249999582767487, "rewards/reward_commands_completeness/std": 0.0515751913189888, "rewards/reward_commands_correctness/mean": 0.05000000074505806, "rewards/reward_commands_correctness/std": 0.030331501737236977, "rewards/reward_diversity/mean": 0.13134442269802094, "rewards/reward_diversity/std": 0.06197904795408249, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.05999999865889549, "rewards/reward_problem_validity/std": 0.035023801028728485, "rewards/reward_solution_effectiveness/mean": 0.08812500536441803, "rewards/reward_solution_effectiveness/std": 0.06241994723677635, "sampling/importance_sampling_ratio/max": 2.9429574012756348, "sampling/importance_sampling_ratio/mean": 1.1008081436157227, "sampling/importance_sampling_ratio/min": 0.05134166032075882, "sampling/sampling_logp_difference/max": 0.8294515609741211, "sampling/sampling_logp_difference/mean": 0.005479263141751289, "step": 11, "step_time": 291.68343139998615 }, { "clip_ratio/high_max": 0.000935730855417205, "clip_ratio/high_mean": 0.000935730855417205, "clip_ratio/low_mean": 0.0003617332113208249, "clip_ratio/low_min": 0.0003617332113208249, "clip_ratio/region_mean": 0.0012974640558240935, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1369.0, "completions/mean_length": 1538.6875, "completions/mean_terminated_length": 948.5385131835938, "completions/min_length": 677.0, "completions/min_terminated_length": 677.0, "entropy": 0.11793313873931766, "epoch": 0.24489795918367346, "frac_reward_zero_std": 0.0, "grad_norm": 0.05878806859254837, "learning_rate": 9.775510204081634e-06, "loss": -0.1236, "num_tokens": 722847.0, "reward": 0.5115697979927063, "reward_std": 0.26389890909194946, "rewards/reward_commands_completeness/mean": 0.0925000011920929, "rewards/reward_commands_completeness/std": 0.06060803681612015, "rewards/reward_commands_correctness/mean": 0.04312499612569809, "rewards/reward_commands_correctness/std": 0.02651257812976837, "rewards/reward_diversity/mean": 0.13281983137130737, "rewards/reward_diversity/std": 0.06332485377788544, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.06187500059604645, "rewards/reward_problem_validity/std": 0.036371923983097076, "rewards/reward_solution_effectiveness/mean": 0.11250000447034836, "rewards/reward_solution_effectiveness/std": 0.08160882443189621, "sampling/importance_sampling_ratio/max": 2.804786205291748, "sampling/importance_sampling_ratio/mean": 0.681554913520813, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0809433460235596, "sampling/sampling_logp_difference/mean": 0.004831742029637098, "step": 12, "step_time": 272.4580768123269 }, { "clip_ratio/high_max": 0.0009597928801667877, "clip_ratio/high_mean": 0.0009597928801667877, "clip_ratio/low_mean": 0.00044249147322261706, "clip_ratio/low_min": 0.00044249147322261706, "clip_ratio/region_mean": 0.0014022843533894047, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1080.0, "completions/mean_length": 1030.875, "completions/mean_terminated_length": 826.5333862304688, "completions/min_length": 708.0, "completions/min_terminated_length": 708.0, "entropy": 0.1641564085148275, "epoch": 0.2653061224489796, "frac_reward_zero_std": 0.0, "grad_norm": 0.10193347930908203, "learning_rate": 9.755102040816327e-06, "loss": -0.0141, "num_tokens": 762741.0, "reward": 0.5652267932891846, "reward_std": 0.11460017412900925, "rewards/reward_commands_completeness/mean": 0.0924999937415123, "rewards/reward_commands_completeness/std": 0.027202941477298737, "rewards/reward_commands_correctness/mean": 0.06749999523162842, "rewards/reward_commands_correctness/std": 0.02323790267109871, "rewards/reward_diversity/mean": 0.14272674918174744, "rewards/reward_diversity/std": 0.040212780237197876, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.018154429271817207, "rewards/reward_solution_effectiveness/mean": 0.09187500178813934, "rewards/reward_solution_effectiveness/std": 0.03709784522652626, "sampling/importance_sampling_ratio/max": 2.3276941776275635, "sampling/importance_sampling_ratio/mean": 0.7623960971832275, "sampling/importance_sampling_ratio/min": 0.08132176846265793, "sampling/sampling_logp_difference/max": 1.1229922771453857, "sampling/sampling_logp_difference/mean": 0.006527320481836796, "step": 13, "step_time": 278.7923781890422 }, { "clip_ratio/high_max": 0.0010044322152680252, "clip_ratio/high_mean": 0.0010044322152680252, "clip_ratio/low_mean": 0.0005497209349414334, "clip_ratio/low_min": 0.0005497209349414334, "clip_ratio/region_mean": 0.0015541531538474374, "completions/clipped_ratio": 0.0, "completions/max_length": 1263.0, "completions/max_terminated_length": 1263.0, "completions/mean_length": 919.375, "completions/mean_terminated_length": 919.375, "completions/min_length": 701.0, "completions/min_terminated_length": 701.0, "entropy": 0.1586097590625286, "epoch": 0.2857142857142857, "frac_reward_zero_std": 0.0, "grad_norm": 0.08789094537496567, "learning_rate": 9.734693877551022e-06, "loss": 0.039, "num_tokens": 804231.0, "reward": 0.46887385845184326, "reward_std": 0.1573835015296936, "rewards/reward_commands_completeness/mean": 0.08374999463558197, "rewards/reward_commands_completeness/std": 0.03739429637789726, "rewards/reward_commands_correctness/mean": 0.06187500059604645, "rewards/reward_commands_correctness/std": 0.03145764768123627, "rewards/reward_diversity/mean": 0.09887385368347168, "rewards/reward_diversity/std": 0.09651327133178711, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.06812499463558197, "rewards/reward_problem_validity/std": 0.029033029451966286, "rewards/reward_solution_effectiveness/mean": 0.07500000298023224, "rewards/reward_solution_effectiveness/std": 0.03464101627469063, "sampling/importance_sampling_ratio/max": 2.370866298675537, "sampling/importance_sampling_ratio/mean": 0.5273747444152832, "sampling/importance_sampling_ratio/min": 0.04127738997340202, "sampling/sampling_logp_difference/max": 0.6848261952400208, "sampling/sampling_logp_difference/mean": 0.007986915297806263, "step": 14, "step_time": 154.36653060093522 }, { "clip_ratio/high_max": 0.0008582247901358642, "clip_ratio/high_mean": 0.0008582247901358642, "clip_ratio/low_mean": 0.00044996882206760347, "clip_ratio/low_min": 0.00044996882206760347, "clip_ratio/region_mean": 0.0013081936122034676, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 1041.1875, "completions/mean_terminated_length": 837.5333862304688, "completions/min_length": 662.0, "completions/min_terminated_length": 662.0, "entropy": 0.1571406340226531, "epoch": 0.30612244897959184, "frac_reward_zero_std": 0.0, "grad_norm": 0.05593395233154297, "learning_rate": 9.714285714285715e-06, "loss": -0.1797, "num_tokens": 845946.0, "reward": 0.6101113557815552, "reward_std": 0.06619545817375183, "rewards/reward_commands_completeness/mean": 0.10000000149011612, "rewards/reward_commands_completeness/std": 0.02529822289943695, "rewards/reward_commands_correctness/mean": 0.07125000655651093, "rewards/reward_commands_correctness/std": 0.015438050031661987, "rewards/reward_diversity/mean": 0.15573638677597046, "rewards/reward_diversity/std": 0.022509120404720306, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08187499642372131, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.10125000029802322, "rewards/reward_solution_effectiveness/std": 0.03774917498230934, "sampling/importance_sampling_ratio/max": 2.4768526554107666, "sampling/importance_sampling_ratio/mean": 0.592642605304718, "sampling/importance_sampling_ratio/min": 0.06595774739980698, "sampling/sampling_logp_difference/max": 1.2705730199813843, "sampling/sampling_logp_difference/mean": 0.006981375627219677, "step": 15, "step_time": 284.8082468509674 }, { "clip_ratio/high_max": 0.0007878091928432696, "clip_ratio/high_mean": 0.0007878091928432696, "clip_ratio/low_mean": 0.0003663848910946399, "clip_ratio/low_min": 0.0003663848910946399, "clip_ratio/region_mean": 0.0011541940693859942, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1232.0, "completions/mean_length": 1552.9375, "completions/mean_terminated_length": 966.0769653320312, "completions/min_length": 739.0, "completions/min_terminated_length": 739.0, "entropy": 0.12056667916476727, "epoch": 0.32653061224489793, "frac_reward_zero_std": 0.0, "grad_norm": 0.04520661383867264, "learning_rate": 9.693877551020408e-06, "loss": 0.2142, "num_tokens": 894185.0, "reward": 0.5522608757019043, "reward_std": 0.1521724909543991, "rewards/reward_commands_completeness/mean": 0.09624999761581421, "rewards/reward_commands_completeness/std": 0.036674246191978455, "rewards/reward_commands_correctness/mean": 0.05687499791383743, "rewards/reward_commands_correctness/std": 0.02151549980044365, "rewards/reward_diversity/mean": 0.14288589358329773, "rewards/reward_diversity/std": 0.040136247873306274, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07124999910593033, "rewards/reward_problem_validity/std": 0.02390955202281475, "rewards/reward_solution_effectiveness/mean": 0.09749999642372131, "rewards/reward_solution_effectiveness/std": 0.043127719312906265, "sampling/importance_sampling_ratio/max": 2.983980655670166, "sampling/importance_sampling_ratio/mean": 0.6557992100715637, "sampling/importance_sampling_ratio/min": 0.04176228120923042, "sampling/sampling_logp_difference/max": 1.3473877906799316, "sampling/sampling_logp_difference/mean": 0.005701254587620497, "step": 16, "step_time": 261.8351999614388 }, { "clip_ratio/high_max": 0.0009209277704940178, "clip_ratio/high_mean": 0.0009209277704940178, "clip_ratio/low_mean": 0.0004276624931662809, "clip_ratio/low_min": 0.0004276624931662809, "clip_ratio/region_mean": 0.0013485902818501927, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1509.0, "completions/mean_length": 1234.3125, "completions/mean_terminated_length": 1043.533447265625, "completions/min_length": 746.0, "completions/min_terminated_length": 746.0, "entropy": 0.12685158103704453, "epoch": 0.3469387755102041, "frac_reward_zero_std": 0.0, "grad_norm": 0.01835990510880947, "learning_rate": 9.673469387755103e-06, "loss": -0.0465, "num_tokens": 937602.0, "reward": 0.5986058712005615, "reward_std": 0.1258731633424759, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.03575379028916359, "rewards/reward_commands_correctness/mean": 0.06687499582767487, "rewards/reward_commands_correctness/std": 0.020564937964081764, "rewards/reward_diversity/mean": 0.12923085689544678, "rewards/reward_diversity/std": 0.09508960694074631, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07999999821186066, "rewards/reward_problem_validity/std": 0.019321836531162262, "rewards/reward_solution_effectiveness/mean": 0.12000000476837158, "rewards/reward_solution_effectiveness/std": 0.04898979887366295, "sampling/importance_sampling_ratio/max": 1.2208675146102905, "sampling/importance_sampling_ratio/mean": 0.25223591923713684, "sampling/importance_sampling_ratio/min": 0.04738251864910126, "sampling/sampling_logp_difference/max": 1.612630844116211, "sampling/sampling_logp_difference/mean": 0.006382284685969353, "step": 17, "step_time": 268.93206201307476 }, { "clip_ratio/high_max": 0.0009086917780223303, "clip_ratio/high_mean": 0.0009086917780223303, "clip_ratio/low_mean": 0.00019617211364675313, "clip_ratio/low_min": 0.00019617211364675313, "clip_ratio/region_mean": 0.0011048638771171682, "completions/clipped_ratio": 0.375, "completions/max_length": 4096.0, "completions/max_terminated_length": 1146.0, "completions/mean_length": 2120.125, "completions/mean_terminated_length": 934.6000366210938, "completions/min_length": 748.0, "completions/min_terminated_length": 748.0, "entropy": 0.1104778004810214, "epoch": 0.3673469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.012434626929461956, "learning_rate": 9.653061224489797e-06, "loss": -0.1211, "num_tokens": 996600.0, "reward": 0.39777863025665283, "reward_std": 0.2959953248500824, "rewards/reward_commands_completeness/mean": 0.07249999791383743, "rewards/reward_commands_completeness/std": 0.053603481501340866, "rewards/reward_commands_correctness/mean": 0.04312499985098839, "rewards/reward_commands_correctness/std": 0.034199174493551254, "rewards/reward_diversity/mean": 0.09590362012386322, "rewards/reward_diversity/std": 0.0705147236585617, "rewards/reward_format/mean": 0.05625000223517418, "rewards/reward_format/std": 0.05123475566506386, "rewards/reward_problem_validity/mean": 0.051249999552965164, "rewards/reward_problem_validity/std": 0.03774917498230934, "rewards/reward_solution_effectiveness/mean": 0.07874999940395355, "rewards/reward_solution_effectiveness/std": 0.06469158083200455, "sampling/importance_sampling_ratio/max": 0.8735832571983337, "sampling/importance_sampling_ratio/mean": 0.18156158924102783, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.386413812637329, "sampling/sampling_logp_difference/mean": 0.004353303462266922, "step": 18, "step_time": 265.55887841433287 }, { "clip_ratio/high_max": 0.0006439428179874085, "clip_ratio/high_mean": 0.0006439428179874085, "clip_ratio/low_mean": 0.0004398750788823236, "clip_ratio/low_min": 0.0004398750788823236, "clip_ratio/region_mean": 0.0010838179005077109, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1426.0, "completions/mean_length": 1529.125, "completions/mean_terminated_length": 936.769287109375, "completions/min_length": 674.0, "completions/min_terminated_length": 674.0, "entropy": 0.1484507117420435, "epoch": 0.3877551020408163, "frac_reward_zero_std": 0.0, "grad_norm": 0.03573344647884369, "learning_rate": 9.63265306122449e-06, "loss": 0.0977, "num_tokens": 1047178.0, "reward": 0.6403510570526123, "reward_std": 0.11918861418962479, "rewards/reward_commands_completeness/mean": 0.11124999821186066, "rewards/reward_commands_completeness/std": 0.037925366312265396, "rewards/reward_commands_correctness/mean": 0.07125000655651093, "rewards/reward_commands_correctness/std": 0.02276693843305111, "rewards/reward_diversity/mean": 0.1609760969877243, "rewards/reward_diversity/std": 0.02897738479077816, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08125000447034836, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.12187500298023224, "rewards/reward_solution_effectiveness/std": 0.05528336390852928, "sampling/importance_sampling_ratio/max": 1.6330736875534058, "sampling/importance_sampling_ratio/mean": 0.29415926337242126, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0042219161987305, "sampling/sampling_logp_difference/mean": 0.005991350393742323, "step": 19, "step_time": 281.19263243861496 }, { "clip_ratio/high_max": 0.0006159703498269664, "clip_ratio/high_mean": 0.0006159703498269664, "clip_ratio/low_mean": 0.0005101647238916485, "clip_ratio/low_min": 0.0005101647238916485, "clip_ratio/region_mean": 0.001126135051890742, "completions/clipped_ratio": 0.375, "completions/max_length": 4096.0, "completions/max_terminated_length": 1074.0, "completions/mean_length": 1993.0, "completions/mean_terminated_length": 911.5, "completions/min_length": 749.0, "completions/min_terminated_length": 749.0, "entropy": 0.12236313801258802, "epoch": 0.40816326530612246, "frac_reward_zero_std": 0.0, "grad_norm": 0.06220225989818573, "learning_rate": 9.612244897959185e-06, "loss": -0.4224, "num_tokens": 1118770.0, "reward": 0.6022971868515015, "reward_std": 0.1267612874507904, "rewards/reward_commands_completeness/mean": 0.10374999791383743, "rewards/reward_commands_completeness/std": 0.03518996015191078, "rewards/reward_commands_correctness/mean": 0.06187500059604645, "rewards/reward_commands_correctness/std": 0.026132674887776375, "rewards/reward_diversity/mean": 0.14854714274406433, "rewards/reward_diversity/std": 0.04224759712815285, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.11249999701976776, "rewards/reward_solution_effectiveness/std": 0.053103674203157425, "sampling/importance_sampling_ratio/max": 2.4879612922668457, "sampling/importance_sampling_ratio/mean": 0.5085237622261047, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.717113733291626, "sampling/sampling_logp_difference/mean": 0.005194623954594135, "step": 20, "step_time": 413.2199177313596 }, { "clip_ratio/high_max": 0.00069506019281107, "clip_ratio/high_mean": 0.00069506019281107, "clip_ratio/low_mean": 0.0004407113028719323, "clip_ratio/low_min": 0.0004407113028719323, "clip_ratio/region_mean": 0.0011357714829500765, "completions/clipped_ratio": 0.5625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1218.0, "completions/mean_length": 2664.1875, "completions/mean_terminated_length": 823.2857666015625, "completions/min_length": 689.0, "completions/min_terminated_length": 689.0, "entropy": 0.06720372103154659, "epoch": 0.42857142857142855, "frac_reward_zero_std": 0.0, "grad_norm": 0.022023798897862434, "learning_rate": 9.591836734693878e-06, "loss": -0.2045, "num_tokens": 1181465.0, "reward": 0.609264075756073, "reward_std": 0.0966130718588829, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.026299558579921722, "rewards/reward_commands_correctness/mean": 0.07249999791383743, "rewards/reward_commands_correctness/std": 0.01653279736638069, "rewards/reward_diversity/mean": 0.1430141031742096, "rewards/reward_diversity/std": 0.0237920843064785, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.07625000178813934, "rewards/reward_problem_validity/std": 0.0145487692207098, "rewards/reward_solution_effectiveness/mean": 0.10875000059604645, "rewards/reward_solution_effectiveness/std": 0.05463515594601631, "sampling/importance_sampling_ratio/max": 2.970540761947632, "sampling/importance_sampling_ratio/mean": 0.41230982542037964, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9363317489624023, "sampling/sampling_logp_difference/mean": 0.00340920127928257, "step": 21, "step_time": 276.46959663182497 }, { "clip_ratio/high_max": 0.0003169288011122262, "clip_ratio/high_mean": 0.0003169288011122262, "clip_ratio/low_mean": 0.0004021947934234049, "clip_ratio/low_min": 0.0004021947934234049, "clip_ratio/region_mean": 0.0007191235945356311, "completions/clipped_ratio": 0.5, "completions/max_length": 4096.0, "completions/max_terminated_length": 3603.0, "completions/mean_length": 2750.3125, "completions/mean_terminated_length": 1404.625, "completions/min_length": 625.0, "completions/min_terminated_length": 625.0, "entropy": 0.08879909664392471, "epoch": 0.4489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.018336452543735504, "learning_rate": 9.571428571428573e-06, "loss": 0.0263, "num_tokens": 1250414.0, "reward": 0.5704278945922852, "reward_std": 0.18650363385677338, "rewards/reward_commands_completeness/mean": 0.10375000536441803, "rewards/reward_commands_completeness/std": 0.04631414636969566, "rewards/reward_commands_correctness/mean": 0.058125000447034836, "rewards/reward_commands_correctness/std": 0.027378519997000694, "rewards/reward_diversity/mean": 0.13667787611484528, "rewards/reward_diversity/std": 0.06349394470453262, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07250000536441803, "rewards/reward_problem_validity/std": 0.03130495548248291, "rewards/reward_solution_effectiveness/mean": 0.11812499910593033, "rewards/reward_solution_effectiveness/std": 0.05844869837164879, "sampling/importance_sampling_ratio/max": 2.7350454330444336, "sampling/importance_sampling_ratio/mean": 0.42821934819221497, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0283496379852295, "sampling/sampling_logp_difference/mean": 0.003998525440692902, "step": 22, "step_time": 285.72589340247214 }, { "clip_ratio/high_max": 0.0004903662993456237, "clip_ratio/high_mean": 0.0004903662993456237, "clip_ratio/low_mean": 0.0004861544366576709, "clip_ratio/low_min": 0.0004861544366576709, "clip_ratio/region_mean": 0.000976520728727337, "completions/clipped_ratio": 0.5, "completions/max_length": 4096.0, "completions/max_terminated_length": 3703.0, "completions/mean_length": 2809.5625, "completions/mean_terminated_length": 1523.125, "completions/min_length": 660.0, "completions/min_terminated_length": 660.0, "entropy": 0.08736256137490273, "epoch": 0.46938775510204084, "frac_reward_zero_std": 0.0, "grad_norm": 0.04664234071969986, "learning_rate": 9.551020408163266e-06, "loss": 0.2907, "num_tokens": 1321215.0, "reward": 0.314258873462677, "reward_std": 0.21143491566181183, "rewards/reward_commands_completeness/mean": 0.04999999701976776, "rewards/reward_commands_completeness/std": 0.038643673062324524, "rewards/reward_commands_correctness/mean": 0.032499998807907104, "rewards/reward_commands_correctness/std": 0.026457514613866806, "rewards/reward_diversity/mean": 0.09238386899232864, "rewards/reward_diversity/std": 0.07607576251029968, "rewards/reward_format/mean": 0.04374999925494194, "rewards/reward_format/std": 0.05123475566506386, "rewards/reward_problem_validity/mean": 0.04124999791383743, "rewards/reward_problem_validity/std": 0.036674242466688156, "rewards/reward_solution_effectiveness/mean": 0.054375000298023224, "rewards/reward_solution_effectiveness/std": 0.03669128194451332, "sampling/importance_sampling_ratio/max": 2.8907248973846436, "sampling/importance_sampling_ratio/mean": 0.4375866651535034, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3424015045166016, "sampling/sampling_logp_difference/mean": 0.0041806455701589584, "step": 23, "step_time": 264.8893793579191 }, { "clip_ratio/high_max": 0.0010396951911388896, "clip_ratio/high_mean": 0.0010396951911388896, "clip_ratio/low_mean": 0.0006363526699715294, "clip_ratio/low_min": 0.0006363526699715294, "clip_ratio/region_mean": 0.001676047861110419, "completions/clipped_ratio": 0.375, "completions/max_length": 4096.0, "completions/max_terminated_length": 2800.0, "completions/mean_length": 2195.625, "completions/mean_terminated_length": 1055.4000244140625, "completions/min_length": 653.0, "completions/min_terminated_length": 653.0, "entropy": 0.11635377490893006, "epoch": 0.4897959183673469, "frac_reward_zero_std": 0.0, "grad_norm": 0.03986644372344017, "learning_rate": 9.53061224489796e-06, "loss": 0.0914, "num_tokens": 1374697.0, "reward": 0.5961976051330566, "reward_std": 0.1379333883523941, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.03180670738220215, "rewards/reward_commands_correctness/mean": 0.0637499988079071, "rewards/reward_commands_correctness/std": 0.020289571955800056, "rewards/reward_diversity/mean": 0.13494761288166046, "rewards/reward_diversity/std": 0.09134822338819504, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.018929695710539818, "rewards/reward_solution_effectiveness/mean": 0.11624999344348907, "rewards/reward_solution_effectiveness/std": 0.04364630579948425, "sampling/importance_sampling_ratio/max": 2.1692745685577393, "sampling/importance_sampling_ratio/mean": 0.452979177236557, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.126286029815674, "sampling/sampling_logp_difference/mean": 0.004836581647396088, "step": 24, "step_time": 274.85543395765126 }, { "clip_ratio/high_max": 0.0004216224442643579, "clip_ratio/high_mean": 0.0004216224442643579, "clip_ratio/low_mean": 8.044210153457243e-05, "clip_ratio/low_min": 8.044210153457243e-05, "clip_ratio/region_mean": 0.0005020645457989303, "completions/clipped_ratio": 0.75, "completions/max_length": 4096.0, "completions/max_terminated_length": 1188.0, "completions/mean_length": 3311.8125, "completions/mean_terminated_length": 959.25, "completions/min_length": 825.0, "completions/min_terminated_length": 825.0, "entropy": 0.04443414695560932, "epoch": 0.5102040816326531, "frac_reward_zero_std": 0.0, "grad_norm": 0.057228196412324905, "learning_rate": 9.510204081632653e-06, "loss": 0.0107, "num_tokens": 1451858.0, "reward": 0.5912448167800903, "reward_std": 0.11123878508806229, "rewards/reward_commands_completeness/mean": 0.10750000178813934, "rewards/reward_commands_completeness/std": 0.03172801434993744, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.02528998628258705, "rewards/reward_diversity/mean": 0.13124483823776245, "rewards/reward_diversity/std": 0.09165968000888824, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.018929695710539818, "rewards/reward_solution_effectiveness/mean": 0.11437500268220901, "rewards/reward_solution_effectiveness/std": 0.04411632940173149, "sampling/importance_sampling_ratio/max": 1.374530553817749, "sampling/importance_sampling_ratio/mean": 0.2884614169597626, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3934192657470703, "sampling/sampling_logp_difference/mean": 0.0025905794464051723, "step": 25, "step_time": 284.8811695948243 }, { "clip_ratio/high_max": 0.0007265311942319386, "clip_ratio/high_mean": 0.0007265311942319386, "clip_ratio/low_mean": 0.00038848833719384857, "clip_ratio/low_min": 0.00038848833719384857, "clip_ratio/region_mean": 0.0011150195314257871, "completions/clipped_ratio": 0.375, "completions/max_length": 4096.0, "completions/max_terminated_length": 1335.0, "completions/mean_length": 2087.0, "completions/mean_terminated_length": 881.6000366210938, "completions/min_length": 673.0, "completions/min_terminated_length": 673.0, "entropy": 0.1199277937412262, "epoch": 0.5306122448979592, "frac_reward_zero_std": 0.0, "grad_norm": 0.027953337877988815, "learning_rate": 9.489795918367348e-06, "loss": 0.1117, "num_tokens": 1503322.0, "reward": 0.4951333999633789, "reward_std": 0.14537927508354187, "rewards/reward_commands_completeness/mean": 0.07999999821186066, "rewards/reward_commands_completeness/std": 0.04320494085550308, "rewards/reward_commands_correctness/mean": 0.05624999850988388, "rewards/reward_commands_correctness/std": 0.03095696121454239, "rewards/reward_diversity/mean": 0.14075839519500732, "rewards/reward_diversity/std": 0.039025988429784775, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06437499821186066, "rewards/reward_problem_validity/std": 0.032653484493494034, "rewards/reward_solution_effectiveness/mean": 0.07874999940395355, "rewards/reward_solution_effectiveness/std": 0.050049975514411926, "sampling/importance_sampling_ratio/max": 1.8586959838867188, "sampling/importance_sampling_ratio/mean": 0.40213942527770996, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.007772445678711, "sampling/sampling_logp_difference/mean": 0.0056335353292524815, "step": 26, "step_time": 262.30709875002503 }, { "clip_ratio/high_max": 0.0009255756358470535, "clip_ratio/high_mean": 0.0009255756358470535, "clip_ratio/low_mean": 0.00013645808394358028, "clip_ratio/low_min": 0.00013645808394358028, "clip_ratio/region_mean": 0.001062033716152655, "completions/clipped_ratio": 0.625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1356.0, "completions/mean_length": 2949.0, "completions/mean_terminated_length": 1037.3333740234375, "completions/min_length": 788.0, "completions/min_terminated_length": 788.0, "entropy": 0.07776289014145732, "epoch": 0.5510204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.01641741208732128, "learning_rate": 9.469387755102041e-06, "loss": 0.1321, "num_tokens": 1585694.0, "reward": 0.6716875433921814, "reward_std": 0.10811514407396317, "rewards/reward_commands_completeness/mean": 0.1287499964237213, "rewards/reward_commands_completeness/std": 0.036492008715867996, "rewards/reward_commands_correctness/mean": 0.06687499582767487, "rewards/reward_commands_correctness/std": 0.01990603096783161, "rewards/reward_diversity/mean": 0.15231254696846008, "rewards/reward_diversity/std": 0.04221150279045105, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08375000208616257, "rewards/reward_problem_validity/std": 0.019958291202783585, "rewards/reward_solution_effectiveness/mean": 0.14624999463558197, "rewards/reward_solution_effectiveness/std": 0.050049975514411926, "sampling/importance_sampling_ratio/max": 0.6462328433990479, "sampling/importance_sampling_ratio/mean": 0.15823262929916382, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.268367767333984, "sampling/sampling_logp_difference/mean": 0.004169633612036705, "step": 27, "step_time": 315.5522811245173 }, { "clip_ratio/high_max": 0.0003941772065445548, "clip_ratio/high_mean": 0.0003941772065445548, "clip_ratio/low_mean": 0.00023360270097327884, "clip_ratio/low_min": 0.00023360270097327884, "clip_ratio/region_mean": 0.0006277799056988442, "completions/clipped_ratio": 0.625, "completions/max_length": 4096.0, "completions/max_terminated_length": 3610.0, "completions/mean_length": 3067.125, "completions/mean_terminated_length": 1352.3333740234375, "completions/min_length": 738.0, "completions/min_terminated_length": 738.0, "entropy": 0.0595466298982501, "epoch": 0.5714285714285714, "frac_reward_zero_std": 0.0, "grad_norm": 0.013559562154114246, "learning_rate": 9.448979591836736e-06, "loss": -0.0673, "num_tokens": 1664296.0, "reward": 0.6858651638031006, "reward_std": 0.11484912037849426, "rewards/reward_commands_completeness/mean": 0.13374999165534973, "rewards/reward_commands_completeness/std": 0.0430309996008873, "rewards/reward_commands_correctness/mean": 0.06749999523162842, "rewards/reward_commands_correctness/std": 0.02408318966627121, "rewards/reward_diversity/mean": 0.13774017989635468, "rewards/reward_diversity/std": 0.09285938739776611, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08250000327825546, "rewards/reward_problem_validity/std": 0.019832635298371315, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.07724151015281677, "sampling/importance_sampling_ratio/max": 1.524442434310913, "sampling/importance_sampling_ratio/mean": 0.32684510946273804, "sampling/importance_sampling_ratio/min": 0.007493465207517147, "sampling/sampling_logp_difference/max": 4.5559282302856445, "sampling/sampling_logp_difference/mean": 0.0037054186686873436, "step": 28, "step_time": 302.8599004242569 }, { "clip_ratio/high_max": 0.00047448845361941494, "clip_ratio/high_mean": 0.00047448845361941494, "clip_ratio/low_mean": 0.00011629541040747426, "clip_ratio/low_min": 0.00011629541040747426, "clip_ratio/region_mean": 0.0005907838640268892, "completions/clipped_ratio": 0.8125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1195.0, "completions/mean_length": 3511.6875, "completions/mean_terminated_length": 979.6666870117188, "completions/min_length": 788.0, "completions/min_terminated_length": 788.0, "entropy": 0.05222143558785319, "epoch": 0.5918367346938775, "frac_reward_zero_std": 0.0, "grad_norm": 0.01108020544052124, "learning_rate": 9.42857142857143e-06, "loss": 0.0124, "num_tokens": 1743783.0, "reward": 0.6330066919326782, "reward_std": 0.10044381022453308, "rewards/reward_commands_completeness/mean": 0.10750000178813934, "rewards/reward_commands_completeness/std": 0.033366650342941284, "rewards/reward_commands_correctness/mean": 0.06874999403953552, "rewards/reward_commands_correctness/std": 0.018211718648672104, "rewards/reward_diversity/mean": 0.1580066680908203, "rewards/reward_diversity/std": 0.03177348151803017, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.1237500011920929, "rewards/reward_solution_effectiveness/std": 0.03930649161338806, "sampling/importance_sampling_ratio/max": 1.1847013235092163, "sampling/importance_sampling_ratio/mean": 0.2318364977836609, "sampling/importance_sampling_ratio/min": 0.0055412896908819675, "sampling/sampling_logp_difference/max": 4.864246368408203, "sampling/sampling_logp_difference/mean": 0.0032831255812197924, "step": 29, "step_time": 311.4429254569113 }, { "clip_ratio/high_max": 0.0002872439526981907, "clip_ratio/high_mean": 0.0002872439526981907, "clip_ratio/low_mean": 0.00023661060913582332, "clip_ratio/low_min": 0.00023661060913582332, "clip_ratio/region_mean": 0.0005238545654719928, "completions/clipped_ratio": 0.5625, "completions/max_length": 4096.0, "completions/max_terminated_length": 2015.0, "completions/mean_length": 2739.6875, "completions/mean_terminated_length": 995.857177734375, "completions/min_length": 655.0, "completions/min_terminated_length": 655.0, "entropy": 0.0643562488257885, "epoch": 0.6122448979591837, "frac_reward_zero_std": 0.0, "grad_norm": 0.037283387035131454, "learning_rate": 9.408163265306123e-06, "loss": -0.1704, "num_tokens": 1808722.0, "reward": 0.35997745394706726, "reward_std": 0.23459148406982422, "rewards/reward_commands_completeness/mean": 0.06624999642372131, "rewards/reward_commands_completeness/std": 0.054999999701976776, "rewards/reward_commands_correctness/mean": 0.03999999910593033, "rewards/reward_commands_correctness/std": 0.03444802761077881, "rewards/reward_diversity/mean": 0.08122745901346207, "rewards/reward_diversity/std": 0.1016627848148346, "rewards/reward_format/mean": 0.05000000074505806, "rewards/reward_format/std": 0.05163978040218353, "rewards/reward_problem_validity/mean": 0.04374999925494194, "rewards/reward_problem_validity/std": 0.037394292652606964, "rewards/reward_solution_effectiveness/mean": 0.07874999940395355, "rewards/reward_solution_effectiveness/std": 0.0708872377872467, "sampling/importance_sampling_ratio/max": 1.9843974113464355, "sampling/importance_sampling_ratio/mean": 0.4650506377220154, "sampling/importance_sampling_ratio/min": 0.003584614023566246, "sampling/sampling_logp_difference/max": 4.744321823120117, "sampling/sampling_logp_difference/mean": 0.0036163341719657183, "step": 30, "step_time": 252.8432511370629 }, { "clip_ratio/high_max": 0.00042260519694536924, "clip_ratio/high_mean": 0.00042260519694536924, "clip_ratio/low_mean": 0.00019871020413120277, "clip_ratio/low_min": 0.00019871020413120277, "clip_ratio/region_mean": 0.000621315401076572, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1379.0, "completions/mean_length": 3160.75, "completions/mean_terminated_length": 1103.2000732421875, "completions/min_length": 849.0, "completions/min_terminated_length": 849.0, "entropy": 0.06377955013886094, "epoch": 0.6326530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.02587207593023777, "learning_rate": 9.387755102040818e-06, "loss": 0.1963, "num_tokens": 1887702.0, "reward": 0.7137311697006226, "reward_std": 0.15415877103805542, "rewards/reward_commands_completeness/mean": 0.14124999940395355, "rewards/reward_commands_completeness/std": 0.04224926233291626, "rewards/reward_commands_correctness/mean": 0.07062499970197678, "rewards/reward_commands_correctness/std": 0.024074537679553032, "rewards/reward_diversity/mean": 0.1468561887741089, "rewards/reward_diversity/std": 0.04431251809000969, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.18000000715255737, "rewards/reward_solution_effectiveness/std": 0.07823043316602707, "sampling/importance_sampling_ratio/max": 2.8438620567321777, "sampling/importance_sampling_ratio/mean": 0.626688539981842, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.63487434387207, "sampling/sampling_logp_difference/mean": 0.0034587776754051447, "step": 31, "step_time": 305.15121217630804 }, { "clip_ratio/high_max": 0.0005317675168043934, "clip_ratio/high_mean": 0.0005317675168043934, "clip_ratio/low_mean": 7.801813262631185e-05, "clip_ratio/low_min": 7.801813262631185e-05, "clip_ratio/region_mean": 0.000609785643973737, "completions/clipped_ratio": 0.8125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1282.0, "completions/mean_length": 3522.875, "completions/mean_terminated_length": 1039.3333740234375, "completions/min_length": 779.0, "completions/min_terminated_length": 779.0, "entropy": 0.045193305471912026, "epoch": 0.6530612244897959, "frac_reward_zero_std": 0.0, "grad_norm": 0.034386537969112396, "learning_rate": 9.367346938775511e-06, "loss": 0.1079, "num_tokens": 1969628.0, "reward": 0.5857187509536743, "reward_std": 0.24112606048583984, "rewards/reward_commands_completeness/mean": 0.11624999344348907, "rewards/reward_commands_completeness/std": 0.05806605890393257, "rewards/reward_commands_correctness/mean": 0.0612499974668026, "rewards/reward_commands_correctness/std": 0.02777888812124729, "rewards/reward_diversity/mean": 0.11821875721216202, "rewards/reward_diversity/std": 0.09869419783353806, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07187499850988388, "rewards/reward_problem_validity/std": 0.03103090077638626, "rewards/reward_solution_effectiveness/mean": 0.1368750035762787, "rewards/reward_solution_effectiveness/std": 0.08483071625232697, "sampling/importance_sampling_ratio/max": 2.4455878734588623, "sampling/importance_sampling_ratio/mean": 0.43046170473098755, "sampling/importance_sampling_ratio/min": 0.0011267994996160269, "sampling/sampling_logp_difference/max": 4.851991176605225, "sampling/sampling_logp_difference/mean": 0.0030236716847866774, "step": 32, "step_time": 289.29324877262115 }, { "clip_ratio/high_max": 0.0005092279643577058, "clip_ratio/high_mean": 0.0005092279643577058, "clip_ratio/low_mean": 0.00013901808415539563, "clip_ratio/low_min": 0.00013901808415539563, "clip_ratio/region_mean": 0.0006482460485131014, "completions/clipped_ratio": 0.8125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 3525.4375, "completions/mean_terminated_length": 1053.0, "completions/min_length": 837.0, "completions/min_terminated_length": 837.0, "entropy": 0.04522319324314594, "epoch": 0.673469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.02885086089372635, "learning_rate": 9.346938775510204e-06, "loss": -0.1346, "num_tokens": 2051771.0, "reward": 0.5978981256484985, "reward_std": 0.17068850994110107, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.042563680559396744, "rewards/reward_commands_correctness/mean": 0.05999999865889549, "rewards/reward_commands_correctness/std": 0.019999999552965164, "rewards/reward_diversity/mean": 0.1347731053829193, "rewards/reward_diversity/std": 0.09300266951322556, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.0230126790702343, "rewards/reward_solution_effectiveness/mean": 0.1237500011920929, "rewards/reward_solution_effectiveness/std": 0.06917369365692139, "sampling/importance_sampling_ratio/max": 2.012239694595337, "sampling/importance_sampling_ratio/mean": 0.4306543469429016, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.031385660171509, "sampling/sampling_logp_difference/mean": 0.0026972698979079723, "step": 33, "step_time": 307.47916670516133 }, { "clip_ratio/high_max": 0.0002029200768447481, "clip_ratio/high_mean": 0.0002029200768447481, "clip_ratio/low_mean": 0.00028075604313926306, "clip_ratio/low_min": 0.00028075604313926306, "clip_ratio/region_mean": 0.00048367612180300057, "completions/clipped_ratio": 0.875, "completions/max_length": 4096.0, "completions/max_terminated_length": 3527.0, "completions/mean_length": 3877.875, "completions/mean_terminated_length": 2351.0, "completions/min_length": 1175.0, "completions/min_terminated_length": 1175.0, "entropy": 0.06988051161170006, "epoch": 0.6938775510204082, "frac_reward_zero_std": 0.0, "grad_norm": 0.02050456590950489, "learning_rate": 9.326530612244898e-06, "loss": 0.1565, "num_tokens": 2150577.0, "reward": 0.518269419670105, "reward_std": 0.18901753425598145, "rewards/reward_commands_completeness/mean": 0.09624999761581421, "rewards/reward_commands_completeness/std": 0.056199051439762115, "rewards/reward_commands_correctness/mean": 0.0456249974668026, "rewards/reward_commands_correctness/std": 0.027560539543628693, "rewards/reward_diversity/mean": 0.13889440894126892, "rewards/reward_diversity/std": 0.07240494340658188, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.0637499988079071, "rewards/reward_problem_validity/std": 0.03757215291261673, "rewards/reward_solution_effectiveness/mean": 0.10500000417232513, "rewards/reward_solution_effectiveness/std": 0.05999999865889549, "sampling/importance_sampling_ratio/max": 2.312840700149536, "sampling/importance_sampling_ratio/mean": 0.4490036070346832, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.18826150894165, "sampling/sampling_logp_difference/mean": 0.0036568031646311283, "step": 34, "step_time": 291.42669360525906 }, { "clip_ratio/high_max": 0.0005322121760400478, "clip_ratio/high_mean": 0.0005322121760400478, "clip_ratio/low_mean": 0.00018369595636613667, "clip_ratio/low_min": 0.00018369595636613667, "clip_ratio/region_mean": 0.0007159081233112374, "completions/clipped_ratio": 0.75, "completions/max_length": 4096.0, "completions/max_terminated_length": 1507.0, "completions/mean_length": 3366.875, "completions/mean_terminated_length": 1179.5, "completions/min_length": 975.0, "completions/min_terminated_length": 975.0, "entropy": 0.06037155631929636, "epoch": 0.7142857142857143, "frac_reward_zero_std": 0.0, "grad_norm": 0.004014275502413511, "learning_rate": 9.306122448979593e-06, "loss": -0.0143, "num_tokens": 2228751.0, "reward": 0.5356021523475647, "reward_std": 0.21957236528396606, "rewards/reward_commands_completeness/mean": 0.09749999642372131, "rewards/reward_commands_completeness/std": 0.05555777996778488, "rewards/reward_commands_correctness/mean": 0.05999999865889549, "rewards/reward_commands_correctness/std": 0.02556038647890091, "rewards/reward_diversity/mean": 0.10997714102268219, "rewards/reward_diversity/std": 0.09874044358730316, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07062499970197678, "rewards/reward_problem_validity/std": 0.03043435700237751, "rewards/reward_solution_effectiveness/mean": 0.11624999344348907, "rewards/reward_solution_effectiveness/std": 0.07813450694084167, "sampling/importance_sampling_ratio/max": 0.8165685534477234, "sampling/importance_sampling_ratio/mean": 0.13378530740737915, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.497438907623291, "sampling/sampling_logp_difference/mean": 0.0034179394133388996, "step": 35, "step_time": 273.1411511898041 }, { "clip_ratio/high_max": 0.00039915164052217733, "clip_ratio/high_mean": 0.00039915164052217733, "clip_ratio/low_mean": 0.00020399999630171806, "clip_ratio/low_min": 0.00020399999630171806, "clip_ratio/region_mean": 0.0006031516313669272, "completions/clipped_ratio": 0.75, "completions/max_length": 4096.0, "completions/max_terminated_length": 1161.0, "completions/mean_length": 3325.5625, "completions/mean_terminated_length": 1014.25, "completions/min_length": 785.0, "completions/min_terminated_length": 785.0, "entropy": 0.05775430053472519, "epoch": 0.7346938775510204, "frac_reward_zero_std": 0.0, "grad_norm": 0.008141259662806988, "learning_rate": 9.285714285714288e-06, "loss": 0.0629, "num_tokens": 2300464.0, "reward": 0.7104589939117432, "reward_std": 0.06281575560569763, "rewards/reward_commands_completeness/mean": 0.1287499964237213, "rewards/reward_commands_completeness/std": 0.020615527406334877, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.010935417376458645, "rewards/reward_diversity/mean": 0.16920900344848633, "rewards/reward_diversity/std": 0.0231935977935791, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.15937501192092896, "rewards/reward_solution_effectiveness/std": 0.032345790416002274, "sampling/importance_sampling_ratio/max": 0.6918980479240417, "sampling/importance_sampling_ratio/mean": 0.13039052486419678, "sampling/importance_sampling_ratio/min": 0.0006432702648453414, "sampling/sampling_logp_difference/max": 5.039969444274902, "sampling/sampling_logp_difference/mean": 0.003765786299481988, "step": 36, "step_time": 274.62222970463336 }, { "clip_ratio/high_max": 0.0008585613923060009, "clip_ratio/high_mean": 0.0008585613923060009, "clip_ratio/low_mean": 0.0002318438855581917, "clip_ratio/low_min": 0.0002318438855581917, "clip_ratio/region_mean": 0.0010904052742262138, "completions/clipped_ratio": 0.625, "completions/max_length": 4096.0, "completions/max_terminated_length": 3117.0, "completions/mean_length": 3118.75, "completions/mean_terminated_length": 1490.0, "completions/min_length": 1087.0, "completions/min_terminated_length": 1087.0, "entropy": 0.0688807968981564, "epoch": 0.7551020408163265, "frac_reward_zero_std": 0.0, "grad_norm": 0.00729407137259841, "learning_rate": 9.26530612244898e-06, "loss": -0.0543, "num_tokens": 2380796.0, "reward": 0.5687947273254395, "reward_std": 0.22221064567565918, "rewards/reward_commands_completeness/mean": 0.11124999821186066, "rewards/reward_commands_completeness/std": 0.04731103032827377, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.027560541406273842, "rewards/reward_diversity/mean": 0.11691973358392715, "rewards/reward_diversity/std": 0.09560585021972656, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07312499731779099, "rewards/reward_problem_validity/std": 0.024958299472928047, "rewards/reward_solution_effectiveness/mean": 0.11437499523162842, "rewards/reward_solution_effectiveness/std": 0.06408002972602844, "sampling/importance_sampling_ratio/max": 0.721924364566803, "sampling/importance_sampling_ratio/mean": 0.15502715110778809, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.117588043212891, "sampling/sampling_logp_difference/mean": 0.0037843273021280766, "step": 37, "step_time": 318.3287899456918 }, { "clip_ratio/high_max": 0.0005066846460977104, "clip_ratio/high_mean": 0.0005066846460977104, "clip_ratio/low_mean": 0.00028226694848854095, "clip_ratio/low_min": 0.00028226694848854095, "clip_ratio/region_mean": 0.0007889515982242301, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 3480.0, "completions/mean_length": 3297.875, "completions/mean_terminated_length": 1542.0, "completions/min_length": 689.0, "completions/min_terminated_length": 689.0, "entropy": 0.06293653161264956, "epoch": 0.7755102040816326, "frac_reward_zero_std": 0.0, "grad_norm": 0.011674280278384686, "learning_rate": 9.244897959183674e-06, "loss": 0.0483, "num_tokens": 2459898.0, "reward": 0.6880128383636475, "reward_std": 0.10378552228212357, "rewards/reward_commands_completeness/mean": 0.13999998569488525, "rewards/reward_commands_completeness/std": 0.04000000283122063, "rewards/reward_commands_correctness/mean": 0.07125000655651093, "rewards/reward_commands_correctness/std": 0.011474610306322575, "rewards/reward_diversity/mean": 0.10926283895969391, "rewards/reward_diversity/std": 0.12470422685146332, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.18000000715255737, "rewards/reward_solution_effectiveness/std": 0.0774596706032753, "sampling/importance_sampling_ratio/max": 1.1708272695541382, "sampling/importance_sampling_ratio/mean": 0.18155837059020996, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.131227493286133, "sampling/sampling_logp_difference/mean": 0.0034633975010365248, "step": 38, "step_time": 286.1622115056962 }, { "clip_ratio/high_max": 0.0004844919531024061, "clip_ratio/high_mean": 0.0004844919531024061, "clip_ratio/low_mean": 0.0002497517198207788, "clip_ratio/low_min": 0.0002497517198207788, "clip_ratio/region_mean": 0.0007342436729231849, "completions/clipped_ratio": 0.8125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1609.0, "completions/mean_length": 3542.75, "completions/mean_terminated_length": 1145.3333740234375, "completions/min_length": 769.0, "completions/min_terminated_length": 769.0, "entropy": 0.10111465840600431, "epoch": 0.7959183673469388, "frac_reward_zero_std": 0.0, "grad_norm": 0.029670054093003273, "learning_rate": 9.224489795918367e-06, "loss": 0.2091, "num_tokens": 2543054.0, "reward": 0.6202192306518555, "reward_std": 0.19285202026367188, "rewards/reward_commands_completeness/mean": 0.11749999225139618, "rewards/reward_commands_completeness/std": 0.05105552822351456, "rewards/reward_commands_correctness/mean": 0.06499999761581421, "rewards/reward_commands_correctness/std": 0.026583204045891762, "rewards/reward_diversity/mean": 0.14146925508975983, "rewards/reward_diversity/std": 0.04546857997775078, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07375000417232513, "rewards/reward_problem_validity/std": 0.02777889184653759, "rewards/reward_solution_effectiveness/mean": 0.13499999046325684, "rewards/reward_solution_effectiveness/std": 0.07668115943670273, "sampling/importance_sampling_ratio/max": 1.888770341873169, "sampling/importance_sampling_ratio/mean": 0.43840497732162476, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.452970027923584, "sampling/sampling_logp_difference/mean": 0.004547507967799902, "step": 39, "step_time": 317.4071712959558 }, { "clip_ratio/high_max": 0.0003468125814833911, "clip_ratio/high_mean": 0.0003468125814833911, "clip_ratio/low_mean": 0.0002933133928308962, "clip_ratio/low_min": 0.0002933133928308962, "clip_ratio/region_mean": 0.0006401259670383297, "completions/clipped_ratio": 0.75, "completions/max_length": 4096.0, "completions/max_terminated_length": 1342.0, "completions/mean_length": 3344.75, "completions/mean_terminated_length": 1091.0, "completions/min_length": 952.0, "completions/min_terminated_length": 952.0, "entropy": 0.06345481402240694, "epoch": 0.8163265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.011182487942278385, "learning_rate": 9.204081632653062e-06, "loss": -0.0357, "num_tokens": 2621878.0, "reward": 0.5191029906272888, "reward_std": 0.13969582319259644, "rewards/reward_commands_completeness/mean": 0.0925000011920929, "rewards/reward_commands_completeness/std": 0.04892170801758766, "rewards/reward_commands_correctness/mean": 0.05312500149011612, "rewards/reward_commands_correctness/std": 0.029147613793611526, "rewards/reward_diversity/mean": 0.1259780079126358, "rewards/reward_diversity/std": 0.057158634066581726, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06562499701976776, "rewards/reward_problem_validity/std": 0.03346017748117447, "rewards/reward_solution_effectiveness/mean": 0.10687500238418579, "rewards/reward_solution_effectiveness/std": 0.0599687434732914, "sampling/importance_sampling_ratio/max": 1.215498685836792, "sampling/importance_sampling_ratio/mean": 0.1628175526857376, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.0276713371276855, "sampling/sampling_logp_difference/mean": 0.0036859659012407064, "step": 40, "step_time": 278.5540202707052 }, { "clip_ratio/high_max": 0.0005436523133539595, "clip_ratio/high_mean": 0.0005436523133539595, "clip_ratio/low_mean": 0.00014399578503798693, "clip_ratio/low_min": 0.00014399578503798693, "clip_ratio/region_mean": 0.0006876480983919464, "completions/clipped_ratio": 0.8125, "completions/max_length": 4096.0, "completions/max_terminated_length": 3059.0, "completions/mean_length": 3766.9375, "completions/mean_terminated_length": 2341.0, "completions/min_length": 1425.0, "completions/min_terminated_length": 1425.0, "entropy": 0.06311355857178569, "epoch": 0.8367346938775511, "frac_reward_zero_std": 0.0, "grad_norm": 0.017930030822753906, "learning_rate": 9.183673469387756e-06, "loss": 0.0945, "num_tokens": 2708405.0, "reward": 0.5915456414222717, "reward_std": 0.21410664916038513, "rewards/reward_commands_completeness/mean": 0.10750000178813934, "rewards/reward_commands_completeness/std": 0.05000000074505806, "rewards/reward_commands_correctness/mean": 0.05562499910593033, "rewards/reward_commands_correctness/std": 0.029432127252221107, "rewards/reward_diversity/mean": 0.14092066884040833, "rewards/reward_diversity/std": 0.06070951744914055, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07312500476837158, "rewards/reward_problem_validity/std": 0.03156343102455139, "rewards/reward_solution_effectiveness/mean": 0.13312500715255737, "rewards/reward_solution_effectiveness/std": 0.08195273578166962, "sampling/importance_sampling_ratio/max": 2.4059524536132812, "sampling/importance_sampling_ratio/mean": 0.4040217399597168, "sampling/importance_sampling_ratio/min": 0.001127766096033156, "sampling/sampling_logp_difference/max": 4.6072187423706055, "sampling/sampling_logp_difference/mean": 0.0034929250832647085, "step": 41, "step_time": 304.20922806300223 }, { "clip_ratio/high_max": 0.00057894674318959, "clip_ratio/high_mean": 0.00057894674318959, "clip_ratio/low_mean": 0.00020284396487113554, "clip_ratio/low_min": 0.00020284396487113554, "clip_ratio/region_mean": 0.000781790709879715, "completions/clipped_ratio": 0.8125, "completions/max_length": 4096.0, "completions/max_terminated_length": 3892.0, "completions/mean_length": 3677.75, "completions/mean_terminated_length": 1865.3333740234375, "completions/min_length": 821.0, "completions/min_terminated_length": 821.0, "entropy": 0.05679405713453889, "epoch": 0.8571428571428571, "frac_reward_zero_std": 0.0, "grad_norm": 0.038993950933218, "learning_rate": 9.163265306122449e-06, "loss": 0.0688, "num_tokens": 2791657.0, "reward": 0.6745936870574951, "reward_std": 0.12621574103832245, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.037594329565763474, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.022425804287195206, "rewards/reward_diversity/mean": 0.16084368526935577, "rewards/reward_diversity/std": 0.023448556661605835, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.15000000596046448, "rewards/reward_solution_effectiveness/std": 0.06480740755796432, "sampling/importance_sampling_ratio/max": 2.4185404777526855, "sampling/importance_sampling_ratio/mean": 0.4293341636657715, "sampling/importance_sampling_ratio/min": 0.0010484012309461832, "sampling/sampling_logp_difference/max": 2.674912452697754, "sampling/sampling_logp_difference/mean": 0.003499051555991173, "step": 42, "step_time": 289.4146448150277 }, { "clip_ratio/high_max": 0.00046234424371505156, "clip_ratio/high_mean": 0.00046234424371505156, "clip_ratio/low_mean": 0.00017535647930344567, "clip_ratio/low_min": 0.00017535647930344567, "clip_ratio/region_mean": 0.0006377007230184972, "completions/clipped_ratio": 0.75, "completions/max_length": 4096.0, "completions/max_terminated_length": 1140.0, "completions/mean_length": 3316.25, "completions/mean_terminated_length": 977.0, "completions/min_length": 799.0, "completions/min_terminated_length": 799.0, "entropy": 0.08515313314273953, "epoch": 0.8775510204081632, "frac_reward_zero_std": 0.25, "grad_norm": 0.012387261725962162, "learning_rate": 9.142857142857144e-06, "loss": -0.1143, "num_tokens": 2877621.0, "reward": 0.44705116748809814, "reward_std": 0.14391204714775085, "rewards/reward_commands_completeness/mean": 0.08249999582767487, "rewards/reward_commands_completeness/std": 0.053603481501340866, "rewards/reward_commands_correctness/mean": 0.036249998956918716, "rewards/reward_commands_correctness/std": 0.02500000037252903, "rewards/reward_diversity/mean": 0.11767619848251343, "rewards/reward_diversity/std": 0.07220287621021271, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.05624999850988388, "rewards/reward_problem_validity/std": 0.037215590476989746, "rewards/reward_solution_effectiveness/mean": 0.09187500178813934, "rewards/reward_solution_effectiveness/std": 0.05741298198699951, "sampling/importance_sampling_ratio/max": 2.6721465587615967, "sampling/importance_sampling_ratio/mean": 0.3743988573551178, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.39186954498291, "sampling/sampling_logp_difference/mean": 0.0042730881832540035, "step": 43, "step_time": 308.49980368465185 }, { "clip_ratio/high_max": 0.0004475098321563564, "clip_ratio/high_mean": 0.0004475098321563564, "clip_ratio/low_mean": 0.0002108911685354542, "clip_ratio/low_min": 0.0002108911685354542, "clip_ratio/region_mean": 0.0006584010006918106, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1315.0, "completions/mean_length": 3155.9375, "completions/mean_terminated_length": 1087.800048828125, "completions/min_length": 908.0, "completions/min_terminated_length": 908.0, "entropy": 0.06764357117936015, "epoch": 0.8979591836734694, "frac_reward_zero_std": 0.0, "grad_norm": 0.018797066062688828, "learning_rate": 9.122448979591837e-06, "loss": 0.0853, "num_tokens": 2958432.0, "reward": 0.48271727561950684, "reward_std": 0.1277233064174652, "rewards/reward_commands_completeness/mean": 0.08124999701976776, "rewards/reward_commands_completeness/std": 0.04470272362232208, "rewards/reward_commands_correctness/mean": 0.04500000178813934, "rewards/reward_commands_correctness/std": 0.027080127969384193, "rewards/reward_diversity/mean": 0.1445922553539276, "rewards/reward_diversity/std": 0.06426016986370087, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.06062500178813934, "rewards/reward_problem_validity/std": 0.03549061343073845, "rewards/reward_solution_effectiveness/mean": 0.08250000327825546, "rewards/reward_solution_effectiveness/std": 0.047116879373788834, "sampling/importance_sampling_ratio/max": 1.6490623950958252, "sampling/importance_sampling_ratio/mean": 0.27712100744247437, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.2397918701171875, "sampling/sampling_logp_difference/mean": 0.0037189216818660498, "step": 44, "step_time": 311.0259733237326 }, { "clip_ratio/high_max": 0.0014143417756713461, "clip_ratio/high_mean": 0.0014143417756713461, "clip_ratio/low_mean": 5.869462984264828e-05, "clip_ratio/low_min": 5.869462984264828e-05, "clip_ratio/region_mean": 0.0014730364055139944, "completions/clipped_ratio": 0.5625, "completions/max_length": 4096.0, "completions/max_terminated_length": 2248.0, "completions/mean_length": 2171.5, "completions/mean_terminated_length": 1242.571533203125, "completions/min_length": 797.0, "completions/min_terminated_length": 797.0, "entropy": 0.09699504775926471, "epoch": 0.9183673469387755, "frac_reward_zero_std": 0.0, "grad_norm": 0.03667191416025162, "learning_rate": 9.102040816326532e-06, "loss": -0.0443, "num_tokens": 3051680.0, "reward": 0.5792335271835327, "reward_std": 0.25547125935554504, "rewards/reward_commands_completeness/mean": 0.10124999284744263, "rewards/reward_commands_completeness/std": 0.04759202152490616, "rewards/reward_commands_correctness/mean": 0.05874999985098839, "rewards/reward_commands_correctness/std": 0.027294687926769257, "rewards/reward_diversity/mean": 0.14673356711864471, "rewards/reward_diversity/std": 0.06736119836568832, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07312500476837158, "rewards/reward_problem_validity/std": 0.03156343102455139, "rewards/reward_solution_effectiveness/mean": 0.11812500655651093, "rewards/reward_solution_effectiveness/std": 0.06705408543348312, "sampling/importance_sampling_ratio/max": 2.859874963760376, "sampling/importance_sampling_ratio/mean": 0.5001544952392578, "sampling/importance_sampling_ratio/min": 0.0024985959753394127, "sampling/sampling_logp_difference/max": 5.44144868850708, "sampling/sampling_logp_difference/mean": 0.006253939121961594, "step": 45, "step_time": 399.92093421146274 }, { "clip_ratio/high_max": 0.0006341082789731445, "clip_ratio/high_mean": 0.0006341082789731445, "clip_ratio/low_mean": 0.00013082689292787109, "clip_ratio/low_min": 0.00013082689292787109, "clip_ratio/region_mean": 0.000764935173720005, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 2319.0, "completions/mean_length": 3294.5625, "completions/mean_terminated_length": 1531.4000244140625, "completions/min_length": 1081.0, "completions/min_terminated_length": 1081.0, "entropy": 0.05868359189480543, "epoch": 0.9387755102040817, "frac_reward_zero_std": 0.0, "grad_norm": 0.01928120292723179, "learning_rate": 9.081632653061225e-06, "loss": -0.0009, "num_tokens": 3132169.0, "reward": 0.5969966650009155, "reward_std": 0.22160235047340393, "rewards/reward_commands_completeness/mean": 0.11749999225139618, "rewards/reward_commands_completeness/std": 0.04892170801758766, "rewards/reward_commands_correctness/mean": 0.05874999985098839, "rewards/reward_commands_correctness/std": 0.02552776224911213, "rewards/reward_diversity/mean": 0.11574666202068329, "rewards/reward_diversity/std": 0.09898463636636734, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.02651257812976837, "rewards/reward_solution_effectiveness/mean": 0.140625, "rewards/reward_solution_effectiveness/std": 0.07075485587120056, "sampling/importance_sampling_ratio/max": 0.8017949461936951, "sampling/importance_sampling_ratio/mean": 0.1840158849954605, "sampling/importance_sampling_ratio/min": 0.0016303281299769878, "sampling/sampling_logp_difference/max": 5.086556434631348, "sampling/sampling_logp_difference/mean": 0.003926153294742107, "step": 46, "step_time": 294.5251908674836 }, { "clip_ratio/high_max": 0.0007992518003447913, "clip_ratio/high_mean": 0.0007992518003447913, "clip_ratio/low_mean": 0.00023160553791967686, "clip_ratio/low_min": 0.00023160553791967686, "clip_ratio/region_mean": 0.0010308573328075, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 3235.0, "completions/mean_length": 3324.25, "completions/mean_terminated_length": 1626.4000244140625, "completions/min_length": 1157.0, "completions/min_terminated_length": 1157.0, "entropy": 0.06290289713069797, "epoch": 0.9591836734693877, "frac_reward_zero_std": 0.0, "grad_norm": 0.012452603317797184, "learning_rate": 9.061224489795919e-06, "loss": 0.0102, "num_tokens": 3216805.0, "reward": 0.6057936549186707, "reward_std": 0.18371538817882538, "rewards/reward_commands_completeness/mean": 0.10499999672174454, "rewards/reward_commands_completeness/std": 0.04472136124968529, "rewards/reward_commands_correctness/mean": 0.0637499988079071, "rewards/reward_commands_correctness/std": 0.02604483626782894, "rewards/reward_diversity/mean": 0.15954366326332092, "rewards/reward_diversity/std": 0.042414866387844086, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07375000417232513, "rewards/reward_problem_validity/std": 0.02526526153087616, "rewards/reward_solution_effectiveness/mean": 0.11625000834465027, "rewards/reward_solution_effectiveness/std": 0.059874873608350754, "sampling/importance_sampling_ratio/max": 0.5164722800254822, "sampling/importance_sampling_ratio/mean": 0.12608909606933594, "sampling/importance_sampling_ratio/min": 0.004791940562427044, "sampling/sampling_logp_difference/max": 4.962502479553223, "sampling/sampling_logp_difference/mean": 0.004032905213534832, "step": 47, "step_time": 322.17987577058375 }, { "clip_ratio/high_max": 0.0005346502621250693, "clip_ratio/high_mean": 0.0005346502621250693, "clip_ratio/low_mean": 0.0006938493097550236, "clip_ratio/low_min": 0.0006938493097550236, "clip_ratio/region_mean": 0.0012284995718800928, "completions/clipped_ratio": 0.5625, "completions/max_length": 4096.0, "completions/max_terminated_length": 2657.0, "completions/mean_length": 2946.375, "completions/mean_terminated_length": 1468.2857666015625, "completions/min_length": 827.0, "completions/min_terminated_length": 827.0, "entropy": 0.06990472320467234, "epoch": 0.9795918367346939, "frac_reward_zero_std": 0.0, "grad_norm": 0.0088637201115489, "learning_rate": 9.040816326530612e-06, "loss": -0.0121, "num_tokens": 3298975.0, "reward": 0.4624011516571045, "reward_std": 0.1091122254729271, "rewards/reward_commands_completeness/mean": 0.08750000596046448, "rewards/reward_commands_completeness/std": 0.06688298285007477, "rewards/reward_commands_correctness/mean": 0.044999998062849045, "rewards/reward_commands_correctness/std": 0.03386246785521507, "rewards/reward_diversity/mean": 0.10740111023187637, "rewards/reward_diversity/std": 0.07242733985185623, "rewards/reward_format/mean": 0.05625000223517418, "rewards/reward_format/std": 0.05123475566506386, "rewards/reward_problem_validity/mean": 0.05375000089406967, "rewards/reward_problem_validity/std": 0.03997916728258133, "rewards/reward_solution_effectiveness/mean": 0.11249999701976776, "rewards/reward_solution_effectiveness/std": 0.0872926115989685, "sampling/importance_sampling_ratio/max": 1.077430009841919, "sampling/importance_sampling_ratio/mean": 0.20154398679733276, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.940724849700928, "sampling/sampling_logp_difference/mean": 0.004297391977161169, "step": 48, "step_time": 300.094816705212 }, { "clip_ratio/high_max": 0.0004193518834654242, "clip_ratio/high_mean": 0.0004193518834654242, "clip_ratio/low_mean": 0.00033688259281916544, "clip_ratio/low_min": 0.00033688259281916544, "clip_ratio/region_mean": 0.0007562344762845896, "completions/clipped_ratio": 0.75, "completions/max_length": 4096.0, "completions/max_terminated_length": 1621.0, "completions/mean_length": 3394.8125, "completions/mean_terminated_length": 1291.25, "completions/min_length": 1048.0, "completions/min_terminated_length": 1048.0, "entropy": 0.06886019092053175, "epoch": 1.0, "frac_reward_zero_std": 0.0, "grad_norm": 0.015568071976304054, "learning_rate": 9.020408163265307e-06, "loss": 0.0488, "num_tokens": 3384548.0, "reward": 0.6711089611053467, "reward_std": 0.05382446199655533, "rewards/reward_commands_completeness/mean": 0.11500000208616257, "rewards/reward_commands_completeness/std": 0.028751811012625694, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.008920948952436447, "rewards/reward_diversity/mean": 0.17548400163650513, "rewards/reward_diversity/std": 0.020916715264320374, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08312499523162842, "rewards/reward_problem_validity/std": 0.0047871386632323265, "rewards/reward_solution_effectiveness/mean": 0.13312500715255737, "rewards/reward_solution_effectiveness/std": 0.040942031890153885, "sampling/importance_sampling_ratio/max": 2.7090840339660645, "sampling/importance_sampling_ratio/mean": 0.3999497890472412, "sampling/importance_sampling_ratio/min": 2.4275977921206504e-05, "sampling/sampling_logp_difference/max": 5.184659004211426, "sampling/sampling_logp_difference/mean": 0.0036499674897640944, "step": 49, "step_time": 321.1302484534681 }, { "clip_ratio/high_max": 0.0004002384448540397, "clip_ratio/high_mean": 0.0004002384448540397, "clip_ratio/low_mean": 0.00047409626677108463, "clip_ratio/low_min": 0.00047409626677108463, "clip_ratio/region_mean": 0.0008743347189010819, "completions/clipped_ratio": 0.625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1208.0, "completions/mean_length": 2932.5, "completions/mean_terminated_length": 993.3333740234375, "completions/min_length": 753.0, "completions/min_terminated_length": 753.0, "entropy": 0.06411427119746804, "epoch": 1.0204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.007600535172969103, "learning_rate": 9e-06, "loss": 0.0067, "num_tokens": 3457280.0, "reward": 0.6505559682846069, "reward_std": 0.04839840903878212, "rewards/reward_commands_completeness/mean": 0.11499999463558197, "rewards/reward_commands_completeness/std": 0.028751811012625694, "rewards/reward_commands_correctness/mean": 0.059999994933605194, "rewards/reward_commands_correctness/std": 0.02160247042775154, "rewards/reward_diversity/mean": 0.1636810004711151, "rewards/reward_diversity/std": 0.015091432258486748, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437499403953552, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.1274999976158142, "rewards/reward_solution_effectiveness/std": 0.04449719190597534, "sampling/importance_sampling_ratio/max": 2.6231682300567627, "sampling/importance_sampling_ratio/mean": 0.26974084973335266, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.274327278137207, "sampling/sampling_logp_difference/mean": 0.004556101281195879, "step": 50, "step_time": 299.6920883934945 }, { "clip_ratio/high_max": 0.0006624599591305014, "clip_ratio/high_mean": 0.0006624599591305014, "clip_ratio/low_mean": 0.000271310236712452, "clip_ratio/low_min": 0.000271310236712452, "clip_ratio/region_mean": 0.0009337702067568898, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 2940.0, "completions/mean_length": 3290.125, "completions/mean_terminated_length": 1517.2000732421875, "completions/min_length": 1022.0, "completions/min_terminated_length": 1022.0, "entropy": 0.06488242419436574, "epoch": 1.0408163265306123, "frac_reward_zero_std": 0.0, "grad_norm": 0.022076308727264404, "learning_rate": 8.979591836734695e-06, "loss": -0.1464, "num_tokens": 3535278.0, "reward": 0.5406180620193481, "reward_std": 0.21121273934841156, "rewards/reward_commands_completeness/mean": 0.09125000238418579, "rewards/reward_commands_completeness/std": 0.042563680559396744, "rewards/reward_commands_correctness/mean": 0.04937499761581421, "rewards/reward_commands_correctness/std": 0.024074537679553032, "rewards/reward_diversity/mean": 0.14749303460121155, "rewards/reward_diversity/std": 0.054375626146793365, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07187500596046448, "rewards/reward_problem_validity/std": 0.03103090077638626, "rewards/reward_solution_effectiveness/mean": 0.09937500208616257, "rewards/reward_solution_effectiveness/std": 0.04864411801099777, "sampling/importance_sampling_ratio/max": 2.4243767261505127, "sampling/importance_sampling_ratio/mean": 0.5623362064361572, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.069698333740234, "sampling/sampling_logp_difference/mean": 0.004553160164505243, "step": 51, "step_time": 288.884364079684 }, { "clip_ratio/high_max": 0.0008890194949344732, "clip_ratio/high_mean": 0.0008890194949344732, "clip_ratio/low_mean": 0.00022924383301869966, "clip_ratio/low_min": 0.00022924383301869966, "clip_ratio/region_mean": 0.0011182633170392364, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 2008.0, "completions/mean_length": 3173.75, "completions/mean_terminated_length": 1144.800048828125, "completions/min_length": 792.0, "completions/min_terminated_length": 792.0, "entropy": 0.08176371641457081, "epoch": 1.0612244897959184, "frac_reward_zero_std": 0.0, "grad_norm": 0.03838660940527916, "learning_rate": 8.959183673469388e-06, "loss": -0.0052, "num_tokens": 3612222.0, "reward": 0.6206926107406616, "reward_std": 0.1709074229001999, "rewards/reward_commands_completeness/mean": 0.11374999582767487, "rewards/reward_commands_completeness/std": 0.04364630579948425, "rewards/reward_commands_correctness/mean": 0.06187500059604645, "rewards/reward_commands_correctness/std": 0.027378521859645844, "rewards/reward_diversity/mean": 0.15569260716438293, "rewards/reward_diversity/std": 0.03895076364278793, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07625000178813934, "rewards/reward_problem_validity/std": 0.026299558579921722, "rewards/reward_solution_effectiveness/mean": 0.12562499940395355, "rewards/reward_solution_effectiveness/std": 0.05609144642949104, "sampling/importance_sampling_ratio/max": 2.5955429077148438, "sampling/importance_sampling_ratio/mean": 0.3951543867588043, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.980913162231445, "sampling/sampling_logp_difference/mean": 0.004676310811191797, "step": 52, "step_time": 294.13910198770463 }, { "clip_ratio/high_max": 0.0006700846679450478, "clip_ratio/high_mean": 0.0006700846679450478, "clip_ratio/low_mean": 0.00020104721261304803, "clip_ratio/low_min": 0.00020104721261304803, "clip_ratio/region_mean": 0.0008711318805580959, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 3951.0, "completions/mean_length": 3607.75, "completions/mean_terminated_length": 2533.60009765625, "completions/min_length": 765.0, "completions/min_terminated_length": 765.0, "entropy": 0.08046893635764718, "epoch": 1.0816326530612246, "frac_reward_zero_std": 0.0, "grad_norm": 0.014512143097817898, "learning_rate": 8.938775510204082e-06, "loss": -0.1266, "num_tokens": 3695994.0, "reward": 0.6009324789047241, "reward_std": 0.2195865958929062, "rewards/reward_commands_completeness/mean": 0.11249999701976776, "rewards/reward_commands_completeness/std": 0.052599117159843445, "rewards/reward_commands_correctness/mean": 0.054375000298023224, "rewards/reward_commands_correctness/std": 0.025024987757205963, "rewards/reward_diversity/mean": 0.14905744791030884, "rewards/reward_diversity/std": 0.04140983521938324, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07250000536441803, "rewards/reward_problem_validity/std": 0.03130495548248291, "rewards/reward_solution_effectiveness/mean": 0.13124999403953552, "rewards/reward_solution_effectiveness/std": 0.06917369365692139, "sampling/importance_sampling_ratio/max": 2.12339448928833, "sampling/importance_sampling_ratio/mean": 0.30844980478286743, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8624980449676514, "sampling/sampling_logp_difference/mean": 0.00403108773753047, "step": 53, "step_time": 304.41410941816866 }, { "clip_ratio/high_max": 0.0003634653548942879, "clip_ratio/high_mean": 0.0003634653548942879, "clip_ratio/low_mean": 0.00023747874001855962, "clip_ratio/low_min": 0.00023747874001855962, "clip_ratio/region_mean": 0.0006009440985508263, "completions/clipped_ratio": 0.8125, "completions/max_length": 4096.0, "completions/max_terminated_length": 2223.0, "completions/mean_length": 3591.625, "completions/mean_terminated_length": 1406.0, "completions/min_length": 919.0, "completions/min_terminated_length": 919.0, "entropy": 0.05690574599429965, "epoch": 1.1020408163265305, "frac_reward_zero_std": 0.0, "grad_norm": 0.02794250287115574, "learning_rate": 8.918367346938777e-06, "loss": -0.0761, "num_tokens": 3784284.0, "reward": 0.7012742757797241, "reward_std": 0.06299592554569244, "rewards/reward_commands_completeness/mean": 0.13499999046325684, "rewards/reward_commands_completeness/std": 0.020000001415610313, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.015916449949145317, "rewards/reward_diversity/mean": 0.155649334192276, "rewards/reward_diversity/std": 0.028012681752443314, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.1537500023841858, "rewards/reward_solution_effectiveness/std": 0.04500000178813934, "sampling/importance_sampling_ratio/max": 2.8250272274017334, "sampling/importance_sampling_ratio/mean": 0.33843570947647095, "sampling/importance_sampling_ratio/min": 0.0005131892394274473, "sampling/sampling_logp_difference/max": 3.3943796157836914, "sampling/sampling_logp_difference/mean": 0.003971092868596315, "step": 54, "step_time": 330.31675680167973 }, { "clip_ratio/high_max": 0.0004491550171223935, "clip_ratio/high_mean": 0.0004491550171223935, "clip_ratio/low_mean": 0.000184700245881686, "clip_ratio/low_min": 0.000184700245881686, "clip_ratio/region_mean": 0.0006338552630040795, "completions/clipped_ratio": 0.875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1242.0, "completions/mean_length": 3381.875, "completions/mean_terminated_length": 1087.5, "completions/min_length": 933.0, "completions/min_terminated_length": 933.0, "entropy": 0.06333990348502994, "epoch": 1.1224489795918366, "frac_reward_zero_std": 0.0, "grad_norm": 0.011271700263023376, "learning_rate": 8.89795918367347e-06, "loss": -0.0454, "num_tokens": 3877194.0, "reward": 0.5776352882385254, "reward_std": 0.19060304760932922, "rewards/reward_commands_completeness/mean": 0.10750000178813934, "rewards/reward_commands_completeness/std": 0.044944413006305695, "rewards/reward_commands_correctness/mean": 0.054375000298023224, "rewards/reward_commands_correctness/std": 0.025811821222305298, "rewards/reward_diversity/mean": 0.1476353108882904, "rewards/reward_diversity/std": 0.09452613443136215, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07187499850988388, "rewards/reward_problem_validity/std": 0.0242813378572464, "rewards/reward_solution_effectiveness/mean": 0.10875000059604645, "rewards/reward_solution_effectiveness/std": 0.056789085268974304, "sampling/importance_sampling_ratio/max": 2.795057773590088, "sampling/importance_sampling_ratio/mean": 0.24920423328876495, "sampling/importance_sampling_ratio/min": 0.00017335043230559677, "sampling/sampling_logp_difference/max": 3.1922545433044434, "sampling/sampling_logp_difference/mean": 0.004150125663727522, "step": 55, "step_time": 408.72752194292843 }, { "clip_ratio/high_max": 0.00099791538741556, "clip_ratio/high_mean": 0.00099791538741556, "clip_ratio/low_mean": 0.00034186618540843483, "clip_ratio/low_min": 0.00034186618540843483, "clip_ratio/region_mean": 0.0013397815491771325, "completions/clipped_ratio": 0.5, "completions/max_length": 4096.0, "completions/max_terminated_length": 3864.0, "completions/mean_length": 2937.125, "completions/mean_terminated_length": 1778.25, "completions/min_length": 1060.0, "completions/min_terminated_length": 1060.0, "entropy": 0.07955974247306585, "epoch": 1.1428571428571428, "frac_reward_zero_std": 0.0, "grad_norm": 0.0026691036764532328, "learning_rate": 8.877551020408163e-06, "loss": -0.0035, "num_tokens": 3946568.0, "reward": 0.6305668950080872, "reward_std": 0.07227049767971039, "rewards/reward_commands_completeness/mean": 0.11499999463558197, "rewards/reward_commands_completeness/std": 0.03386246785521507, "rewards/reward_commands_correctness/mean": 0.06062500178813934, "rewards/reward_commands_correctness/std": 0.02143789641559124, "rewards/reward_diversity/mean": 0.1624419093132019, "rewards/reward_diversity/std": 0.025596236810088158, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.018929695710539818, "rewards/reward_solution_effectiveness/mean": 0.12000000476837158, "rewards/reward_solution_effectiveness/std": 0.05019960179924965, "sampling/importance_sampling_ratio/max": 0.4733484387397766, "sampling/importance_sampling_ratio/mean": 0.06739678233861923, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.886334419250488, "sampling/sampling_logp_difference/mean": 0.005053211934864521, "step": 56, "step_time": 267.08980337530375 }, { "clip_ratio/high_max": 0.0004265293464413844, "clip_ratio/high_mean": 0.0004265293464413844, "clip_ratio/low_mean": 0.0002262369598611258, "clip_ratio/low_min": 0.0002262369598611258, "clip_ratio/region_mean": 0.0006527663063025102, "completions/clipped_ratio": 0.5625, "completions/max_length": 4096.0, "completions/max_terminated_length": 2764.0, "completions/mean_length": 2959.1875, "completions/mean_terminated_length": 1497.571533203125, "completions/min_length": 820.0, "completions/min_terminated_length": 820.0, "entropy": 0.07540672086179256, "epoch": 1.163265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.008442986756563187, "learning_rate": 8.857142857142858e-06, "loss": -0.0285, "num_tokens": 4028295.0, "reward": 0.6436769962310791, "reward_std": 0.1303529590368271, "rewards/reward_commands_completeness/mean": 0.11999999731779099, "rewards/reward_commands_completeness/std": 0.03425395488739014, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.021823154762387276, "rewards/reward_diversity/mean": 0.15555205941200256, "rewards/reward_diversity/std": 0.043716832995414734, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.078125, "rewards/reward_problem_validity/std": 0.02197536826133728, "rewards/reward_solution_effectiveness/mean": 0.13312500715255737, "rewards/reward_solution_effectiveness/std": 0.04771006479859352, "sampling/importance_sampling_ratio/max": 2.941014051437378, "sampling/importance_sampling_ratio/mean": 0.1995985060930252, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.905497074127197, "sampling/sampling_logp_difference/mean": 0.00448769424110651, "step": 57, "step_time": 312.05014408938587 }, { "clip_ratio/high_max": 0.0009679936883912887, "clip_ratio/high_mean": 0.0009679936883912887, "clip_ratio/low_mean": 0.0002178508075303398, "clip_ratio/low_min": 0.0002178508075303398, "clip_ratio/region_mean": 0.0011858445031975862, "completions/clipped_ratio": 0.4375, "completions/max_length": 4096.0, "completions/max_terminated_length": 2840.0, "completions/mean_length": 2721.5, "completions/mean_terminated_length": 1652.4444580078125, "completions/min_length": 662.0, "completions/min_terminated_length": 662.0, "entropy": 0.08657872909680009, "epoch": 1.183673469387755, "frac_reward_zero_std": 0.0, "grad_norm": 0.031231900677084923, "learning_rate": 8.836734693877552e-06, "loss": 0.0904, "num_tokens": 4096599.0, "reward": 0.5934181809425354, "reward_std": 0.15705308318138123, "rewards/reward_commands_completeness/mean": 0.09624999761581421, "rewards/reward_commands_completeness/std": 0.042720019817352295, "rewards/reward_commands_correctness/mean": 0.0612499974668026, "rewards/reward_commands_correctness/std": 0.027294689789414406, "rewards/reward_diversity/mean": 0.16841818392276764, "rewards/reward_diversity/std": 0.021629698574543, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07312500476837158, "rewards/reward_problem_validity/std": 0.024958299472928047, "rewards/reward_solution_effectiveness/mean": 0.10687500238418579, "rewards/reward_solution_effectiveness/std": 0.060961052775382996, "sampling/importance_sampling_ratio/max": 2.0359787940979004, "sampling/importance_sampling_ratio/mean": 0.3307073712348938, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.2933197021484375, "sampling/sampling_logp_difference/mean": 0.00484894122928381, "step": 58, "step_time": 304.2847150750458 }, { "clip_ratio/high_max": 0.0004230737795296591, "clip_ratio/high_mean": 0.0004230737795296591, "clip_ratio/low_mean": 0.00015271391384885646, "clip_ratio/low_min": 0.00015271391384885646, "clip_ratio/region_mean": 0.0005757876933785155, "completions/clipped_ratio": 0.8125, "completions/max_length": 4096.0, "completions/max_terminated_length": 2734.0, "completions/mean_length": 3640.25, "completions/mean_terminated_length": 1665.3333740234375, "completions/min_length": 994.0, "completions/min_terminated_length": 994.0, "entropy": 0.061416428070515394, "epoch": 1.2040816326530612, "frac_reward_zero_std": 0.0, "grad_norm": 0.003966862335801125, "learning_rate": 8.816326530612247e-06, "loss": 0.0107, "num_tokens": 4175427.0, "reward": 0.5708759427070618, "reward_std": 0.2514386475086212, "rewards/reward_commands_completeness/mean": 0.10999999940395355, "rewards/reward_commands_completeness/std": 0.05977736786007881, "rewards/reward_commands_correctness/mean": 0.054999999701976776, "rewards/reward_commands_correctness/std": 0.029888683930039406, "rewards/reward_diversity/mean": 0.12712594866752625, "rewards/reward_diversity/std": 0.10547088086605072, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06875000149011612, "rewards/reward_problem_validity/std": 0.03518996387720108, "rewards/reward_solution_effectiveness/mean": 0.13499999046325684, "rewards/reward_solution_effectiveness/std": 0.08414273709058762, "sampling/importance_sampling_ratio/max": 0.5170027017593384, "sampling/importance_sampling_ratio/mean": 0.06860493123531342, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.821366786956787, "sampling/sampling_logp_difference/mean": 0.00401478074491024, "step": 59, "step_time": 264.13127064332366 }, { "clip_ratio/high_max": 0.0005036788425059058, "clip_ratio/high_mean": 0.0005036788425059058, "clip_ratio/low_mean": 0.00019958626216975972, "clip_ratio/low_min": 0.00019958626216975972, "clip_ratio/region_mean": 0.0007032650973997079, "completions/clipped_ratio": 0.75, "completions/max_length": 4096.0, "completions/max_terminated_length": 3883.0, "completions/mean_length": 3623.625, "completions/mean_terminated_length": 2206.5, "completions/min_length": 855.0, "completions/min_terminated_length": 855.0, "entropy": 0.08105066465213895, "epoch": 1.2244897959183674, "frac_reward_zero_std": 0.0, "grad_norm": 0.004510645754635334, "learning_rate": 8.79591836734694e-06, "loss": -0.0314, "num_tokens": 4253365.0, "reward": 0.5781620740890503, "reward_std": 0.16519533097743988, "rewards/reward_commands_completeness/mean": 0.10124999284744263, "rewards/reward_commands_completeness/std": 0.04529532790184021, "rewards/reward_commands_correctness/mean": 0.05562499910593033, "rewards/reward_commands_correctness/std": 0.026575367897748947, "rewards/reward_diversity/mean": 0.15566208958625793, "rewards/reward_diversity/std": 0.05408244580030441, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07000000029802322, "rewards/reward_problem_validity/std": 0.030110908672213554, "rewards/reward_solution_effectiveness/mean": 0.11437500268220901, "rewards/reward_solution_effectiveness/std": 0.05046038329601288, "sampling/importance_sampling_ratio/max": 0.3423909842967987, "sampling/importance_sampling_ratio/mean": 0.06351517885923386, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5762076377868652, "sampling/sampling_logp_difference/mean": 0.004277021158486605, "step": 60, "step_time": 278.9350563939661 }, { "clip_ratio/high_max": 0.0007569331719423644, "clip_ratio/high_mean": 0.0007569331719423644, "clip_ratio/low_mean": 0.00039150994780356996, "clip_ratio/low_min": 0.00039150994780356996, "clip_ratio/region_mean": 0.001148443108831998, "completions/clipped_ratio": 0.4375, "completions/max_length": 4096.0, "completions/max_terminated_length": 2472.0, "completions/mean_length": 2491.75, "completions/mean_terminated_length": 1244.0, "completions/min_length": 775.0, "completions/min_terminated_length": 775.0, "entropy": 0.11021966114640236, "epoch": 1.2448979591836735, "frac_reward_zero_std": 0.0, "grad_norm": 0.05635986104607582, "learning_rate": 8.775510204081633e-06, "loss": 0.3239, "num_tokens": 4312681.0, "reward": 0.7057018876075745, "reward_std": 0.04627513885498047, "rewards/reward_commands_completeness/mean": 0.1262499988079071, "rewards/reward_commands_completeness/std": 0.02276693657040596, "rewards/reward_commands_correctness/mean": 0.07312500476837158, "rewards/reward_commands_correctness/std": 0.00704154372215271, "rewards/reward_diversity/mean": 0.17445188760757446, "rewards/reward_diversity/std": 0.0201640035957098, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.03829164430499077, "sampling/importance_sampling_ratio/max": 2.6945900917053223, "sampling/importance_sampling_ratio/mean": 0.5526204109191895, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.299505710601807, "sampling/sampling_logp_difference/mean": 0.0052790469489991665, "step": 61, "step_time": 272.96927439048886 }, { "clip_ratio/high_max": 0.0003161896202072967, "clip_ratio/high_mean": 0.0003161896202072967, "clip_ratio/low_mean": 0.000296364367386559, "clip_ratio/low_min": 0.000296364367386559, "clip_ratio/region_mean": 0.0006125539839558769, "completions/clipped_ratio": 0.75, "completions/max_length": 4096.0, "completions/max_terminated_length": 2417.0, "completions/mean_length": 3452.5625, "completions/mean_terminated_length": 1522.25, "completions/min_length": 1211.0, "completions/min_terminated_length": 1211.0, "entropy": 0.06178431631997228, "epoch": 1.2653061224489797, "frac_reward_zero_std": 0.0, "grad_norm": 0.0035453778691589832, "learning_rate": 8.755102040816326e-06, "loss": 0.0381, "num_tokens": 4389050.0, "reward": 0.6381083130836487, "reward_std": 0.08493023365736008, "rewards/reward_commands_completeness/mean": 0.11999999731779099, "rewards/reward_commands_completeness/std": 0.02828427217900753, "rewards/reward_commands_correctness/mean": 0.07625000178813934, "rewards/reward_commands_correctness/std": 0.013601471669971943, "rewards/reward_diversity/mean": 0.10810832679271698, "rewards/reward_diversity/std": 0.1230706050992012, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.14812499284744263, "rewards/reward_solution_effectiveness/std": 0.04956056922674179, "sampling/importance_sampling_ratio/max": 0.916253924369812, "sampling/importance_sampling_ratio/mean": 0.15942007303237915, "sampling/importance_sampling_ratio/min": 0.0018353117629885674, "sampling/sampling_logp_difference/max": 5.060945510864258, "sampling/sampling_logp_difference/mean": 0.00377137353643775, "step": 62, "step_time": 280.6523254606873 }, { "clip_ratio/high_max": 0.0007883955986471847, "clip_ratio/high_mean": 0.0007883955986471847, "clip_ratio/low_mean": 0.0002871088363463059, "clip_ratio/low_min": 0.0002871088363463059, "clip_ratio/region_mean": 0.0010755044422694482, "completions/clipped_ratio": 0.5625, "completions/max_length": 4096.0, "completions/max_terminated_length": 3071.0, "completions/mean_length": 2950.9375, "completions/mean_terminated_length": 1478.71435546875, "completions/min_length": 663.0, "completions/min_terminated_length": 663.0, "entropy": 0.09609538642689586, "epoch": 1.2857142857142856, "frac_reward_zero_std": 0.0, "grad_norm": 0.04004299268126488, "learning_rate": 8.734693877551021e-06, "loss": 0.0961, "num_tokens": 4460385.0, "reward": 0.576536238193512, "reward_std": 0.1539316475391388, "rewards/reward_commands_completeness/mean": 0.10249999165534973, "rewards/reward_commands_completeness/std": 0.03924283757805824, "rewards/reward_commands_correctness/mean": 0.05812499672174454, "rewards/reward_commands_correctness/std": 0.028570091351866722, "rewards/reward_diversity/mean": 0.15091121196746826, "rewards/reward_diversity/std": 0.058055564761161804, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07437500357627869, "rewards/reward_problem_validity/std": 0.025552237406373024, "rewards/reward_solution_effectiveness/mean": 0.10312500596046448, "rewards/reward_solution_effectiveness/std": 0.052500005811452866, "sampling/importance_sampling_ratio/max": 2.805809259414673, "sampling/importance_sampling_ratio/mean": 0.2999890446662903, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.528663635253906, "sampling/sampling_logp_difference/mean": 0.00595885282382369, "step": 63, "step_time": 302.0221939291805 }, { "clip_ratio/high_max": 0.0002722281187743647, "clip_ratio/high_mean": 0.0002722281187743647, "clip_ratio/low_mean": 0.0004708624037448317, "clip_ratio/low_min": 0.0004708624037448317, "clip_ratio/region_mean": 0.0007430905170622282, "completions/clipped_ratio": 0.625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1349.0, "completions/mean_length": 2922.875, "completions/mean_terminated_length": 967.6666870117188, "completions/min_length": 680.0, "completions/min_terminated_length": 680.0, "entropy": 0.0805862839333713, "epoch": 1.306122448979592, "frac_reward_zero_std": 0.0, "grad_norm": 0.013436314649879932, "learning_rate": 8.714285714285715e-06, "loss": 0.0189, "num_tokens": 4529843.0, "reward": 0.48278170824050903, "reward_std": 0.1547449827194214, "rewards/reward_commands_completeness/mean": 0.10249999165534973, "rewards/reward_commands_completeness/std": 0.044347118586301804, "rewards/reward_commands_correctness/mean": 0.0637499988079071, "rewards/reward_commands_correctness/std": 0.02801785245537758, "rewards/reward_diversity/mean": 0.04403170570731163, "rewards/reward_diversity/std": 0.1531493216753006, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07437500357627869, "rewards/reward_problem_validity/std": 0.025552235543727875, "rewards/reward_solution_effectiveness/mean": 0.1106249988079071, "rewards/reward_solution_effectiveness/std": 0.0690380334854126, "sampling/importance_sampling_ratio/max": 0.9514508247375488, "sampling/importance_sampling_ratio/mean": 0.12937107682228088, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.5551066398620605, "sampling/sampling_logp_difference/mean": 0.004187798593193293, "step": 64, "step_time": 271.86333022639155 }, { "clip_ratio/high_max": 0.00116935031110188, "clip_ratio/high_mean": 0.00116935031110188, "clip_ratio/low_mean": 0.00023925451750983484, "clip_ratio/low_min": 0.00023925451750983484, "clip_ratio/region_mean": 0.0014086048322496936, "completions/clipped_ratio": 0.625, "completions/max_length": 4096.0, "completions/max_terminated_length": 2954.0, "completions/mean_length": 3112.1875, "completions/mean_terminated_length": 1472.5, "completions/min_length": 651.0, "completions/min_terminated_length": 651.0, "entropy": 0.07742350827902555, "epoch": 1.3265306122448979, "frac_reward_zero_std": 0.0, "grad_norm": 0.02805636078119278, "learning_rate": 8.69387755102041e-06, "loss": -0.2379, "num_tokens": 4602602.0, "reward": 0.4847434163093567, "reward_std": 0.16254116594791412, "rewards/reward_commands_completeness/mean": 0.08249999582767487, "rewards/reward_commands_completeness/std": 0.04553386941552162, "rewards/reward_commands_correctness/mean": 0.05000000074505806, "rewards/reward_commands_correctness/std": 0.029664795845746994, "rewards/reward_diversity/mean": 0.13786840438842773, "rewards/reward_diversity/std": 0.06025165691971779, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06062500178813934, "rewards/reward_problem_validity/std": 0.031930916011333466, "rewards/reward_solution_effectiveness/mean": 0.07874999940395355, "rewards/reward_solution_effectiveness/std": 0.05123475193977356, "sampling/importance_sampling_ratio/max": 2.768185615539551, "sampling/importance_sampling_ratio/mean": 0.4301525950431824, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.786402702331543, "sampling/sampling_logp_difference/mean": 0.0047508832067251205, "step": 65, "step_time": 273.7304587904364 }, { "clip_ratio/high_max": 0.0005359275237424299, "clip_ratio/high_mean": 0.0005359275237424299, "clip_ratio/low_mean": 0.0001942571107065305, "clip_ratio/low_min": 0.0001942571107065305, "clip_ratio/region_mean": 0.0007301846344489604, "completions/clipped_ratio": 0.875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1237.0, "completions/mean_length": 3733.25, "completions/mean_terminated_length": 1194.0, "completions/min_length": 1151.0, "completions/min_terminated_length": 1151.0, "entropy": 0.047608180437237024, "epoch": 1.346938775510204, "frac_reward_zero_std": 0.0, "grad_norm": 0.02023443393409252, "learning_rate": 8.673469387755103e-06, "loss": -0.0432, "num_tokens": 4693962.0, "reward": 0.4055063724517822, "reward_std": 0.2738129794597626, "rewards/reward_commands_completeness/mean": 0.07124999910593033, "rewards/reward_commands_completeness/std": 0.05463515222072601, "rewards/reward_commands_correctness/mean": 0.03500000014901161, "rewards/reward_commands_correctness/std": 0.02828427217900753, "rewards/reward_diversity/mean": 0.12488134205341339, "rewards/reward_diversity/std": 0.0753355398774147, "rewards/reward_format/mean": 0.05000000074505806, "rewards/reward_format/std": 0.05163978040218353, "rewards/reward_problem_validity/mean": 0.04749999940395355, "rewards/reward_problem_validity/std": 0.03890158608555794, "rewards/reward_solution_effectiveness/mean": 0.0768750011920929, "rewards/reward_solution_effectiveness/std": 0.054738011211156845, "sampling/importance_sampling_ratio/max": 2.966539144515991, "sampling/importance_sampling_ratio/mean": 0.2671508491039276, "sampling/importance_sampling_ratio/min": 0.0011846460402011871, "sampling/sampling_logp_difference/max": 5.698859214782715, "sampling/sampling_logp_difference/mean": 0.0035032941959798336, "step": 66, "step_time": 279.69152923859656 }, { "clip_ratio/high_max": 0.0006918287872395013, "clip_ratio/high_mean": 0.0006918287872395013, "clip_ratio/low_mean": 0.00031928938915370964, "clip_ratio/low_min": 0.00031928938915370964, "clip_ratio/region_mean": 0.0010111181763932109, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 2587.0, "completions/mean_length": 3231.625, "completions/mean_terminated_length": 1330.0, "completions/min_length": 788.0, "completions/min_terminated_length": 788.0, "entropy": 0.056649595499038696, "epoch": 1.3673469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.03674064576625824, "learning_rate": 8.653061224489798e-06, "loss": 0.0862, "num_tokens": 4775504.0, "reward": 0.636497437953949, "reward_std": 0.14369705319404602, "rewards/reward_commands_completeness/mean": 0.11375000327825546, "rewards/reward_commands_completeness/std": 0.04240676388144493, "rewards/reward_commands_correctness/mean": 0.06187500059604645, "rewards/reward_commands_correctness/std": 0.01939716562628746, "rewards/reward_diversity/mean": 0.15774744749069214, "rewards/reward_diversity/std": 0.043725840747356415, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07999999821186066, "rewards/reward_problem_validity/std": 0.01932183839380741, "rewards/reward_solution_effectiveness/mean": 0.12937501072883606, "rewards/reward_solution_effectiveness/std": 0.06454649567604065, "sampling/importance_sampling_ratio/max": 2.2248880863189697, "sampling/importance_sampling_ratio/mean": 0.44145190715789795, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6686811447143555, "sampling/sampling_logp_difference/mean": 0.004170896485447884, "step": 67, "step_time": 310.408379804343 }, { "clip_ratio/high_max": 0.001201090783069958, "clip_ratio/high_mean": 0.001201090783069958, "clip_ratio/low_mean": 0.00027606366529653315, "clip_ratio/low_min": 0.00027606366529653315, "clip_ratio/region_mean": 0.0014771544556424487, "completions/clipped_ratio": 0.75, "completions/max_length": 4096.0, "completions/max_terminated_length": 2752.0, "completions/mean_length": 3400.0625, "completions/mean_terminated_length": 1312.25, "completions/min_length": 724.0, "completions/min_terminated_length": 724.0, "entropy": 0.08494243584573269, "epoch": 1.3877551020408163, "frac_reward_zero_std": 0.0, "grad_norm": 0.024552488699555397, "learning_rate": 8.632653061224491e-06, "loss": 0.1383, "num_tokens": 4847337.0, "reward": 0.6596410274505615, "reward_std": 0.1099739670753479, "rewards/reward_commands_completeness/mean": 0.11749999970197678, "rewards/reward_commands_completeness/std": 0.04374166578054428, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.026323311030864716, "rewards/reward_diversity/mean": 0.1677660495042801, "rewards/reward_diversity/std": 0.014997163787484169, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.02651257812976837, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.05921359732747078, "sampling/importance_sampling_ratio/max": 2.328782081604004, "sampling/importance_sampling_ratio/mean": 0.2892916202545166, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.376859664916992, "sampling/sampling_logp_difference/mean": 0.004694750066846609, "step": 68, "step_time": 260.74857317470014 }, { "clip_ratio/high_max": 0.0002805067924782634, "clip_ratio/high_mean": 0.0002805067924782634, "clip_ratio/low_mean": 0.00020878454597550444, "clip_ratio/low_min": 0.00020878454597550444, "clip_ratio/region_mean": 0.0004892913384537678, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 2267.0, "completions/mean_length": 3237.0, "completions/mean_terminated_length": 1347.2000732421875, "completions/min_length": 863.0, "completions/min_terminated_length": 863.0, "entropy": 0.06727993302047253, "epoch": 1.4081632653061225, "frac_reward_zero_std": 0.0, "grad_norm": 0.00599661935120821, "learning_rate": 8.612244897959184e-06, "loss": -0.0108, "num_tokens": 4927725.0, "reward": 0.5451360940933228, "reward_std": 0.19110426306724548, "rewards/reward_commands_completeness/mean": 0.10125000774860382, "rewards/reward_commands_completeness/std": 0.05389805883169174, "rewards/reward_commands_correctness/mean": 0.054375000298023224, "rewards/reward_commands_correctness/std": 0.029657769948244095, "rewards/reward_diversity/mean": 0.12638607621192932, "rewards/reward_diversity/std": 0.06847211718559265, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06812500208616257, "rewards/reward_problem_validity/std": 0.03487477824091911, "rewards/reward_solution_effectiveness/mean": 0.12000000476837158, "rewards/reward_solution_effectiveness/std": 0.07183314114809036, "sampling/importance_sampling_ratio/max": 0.4410887062549591, "sampling/importance_sampling_ratio/mean": 0.07685910910367966, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.852258682250977, "sampling/sampling_logp_difference/mean": 0.004309697542339563, "step": 69, "step_time": 294.3131929412484 }, { "clip_ratio/high_max": 0.0006678451954940101, "clip_ratio/high_mean": 0.0006678451954940101, "clip_ratio/low_mean": 0.0003319854185974691, "clip_ratio/low_min": 0.0003319854185974691, "clip_ratio/region_mean": 0.0009998306159104686, "completions/clipped_ratio": 0.5, "completions/max_length": 4096.0, "completions/max_terminated_length": 3014.0, "completions/mean_length": 2950.25, "completions/mean_terminated_length": 1804.5, "completions/min_length": 724.0, "completions/min_terminated_length": 724.0, "entropy": 0.087884655687958, "epoch": 1.4285714285714286, "frac_reward_zero_std": 0.0, "grad_norm": 0.013498477637767792, "learning_rate": 8.591836734693878e-06, "loss": -0.0479, "num_tokens": 4999621.0, "reward": 0.6187708377838135, "reward_std": 0.1107804924249649, "rewards/reward_commands_completeness/mean": 0.11249999701976776, "rewards/reward_commands_completeness/std": 0.039242833852767944, "rewards/reward_commands_correctness/mean": 0.060624998062849045, "rewards/reward_commands_correctness/std": 0.0201556459069252, "rewards/reward_diversity/mean": 0.15564581751823425, "rewards/reward_diversity/std": 0.03797208517789841, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07999999821186066, "rewards/reward_problem_validity/std": 0.019321836531162262, "rewards/reward_solution_effectiveness/mean": 0.11625000834465027, "rewards/reward_solution_effectiveness/std": 0.05123475566506386, "sampling/importance_sampling_ratio/max": 0.9894089698791504, "sampling/importance_sampling_ratio/mean": 0.1742313951253891, "sampling/importance_sampling_ratio/min": 0.0005159779102541506, "sampling/sampling_logp_difference/max": 5.101154804229736, "sampling/sampling_logp_difference/mean": 0.005322219803929329, "step": 70, "step_time": 312.0712188743055 }, { "clip_ratio/high_max": 0.0006088783266022801, "clip_ratio/high_mean": 0.0006088783266022801, "clip_ratio/low_mean": 0.00020120710541959852, "clip_ratio/low_min": 0.00020120710541959852, "clip_ratio/region_mean": 0.0008100854320218787, "completions/clipped_ratio": 0.75, "completions/max_length": 4096.0, "completions/max_terminated_length": 1174.0, "completions/mean_length": 3348.75, "completions/mean_terminated_length": 1107.0, "completions/min_length": 1035.0, "completions/min_terminated_length": 1035.0, "entropy": 0.0659108052495867, "epoch": 1.4489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.015145080164074898, "learning_rate": 8.571428571428571e-06, "loss": 0.0128, "num_tokens": 5082169.0, "reward": 0.5566632151603699, "reward_std": 0.1743772327899933, "rewards/reward_commands_completeness/mean": 0.10249999165534973, "rewards/reward_commands_completeness/std": 0.05555777996778488, "rewards/reward_commands_correctness/mean": 0.05625000223517418, "rewards/reward_commands_correctness/std": 0.03180670738220215, "rewards/reward_diversity/mean": 0.1385382115840912, "rewards/reward_diversity/std": 0.06127701699733734, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06437499821186066, "rewards/reward_problem_validity/std": 0.032653484493494034, "rewards/reward_solution_effectiveness/mean": 0.12000000476837158, "rewards/reward_solution_effectiveness/std": 0.07823042571544647, "sampling/importance_sampling_ratio/max": 1.4301648139953613, "sampling/importance_sampling_ratio/mean": 0.24484702944755554, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.5854620933532715, "sampling/sampling_logp_difference/mean": 0.004022589884698391, "step": 71, "step_time": 294.29247535951436 }, { "clip_ratio/high_max": 0.0008574603889428545, "clip_ratio/high_mean": 0.0008574603889428545, "clip_ratio/low_mean": 0.00025998199816967826, "clip_ratio/low_min": 0.00025998199816967826, "clip_ratio/region_mean": 0.0011174423889315221, "completions/clipped_ratio": 0.625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1549.0, "completions/mean_length": 2951.0, "completions/mean_terminated_length": 1042.666748046875, "completions/min_length": 863.0, "completions/min_terminated_length": 863.0, "entropy": 0.06956539256498218, "epoch": 1.469387755102041, "frac_reward_zero_std": 0.0, "grad_norm": 0.015179642476141453, "learning_rate": 8.551020408163266e-06, "loss": -0.0019, "num_tokens": 5156237.0, "reward": 0.6961989402770996, "reward_std": 0.06562280654907227, "rewards/reward_commands_completeness/mean": 0.1287499964237213, "rewards/reward_commands_completeness/std": 0.02526525966823101, "rewards/reward_commands_correctness/mean": 0.06624999642372131, "rewards/reward_commands_correctness/std": 0.013601471669971943, "rewards/reward_diversity/mean": 0.17244893312454224, "rewards/reward_diversity/std": 0.028820538893342018, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.04545602202415466, "sampling/importance_sampling_ratio/max": 1.2007004022598267, "sampling/importance_sampling_ratio/mean": 0.15536215901374817, "sampling/importance_sampling_ratio/min": 3.4878135011240374e-06, "sampling/sampling_logp_difference/max": 5.614442825317383, "sampling/sampling_logp_difference/mean": 0.005210923962295055, "step": 72, "step_time": 306.9023040551692 }, { "clip_ratio/high_max": 0.00063867456628941, "clip_ratio/high_mean": 0.00063867456628941, "clip_ratio/low_mean": 0.0001452590586268343, "clip_ratio/low_min": 0.0001452590586268343, "clip_ratio/region_mean": 0.0007839336249162443, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4078.0, "completions/mean_length": 3577.5, "completions/mean_terminated_length": 2436.800048828125, "completions/min_length": 930.0, "completions/min_terminated_length": 930.0, "entropy": 0.09170200722292066, "epoch": 1.489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.004126963671296835, "learning_rate": 8.530612244897961e-06, "loss": -0.0315, "num_tokens": 5237929.0, "reward": 0.5737391114234924, "reward_std": 0.1529652625322342, "rewards/reward_commands_completeness/mean": 0.10374999046325684, "rewards/reward_commands_completeness/std": 0.038100745528936386, "rewards/reward_commands_correctness/mean": 0.054999999701976776, "rewards/reward_commands_correctness/std": 0.023380905389785767, "rewards/reward_diversity/mean": 0.14498907327651978, "rewards/reward_diversity/std": 0.05490034073591232, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07562500238418579, "rewards/reward_problem_validity/std": 0.026068823412060738, "rewards/reward_solution_effectiveness/mean": 0.1068749949336052, "rewards/reward_solution_effectiveness/std": 0.04771006479859352, "sampling/importance_sampling_ratio/max": 1.1376932859420776, "sampling/importance_sampling_ratio/mean": 0.18648874759674072, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.699950218200684, "sampling/sampling_logp_difference/mean": 0.004506778437644243, "step": 73, "step_time": 278.6857324875891 }, { "clip_ratio/high_max": 0.0003569501714082435, "clip_ratio/high_mean": 0.0003569501714082435, "clip_ratio/low_mean": 0.0001934605788846966, "clip_ratio/low_min": 0.0001934605788846966, "clip_ratio/region_mean": 0.0005504107466549613, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1371.0, "completions/mean_length": 3194.875, "completions/mean_terminated_length": 1212.4000244140625, "completions/min_length": 1092.0, "completions/min_terminated_length": 1092.0, "entropy": 0.058100983034819365, "epoch": 1.510204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.009642686694860458, "learning_rate": 8.510204081632654e-06, "loss": -0.0537, "num_tokens": 5319519.0, "reward": 0.6139786243438721, "reward_std": 0.0656280592083931, "rewards/reward_commands_completeness/mean": 0.10374999046325684, "rewards/reward_commands_completeness/std": 0.023345235735177994, "rewards/reward_commands_correctness/mean": 0.06874999403953552, "rewards/reward_commands_correctness/std": 0.012041596695780754, "rewards/reward_diversity/mean": 0.15335358679294586, "rewards/reward_diversity/std": 0.03188859671354294, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.10687500238418579, "rewards/reward_solution_effectiveness/std": 0.03458684682846069, "sampling/importance_sampling_ratio/max": 1.6030054092407227, "sampling/importance_sampling_ratio/mean": 0.2178974747657776, "sampling/importance_sampling_ratio/min": 3.3444584914832376e-06, "sampling/sampling_logp_difference/max": 5.806643962860107, "sampling/sampling_logp_difference/mean": 0.004351528827100992, "step": 74, "step_time": 312.5589562896639 }, { "clip_ratio/high_max": 0.0006018233543727547, "clip_ratio/high_mean": 0.0006018233543727547, "clip_ratio/low_mean": 0.0004216489032842219, "clip_ratio/low_min": 0.0004216489032842219, "clip_ratio/region_mean": 0.0010234722576569766, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1901.0, "completions/mean_length": 3221.1875, "completions/mean_terminated_length": 1296.5999755859375, "completions/min_length": 1058.0, "completions/min_terminated_length": 1058.0, "entropy": 0.06113731465302408, "epoch": 1.5306122448979593, "frac_reward_zero_std": 0.0, "grad_norm": 0.03301624953746796, "learning_rate": 8.489795918367347e-06, "loss": 0.1141, "num_tokens": 5402066.0, "reward": 0.6466319561004639, "reward_std": 0.11624734103679657, "rewards/reward_commands_completeness/mean": 0.11874999850988388, "rewards/reward_commands_completeness/std": 0.034616950899362564, "rewards/reward_commands_correctness/mean": 0.05999999865889549, "rewards/reward_commands_correctness/std": 0.019663842394948006, "rewards/reward_diversity/mean": 0.16163194179534912, "rewards/reward_diversity/std": 0.04897584766149521, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.13124999403953552, "rewards/reward_solution_effectiveness/std": 0.04631414636969566, "sampling/importance_sampling_ratio/max": 1.9783679246902466, "sampling/importance_sampling_ratio/mean": 0.3020472526550293, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7572267055511475, "sampling/sampling_logp_difference/mean": 0.003914861474186182, "step": 75, "step_time": 295.29917506314814 }, { "clip_ratio/high_max": 0.00033214672293979675, "clip_ratio/high_mean": 0.00033214672293979675, "clip_ratio/low_mean": 0.00022608707877225243, "clip_ratio/low_min": 0.00022608707877225243, "clip_ratio/region_mean": 0.0005582337980740704, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4056.0, "completions/mean_length": 3672.375, "completions/mean_terminated_length": 2740.400146484375, "completions/min_length": 1106.0, "completions/min_terminated_length": 1106.0, "entropy": 0.04747932916507125, "epoch": 1.5510204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.014093806967139244, "learning_rate": 8.469387755102042e-06, "loss": -0.0383, "num_tokens": 5488256.0, "reward": 0.568668007850647, "reward_std": 0.05679650604724884, "rewards/reward_commands_completeness/mean": 0.10624999552965164, "rewards/reward_commands_completeness/std": 0.04883646219968796, "rewards/reward_commands_correctness/mean": 0.0625, "rewards/reward_commands_correctness/std": 0.027446920052170753, "rewards/reward_diversity/mean": 0.11554297804832458, "rewards/reward_diversity/std": 0.10441841930150986, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07375000417232513, "rewards/reward_problem_validity/std": 0.03180671110749245, "rewards/reward_solution_effectiveness/mean": 0.12937501072883606, "rewards/reward_solution_effectiveness/std": 0.06265980005264282, "sampling/importance_sampling_ratio/max": 0.9663150906562805, "sampling/importance_sampling_ratio/mean": 0.21249917149543762, "sampling/importance_sampling_ratio/min": 0.0008680393802933395, "sampling/sampling_logp_difference/max": 2.8576297760009766, "sampling/sampling_logp_difference/mean": 0.0036104475148022175, "step": 76, "step_time": 292.7969306334853 }, { "clip_ratio/high_max": 0.0008612427627667785, "clip_ratio/high_mean": 0.0008612427627667785, "clip_ratio/low_mean": 0.00034083211721736006, "clip_ratio/low_min": 0.00034083211721736006, "clip_ratio/region_mean": 0.0012020748836221173, "completions/clipped_ratio": 0.625, "completions/max_length": 4096.0, "completions/max_terminated_length": 2881.0, "completions/mean_length": 3050.9375, "completions/mean_terminated_length": 1309.166748046875, "completions/min_length": 713.0, "completions/min_terminated_length": 713.0, "entropy": 0.06993644451722503, "epoch": 1.5714285714285714, "frac_reward_zero_std": 0.0, "grad_norm": 0.017438679933547974, "learning_rate": 8.448979591836736e-06, "loss": -0.1001, "num_tokens": 5560931.0, "reward": 0.5465856790542603, "reward_std": 0.19508206844329834, "rewards/reward_commands_completeness/mean": 0.08874999731779099, "rewards/reward_commands_completeness/std": 0.04616997390985489, "rewards/reward_commands_correctness/mean": 0.04937499761581421, "rewards/reward_commands_correctness/std": 0.0271952822804451, "rewards/reward_diversity/mean": 0.17033566534519196, "rewards/reward_diversity/std": 0.042643651366233826, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06562499701976776, "rewards/reward_problem_validity/std": 0.03346017748117447, "rewards/reward_solution_effectiveness/mean": 0.09750000387430191, "rewards/reward_solution_effectiveness/std": 0.05639148876070976, "sampling/importance_sampling_ratio/max": 1.7802414894104004, "sampling/importance_sampling_ratio/mean": 0.26409879326820374, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.971156597137451, "sampling/sampling_logp_difference/mean": 0.004944062791764736, "step": 77, "step_time": 285.01007027737796 }, { "clip_ratio/high_max": 0.0005660887400154024, "clip_ratio/high_mean": 0.0005660887400154024, "clip_ratio/low_mean": 0.00022658427405986004, "clip_ratio/low_min": 0.00022658427405986004, "clip_ratio/region_mean": 0.0007926730140752625, "completions/clipped_ratio": 0.75, "completions/max_length": 4096.0, "completions/max_terminated_length": 1501.0, "completions/mean_length": 3373.5625, "completions/mean_terminated_length": 1206.25, "completions/min_length": 987.0, "completions/min_terminated_length": 987.0, "entropy": 0.06403050385415554, "epoch": 1.5918367346938775, "frac_reward_zero_std": 0.0, "grad_norm": 0.03252089023590088, "learning_rate": 8.428571428571429e-06, "loss": 0.1084, "num_tokens": 5643116.0, "reward": 0.6620949506759644, "reward_std": 0.1941491663455963, "rewards/reward_commands_completeness/mean": 0.13124999403953552, "rewards/reward_commands_completeness/std": 0.05110447108745575, "rewards/reward_commands_correctness/mean": 0.07125000655651093, "rewards/reward_commands_correctness/std": 0.02526526153087616, "rewards/reward_diversity/mean": 0.12272001802921295, "rewards/reward_diversity/std": 0.0954846441745758, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.027049340307712555, "rewards/reward_solution_effectiveness/mean": 0.1706250160932541, "rewards/reward_solution_effectiveness/std": 0.08028855174779892, "sampling/importance_sampling_ratio/max": 2.212416887283325, "sampling/importance_sampling_ratio/mean": 0.29232728481292725, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.380084037780762, "sampling/sampling_logp_difference/mean": 0.004125856328755617, "step": 78, "step_time": 296.9687467161566 }, { "clip_ratio/high_max": 0.00044718260323861614, "clip_ratio/high_mean": 0.00044718260323861614, "clip_ratio/low_mean": 0.00023622615117346868, "clip_ratio/low_min": 0.00023622615117346868, "clip_ratio/region_mean": 0.000683408750774106, "completions/clipped_ratio": 0.8125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1345.0, "completions/mean_length": 3515.4375, "completions/mean_terminated_length": 999.6666870117188, "completions/min_length": 795.0, "completions/min_terminated_length": 795.0, "entropy": 0.056172434240579605, "epoch": 1.6122448979591837, "frac_reward_zero_std": 0.0, "grad_norm": 0.020272865891456604, "learning_rate": 8.408163265306122e-06, "loss": -0.0449, "num_tokens": 5723475.0, "reward": 0.5130505561828613, "reward_std": 0.19853252172470093, "rewards/reward_commands_completeness/mean": 0.09000000357627869, "rewards/reward_commands_completeness/std": 0.05163978040218353, "rewards/reward_commands_correctness/mean": 0.04874999821186066, "rewards/reward_commands_correctness/std": 0.030956963077187538, "rewards/reward_diversity/mean": 0.14680053293704987, "rewards/reward_diversity/std": 0.052752844989299774, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.05937499925494194, "rewards/reward_problem_validity/std": 0.0345386266708374, "rewards/reward_solution_effectiveness/mean": 0.09937499463558197, "rewards/reward_solution_effectiveness/std": 0.05767365172505379, "sampling/importance_sampling_ratio/max": 2.1261539459228516, "sampling/importance_sampling_ratio/mean": 0.3367405831813812, "sampling/importance_sampling_ratio/min": 0.0011949505424126983, "sampling/sampling_logp_difference/max": 3.2391715049743652, "sampling/sampling_logp_difference/mean": 0.003759247250854969, "step": 79, "step_time": 275.89399835281074 }, { "clip_ratio/high_max": 0.0007978175635798834, "clip_ratio/high_mean": 0.0007978175635798834, "clip_ratio/low_mean": 9.903584577841684e-05, "clip_ratio/low_min": 9.903584577841684e-05, "clip_ratio/region_mean": 0.0008968534166342579, "completions/clipped_ratio": 0.5, "completions/max_length": 4096.0, "completions/max_terminated_length": 1299.0, "completions/mean_length": 2599.375, "completions/mean_terminated_length": 1102.75, "completions/min_length": 790.0, "completions/min_terminated_length": 790.0, "entropy": 0.06733405170962214, "epoch": 1.6326530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.04753265529870987, "learning_rate": 8.387755102040817e-06, "loss": 0.1485, "num_tokens": 5788669.0, "reward": 0.5233392715454102, "reward_std": 0.2140418291091919, "rewards/reward_commands_completeness/mean": 0.10124999284744263, "rewards/reward_commands_completeness/std": 0.05031567066907883, "rewards/reward_commands_correctness/mean": 0.0481250025331974, "rewards/reward_commands_correctness/std": 0.02713392674922943, "rewards/reward_diversity/mean": 0.11583924293518066, "rewards/reward_diversity/std": 0.10106873512268066, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06499999761581421, "rewards/reward_problem_validity/std": 0.03306559473276138, "rewards/reward_solution_effectiveness/mean": 0.11812499910593033, "rewards/reward_solution_effectiveness/std": 0.05844869837164879, "sampling/importance_sampling_ratio/max": 2.691962957382202, "sampling/importance_sampling_ratio/mean": 0.505861759185791, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.6060869693756104, "sampling/sampling_logp_difference/mean": 0.004711420275270939, "step": 80, "step_time": 262.13977718167007 }, { "clip_ratio/high_max": 0.0007023116340860724, "clip_ratio/high_mean": 0.0007023116340860724, "clip_ratio/low_mean": 0.0005480952968355268, "clip_ratio/low_min": 0.0005480952968355268, "clip_ratio/region_mean": 0.0012504069309215993, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 2260.0, "completions/mean_length": 3230.0625, "completions/mean_terminated_length": 1325.0, "completions/min_length": 879.0, "completions/min_terminated_length": 879.0, "entropy": 0.07669224916025996, "epoch": 1.6530612244897958, "frac_reward_zero_std": 0.0, "grad_norm": 0.0048389434814453125, "learning_rate": 8.36734693877551e-06, "loss": 0.0039, "num_tokens": 5861586.0, "reward": 0.5822635293006897, "reward_std": 0.16302241384983063, "rewards/reward_commands_completeness/mean": 0.10375000536441803, "rewards/reward_commands_completeness/std": 0.049648769199848175, "rewards/reward_commands_correctness/mean": 0.054999999701976776, "rewards/reward_commands_correctness/std": 0.02780887298285961, "rewards/reward_diversity/mean": 0.1603885143995285, "rewards/reward_diversity/std": 0.042059022933244705, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.06937499344348907, "rewards/reward_problem_validity/std": 0.029769953340291977, "rewards/reward_solution_effectiveness/mean": 0.11249999701976776, "rewards/reward_solution_effectiveness/std": 0.05848076939582825, "sampling/importance_sampling_ratio/max": 2.2502529621124268, "sampling/importance_sampling_ratio/mean": 0.21330654621124268, "sampling/importance_sampling_ratio/min": 0.00012683858221862465, "sampling/sampling_logp_difference/max": 5.345641613006592, "sampling/sampling_logp_difference/mean": 0.004568898119032383, "step": 81, "step_time": 273.21777475625277 }, { "clip_ratio/high_max": 0.0008291215417557396, "clip_ratio/high_mean": 0.0008291215417557396, "clip_ratio/low_mean": 0.00034485487049096264, "clip_ratio/low_min": 0.00034485487049096264, "clip_ratio/region_mean": 0.0011739764086087234, "completions/clipped_ratio": 0.625, "completions/max_length": 4096.0, "completions/max_terminated_length": 2923.0, "completions/mean_length": 3143.625, "completions/mean_terminated_length": 1556.3333740234375, "completions/min_length": 816.0, "completions/min_terminated_length": 816.0, "entropy": 0.08722012210637331, "epoch": 1.6734693877551021, "frac_reward_zero_std": 0.0, "grad_norm": 0.029118521139025688, "learning_rate": 8.346938775510205e-06, "loss": 0.1344, "num_tokens": 5942384.0, "reward": 0.5952996015548706, "reward_std": 0.09199190139770508, "rewards/reward_commands_completeness/mean": 0.09874999523162842, "rewards/reward_commands_completeness/std": 0.03304038196802139, "rewards/reward_commands_correctness/mean": 0.054999999701976776, "rewards/reward_commands_correctness/std": 0.019321836531162262, "rewards/reward_diversity/mean": 0.1671745628118515, "rewards/reward_diversity/std": 0.0241295974701643, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.019137661904096603, "rewards/reward_solution_effectiveness/mean": 0.10125000029802322, "rewards/reward_solution_effectiveness/std": 0.04364630952477455, "sampling/importance_sampling_ratio/max": 2.6734635829925537, "sampling/importance_sampling_ratio/mean": 0.5045737028121948, "sampling/importance_sampling_ratio/min": 2.4214219592977315e-05, "sampling/sampling_logp_difference/max": 3.0361111164093018, "sampling/sampling_logp_difference/mean": 0.0051016914658248425, "step": 82, "step_time": 324.1210765466094 }, { "clip_ratio/high_max": 0.0003509521484375, "clip_ratio/high_mean": 0.0003509521484375, "clip_ratio/low_mean": 0.0002593994140625, "clip_ratio/low_min": 0.0002593994140625, "clip_ratio/region_mean": 0.0006103515625, "completions/clipped_ratio": 1.0, "completions/max_length": 4096.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 4096.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 4096.0, "completions/min_terminated_length": 0.0, "entropy": 0.0348891222383827, "epoch": 1.693877551020408, "frac_reward_zero_std": 0.0, "grad_norm": 0.007231777999550104, "learning_rate": 8.326530612244899e-06, "loss": -0.0654, "num_tokens": 6038524.0, "reward": 0.5577186346054077, "reward_std": 0.22991439700126648, "rewards/reward_commands_completeness/mean": 0.10624999552965164, "rewards/reward_commands_completeness/std": 0.05920303612947464, "rewards/reward_commands_correctness/mean": 0.04874999821186066, "rewards/reward_commands_correctness/std": 0.02604483626782894, "rewards/reward_diversity/mean": 0.13896861672401428, "rewards/reward_diversity/std": 0.06199067458510399, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06875000149011612, "rewards/reward_problem_validity/std": 0.03518996387720108, "rewards/reward_solution_effectiveness/mean": 0.11999999731779099, "rewards/reward_solution_effectiveness/std": 0.07589466869831085, "sampling/importance_sampling_ratio/max": 0.6328381299972534, "sampling/importance_sampling_ratio/mean": 0.12428495287895203, "sampling/importance_sampling_ratio/min": 0.0012292598839849234, "sampling/sampling_logp_difference/max": 3.140415906906128, "sampling/sampling_logp_difference/mean": 0.002999559510499239, "step": 83, "step_time": 288.794145969674 }, { "clip_ratio/high_max": 0.0006513619446195662, "clip_ratio/high_mean": 0.0006513619446195662, "clip_ratio/low_mean": 0.00028940193442394957, "clip_ratio/low_min": 0.00028940193442394957, "clip_ratio/region_mean": 0.0009407638790435158, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 3772.0, "completions/mean_length": 3570.875, "completions/mean_terminated_length": 2415.60009765625, "completions/min_length": 851.0, "completions/min_terminated_length": 851.0, "entropy": 0.07817555870860815, "epoch": 1.7142857142857144, "frac_reward_zero_std": 0.0, "grad_norm": 0.014831035397946835, "learning_rate": 8.306122448979592e-06, "loss": 0.0706, "num_tokens": 6125726.0, "reward": 0.7276594638824463, "reward_std": 0.09796900302171707, "rewards/reward_commands_completeness/mean": 0.13374999165534973, "rewards/reward_commands_completeness/std": 0.03774917498230934, "rewards/reward_commands_correctness/mean": 0.07500000298023224, "rewards/reward_commands_correctness/std": 0.01095445267856121, "rewards/reward_diversity/mean": 0.16828443109989166, "rewards/reward_diversity/std": 0.02071390673518181, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.16499999165534973, "rewards/reward_solution_effectiveness/std": 0.06663332879543304, "sampling/importance_sampling_ratio/max": 2.5889651775360107, "sampling/importance_sampling_ratio/mean": 0.24506041407585144, "sampling/importance_sampling_ratio/min": 0.00020322285126894712, "sampling/sampling_logp_difference/max": 3.6512115001678467, "sampling/sampling_logp_difference/mean": 0.004912790842354298, "step": 84, "step_time": 323.3765970040113 }, { "clip_ratio/high_max": 0.0010538409551372752, "clip_ratio/high_mean": 0.0010538409551372752, "clip_ratio/low_mean": 0.0003226937842555344, "clip_ratio/low_min": 0.0003226937842555344, "clip_ratio/region_mean": 0.0013765347393928096, "completions/clipped_ratio": 0.5625, "completions/max_length": 4096.0, "completions/max_terminated_length": 2878.0, "completions/mean_length": 2890.4375, "completions/mean_terminated_length": 1340.4285888671875, "completions/min_length": 763.0, "completions/min_terminated_length": 763.0, "entropy": 0.10758195538073778, "epoch": 1.7346938775510203, "frac_reward_zero_std": 0.0, "grad_norm": 0.025073185563087463, "learning_rate": 8.285714285714287e-06, "loss": 0.0324, "num_tokens": 6195025.0, "reward": 0.6351572871208191, "reward_std": 0.12171205878257751, "rewards/reward_commands_completeness/mean": 0.11249999701976776, "rewards/reward_commands_completeness/std": 0.03924283757805824, "rewards/reward_commands_correctness/mean": 0.06812500208616257, "rewards/reward_commands_correctness/std": 0.02257395349442959, "rewards/reward_diversity/mean": 0.16015729308128357, "rewards/reward_diversity/std": 0.037622544914484024, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0768749937415123, "rewards/reward_problem_validity/std": 0.01922455057501793, "rewards/reward_solution_effectiveness/mean": 0.1237500011920929, "rewards/reward_solution_effectiveness/std": 0.06375735253095627, "sampling/importance_sampling_ratio/max": 2.792555093765259, "sampling/importance_sampling_ratio/mean": 0.5169107913970947, "sampling/importance_sampling_ratio/min": 9.762388799572363e-05, "sampling/sampling_logp_difference/max": 5.220487594604492, "sampling/sampling_logp_difference/mean": 0.006469461601227522, "step": 85, "step_time": 284.96621115878224 }, { "clip_ratio/high_max": 0.0005238394078332931, "clip_ratio/high_mean": 0.0005238394078332931, "clip_ratio/low_mean": 0.000244140625, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.0007679800328332931, "completions/clipped_ratio": 0.75, "completions/max_length": 4096.0, "completions/max_terminated_length": 2743.0, "completions/mean_length": 3542.0625, "completions/mean_terminated_length": 1880.25, "completions/min_length": 904.0, "completions/min_terminated_length": 904.0, "entropy": 0.08146404568105936, "epoch": 1.7551020408163265, "frac_reward_zero_std": 0.0, "grad_norm": 0.0033594942651689053, "learning_rate": 8.26530612244898e-06, "loss": -0.0158, "num_tokens": 6275310.0, "reward": 0.6493115425109863, "reward_std": 0.04973287135362625, "rewards/reward_commands_completeness/mean": 0.11874999850988388, "rewards/reward_commands_completeness/std": 0.022472204640507698, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.01778342016041279, "rewards/reward_diversity/mean": 0.15868651866912842, "rewards/reward_diversity/std": 0.028390493243932724, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08312499523162842, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.12562501430511475, "rewards/reward_solution_effectiveness/std": 0.042734649032354355, "sampling/importance_sampling_ratio/max": 0.4950502812862396, "sampling/importance_sampling_ratio/mean": 0.06587924063205719, "sampling/importance_sampling_ratio/min": 0.0001332793472101912, "sampling/sampling_logp_difference/max": 4.005795001983643, "sampling/sampling_logp_difference/mean": 0.005069937091320753, "step": 86, "step_time": 294.0310796108097 }, { "clip_ratio/high_max": 0.0005444007874757517, "clip_ratio/high_mean": 0.0005444007874757517, "clip_ratio/low_mean": 0.0005710121749871178, "clip_ratio/low_min": 0.0005710121749871178, "clip_ratio/region_mean": 0.0011154129460919648, "completions/clipped_ratio": 0.625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1322.0, "completions/mean_length": 2947.125, "completions/mean_terminated_length": 1032.3333740234375, "completions/min_length": 726.0, "completions/min_terminated_length": 726.0, "entropy": 0.07005228684283793, "epoch": 1.7755102040816326, "frac_reward_zero_std": 0.0, "grad_norm": 0.006664608605206013, "learning_rate": 8.244897959183674e-06, "loss": 0.0087, "num_tokens": 6342564.0, "reward": 0.5395588278770447, "reward_std": 0.1557241976261139, "rewards/reward_commands_completeness/mean": 0.10249999910593033, "rewards/reward_commands_completeness/std": 0.05000000074505806, "rewards/reward_commands_correctness/mean": 0.045625001192092896, "rewards/reward_commands_correctness/std": 0.027317577973008156, "rewards/reward_diversity/mean": 0.15080882608890533, "rewards/reward_diversity/std": 0.06869422644376755, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.0625, "rewards/reward_problem_validity/std": 0.033366650342941284, "rewards/reward_solution_effectiveness/mean": 0.10312500596046448, "rewards/reward_solution_effectiveness/std": 0.053630683571100235, "sampling/importance_sampling_ratio/max": 0.5715231895446777, "sampling/importance_sampling_ratio/mean": 0.08670490235090256, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.515517234802246, "sampling/sampling_logp_difference/mean": 0.0052073742263019085, "step": 87, "step_time": 261.4219650346786 }, { "clip_ratio/high_max": 0.0007436767664330546, "clip_ratio/high_mean": 0.0007436767664330546, "clip_ratio/low_mean": 0.0001426280887244502, "clip_ratio/low_min": 0.0001426280887244502, "clip_ratio/region_mean": 0.0008863048460625578, "completions/clipped_ratio": 0.5, "completions/max_length": 4096.0, "completions/max_terminated_length": 2984.0, "completions/mean_length": 2387.75, "completions/mean_terminated_length": 1355.625, "completions/min_length": 884.0, "completions/min_terminated_length": 884.0, "entropy": 0.07279730588197708, "epoch": 1.7959183673469388, "frac_reward_zero_std": 0.0, "grad_norm": 0.01897869072854519, "learning_rate": 8.224489795918369e-06, "loss": 0.0578, "num_tokens": 6424168.0, "reward": 0.5727201700210571, "reward_std": 0.1892101913690567, "rewards/reward_commands_completeness/mean": 0.09624999761581421, "rewards/reward_commands_completeness/std": 0.04514790698885918, "rewards/reward_commands_correctness/mean": 0.0625, "rewards/reward_commands_correctness/std": 0.022360682487487793, "rewards/reward_diversity/mean": 0.14959518611431122, "rewards/reward_diversity/std": 0.04280656948685646, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07187500596046448, "rewards/reward_problem_validity/std": 0.028099527582526207, "rewards/reward_solution_effectiveness/mean": 0.10500000417232513, "rewards/reward_solution_effectiveness/std": 0.06387487798929214, "sampling/importance_sampling_ratio/max": 1.699948787689209, "sampling/importance_sampling_ratio/mean": 0.3139578402042389, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.0987229347229, "sampling/sampling_logp_difference/mean": 0.005519349593669176, "step": 88, "step_time": 398.6606902331114 }, { "clip_ratio/high_max": 0.0004904243232886074, "clip_ratio/high_mean": 0.0004904243232886074, "clip_ratio/low_mean": 0.00020434401267266367, "clip_ratio/low_min": 0.00020434401267266367, "clip_ratio/region_mean": 0.0006947683323232923, "completions/clipped_ratio": 0.8125, "completions/max_length": 4096.0, "completions/max_terminated_length": 3533.0, "completions/mean_length": 3676.375, "completions/mean_terminated_length": 1858.0, "completions/min_length": 889.0, "completions/min_terminated_length": 889.0, "entropy": 0.04487663647159934, "epoch": 1.816326530612245, "frac_reward_zero_std": 0.0, "grad_norm": 0.022983809933066368, "learning_rate": 8.204081632653062e-06, "loss": -0.0403, "num_tokens": 6509358.0, "reward": 0.5641312599182129, "reward_std": 0.24235489964485168, "rewards/reward_commands_completeness/mean": 0.10374999791383743, "rewards/reward_commands_completeness/std": 0.05475704371929169, "rewards/reward_commands_correctness/mean": 0.05687499791383743, "rewards/reward_commands_correctness/std": 0.03048907034099102, "rewards/reward_diversity/mean": 0.13600629568099976, "rewards/reward_diversity/std": 0.053292520344257355, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.06624999642372131, "rewards/reward_problem_validity/std": 0.030083218589425087, "rewards/reward_solution_effectiveness/mean": 0.11999999731779099, "rewards/reward_solution_effectiveness/std": 0.08049844950437546, "sampling/importance_sampling_ratio/max": 2.7982888221740723, "sampling/importance_sampling_ratio/mean": 0.2070268988609314, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.077340126037598, "sampling/sampling_logp_difference/mean": 0.004004518501460552, "step": 89, "step_time": 281.06603773497045 }, { "clip_ratio/high_max": 0.0010677793052309426, "clip_ratio/high_mean": 0.0010677793052309426, "clip_ratio/low_mean": 0.00028939649564563297, "clip_ratio/low_min": 0.00028939649564563297, "clip_ratio/region_mean": 0.0013571757972385967, "completions/clipped_ratio": 0.625, "completions/max_length": 4096.0, "completions/max_terminated_length": 2483.0, "completions/mean_length": 2642.5, "completions/mean_terminated_length": 1422.0, "completions/min_length": 876.0, "completions/min_terminated_length": 876.0, "entropy": 0.06643947819247842, "epoch": 1.836734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.021896682679653168, "learning_rate": 8.183673469387757e-06, "loss": 0.0453, "num_tokens": 6595390.0, "reward": 0.5939116477966309, "reward_std": 0.156762033700943, "rewards/reward_commands_completeness/mean": 0.11124999821186066, "rewards/reward_commands_completeness/std": 0.04318564757704735, "rewards/reward_commands_correctness/mean": 0.05874999612569809, "rewards/reward_commands_correctness/std": 0.020615529268980026, "rewards/reward_diversity/mean": 0.13641169667243958, "rewards/reward_diversity/std": 0.09797324985265732, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07625000178813934, "rewards/reward_problem_validity/std": 0.026299558579921722, "rewards/reward_solution_effectiveness/mean": 0.1237499937415123, "rewards/reward_solution_effectiveness/std": 0.05352569743990898, "sampling/importance_sampling_ratio/max": 1.7970669269561768, "sampling/importance_sampling_ratio/mean": 0.219811350107193, "sampling/importance_sampling_ratio/min": 1.1213120160391554e-05, "sampling/sampling_logp_difference/max": 5.368621826171875, "sampling/sampling_logp_difference/mean": 0.005665035452693701, "step": 90, "step_time": 405.54748310521245 }, { "clip_ratio/high_max": 0.0004535426414804533, "clip_ratio/high_mean": 0.0004535426414804533, "clip_ratio/low_mean": 0.00043867881322512403, "clip_ratio/low_min": 0.00043867881322512403, "clip_ratio/region_mean": 0.000892221461981535, "completions/clipped_ratio": 0.6875, "completions/max_length": 4096.0, "completions/max_terminated_length": 3085.0, "completions/mean_length": 3342.0625, "completions/mean_terminated_length": 1683.4000244140625, "completions/min_length": 1003.0, "completions/min_terminated_length": 1003.0, "entropy": 0.057746287900954485, "epoch": 1.8571428571428572, "frac_reward_zero_std": 0.0, "grad_norm": 0.0050527919083833694, "learning_rate": 8.16326530612245e-06, "loss": 0.0111, "num_tokens": 6682015.0, "reward": 0.6782625913619995, "reward_std": 0.11476945877075195, "rewards/reward_commands_completeness/mean": 0.11625000089406967, "rewards/reward_commands_completeness/std": 0.0408044159412384, "rewards/reward_commands_correctness/mean": 0.07187500596046448, "rewards/reward_commands_correctness/std": 0.020726393908262253, "rewards/reward_diversity/mean": 0.16076260805130005, "rewards/reward_diversity/std": 0.027744991704821587, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.005163980647921562, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.07201562076807022, "sampling/importance_sampling_ratio/max": 1.7059260606765747, "sampling/importance_sampling_ratio/mean": 0.17362377047538757, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.403493881225586, "sampling/sampling_logp_difference/mean": 0.004758185241371393, "step": 91, "step_time": 330.07668631337583 }, { "clip_ratio/high_max": 0.0009332406880275812, "clip_ratio/high_mean": 0.0009332406880275812, "clip_ratio/low_mean": 0.0003920886338164564, "clip_ratio/low_min": 0.0003920886338164564, "clip_ratio/region_mean": 0.0013253293473098893, "completions/clipped_ratio": 0.5, "completions/max_length": 4096.0, "completions/max_terminated_length": 1254.0, "completions/mean_length": 2574.4375, "completions/mean_terminated_length": 1052.875, "completions/min_length": 715.0, "completions/min_terminated_length": 715.0, "entropy": 0.09162968304008245, "epoch": 1.8775510204081631, "frac_reward_zero_std": 0.0, "grad_norm": 0.041289906948804855, "learning_rate": 8.142857142857143e-06, "loss": 0.1576, "num_tokens": 6751074.0, "reward": 0.7057737112045288, "reward_std": 0.1550482213497162, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.04187282547354698, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.021592825651168823, "rewards/reward_diversity/mean": 0.1613987684249878, "rewards/reward_diversity/std": 0.04063490778207779, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.07325468957424164, "sampling/importance_sampling_ratio/max": 1.4967355728149414, "sampling/importance_sampling_ratio/mean": 0.2399197518825531, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.874111175537109, "sampling/sampling_logp_difference/mean": 0.006041223648935556, "step": 92, "step_time": 299.9226195849478 }, { "clip_ratio/high_max": 0.0005883581761736423, "clip_ratio/high_mean": 0.0005883581761736423, "clip_ratio/low_mean": 0.00035722496431844775, "clip_ratio/low_min": 0.00035722496431844775, "clip_ratio/region_mean": 0.0009455831386731006, "completions/clipped_ratio": 0.5625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1271.0, "completions/mean_length": 2766.6875, "completions/mean_terminated_length": 1057.571533203125, "completions/min_length": 818.0, "completions/min_terminated_length": 818.0, "entropy": 0.07063361955806613, "epoch": 1.8979591836734695, "frac_reward_zero_std": 0.0, "grad_norm": 0.015435873530805111, "learning_rate": 8.122448979591837e-06, "loss": -0.0075, "num_tokens": 6827541.0, "reward": 0.6165577173233032, "reward_std": 0.22690072655677795, "rewards/reward_commands_completeness/mean": 0.11499999463558197, "rewards/reward_commands_completeness/std": 0.05341660603880882, "rewards/reward_commands_correctness/mean": 0.05562499910593033, "rewards/reward_commands_correctness/std": 0.02682505175471306, "rewards/reward_diversity/mean": 0.1496826857328415, "rewards/reward_diversity/std": 0.06613495945930481, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07249999791383743, "rewards/reward_problem_validity/std": 0.03130495548248291, "rewards/reward_solution_effectiveness/mean": 0.14250001311302185, "rewards/reward_solution_effectiveness/std": 0.08306624740362167, "sampling/importance_sampling_ratio/max": 2.964853048324585, "sampling/importance_sampling_ratio/mean": 0.31923070549964905, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.397058010101318, "sampling/sampling_logp_difference/mean": 0.005486533511430025, "step": 93, "step_time": 294.95395783521235 }, { "clip_ratio/high_max": 0.0011504505910124863, "clip_ratio/high_mean": 0.0011504505910124863, "clip_ratio/low_mean": 0.00042019844477181323, "clip_ratio/low_min": 0.00042019844477181323, "clip_ratio/region_mean": 0.0015706490376032889, "completions/clipped_ratio": 0.3125, "completions/max_length": 4096.0, "completions/max_terminated_length": 2185.0, "completions/mean_length": 2152.75, "completions/mean_terminated_length": 1269.45458984375, "completions/min_length": 1047.0, "completions/min_terminated_length": 1047.0, "entropy": 0.08743211906403303, "epoch": 1.9183673469387754, "frac_reward_zero_std": 0.0, "grad_norm": 0.030348658561706543, "learning_rate": 8.102040816326532e-06, "loss": 0.0256, "num_tokens": 6897825.0, "reward": 0.6054720878601074, "reward_std": 0.18246479332447052, "rewards/reward_commands_completeness/mean": 0.10999999940395355, "rewards/reward_commands_completeness/std": 0.051639776676893234, "rewards/reward_commands_correctness/mean": 0.05687500163912773, "rewards/reward_commands_correctness/std": 0.028217902407050133, "rewards/reward_diversity/mean": 0.15984711050987244, "rewards/reward_diversity/std": 0.056493259966373444, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07187500596046448, "rewards/reward_problem_validity/std": 0.03103090077638626, "rewards/reward_solution_effectiveness/mean": 0.12562501430511475, "rewards/reward_solution_effectiveness/std": 0.06772186607122421, "sampling/importance_sampling_ratio/max": 1.7591813802719116, "sampling/importance_sampling_ratio/mean": 0.3960864245891571, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.019026756286621, "sampling/sampling_logp_difference/mean": 0.005976421758532524, "step": 94, "step_time": 304.4661788903177 }, { "clip_ratio/high_max": 0.0008176477367669577, "clip_ratio/high_mean": 0.0008176477367669577, "clip_ratio/low_mean": 0.00031928761018207297, "clip_ratio/low_min": 0.00031928761018207297, "clip_ratio/region_mean": 0.0011369353524059989, "completions/clipped_ratio": 0.375, "completions/max_length": 4096.0, "completions/max_terminated_length": 2333.0, "completions/mean_length": 2070.25, "completions/mean_terminated_length": 1215.4000244140625, "completions/min_length": 707.0, "completions/min_terminated_length": 707.0, "entropy": 0.08488424820825458, "epoch": 1.9387755102040818, "frac_reward_zero_std": 0.0, "grad_norm": 0.019794953987002373, "learning_rate": 8.081632653061225e-06, "loss": 0.0436, "num_tokens": 6979909.0, "reward": 0.5817437767982483, "reward_std": 0.2495093196630478, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.05840947479009628, "rewards/reward_commands_correctness/mean": 0.054999999701976776, "rewards/reward_commands_correctness/std": 0.029664795845746994, "rewards/reward_diversity/mean": 0.14611877501010895, "rewards/reward_diversity/std": 0.0655924528837204, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06750000268220901, "rewards/reward_problem_validity/std": 0.03454466164112091, "rewards/reward_solution_effectiveness/mean": 0.12937501072883606, "rewards/reward_solution_effectiveness/std": 0.07243099808692932, "sampling/importance_sampling_ratio/max": 1.811021089553833, "sampling/importance_sampling_ratio/mean": 0.14577345550060272, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.0428290367126465, "sampling/sampling_logp_difference/mean": 0.006551974918693304, "step": 95, "step_time": 393.4578711614013 }, { "clip_ratio/high_max": 0.0009958224836736917, "clip_ratio/high_mean": 0.0009958224836736917, "clip_ratio/low_mean": 0.00029663340319530107, "clip_ratio/low_min": 0.00029663340319530107, "clip_ratio/region_mean": 0.001292455883231014, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4057.0, "completions/mean_length": 1986.125, "completions/mean_terminated_length": 1499.2308349609375, "completions/min_length": 804.0, "completions/min_terminated_length": 804.0, "entropy": 0.0905060120858252, "epoch": 1.9591836734693877, "frac_reward_zero_std": 0.0, "grad_norm": 0.024213355034589767, "learning_rate": 8.06122448979592e-06, "loss": 0.0295, "num_tokens": 7032319.0, "reward": 0.6493595838546753, "reward_std": 0.1187785267829895, "rewards/reward_commands_completeness/mean": 0.1274999976158142, "rewards/reward_commands_completeness/std": 0.047258161008358, "rewards/reward_commands_correctness/mean": 0.07250000536441803, "rewards/reward_commands_correctness/std": 0.02113449200987816, "rewards/reward_diversity/mean": 0.13060957193374634, "rewards/reward_diversity/std": 0.09926006197929382, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.018929695710539818, "rewards/reward_solution_effectiveness/mean": 0.14625000953674316, "rewards/reward_solution_effectiveness/std": 0.06469157338142395, "sampling/importance_sampling_ratio/max": 1.5593173503875732, "sampling/importance_sampling_ratio/mean": 0.4006228744983673, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.887530326843262, "sampling/sampling_logp_difference/mean": 0.006687031127512455, "step": 96, "step_time": 274.4686618875712 }, { "clip_ratio/high_max": 0.0006090926926844986, "clip_ratio/high_mean": 0.0006090926926844986, "clip_ratio/low_mean": 0.0003191844007233158, "clip_ratio/low_min": 0.0003191844007233158, "clip_ratio/region_mean": 0.0009282770970457932, "completions/clipped_ratio": 0.25, "completions/max_length": 4096.0, "completions/max_terminated_length": 1381.0, "completions/mean_length": 1916.375, "completions/mean_terminated_length": 1189.8333740234375, "completions/min_length": 928.0, "completions/min_terminated_length": 928.0, "entropy": 0.0885073496028781, "epoch": 1.9795918367346939, "frac_reward_zero_std": 0.0, "grad_norm": 0.0823521837592125, "learning_rate": 8.040816326530613e-06, "loss": -0.2441, "num_tokens": 7086557.0, "reward": 0.6950962543487549, "reward_std": 0.1103619709610939, "rewards/reward_commands_completeness/mean": 0.12999999523162842, "rewards/reward_commands_completeness/std": 0.04676181077957153, "rewards/reward_commands_correctness/mean": 0.06624999642372131, "rewards/reward_commands_correctness/std": 0.02093641273677349, "rewards/reward_diversity/mean": 0.16634628176689148, "rewards/reward_diversity/std": 0.02207527868449688, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.019137661904096603, "rewards/reward_solution_effectiveness/mean": 0.15937501192092896, "rewards/reward_solution_effectiveness/std": 0.0832240954041481, "sampling/importance_sampling_ratio/max": 2.9823861122131348, "sampling/importance_sampling_ratio/mean": 0.6207504868507385, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.160210609436035, "sampling/sampling_logp_difference/mean": 0.006161704193800688, "step": 97, "step_time": 287.9140160474926 }, { "clip_ratio/high_max": 0.0006706792810291518, "clip_ratio/high_mean": 0.0006706792810291518, "clip_ratio/low_mean": 0.0004907059319521068, "clip_ratio/low_min": 0.0004907059319521068, "clip_ratio/region_mean": 0.0011613851911533857, "completions/clipped_ratio": 0.375, "completions/max_length": 4096.0, "completions/max_terminated_length": 3659.0, "completions/mean_length": 2317.625, "completions/mean_terminated_length": 1250.5999755859375, "completions/min_length": 724.0, "completions/min_terminated_length": 724.0, "entropy": 0.10600731987506151, "epoch": 2.0, "frac_reward_zero_std": 0.0, "grad_norm": 0.01857270672917366, "learning_rate": 8.020408163265306e-06, "loss": 0.0105, "num_tokens": 7145107.0, "reward": 0.7017594575881958, "reward_std": 0.10908284783363342, "rewards/reward_commands_completeness/mean": 0.13499999046325684, "rewards/reward_commands_completeness/std": 0.048716869205236435, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.025552235543727875, "rewards/reward_diversity/mean": 0.1667594313621521, "rewards/reward_diversity/std": 0.04468925669789314, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.027195284143090248, "rewards/reward_solution_effectiveness/mean": 0.16875001788139343, "rewards/reward_solution_effectiveness/std": 0.0717286542057991, "sampling/importance_sampling_ratio/max": 2.3610494136810303, "sampling/importance_sampling_ratio/mean": 0.34209996461868286, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.203127384185791, "sampling/sampling_logp_difference/mean": 0.007213914766907692, "step": 98, "step_time": 304.57390940748155 }, { "clip_ratio/high_max": 0.0013002635787415784, "clip_ratio/high_mean": 0.0013002635787415784, "clip_ratio/low_mean": 0.00035834626760333776, "clip_ratio/low_min": 0.00035834626760333776, "clip_ratio/region_mean": 0.0016586098172410857, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1577.0, "completions/mean_length": 1711.75, "completions/mean_terminated_length": 1161.5384521484375, "completions/min_length": 700.0, "completions/min_terminated_length": 700.0, "entropy": 0.16005569137632847, "epoch": 2.020408163265306, "frac_reward_zero_std": 0.25, "grad_norm": 0.08997897803783417, "learning_rate": 8.000000000000001e-06, "loss": 0.1294, "num_tokens": 7195775.0, "reward": 0.4578756093978882, "reward_std": 0.1642255187034607, "rewards/reward_commands_completeness/mean": 0.08499999344348907, "rewards/reward_commands_completeness/std": 0.055856965482234955, "rewards/reward_commands_correctness/mean": 0.041875001043081284, "rewards/reward_commands_correctness/std": 0.029713913798332214, "rewards/reward_diversity/mean": 0.1072506308555603, "rewards/reward_diversity/std": 0.06574707478284836, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.05812499672174454, "rewards/reward_problem_validity/std": 0.03868139535188675, "rewards/reward_solution_effectiveness/mean": 0.10312499850988388, "rewards/reward_solution_effectiveness/std": 0.06925496459007263, "sampling/importance_sampling_ratio/max": 2.762000560760498, "sampling/importance_sampling_ratio/mean": 0.7385408878326416, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.939435005187988, "sampling/sampling_logp_difference/mean": 0.009341382421553135, "step": 99, "step_time": 258.37574983015656 }, { "clip_ratio/high_max": 0.0010414136449981015, "clip_ratio/high_mean": 0.0010414136449981015, "clip_ratio/low_mean": 0.0003963466915593017, "clip_ratio/low_min": 0.0003963466915593017, "clip_ratio/region_mean": 0.0014377603365574032, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 2707.0, "completions/mean_length": 1759.3125, "completions/mean_terminated_length": 1220.0770263671875, "completions/min_length": 885.0, "completions/min_terminated_length": 885.0, "entropy": 0.10476146638393402, "epoch": 2.0408163265306123, "frac_reward_zero_std": 0.0, "grad_norm": 0.02623097226023674, "learning_rate": 7.979591836734695e-06, "loss": -0.1508, "num_tokens": 7256912.0, "reward": 0.6619293689727783, "reward_std": 0.1795407086610794, "rewards/reward_commands_completeness/mean": 0.12124999612569809, "rewards/reward_commands_completeness/std": 0.04978286474943161, "rewards/reward_commands_correctness/mean": 0.054999999701976776, "rewards/reward_commands_correctness/std": 0.02250925824046135, "rewards/reward_diversity/mean": 0.1844293475151062, "rewards/reward_diversity/std": 0.015752309933304787, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.02651257812976837, "rewards/reward_solution_effectiveness/mean": 0.1368750035762787, "rewards/reward_solution_effectiveness/std": 0.06477846950292587, "sampling/importance_sampling_ratio/max": 1.1294270753860474, "sampling/importance_sampling_ratio/mean": 0.4157540202140808, "sampling/importance_sampling_ratio/min": 1.5003810403868556e-05, "sampling/sampling_logp_difference/max": 6.590065002441406, "sampling/sampling_logp_difference/mean": 0.007571840658783913, "step": 100, "step_time": 316.6972904205322 }, { "clip_ratio/high_max": 0.00054750432173023, "clip_ratio/high_mean": 0.00054750432173023, "clip_ratio/low_mean": 0.0002269865399284754, "clip_ratio/low_min": 0.0002269865399284754, "clip_ratio/region_mean": 0.0007744908580207266, "completions/clipped_ratio": 0.25, "completions/max_length": 4096.0, "completions/max_terminated_length": 1385.0, "completions/mean_length": 1880.5625, "completions/mean_terminated_length": 1142.0833740234375, "completions/min_length": 976.0, "completions/min_terminated_length": 976.0, "entropy": 0.08300714753568172, "epoch": 2.061224489795918, "frac_reward_zero_std": 0.0, "grad_norm": 0.0200638547539711, "learning_rate": 7.959183673469388e-06, "loss": 0.0185, "num_tokens": 7312093.0, "reward": 0.5786556601524353, "reward_std": 0.11975744366645813, "rewards/reward_commands_completeness/mean": 0.10499998927116394, "rewards/reward_commands_completeness/std": 0.051380932331085205, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.02827690728008747, "rewards/reward_diversity/mean": 0.12490566819906235, "rewards/reward_diversity/std": 0.054909732192754745, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07187500596046448, "rewards/reward_problem_validity/std": 0.03103090077638626, "rewards/reward_solution_effectiveness/mean": 0.13125000894069672, "rewards/reward_solution_effectiveness/std": 0.06917369365692139, "sampling/importance_sampling_ratio/max": 1.3541918992996216, "sampling/importance_sampling_ratio/mean": 0.24367383122444153, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.951627731323242, "sampling/sampling_logp_difference/mean": 0.006120084319263697, "step": 101, "step_time": 287.30113172344863 }, { "clip_ratio/high_max": 0.0011647166174952872, "clip_ratio/high_mean": 0.0011647166174952872, "clip_ratio/low_mean": 0.0003883776553266216, "clip_ratio/low_min": 0.0003883776553266216, "clip_ratio/region_mean": 0.0015530942619079724, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1503.0, "completions/mean_length": 1422.3125, "completions/mean_terminated_length": 1082.6923828125, "completions/min_length": 826.0, "completions/min_terminated_length": 826.0, "entropy": 0.10050705773755908, "epoch": 2.0816326530612246, "frac_reward_zero_std": 0.0, "grad_norm": 0.07397327572107315, "learning_rate": 7.938775510204081e-06, "loss": -0.3058, "num_tokens": 7372194.0, "reward": 0.4845450520515442, "reward_std": 0.16057103872299194, "rewards/reward_commands_completeness/mean": 0.08124999701976776, "rewards/reward_commands_completeness/std": 0.0486997626721859, "rewards/reward_commands_correctness/mean": 0.05375000089406967, "rewards/reward_commands_correctness/std": 0.03201562166213989, "rewards/reward_diversity/mean": 0.12704505026340485, "rewards/reward_diversity/std": 0.0627424344420433, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.05999999865889549, "rewards/reward_problem_validity/std": 0.035023801028728485, "rewards/reward_solution_effectiveness/mean": 0.09375, "rewards/reward_solution_effectiveness/std": 0.056789081543684006, "sampling/importance_sampling_ratio/max": 2.855053186416626, "sampling/importance_sampling_ratio/mean": 0.5305854678153992, "sampling/importance_sampling_ratio/min": 0.0001898205664474517, "sampling/sampling_logp_difference/max": 6.681371212005615, "sampling/sampling_logp_difference/mean": 0.007867831736803055, "step": 102, "step_time": 386.08179751969874 }, { "clip_ratio/high_max": 0.0002876723010558635, "clip_ratio/high_mean": 0.0002876723010558635, "clip_ratio/low_mean": 0.000719945566743263, "clip_ratio/low_min": 0.000719945566743263, "clip_ratio/region_mean": 0.001007617869618116, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1238.0, "completions/mean_length": 1391.625, "completions/mean_terminated_length": 1005.2857666015625, "completions/min_length": 723.0, "completions/min_terminated_length": 723.0, "entropy": 0.1134812687523663, "epoch": 2.1020408163265305, "frac_reward_zero_std": 0.0, "grad_norm": 0.0766594335436821, "learning_rate": 7.918367346938776e-06, "loss": -0.1358, "num_tokens": 7420004.0, "reward": 0.6897814273834229, "reward_std": 0.07567985355854034, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.028751812875270844, "rewards/reward_commands_correctness/mean": 0.07000000774860382, "rewards/reward_commands_correctness/std": 0.020330602303147316, "rewards/reward_diversity/mean": 0.17415645718574524, "rewards/reward_diversity/std": 0.01879246160387993, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08374999463558197, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.1368750035762787, "rewards/reward_solution_effectiveness/std": 0.053630683571100235, "sampling/importance_sampling_ratio/max": 2.893746852874756, "sampling/importance_sampling_ratio/mean": 0.9452985525131226, "sampling/importance_sampling_ratio/min": 0.00019187797443009913, "sampling/sampling_logp_difference/max": 5.862636566162109, "sampling/sampling_logp_difference/mean": 0.007532115560024977, "step": 103, "step_time": 284.33758616261184 }, { "clip_ratio/high_max": 0.0013191102880227845, "clip_ratio/high_mean": 0.0013191102880227845, "clip_ratio/low_mean": 0.00036712471046485007, "clip_ratio/low_min": 0.00036712471046485007, "clip_ratio/region_mean": 0.0016862350021256134, "completions/clipped_ratio": 0.0, "completions/max_length": 2809.0, "completions/max_terminated_length": 2809.0, "completions/mean_length": 1281.1875, "completions/mean_terminated_length": 1281.1875, "completions/min_length": 728.0, "completions/min_terminated_length": 728.0, "entropy": 0.12567255552858114, "epoch": 2.122448979591837, "frac_reward_zero_std": 0.0, "grad_norm": 0.09362953901290894, "learning_rate": 7.897959183673471e-06, "loss": 0.0332, "num_tokens": 7461567.0, "reward": 0.6173664331436157, "reward_std": 0.09464040398597717, "rewards/reward_commands_completeness/mean": 0.1262499988079071, "rewards/reward_commands_completeness/std": 0.027049336582422256, "rewards/reward_commands_correctness/mean": 0.06874999403953552, "rewards/reward_commands_correctness/std": 0.020289571955800056, "rewards/reward_diversity/mean": 0.10111647844314575, "rewards/reward_diversity/std": 0.12160677462816238, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08062499761581421, "rewards/reward_problem_validity/std": 0.0118145402520895, "rewards/reward_solution_effectiveness/mean": 0.140625, "rewards/reward_solution_effectiveness/std": 0.05662376061081886, "sampling/importance_sampling_ratio/max": 1.9935510158538818, "sampling/importance_sampling_ratio/mean": 0.7674080729484558, "sampling/importance_sampling_ratio/min": 0.00013108071289025247, "sampling/sampling_logp_difference/max": 7.012292861938477, "sampling/sampling_logp_difference/mean": 0.008153232745826244, "step": 104, "step_time": 213.05051031894982 }, { "clip_ratio/high_max": 0.0009229885508830193, "clip_ratio/high_mean": 0.0009229885508830193, "clip_ratio/low_mean": 0.00079779958105064, "clip_ratio/low_min": 0.00079779958105064, "clip_ratio/region_mean": 0.0017207881246577017, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1609.0, "completions/mean_length": 1511.0, "completions/mean_terminated_length": 1141.71435546875, "completions/min_length": 675.0, "completions/min_terminated_length": 675.0, "entropy": 0.11199420085176826, "epoch": 2.142857142857143, "frac_reward_zero_std": 0.0, "grad_norm": 0.059470999985933304, "learning_rate": 7.877551020408164e-06, "loss": -0.0715, "num_tokens": 7512795.0, "reward": 0.6284329295158386, "reward_std": 0.10925901681184769, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.02918332815170288, "rewards/reward_commands_correctness/mean": 0.06937500089406967, "rewards/reward_commands_correctness/std": 0.010626226663589478, "rewards/reward_diversity/mean": 0.14655792713165283, "rewards/reward_diversity/std": 0.09804502129554749, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08374999463558197, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.12000000476837158, "rewards/reward_solution_effectiveness/std": 0.043817803263664246, "sampling/importance_sampling_ratio/max": 2.3774564266204834, "sampling/importance_sampling_ratio/mean": 0.5037097930908203, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.133633613586426, "sampling/sampling_logp_difference/mean": 0.007419314701110125, "step": 105, "step_time": 307.9808116443455 }, { "clip_ratio/high_max": 0.0014909892051946372, "clip_ratio/high_mean": 0.0014909892051946372, "clip_ratio/low_mean": 0.0007843854909879155, "clip_ratio/low_min": 0.0007843854909879155, "clip_ratio/region_mean": 0.002275374688906595, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 2468.0, "completions/mean_length": 1612.3125, "completions/mean_terminated_length": 1257.5, "completions/min_length": 797.0, "completions/min_terminated_length": 797.0, "entropy": 0.12316694715991616, "epoch": 2.163265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.05563012883067131, "learning_rate": 7.857142857142858e-06, "loss": -0.073, "num_tokens": 7562908.0, "reward": 0.6248592734336853, "reward_std": 0.1508830487728119, "rewards/reward_commands_completeness/mean": 0.10874999314546585, "rewards/reward_commands_completeness/std": 0.04731103032827377, "rewards/reward_commands_correctness/mean": 0.06687499582767487, "rewards/reward_commands_correctness/std": 0.02358495444059372, "rewards/reward_diversity/mean": 0.1548592746257782, "rewards/reward_diversity/std": 0.054109904915094376, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07750000059604645, "rewards/reward_problem_validity/std": 0.02670830301940441, "rewards/reward_solution_effectiveness/mean": 0.12937501072883606, "rewards/reward_solution_effectiveness/std": 0.07567199319601059, "sampling/importance_sampling_ratio/max": 2.1816515922546387, "sampling/importance_sampling_ratio/mean": 0.6898515820503235, "sampling/importance_sampling_ratio/min": 2.498488720448222e-05, "sampling/sampling_logp_difference/max": 7.365365982055664, "sampling/sampling_logp_difference/mean": 0.008491815067827702, "step": 106, "step_time": 287.2746147159487 }, { "clip_ratio/high_max": 0.0009591217094566673, "clip_ratio/high_mean": 0.0009591217094566673, "clip_ratio/low_mean": 0.0005478589755512075, "clip_ratio/low_min": 0.0005478589755512075, "clip_ratio/region_mean": 0.0015069806613610126, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 2069.0, "completions/mean_length": 1502.3125, "completions/mean_terminated_length": 1131.7857666015625, "completions/min_length": 786.0, "completions/min_terminated_length": 786.0, "entropy": 0.10851563373580575, "epoch": 2.183673469387755, "frac_reward_zero_std": 0.0, "grad_norm": 0.03150688484311104, "learning_rate": 7.836734693877551e-06, "loss": 0.0246, "num_tokens": 7606121.0, "reward": 0.6591163873672485, "reward_std": 0.08618032187223434, "rewards/reward_commands_completeness/mean": 0.1237500011920929, "rewards/reward_commands_completeness/std": 0.0344238318502903, "rewards/reward_commands_correctness/mean": 0.07249999791383743, "rewards/reward_commands_correctness/std": 0.021756228059530258, "rewards/reward_diversity/mean": 0.15161636471748352, "rewards/reward_diversity/std": 0.03567955270409584, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08062499761581421, "rewards/reward_problem_validity/std": 0.019482901319861412, "rewards/reward_solution_effectiveness/mean": 0.1368750035762787, "rewards/reward_solution_effectiveness/std": 0.050162237137556076, "sampling/importance_sampling_ratio/max": 1.1984182596206665, "sampling/importance_sampling_ratio/mean": 0.29243630170822144, "sampling/importance_sampling_ratio/min": 4.24759446104872e-06, "sampling/sampling_logp_difference/max": 7.491268157958984, "sampling/sampling_logp_difference/mean": 0.008351000025868416, "step": 107, "step_time": 263.0880236327648 }, { "clip_ratio/high_max": 0.0006431867732317187, "clip_ratio/high_mean": 0.0006431867732317187, "clip_ratio/low_mean": 0.0004921317140542669, "clip_ratio/low_min": 0.0004921317140542669, "clip_ratio/region_mean": 0.001135318489104975, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1513.0, "completions/mean_length": 1671.125, "completions/mean_terminated_length": 1111.5384521484375, "completions/min_length": 792.0, "completions/min_terminated_length": 792.0, "entropy": 0.12323458679020405, "epoch": 2.204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.04224713519215584, "learning_rate": 7.816326530612246e-06, "loss": 0.1566, "num_tokens": 7661899.0, "reward": 0.6430992484092712, "reward_std": 0.05244765430688858, "rewards/reward_commands_completeness/mean": 0.11624999344348907, "rewards/reward_commands_completeness/std": 0.019621416926383972, "rewards/reward_commands_correctness/mean": 0.05937499925494194, "rewards/reward_commands_correctness/std": 0.016918925568461418, "rewards/reward_diversity/mean": 0.16997428238391876, "rewards/reward_diversity/std": 0.028984442353248596, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08312499523162842, "rewards/reward_problem_validity/std": 0.006020799279212952, "rewards/reward_solution_effectiveness/mean": 0.11437500268220901, "rewards/reward_solution_effectiveness/std": 0.03326033800840378, "sampling/importance_sampling_ratio/max": 2.7424187660217285, "sampling/importance_sampling_ratio/mean": 0.6088576912879944, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.940525531768799, "sampling/sampling_logp_difference/mean": 0.008144337683916092, "step": 108, "step_time": 318.4476497322321 }, { "clip_ratio/high_max": 0.0017044886626536027, "clip_ratio/high_mean": 0.0017044886626536027, "clip_ratio/low_mean": 0.00041555775169399567, "clip_ratio/low_min": 0.00041555775169399567, "clip_ratio/region_mean": 0.0021200464070716407, "completions/clipped_ratio": 0.0, "completions/max_length": 1462.0, "completions/max_terminated_length": 1462.0, "completions/mean_length": 1210.3125, "completions/mean_terminated_length": 1210.3125, "completions/min_length": 1070.0, "completions/min_terminated_length": 1070.0, "entropy": 0.11482389457523823, "epoch": 2.2244897959183674, "frac_reward_zero_std": 0.0, "grad_norm": 0.13300004601478577, "learning_rate": 7.79591836734694e-06, "loss": 0.1629, "num_tokens": 7711884.0, "reward": 0.6690410375595093, "reward_std": 0.06767591834068298, "rewards/reward_commands_completeness/mean": 0.12125000357627869, "rewards/reward_commands_completeness/std": 0.028722815215587616, "rewards/reward_commands_correctness/mean": 0.07187499850988388, "rewards/reward_commands_correctness/std": 0.015152009204030037, "rewards/reward_diversity/mean": 0.15716606378555298, "rewards/reward_diversity/std": 0.025906827300786972, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08374999463558197, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.13500000536441803, "rewards/reward_solution_effectiveness/std": 0.043817806988954544, "sampling/importance_sampling_ratio/max": 2.9607434272766113, "sampling/importance_sampling_ratio/mean": 0.9221088290214539, "sampling/importance_sampling_ratio/min": 0.026269113644957542, "sampling/sampling_logp_difference/max": 1.5849623680114746, "sampling/sampling_logp_difference/mean": 0.008368194103240967, "step": 109, "step_time": 165.02090289629996 }, { "clip_ratio/high_max": 0.0013438196765491739, "clip_ratio/high_mean": 0.0013438196765491739, "clip_ratio/low_mean": 0.0003678616412798874, "clip_ratio/low_min": 0.0003678616412798874, "clip_ratio/region_mean": 0.0017116813178290613, "completions/clipped_ratio": 0.3125, "completions/max_length": 4096.0, "completions/max_terminated_length": 3729.0, "completions/mean_length": 2216.3125, "completions/mean_terminated_length": 1361.9091796875, "completions/min_length": 1038.0, "completions/min_terminated_length": 1038.0, "entropy": 0.0845858920365572, "epoch": 2.2448979591836733, "frac_reward_zero_std": 0.0, "grad_norm": 0.03671112656593323, "learning_rate": 7.775510204081632e-06, "loss": -0.0533, "num_tokens": 7792149.0, "reward": 0.7069706916809082, "reward_std": 0.12788981199264526, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.037859391421079636, "rewards/reward_commands_correctness/mean": 0.06812500208616257, "rewards/reward_commands_correctness/std": 0.019737865775823593, "rewards/reward_diversity/mean": 0.17197071015834808, "rewards/reward_diversity/std": 0.04731074348092079, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.15937499701976776, "rewards/reward_solution_effectiveness/std": 0.05446328967809677, "sampling/importance_sampling_ratio/max": 1.4870035648345947, "sampling/importance_sampling_ratio/mean": 0.2935619056224823, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.859932899475098, "sampling/sampling_logp_difference/mean": 0.006861168425530195, "step": 110, "step_time": 414.00961847789586 }, { "clip_ratio/high_max": 0.0012052204438077752, "clip_ratio/high_mean": 0.0012052204438077752, "clip_ratio/low_mean": 0.00040814011299517006, "clip_ratio/low_min": 0.00040814011299517006, "clip_ratio/region_mean": 0.0016133605386130512, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 1378.75, "completions/mean_terminated_length": 1197.60009765625, "completions/min_length": 923.0, "completions/min_terminated_length": 923.0, "entropy": 0.12365362327545881, "epoch": 2.2653061224489797, "frac_reward_zero_std": 0.0, "grad_norm": 0.04858901724219322, "learning_rate": 7.755102040816327e-06, "loss": -0.0698, "num_tokens": 7844285.0, "reward": 0.5474109649658203, "reward_std": 0.20932485163211823, "rewards/reward_commands_completeness/mean": 0.09624999761581421, "rewards/reward_commands_completeness/std": 0.05377422273159027, "rewards/reward_commands_correctness/mean": 0.046875, "rewards/reward_commands_correctness/std": 0.028453178703784943, "rewards/reward_diversity/mean": 0.15803596377372742, "rewards/reward_diversity/std": 0.05511775240302086, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06624999642372131, "rewards/reward_problem_validity/std": 0.033837851136922836, "rewards/reward_solution_effectiveness/mean": 0.10500000417232513, "rewards/reward_solution_effectiveness/std": 0.07099296152591705, "sampling/importance_sampling_ratio/max": 2.677750587463379, "sampling/importance_sampling_ratio/mean": 0.6084073781967163, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.060493469238281, "sampling/sampling_logp_difference/mean": 0.008399488404393196, "step": 111, "step_time": 291.45535585843027 }, { "clip_ratio/high_max": 0.0011127526631753426, "clip_ratio/high_mean": 0.0011127526631753426, "clip_ratio/low_mean": 0.00028251826734049246, "clip_ratio/low_min": 0.00028251826734049246, "clip_ratio/region_mean": 0.0013952709341538139, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1365.0, "completions/mean_length": 1372.875, "completions/mean_terminated_length": 1191.3333740234375, "completions/min_length": 744.0, "completions/min_terminated_length": 744.0, "entropy": 0.11027167038992047, "epoch": 2.2857142857142856, "frac_reward_zero_std": 0.0, "grad_norm": 0.06305532902479172, "learning_rate": 7.73469387755102e-06, "loss": -0.0032, "num_tokens": 7890759.0, "reward": 0.6670430898666382, "reward_std": 0.07192687690258026, "rewards/reward_commands_completeness/mean": 0.1274999976158142, "rewards/reward_commands_completeness/std": 0.033366650342941284, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.011954776011407375, "rewards/reward_diversity/mean": 0.15704306960105896, "rewards/reward_diversity/std": 0.039095211774110794, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437499403953552, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.13500000536441803, "rewards/reward_solution_effectiveness/std": 0.04898979514837265, "sampling/importance_sampling_ratio/max": 2.682359218597412, "sampling/importance_sampling_ratio/mean": 0.4893975853919983, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.140554428100586, "sampling/sampling_logp_difference/mean": 0.00815634522587061, "step": 112, "step_time": 292.12671959400177 }, { "clip_ratio/high_max": 0.0010170819441555068, "clip_ratio/high_mean": 0.0010170819441555068, "clip_ratio/low_mean": 0.00017958673197426833, "clip_ratio/low_min": 0.00017958673197426833, "clip_ratio/region_mean": 0.0011966686834057327, "completions/clipped_ratio": 0.25, "completions/max_length": 4096.0, "completions/max_terminated_length": 3269.0, "completions/mean_length": 2064.8125, "completions/mean_terminated_length": 1387.75, "completions/min_length": 862.0, "completions/min_terminated_length": 862.0, "entropy": 0.11309266835451126, "epoch": 2.306122448979592, "frac_reward_zero_std": 0.0, "grad_norm": 0.05020181089639664, "learning_rate": 7.714285714285716e-06, "loss": -0.1336, "num_tokens": 7949700.0, "reward": 0.5596164464950562, "reward_std": 0.17090614140033722, "rewards/reward_commands_completeness/mean": 0.10374999791383743, "rewards/reward_commands_completeness/std": 0.04209117218852043, "rewards/reward_commands_correctness/mean": 0.051249999552965164, "rewards/reward_commands_correctness/std": 0.02500000037252903, "rewards/reward_diversity/mean": 0.14774146676063538, "rewards/reward_diversity/std": 0.06546130031347275, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.06875000149011612, "rewards/reward_problem_validity/std": 0.029410885646939278, "rewards/reward_solution_effectiveness/mean": 0.10687500238418579, "rewards/reward_solution_effectiveness/std": 0.04377499222755432, "sampling/importance_sampling_ratio/max": 2.567847967147827, "sampling/importance_sampling_ratio/mean": 0.6195127367973328, "sampling/importance_sampling_ratio/min": 5.014699695493618e-07, "sampling/sampling_logp_difference/max": 7.703431606292725, "sampling/sampling_logp_difference/mean": 0.008241436444222927, "step": 113, "step_time": 278.5381249934435 }, { "clip_ratio/high_max": 0.0017408624371455517, "clip_ratio/high_mean": 0.0017408624371455517, "clip_ratio/low_mean": 0.00027589566707320046, "clip_ratio/low_min": 0.00027589566707320046, "clip_ratio/region_mean": 0.0020167581096757203, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 3061.0, "completions/mean_length": 1533.4375, "completions/mean_terminated_length": 1362.60009765625, "completions/min_length": 983.0, "completions/min_terminated_length": 983.0, "entropy": 0.12835211772471666, "epoch": 2.326530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.08867936581373215, "learning_rate": 7.693877551020409e-06, "loss": -0.0018, "num_tokens": 7996711.0, "reward": 0.5758950710296631, "reward_std": 0.21416692435741425, "rewards/reward_commands_completeness/mean": 0.10625000298023224, "rewards/reward_commands_completeness/std": 0.049379486590623856, "rewards/reward_commands_correctness/mean": 0.05562499910593033, "rewards/reward_commands_correctness/std": 0.030103987082839012, "rewards/reward_diversity/mean": 0.1383950561285019, "rewards/reward_diversity/std": 0.06591036915779114, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07062499970197678, "rewards/reward_problem_validity/std": 0.03043435700237751, "rewards/reward_solution_effectiveness/mean": 0.1237500011920929, "rewards/reward_solution_effectiveness/std": 0.06652067601680756, "sampling/importance_sampling_ratio/max": 2.781235933303833, "sampling/importance_sampling_ratio/mean": 0.7159072160720825, "sampling/importance_sampling_ratio/min": 8.978383266367018e-05, "sampling/sampling_logp_difference/max": 6.968435764312744, "sampling/sampling_logp_difference/mean": 0.00987367145717144, "step": 114, "step_time": 269.61815360188484 }, { "clip_ratio/high_max": 0.0016049001205828972, "clip_ratio/high_mean": 0.0016049001205828972, "clip_ratio/low_mean": 0.000240339977608528, "clip_ratio/low_min": 0.000240339977608528, "clip_ratio/region_mean": 0.0018452401054673828, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1635.0, "completions/mean_length": 1505.5, "completions/mean_terminated_length": 1135.4285888671875, "completions/min_length": 856.0, "completions/min_terminated_length": 856.0, "entropy": 0.10802521416917443, "epoch": 2.3469387755102042, "frac_reward_zero_std": 0.0, "grad_norm": 0.042330022901296616, "learning_rate": 7.673469387755102e-06, "loss": 0.0943, "num_tokens": 8049775.0, "reward": 0.6404848694801331, "reward_std": 0.11441751569509506, "rewards/reward_commands_completeness/mean": 0.11874999850988388, "rewards/reward_commands_completeness/std": 0.04224926233291626, "rewards/reward_commands_correctness/mean": 0.05687500163912773, "rewards/reward_commands_correctness/std": 0.022425804287195206, "rewards/reward_diversity/mean": 0.1623598337173462, "rewards/reward_diversity/std": 0.04154520481824875, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.019137661904096603, "rewards/reward_solution_effectiveness/mean": 0.12937499582767487, "rewards/reward_solution_effectiveness/std": 0.06546945869922638, "sampling/importance_sampling_ratio/max": 1.1609864234924316, "sampling/importance_sampling_ratio/mean": 0.339516282081604, "sampling/importance_sampling_ratio/min": 5.6266842875629663e-05, "sampling/sampling_logp_difference/max": 7.17613410949707, "sampling/sampling_logp_difference/mean": 0.009385545738041401, "step": 115, "step_time": 306.6911676041782 }, { "clip_ratio/high_max": 0.0017044933920260519, "clip_ratio/high_mean": 0.0017044933920260519, "clip_ratio/low_mean": 0.000648060751700541, "clip_ratio/low_min": 0.000648060751700541, "clip_ratio/region_mean": 0.002352554161916487, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 1422.1875, "completions/mean_terminated_length": 1040.21435546875, "completions/min_length": 796.0, "completions/min_terminated_length": 796.0, "entropy": 0.14463330153375864, "epoch": 2.36734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.04170307517051697, "learning_rate": 7.653061224489796e-06, "loss": -0.0233, "num_tokens": 8095110.0, "reward": 0.6744627952575684, "reward_std": 0.15898263454437256, "rewards/reward_commands_completeness/mean": 0.12125000357627869, "rewards/reward_commands_completeness/std": 0.0403112918138504, "rewards/reward_commands_correctness/mean": 0.06499999761581421, "rewards/reward_commands_correctness/std": 0.02250925824046135, "rewards/reward_diversity/mean": 0.1688377559185028, "rewards/reward_diversity/std": 0.04500337690114975, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.1443750113248825, "rewards/reward_solution_effectiveness/std": 0.06217917427420616, "sampling/importance_sampling_ratio/max": 2.1750967502593994, "sampling/importance_sampling_ratio/mean": 0.3916915953159332, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.9287824630737305, "sampling/sampling_logp_difference/mean": 0.011360912583768368, "step": 116, "step_time": 266.2723757904023 }, { "clip_ratio/high_max": 0.0006241799201234244, "clip_ratio/high_mean": 0.0006241799201234244, "clip_ratio/low_mean": 0.0006747900297341403, "clip_ratio/low_min": 0.0006747900297341403, "clip_ratio/region_mean": 0.0012989699498575646, "completions/clipped_ratio": 0.375, "completions/max_length": 4096.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 2265.25, "completions/mean_terminated_length": 1166.800048828125, "completions/min_length": 969.0, "completions/min_terminated_length": 969.0, "entropy": 0.08615696849301457, "epoch": 2.387755102040816, "frac_reward_zero_std": 0.0, "grad_norm": 0.026073575019836426, "learning_rate": 7.63265306122449e-06, "loss": 0.0574, "num_tokens": 8160250.0, "reward": 0.48466792702674866, "reward_std": 0.20984819531440735, "rewards/reward_commands_completeness/mean": 0.08875000476837158, "rewards/reward_commands_completeness/std": 0.056080304086208344, "rewards/reward_commands_correctness/mean": 0.04374999552965164, "rewards/reward_commands_correctness/std": 0.02753785438835621, "rewards/reward_diversity/mean": 0.12904292345046997, "rewards/reward_diversity/std": 0.07790575921535492, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.06062500178813934, "rewards/reward_problem_validity/std": 0.03549061343073845, "rewards/reward_solution_effectiveness/mean": 0.09375, "rewards/reward_solution_effectiveness/std": 0.06652066856622696, "sampling/importance_sampling_ratio/max": 1.8915516138076782, "sampling/importance_sampling_ratio/mean": 0.18105056881904602, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.3518877029418945, "sampling/sampling_logp_difference/mean": 0.006879795342683792, "step": 117, "step_time": 273.3633923344314 }, { "clip_ratio/high_max": 0.0009333172038168414, "clip_ratio/high_mean": 0.0009333172038168414, "clip_ratio/low_mean": 0.000555081176571548, "clip_ratio/low_min": 0.000555081176571548, "clip_ratio/region_mean": 0.0014883983894833364, "completions/clipped_ratio": 0.3125, "completions/max_length": 4096.0, "completions/max_terminated_length": 3813.0, "completions/mean_length": 2210.9375, "completions/mean_terminated_length": 1354.0909423828125, "completions/min_length": 789.0, "completions/min_terminated_length": 789.0, "entropy": 0.08527735015377402, "epoch": 2.4081632653061225, "frac_reward_zero_std": 0.0, "grad_norm": 0.05763768404722214, "learning_rate": 7.612244897959185e-06, "loss": 0.1426, "num_tokens": 8226057.0, "reward": 0.6899881958961487, "reward_std": 0.0626598373055458, "rewards/reward_commands_completeness/mean": 0.1262499988079071, "rewards/reward_commands_completeness/std": 0.027049338445067406, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.016214706003665924, "rewards/reward_diversity/mean": 0.17061319947242737, "rewards/reward_diversity/std": 0.01564139500260353, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.14250001311302185, "rewards/reward_solution_effectiveness/std": 0.05079369992017746, "sampling/importance_sampling_ratio/max": 2.881291389465332, "sampling/importance_sampling_ratio/mean": 0.6273714303970337, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.620521545410156, "sampling/sampling_logp_difference/mean": 0.007111944258213043, "step": 118, "step_time": 316.3039541915059 }, { "clip_ratio/high_max": 0.001003008997940924, "clip_ratio/high_mean": 0.001003008997940924, "clip_ratio/low_mean": 0.0012257019443495665, "clip_ratio/low_min": 0.0012257019443495665, "clip_ratio/region_mean": 0.0022287109750322998, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1714.0, "completions/mean_length": 1537.5, "completions/mean_terminated_length": 1172.0, "completions/min_length": 805.0, "completions/min_terminated_length": 805.0, "entropy": 0.10439159302040935, "epoch": 2.4285714285714284, "frac_reward_zero_std": 0.0, "grad_norm": 0.029353564605116844, "learning_rate": 7.591836734693878e-06, "loss": 0.0484, "num_tokens": 8282949.0, "reward": 0.5648998618125916, "reward_std": 0.09805238991975784, "rewards/reward_commands_completeness/mean": 0.09999999403953552, "rewards/reward_commands_completeness/std": 0.044422220438718796, "rewards/reward_commands_correctness/mean": 0.05000000074505806, "rewards/reward_commands_correctness/std": 0.0236643198877573, "rewards/reward_diversity/mean": 0.15614986419677734, "rewards/reward_diversity/std": 0.0682779997587204, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07062500715255737, "rewards/reward_problem_validity/std": 0.03043435700237751, "rewards/reward_solution_effectiveness/mean": 0.10687500238418579, "rewards/reward_solution_effectiveness/std": 0.05582338571548462, "sampling/importance_sampling_ratio/max": 1.4527883529663086, "sampling/importance_sampling_ratio/mean": 0.20290668308734894, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.213916778564453, "sampling/sampling_logp_difference/mean": 0.009653002955019474, "step": 119, "step_time": 303.38850847631693 }, { "clip_ratio/high_max": 0.0011909893037227448, "clip_ratio/high_mean": 0.0011909893037227448, "clip_ratio/low_mean": 0.0005352508051146287, "clip_ratio/low_min": 0.0005352508051146287, "clip_ratio/region_mean": 0.0017262400870095007, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1425.0, "completions/mean_length": 1486.875, "completions/mean_terminated_length": 1114.1429443359375, "completions/min_length": 705.0, "completions/min_terminated_length": 705.0, "entropy": 0.1124047813937068, "epoch": 2.4489795918367347, "frac_reward_zero_std": 0.25, "grad_norm": 0.008210253901779652, "learning_rate": 7.571428571428572e-06, "loss": -0.0228, "num_tokens": 8329243.0, "reward": 0.540810227394104, "reward_std": 0.07844720035791397, "rewards/reward_commands_completeness/mean": 0.10124999284744263, "rewards/reward_commands_completeness/std": 0.054878655821084976, "rewards/reward_commands_correctness/mean": 0.048124998807907104, "rewards/reward_commands_correctness/std": 0.02786126546561718, "rewards/reward_diversity/mean": 0.13393527269363403, "rewards/reward_diversity/std": 0.06929555535316467, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06437499821186066, "rewards/reward_problem_validity/std": 0.03463500365614891, "rewards/reward_solution_effectiveness/mean": 0.11812500655651093, "rewards/reward_solution_effectiveness/std": 0.0730496421456337, "sampling/importance_sampling_ratio/max": 2.672110080718994, "sampling/importance_sampling_ratio/mean": 0.26754507422447205, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.046036720275879, "sampling/sampling_logp_difference/mean": 0.010258873924612999, "step": 120, "step_time": 258.26424797810614 }, { "clip_ratio/high_max": 0.0013745608302997425, "clip_ratio/high_mean": 0.0013745608302997425, "clip_ratio/low_mean": 0.0007846858497941867, "clip_ratio/low_min": 0.0007846858497941867, "clip_ratio/region_mean": 0.002159246643714141, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1524.0, "completions/mean_length": 1522.8125, "completions/mean_terminated_length": 1155.21435546875, "completions/min_length": 798.0, "completions/min_terminated_length": 798.0, "entropy": 0.12107107788324356, "epoch": 2.4693877551020407, "frac_reward_zero_std": 0.0, "grad_norm": 0.03268000856041908, "learning_rate": 7.551020408163265e-06, "loss": -0.0948, "num_tokens": 8385040.0, "reward": 0.7659653425216675, "reward_std": 0.10468654334545135, "rewards/reward_commands_completeness/mean": 0.14625000953674316, "rewards/reward_commands_completeness/std": 0.03403430059552193, "rewards/reward_commands_correctness/mean": 0.07125000655651093, "rewards/reward_commands_correctness/std": 0.01408308744430542, "rewards/reward_diversity/mean": 0.1722153127193451, "rewards/reward_diversity/std": 0.028726808726787567, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08875000476837158, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.1875, "rewards/reward_solution_effectiveness/std": 0.06244998052716255, "sampling/importance_sampling_ratio/max": 1.8821271657943726, "sampling/importance_sampling_ratio/mean": 0.3196442723274231, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.473867893218994, "sampling/sampling_logp_difference/mean": 0.010499272495508194, "step": 121, "step_time": 311.966731434688 }, { "clip_ratio/high_max": 0.0015590297552989796, "clip_ratio/high_mean": 0.0015590297552989796, "clip_ratio/low_mean": 0.0006989716384850908, "clip_ratio/low_min": 0.0006989716384850908, "clip_ratio/region_mean": 0.0022580013719561975, "completions/clipped_ratio": 0.0, "completions/max_length": 3045.0, "completions/max_terminated_length": 3045.0, "completions/mean_length": 1247.1875, "completions/mean_terminated_length": 1247.1875, "completions/min_length": 919.0, "completions/min_terminated_length": 919.0, "entropy": 0.1199248656630516, "epoch": 2.489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.05293029919266701, "learning_rate": 7.53061224489796e-06, "loss": -0.0974, "num_tokens": 8427115.0, "reward": 0.5399748682975769, "reward_std": 0.11479562520980835, "rewards/reward_commands_completeness/mean": 0.09375, "rewards/reward_commands_completeness/std": 0.04485160857439041, "rewards/reward_commands_correctness/mean": 0.06187500059604645, "rewards/reward_commands_correctness/std": 0.033109668642282486, "rewards/reward_diversity/mean": 0.13809987902641296, "rewards/reward_diversity/std": 0.062104519456624985, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.06749999523162842, "rewards/reward_problem_validity/std": 0.030876100063323975, "rewards/reward_solution_effectiveness/mean": 0.09750000387430191, "rewards/reward_solution_effectiveness/std": 0.06434284150600433, "sampling/importance_sampling_ratio/max": 2.6228909492492676, "sampling/importance_sampling_ratio/mean": 0.3969811201095581, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.202175140380859, "sampling/sampling_logp_difference/mean": 0.012103374116122723, "step": 122, "step_time": 205.48114044964314 }, { "clip_ratio/high_max": 0.0013496553692675661, "clip_ratio/high_mean": 0.0013496553692675661, "clip_ratio/low_mean": 0.0007694323467148934, "clip_ratio/low_min": 0.0007694323467148934, "clip_ratio/region_mean": 0.00211908774508629, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1244.0, "completions/mean_length": 1260.1875, "completions/mean_terminated_length": 1071.1334228515625, "completions/min_length": 856.0, "completions/min_terminated_length": 856.0, "entropy": 0.1285249199718237, "epoch": 2.510204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.16368088126182556, "learning_rate": 7.5102040816326536e-06, "loss": -0.0331, "num_tokens": 8470426.0, "reward": 0.700266420841217, "reward_std": 0.03928660601377487, "rewards/reward_commands_completeness/mean": 0.13124999403953552, "rewards/reward_commands_completeness/std": 0.02629956044256687, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.012632629834115505, "rewards/reward_diversity/mean": 0.16651645302772522, "rewards/reward_diversity/std": 0.026171546429395676, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.005163980647921562, "rewards/reward_solution_effectiveness/mean": 0.15187500417232513, "rewards/reward_solution_effectiveness/std": 0.050756778568029404, "sampling/importance_sampling_ratio/max": 2.7123472690582275, "sampling/importance_sampling_ratio/mean": 0.8215821981430054, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.2624077796936035, "sampling/sampling_logp_difference/mean": 0.011900356970727444, "step": 123, "step_time": 276.424889203161 }, { "clip_ratio/high_max": 0.0016354605468222871, "clip_ratio/high_mean": 0.0016354605468222871, "clip_ratio/low_mean": 0.0007337550341617316, "clip_ratio/low_min": 0.0007337550341617316, "clip_ratio/region_mean": 0.002369215595535934, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1431.0, "completions/mean_length": 1421.625, "completions/mean_terminated_length": 1039.571533203125, "completions/min_length": 715.0, "completions/min_terminated_length": 715.0, "entropy": 0.12483634194359183, "epoch": 2.5306122448979593, "frac_reward_zero_std": 0.0, "grad_norm": 0.06806426495313644, "learning_rate": 7.489795918367348e-06, "loss": -0.1364, "num_tokens": 8513028.0, "reward": 0.6765360236167908, "reward_std": 0.14898189902305603, "rewards/reward_commands_completeness/mean": 0.1237500011920929, "rewards/reward_commands_completeness/std": 0.040804412215948105, "rewards/reward_commands_correctness/mean": 0.06937499344348907, "rewards/reward_commands_correctness/std": 0.020806651562452316, "rewards/reward_diversity/mean": 0.16528598964214325, "rewards/reward_diversity/std": 0.04407185688614845, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.078125, "rewards/reward_problem_validity/std": 0.02197536826133728, "rewards/reward_solution_effectiveness/mean": 0.14625000953674316, "rewards/reward_solution_effectiveness/std": 0.06280923634767532, "sampling/importance_sampling_ratio/max": 1.6687430143356323, "sampling/importance_sampling_ratio/mean": 0.2217201441526413, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.2293806076049805, "sampling/sampling_logp_difference/mean": 0.011299953795969486, "step": 124, "step_time": 263.73267719335854 }, { "clip_ratio/high_max": 0.0014543539691658225, "clip_ratio/high_mean": 0.0014543539691658225, "clip_ratio/low_mean": 4.7837733291089535e-05, "clip_ratio/low_min": 4.7837733291089535e-05, "clip_ratio/region_mean": 0.001502191702456912, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1263.0, "completions/mean_length": 1226.9375, "completions/mean_terminated_length": 1035.666748046875, "completions/min_length": 673.0, "completions/min_terminated_length": 673.0, "entropy": 0.1321640256792307, "epoch": 2.5510204081632653, "frac_reward_zero_std": 0.25, "grad_norm": 0.09660740941762924, "learning_rate": 7.469387755102041e-06, "loss": 0.1182, "num_tokens": 8557739.0, "reward": 0.5235869288444519, "reward_std": 0.11856910586357117, "rewards/reward_commands_completeness/mean": 0.09624999761581421, "rewards/reward_commands_completeness/std": 0.05760497972369194, "rewards/reward_commands_correctness/mean": 0.051249999552965164, "rewards/reward_commands_correctness/std": 0.031171569600701332, "rewards/reward_diversity/mean": 0.12858694791793823, "rewards/reward_diversity/std": 0.07713016122579575, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.0625, "rewards/reward_problem_validity/std": 0.03678768128156662, "rewards/reward_solution_effectiveness/mean": 0.11625000834465027, "rewards/reward_solution_effectiveness/std": 0.0717286542057991, "sampling/importance_sampling_ratio/max": 1.8680007457733154, "sampling/importance_sampling_ratio/mean": 0.2500653862953186, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.588035583496094, "sampling/sampling_logp_difference/mean": 0.01293514110147953, "step": 125, "step_time": 257.4270004350692 }, { "clip_ratio/high_max": 0.0010968167407554574, "clip_ratio/high_mean": 0.0010968167407554574, "clip_ratio/low_mean": 0.0008949727616709424, "clip_ratio/low_min": 0.0008949727616709424, "clip_ratio/region_mean": 0.0019917895042453893, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1530.0, "completions/mean_length": 1544.75, "completions/mean_terminated_length": 1180.2857666015625, "completions/min_length": 894.0, "completions/min_terminated_length": 894.0, "entropy": 0.10591010516509414, "epoch": 2.571428571428571, "frac_reward_zero_std": 0.0, "grad_norm": 0.06617513298988342, "learning_rate": 7.448979591836736e-06, "loss": -0.1945, "num_tokens": 8611063.0, "reward": 0.49511265754699707, "reward_std": 0.2397453784942627, "rewards/reward_commands_completeness/mean": 0.09125000238418579, "rewards/reward_commands_completeness/std": 0.057023387402296066, "rewards/reward_commands_correctness/mean": 0.04749999940395355, "rewards/reward_commands_correctness/std": 0.028635641559958458, "rewards/reward_diversity/mean": 0.12573768198490143, "rewards/reward_diversity/std": 0.06480135768651962, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.06062500178813934, "rewards/reward_problem_validity/std": 0.03549061343073845, "rewards/reward_solution_effectiveness/mean": 0.10124999284744263, "rewards/reward_solution_effectiveness/std": 0.06652067601680756, "sampling/importance_sampling_ratio/max": 2.469914674758911, "sampling/importance_sampling_ratio/mean": 0.5850806832313538, "sampling/importance_sampling_ratio/min": 6.0382666561054066e-05, "sampling/sampling_logp_difference/max": 7.336891174316406, "sampling/sampling_logp_difference/mean": 0.010148867964744568, "step": 126, "step_time": 286.59335492551327 }, { "clip_ratio/high_max": 0.0013641822733916342, "clip_ratio/high_mean": 0.0013641822733916342, "clip_ratio/low_mean": 0.0002711031229409855, "clip_ratio/low_min": 0.0002711031229409855, "clip_ratio/region_mean": 0.001635285392694641, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1253.0, "completions/mean_length": 1635.9375, "completions/mean_terminated_length": 1068.2308349609375, "completions/min_length": 794.0, "completions/min_terminated_length": 794.0, "entropy": 0.10120017407462001, "epoch": 2.5918367346938775, "frac_reward_zero_std": 0.25, "grad_norm": 0.014045675285160542, "learning_rate": 7.428571428571429e-06, "loss": -0.0252, "num_tokens": 8661426.0, "reward": 0.4919400215148926, "reward_std": 0.12619812786579132, "rewards/reward_commands_completeness/mean": 0.08624999970197678, "rewards/reward_commands_completeness/std": 0.05920303985476494, "rewards/reward_commands_correctness/mean": 0.048124998807907104, "rewards/reward_commands_correctness/std": 0.0325000025331974, "rewards/reward_diversity/mean": 0.1363150179386139, "rewards/reward_diversity/std": 0.07023998349905014, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.057499997317790985, "rewards/reward_problem_validity/std": 0.0382099486887455, "rewards/reward_solution_effectiveness/mean": 0.10125000774860382, "rewards/reward_solution_effectiveness/std": 0.07499999552965164, "sampling/importance_sampling_ratio/max": 0.6896750926971436, "sampling/importance_sampling_ratio/mean": 0.1548951417207718, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.859101295471191, "sampling/sampling_logp_difference/mean": 0.010033399797976017, "step": 127, "step_time": 270.4859570134431 }, { "clip_ratio/high_max": 0.001771183258824749, "clip_ratio/high_mean": 0.001771183258824749, "clip_ratio/low_mean": 0.0005778676786576398, "clip_ratio/low_min": 0.0005778676786576398, "clip_ratio/region_mean": 0.0023490509265684523, "completions/clipped_ratio": 0.0, "completions/max_length": 1301.0, "completions/max_terminated_length": 1301.0, "completions/mean_length": 1087.75, "completions/mean_terminated_length": 1087.75, "completions/min_length": 706.0, "completions/min_terminated_length": 706.0, "entropy": 0.13795570190995932, "epoch": 2.612244897959184, "frac_reward_zero_std": 0.0, "grad_norm": 0.06508559733629227, "learning_rate": 7.408163265306123e-06, "loss": -0.1395, "num_tokens": 8707678.0, "reward": 0.6621281504631042, "reward_std": 0.12027253210544586, "rewards/reward_commands_completeness/mean": 0.11749999225139618, "rewards/reward_commands_completeness/std": 0.03855732083320618, "rewards/reward_commands_correctness/mean": 0.07312500476837158, "rewards/reward_commands_correctness/std": 0.01956825703382492, "rewards/reward_diversity/mean": 0.16837815940380096, "rewards/reward_diversity/std": 0.046043310314416885, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07999999821186066, "rewards/reward_problem_validity/std": 0.019321836531162262, "rewards/reward_solution_effectiveness/mean": 0.12937501072883606, "rewards/reward_solution_effectiveness/std": 0.05446328967809677, "sampling/importance_sampling_ratio/max": 2.461183547973633, "sampling/importance_sampling_ratio/mean": 0.3831317126750946, "sampling/importance_sampling_ratio/min": 2.3105369109543972e-05, "sampling/sampling_logp_difference/max": 2.2164885997772217, "sampling/sampling_logp_difference/mean": 0.015960829332470894, "step": 128, "step_time": 143.15919862501323 }, { "clip_ratio/high_max": 0.0014492677837552037, "clip_ratio/high_mean": 0.0014492677837552037, "clip_ratio/low_mean": 0.0007211145129986107, "clip_ratio/low_min": 0.0007211145129986107, "clip_ratio/region_mean": 0.002170382282201899, "completions/clipped_ratio": 0.0, "completions/max_length": 1479.0, "completions/max_terminated_length": 1479.0, "completions/mean_length": 1166.5, "completions/mean_terminated_length": 1166.5, "completions/min_length": 1002.0, "completions/min_terminated_length": 1002.0, "entropy": 0.12885688059031963, "epoch": 2.63265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.021276237443089485, "learning_rate": 7.387755102040817e-06, "loss": 0.069, "num_tokens": 8754534.0, "reward": 0.7604231238365173, "reward_std": 0.048414651304483414, "rewards/reward_commands_completeness/mean": 0.14875000715255737, "rewards/reward_commands_completeness/std": 0.02418677695095539, "rewards/reward_commands_correctness/mean": 0.06937500089406967, "rewards/reward_commands_correctness/std": 0.02205107919871807, "rewards/reward_diversity/mean": 0.17479810118675232, "rewards/reward_diversity/std": 0.01963280327618122, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500774860382, "rewards/reward_problem_validity/std": 0.008139412850141525, "rewards/reward_solution_effectiveness/mean": 0.18187500536441803, "rewards/reward_solution_effectiveness/std": 0.06337389349937439, "sampling/importance_sampling_ratio/max": 2.2027459144592285, "sampling/importance_sampling_ratio/mean": 0.3196198344230652, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1636435985565186, "sampling/sampling_logp_difference/mean": 0.013241343200206757, "step": 129, "step_time": 169.04363920912147 }, { "clip_ratio/high_max": 0.001592340602655895, "clip_ratio/high_mean": 0.001592340602655895, "clip_ratio/low_mean": 0.000708177627529949, "clip_ratio/low_min": 0.000708177627529949, "clip_ratio/region_mean": 0.002300518252013717, "completions/clipped_ratio": 0.0, "completions/max_length": 2233.0, "completions/max_terminated_length": 2233.0, "completions/mean_length": 1211.0, "completions/mean_terminated_length": 1211.0, "completions/min_length": 734.0, "completions/min_terminated_length": 734.0, "entropy": 0.12296013161540031, "epoch": 2.6530612244897958, "frac_reward_zero_std": 0.0, "grad_norm": 0.09504980593919754, "learning_rate": 7.367346938775511e-06, "loss": 0.0132, "num_tokens": 8804122.0, "reward": 0.6993801593780518, "reward_std": 0.08761599659919739, "rewards/reward_commands_completeness/mean": 0.1287499964237213, "rewards/reward_commands_completeness/std": 0.03180671110749245, "rewards/reward_commands_correctness/mean": 0.07124999910593033, "rewards/reward_commands_correctness/std": 0.018574178218841553, "rewards/reward_diversity/mean": 0.17063018679618835, "rewards/reward_diversity/std": 0.021730652078986168, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08249999582767487, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.14624999463558197, "rewards/reward_solution_effectiveness/std": 0.056789085268974304, "sampling/importance_sampling_ratio/max": 1.5711259841918945, "sampling/importance_sampling_ratio/mean": 0.3822461664676666, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2770092487335205, "sampling/sampling_logp_difference/mean": 0.013725845143198967, "step": 130, "step_time": 217.02253586612642 }, { "clip_ratio/high_max": 0.0016636162908980623, "clip_ratio/high_mean": 0.0016636162908980623, "clip_ratio/low_mean": 0.0006306888317340054, "clip_ratio/low_min": 0.0006306888317340054, "clip_ratio/region_mean": 0.002294305100804195, "completions/clipped_ratio": 0.25, "completions/max_length": 4096.0, "completions/max_terminated_length": 1399.0, "completions/mean_length": 1869.0, "completions/mean_terminated_length": 1126.666748046875, "completions/min_length": 810.0, "completions/min_terminated_length": 810.0, "entropy": 0.09611077699810266, "epoch": 2.673469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.031438443809747696, "learning_rate": 7.346938775510205e-06, "loss": -0.0547, "num_tokens": 8865010.0, "reward": 0.6117055416107178, "reward_std": 0.16532166302204132, "rewards/reward_commands_completeness/mean": 0.10750000923871994, "rewards/reward_commands_completeness/std": 0.05105552449822426, "rewards/reward_commands_correctness/mean": 0.05624999850988388, "rewards/reward_commands_correctness/std": 0.02753785438835621, "rewards/reward_diversity/mean": 0.1698305606842041, "rewards/reward_diversity/std": 0.040903519839048386, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07125000655651093, "rewards/reward_problem_validity/std": 0.030740855261683464, "rewards/reward_solution_effectiveness/mean": 0.12562499940395355, "rewards/reward_solution_effectiveness/std": 0.06947121769189835, "sampling/importance_sampling_ratio/max": 1.8856419324874878, "sampling/importance_sampling_ratio/mean": 0.1943281590938568, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.091229438781738, "sampling/sampling_logp_difference/mean": 0.010327858850359917, "step": 131, "step_time": 288.4686396624893 }, { "clip_ratio/high_max": 0.0012020253197988495, "clip_ratio/high_mean": 0.0012020253197988495, "clip_ratio/low_mean": 0.000679746732203057, "clip_ratio/low_min": 0.000679746732203057, "clip_ratio/region_mean": 0.0018817720592778642, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 1301.6875, "completions/mean_terminated_length": 1115.4000244140625, "completions/min_length": 753.0, "completions/min_terminated_length": 753.0, "entropy": 0.10191709687933326, "epoch": 2.693877551020408, "frac_reward_zero_std": 0.0, "grad_norm": 0.08453759551048279, "learning_rate": 7.326530612244899e-06, "loss": -0.2936, "num_tokens": 8911589.0, "reward": 0.7296097278594971, "reward_std": 0.061356112360954285, "rewards/reward_commands_completeness/mean": 0.13499999046325684, "rewards/reward_commands_completeness/std": 0.026832817122340202, "rewards/reward_commands_correctness/mean": 0.07375000417232513, "rewards/reward_commands_correctness/std": 0.0145487692207098, "rewards/reward_diversity/mean": 0.17523470520973206, "rewards/reward_diversity/std": 0.012096515856683254, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.1575000137090683, "rewards/reward_solution_effectiveness/std": 0.041713304817676544, "sampling/importance_sampling_ratio/max": 2.4702630043029785, "sampling/importance_sampling_ratio/mean": 0.41047707200050354, "sampling/importance_sampling_ratio/min": 6.374664371833205e-05, "sampling/sampling_logp_difference/max": 7.3798346519470215, "sampling/sampling_logp_difference/mean": 0.012297592125833035, "step": 132, "step_time": 293.44757851213217 }, { "clip_ratio/high_max": 0.0012337872794887517, "clip_ratio/high_mean": 0.0012337872794887517, "clip_ratio/low_mean": 0.000994388454273576, "clip_ratio/low_min": 0.000994388454273576, "clip_ratio/region_mean": 0.0022281757301243488, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1395.0, "completions/mean_length": 1310.9375, "completions/mean_terminated_length": 1125.2667236328125, "completions/min_length": 872.0, "completions/min_terminated_length": 872.0, "entropy": 0.11465141363441944, "epoch": 2.7142857142857144, "frac_reward_zero_std": 0.0, "grad_norm": 0.04489774629473686, "learning_rate": 7.306122448979592e-06, "loss": -0.0779, "num_tokens": 8954156.0, "reward": 0.6887465715408325, "reward_std": 0.0757080465555191, "rewards/reward_commands_completeness/mean": 0.11875000596046448, "rewards/reward_commands_completeness/std": 0.030523216351866722, "rewards/reward_commands_correctness/mean": 0.07249999791383743, "rewards/reward_commands_correctness/std": 0.012909946031868458, "rewards/reward_diversity/mean": 0.16999655961990356, "rewards/reward_diversity/std": 0.014323608949780464, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08312500268220901, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.0516357496380806, "sampling/importance_sampling_ratio/max": 1.3987213373184204, "sampling/importance_sampling_ratio/mean": 0.25242263078689575, "sampling/importance_sampling_ratio/min": 4.414760951476637e-06, "sampling/sampling_logp_difference/max": 7.25978422164917, "sampling/sampling_logp_difference/mean": 0.012925583869218826, "step": 133, "step_time": 280.6408644504845 }, { "clip_ratio/high_max": 0.0015583741987938993, "clip_ratio/high_mean": 0.0015583741987938993, "clip_ratio/low_mean": 0.00048229652747977525, "clip_ratio/low_min": 0.00048229652747977525, "clip_ratio/region_mean": 0.002040670718997717, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1291.0, "completions/mean_length": 1233.6875, "completions/mean_terminated_length": 1042.86669921875, "completions/min_length": 776.0, "completions/min_terminated_length": 776.0, "entropy": 0.11391079798340797, "epoch": 2.7346938775510203, "frac_reward_zero_std": 0.0, "grad_norm": 0.0422009713947773, "learning_rate": 7.285714285714286e-06, "loss": 0.0582, "num_tokens": 8998999.0, "reward": 0.5032713413238525, "reward_std": 0.2471754401922226, "rewards/reward_commands_completeness/mean": 0.0925000011920929, "rewards/reward_commands_completeness/std": 0.05744563043117523, "rewards/reward_commands_correctness/mean": 0.04625000059604645, "rewards/reward_commands_correctness/std": 0.02825479581952095, "rewards/reward_diversity/mean": 0.13577133417129517, "rewards/reward_diversity/std": 0.07185353338718414, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.06062500178813934, "rewards/reward_problem_validity/std": 0.03549061343073845, "rewards/reward_solution_effectiveness/mean": 0.09937500208616257, "rewards/reward_solution_effectiveness/std": 0.0663795918226242, "sampling/importance_sampling_ratio/max": 1.1955621242523193, "sampling/importance_sampling_ratio/mean": 0.14802247285842896, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.615015983581543, "sampling/sampling_logp_difference/mean": 0.014035052619874477, "step": 134, "step_time": 273.63378514908254 }, { "clip_ratio/high_max": 0.0016802971877041273, "clip_ratio/high_mean": 0.0016802971877041273, "clip_ratio/low_mean": 0.0009344261634396389, "clip_ratio/low_min": 0.0009344261634396389, "clip_ratio/region_mean": 0.0026147233438678086, "completions/clipped_ratio": 0.0, "completions/max_length": 1327.0, "completions/max_terminated_length": 1327.0, "completions/mean_length": 1100.6875, "completions/mean_terminated_length": 1100.6875, "completions/min_length": 714.0, "completions/min_terminated_length": 714.0, "entropy": 0.10656317975372076, "epoch": 2.7551020408163263, "frac_reward_zero_std": 0.0, "grad_norm": 0.04246143624186516, "learning_rate": 7.2653061224489805e-06, "loss": -0.1361, "num_tokens": 9041442.0, "reward": 0.6831507682800293, "reward_std": 0.1525394469499588, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.03855732083320618, "rewards/reward_commands_correctness/mean": 0.060624994337558746, "rewards/reward_commands_correctness/std": 0.020806651562452316, "rewards/reward_diversity/mean": 0.1569007933139801, "rewards/reward_diversity/std": 0.04271286353468895, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07999999821186066, "rewards/reward_problem_validity/std": 0.019321836531162262, "rewards/reward_solution_effectiveness/mean": 0.15937501192092896, "rewards/reward_solution_effectiveness/std": 0.05870477482676506, "sampling/importance_sampling_ratio/max": 2.1860010623931885, "sampling/importance_sampling_ratio/mean": 0.33673787117004395, "sampling/importance_sampling_ratio/min": 0.0007414421415887773, "sampling/sampling_logp_difference/max": 3.3749866485595703, "sampling/sampling_logp_difference/mean": 0.015588561072945595, "step": 135, "step_time": 144.37000884860754 }, { "clip_ratio/high_max": 0.002449961975798942, "clip_ratio/high_mean": 0.002449961975798942, "clip_ratio/low_mean": 0.0007306372026505414, "clip_ratio/low_min": 0.0007306372026505414, "clip_ratio/region_mean": 0.0031805991748115048, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1455.0, "completions/mean_length": 1376.5, "completions/mean_terminated_length": 1195.2000732421875, "completions/min_length": 837.0, "completions/min_terminated_length": 837.0, "entropy": 0.12591787334531546, "epoch": 2.7755102040816326, "frac_reward_zero_std": 0.0, "grad_norm": 0.011867190711200237, "learning_rate": 7.244897959183675e-06, "loss": 0.0229, "num_tokens": 9089930.0, "reward": 0.521333634853363, "reward_std": 0.17083388566970825, "rewards/reward_commands_completeness/mean": 0.08624999970197678, "rewards/reward_commands_completeness/std": 0.03844910115003586, "rewards/reward_commands_correctness/mean": 0.05624999850988388, "rewards/reward_commands_correctness/std": 0.02777889184653759, "rewards/reward_diversity/mean": 0.13820862770080566, "rewards/reward_diversity/std": 0.06274054944515228, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.06937500089406967, "rewards/reward_problem_validity/std": 0.029769953340291977, "rewards/reward_solution_effectiveness/mean": 0.09000000357627869, "rewards/reward_solution_effectiveness/std": 0.03949683532118797, "sampling/importance_sampling_ratio/max": 0.9851588606834412, "sampling/importance_sampling_ratio/mean": 0.10059038549661636, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0128345489501953, "sampling/sampling_logp_difference/mean": 0.01465817354619503, "step": 136, "step_time": 288.35747206397355 }, { "clip_ratio/high_max": 0.0015370295805041678, "clip_ratio/high_mean": 0.0015370295805041678, "clip_ratio/low_mean": 0.0009402833238709718, "clip_ratio/low_min": 0.0009402833238709718, "clip_ratio/region_mean": 0.0024773129043751396, "completions/clipped_ratio": 0.0, "completions/max_length": 1479.0, "completions/max_terminated_length": 1479.0, "completions/mean_length": 1225.375, "completions/mean_terminated_length": 1225.375, "completions/min_length": 1096.0, "completions/min_terminated_length": 1096.0, "entropy": 0.12248317711055279, "epoch": 2.795918367346939, "frac_reward_zero_std": 0.0, "grad_norm": 0.00400357972830534, "learning_rate": 7.224489795918368e-06, "loss": 0.0078, "num_tokens": 9142044.0, "reward": 0.6722906827926636, "reward_std": 0.07412385195493698, "rewards/reward_commands_completeness/mean": 0.11749999225139618, "rewards/reward_commands_completeness/std": 0.025166114792227745, "rewards/reward_commands_correctness/mean": 0.06874999403953552, "rewards/reward_commands_correctness/std": 0.017078252509236336, "rewards/reward_diversity/mean": 0.1735406368970871, "rewards/reward_diversity/std": 0.021537980064749718, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08312500268220901, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.12937499582767487, "rewards/reward_solution_effectiveness/std": 0.03906725347042084, "sampling/importance_sampling_ratio/max": 0.6803454756736755, "sampling/importance_sampling_ratio/mean": 0.09110428392887115, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4398369789123535, "sampling/sampling_logp_difference/mean": 0.015568523667752743, "step": 137, "step_time": 166.12361540459096 }, { "clip_ratio/high_max": 0.0019159889197908342, "clip_ratio/high_mean": 0.0019159889197908342, "clip_ratio/low_mean": 0.0007984543008205947, "clip_ratio/low_min": 0.0007984543008205947, "clip_ratio/region_mean": 0.0027144432242494076, "completions/clipped_ratio": 0.0, "completions/max_length": 1452.0, "completions/max_terminated_length": 1452.0, "completions/mean_length": 1202.0, "completions/mean_terminated_length": 1202.0, "completions/min_length": 1044.0, "completions/min_terminated_length": 1044.0, "entropy": 0.10859791655093431, "epoch": 2.816326530612245, "frac_reward_zero_std": 0.0, "grad_norm": 0.02904621884226799, "learning_rate": 7.204081632653061e-06, "loss": -0.054, "num_tokens": 9186084.0, "reward": 0.6244438886642456, "reward_std": 0.16857236623764038, "rewards/reward_commands_completeness/mean": 0.11625000089406967, "rewards/reward_commands_completeness/std": 0.0501830019056797, "rewards/reward_commands_correctness/mean": 0.06187500059604645, "rewards/reward_commands_correctness/std": 0.024281339719891548, "rewards/reward_diversity/mean": 0.14819395542144775, "rewards/reward_diversity/std": 0.052674125880002975, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07562500238418579, "rewards/reward_problem_validity/std": 0.02606882154941559, "rewards/reward_solution_effectiveness/mean": 0.13499999046325684, "rewards/reward_solution_effectiveness/std": 0.06480740755796432, "sampling/importance_sampling_ratio/max": 1.0864386558532715, "sampling/importance_sampling_ratio/mean": 0.16003571450710297, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4920897483825684, "sampling/sampling_logp_difference/mean": 0.01478666253387928, "step": 138, "step_time": 138.4058599267155 }, { "clip_ratio/high_max": 0.0018657622567843646, "clip_ratio/high_mean": 0.0018657622567843646, "clip_ratio/low_mean": 0.0006727770705765579, "clip_ratio/low_min": 0.0006727770705765579, "clip_ratio/region_mean": 0.0025385393091710284, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1257.0, "completions/mean_length": 1275.5625, "completions/mean_terminated_length": 1087.533447265625, "completions/min_length": 876.0, "completions/min_terminated_length": 876.0, "entropy": 0.09930714964866638, "epoch": 2.836734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.03136887401342392, "learning_rate": 7.183673469387755e-06, "loss": 0.0144, "num_tokens": 9233769.0, "reward": 0.6772017478942871, "reward_std": 0.17981630563735962, "rewards/reward_commands_completeness/mean": 0.1274999976158142, "rewards/reward_commands_completeness/std": 0.05000000074505806, "rewards/reward_commands_correctness/mean": 0.06687499582767487, "rewards/reward_commands_correctness/std": 0.024689743295311928, "rewards/reward_diversity/mean": 0.16345173120498657, "rewards/reward_diversity/std": 0.045087989419698715, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07625000178813934, "rewards/reward_problem_validity/std": 0.026299558579921722, "rewards/reward_solution_effectiveness/mean": 0.15562500059604645, "rewards/reward_solution_effectiveness/std": 0.07762892544269562, "sampling/importance_sampling_ratio/max": 0.8491856455802917, "sampling/importance_sampling_ratio/mean": 0.16895148158073425, "sampling/importance_sampling_ratio/min": 0.0009651631116867065, "sampling/sampling_logp_difference/max": 7.182512283325195, "sampling/sampling_logp_difference/mean": 0.0136177446693182, "step": 139, "step_time": 291.3772333562374 }, { "clip_ratio/high_max": 0.0007936566144053359, "clip_ratio/high_mean": 0.0007936566144053359, "clip_ratio/low_mean": 0.001400313889462268, "clip_ratio/low_min": 0.001400313889462268, "clip_ratio/region_mean": 0.002193970500229625, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 3943.0, "completions/mean_length": 1481.0625, "completions/mean_terminated_length": 1306.7333984375, "completions/min_length": 747.0, "completions/min_terminated_length": 747.0, "entropy": 0.11331060389056802, "epoch": 2.857142857142857, "frac_reward_zero_std": 0.0, "grad_norm": 0.024875551462173462, "learning_rate": 7.16326530612245e-06, "loss": 0.0582, "num_tokens": 9282230.0, "reward": 0.46843719482421875, "reward_std": 0.2024804949760437, "rewards/reward_commands_completeness/mean": 0.08375000208616257, "rewards/reward_commands_completeness/std": 0.05760497599840164, "rewards/reward_commands_correctness/mean": 0.04749999940395355, "rewards/reward_commands_correctness/std": 0.033366650342941284, "rewards/reward_diversity/mean": 0.11656222492456436, "rewards/reward_diversity/std": 0.07785309106111526, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.058750003576278687, "rewards/reward_problem_validity/std": 0.03913651406764984, "rewards/reward_solution_effectiveness/mean": 0.09937499463558197, "rewards/reward_solution_effectiveness/std": 0.0748748928308487, "sampling/importance_sampling_ratio/max": 1.7361431121826172, "sampling/importance_sampling_ratio/mean": 0.22162535786628723, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.282451629638672, "sampling/sampling_logp_difference/mean": 0.013193429447710514, "step": 140, "step_time": 261.6500143315643 }, { "clip_ratio/high_max": 0.0012266030971659347, "clip_ratio/high_mean": 0.0012266030971659347, "clip_ratio/low_mean": 0.0005369119098759256, "clip_ratio/low_min": 0.0005369119098759256, "clip_ratio/region_mean": 0.0017635150070418604, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 1300.0, "completions/mean_length": 1689.25, "completions/mean_terminated_length": 1133.84619140625, "completions/min_length": 1016.0, "completions/min_terminated_length": 1016.0, "entropy": 0.07963403314352036, "epoch": 2.877551020408163, "frac_reward_zero_std": 0.0, "grad_norm": 0.015368693508207798, "learning_rate": 7.1428571428571436e-06, "loss": -0.0613, "num_tokens": 9337430.0, "reward": 0.5413197875022888, "reward_std": 0.23557788133621216, "rewards/reward_commands_completeness/mean": 0.09999999403953552, "rewards/reward_commands_completeness/std": 0.05887841060757637, "rewards/reward_commands_correctness/mean": 0.05312499776482582, "rewards/reward_commands_correctness/std": 0.030706949532032013, "rewards/reward_diversity/mean": 0.13694477081298828, "rewards/reward_diversity/std": 0.07131960242986679, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.0625, "rewards/reward_problem_validity/std": 0.03678768128156662, "rewards/reward_solution_effectiveness/mean": 0.12000000476837158, "rewards/reward_solution_effectiveness/std": 0.07266361266374588, "sampling/importance_sampling_ratio/max": 1.1179906129837036, "sampling/importance_sampling_ratio/mean": 0.180836021900177, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.39187479019165, "sampling/sampling_logp_difference/mean": 0.010773377493023872, "step": 141, "step_time": 283.6576902717352 }, { "clip_ratio/high_max": 0.0020478046717471443, "clip_ratio/high_mean": 0.0020478046717471443, "clip_ratio/low_mean": 0.0010532031010370702, "clip_ratio/low_min": 0.0010532031010370702, "clip_ratio/region_mean": 0.0031010077873361297, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1363.0, "completions/mean_length": 1274.75, "completions/mean_terminated_length": 1086.666748046875, "completions/min_length": 698.0, "completions/min_terminated_length": 698.0, "entropy": 0.10528827691450715, "epoch": 2.8979591836734695, "frac_reward_zero_std": 0.0, "grad_norm": 0.016252301633358, "learning_rate": 7.122448979591837e-06, "loss": -0.0061, "num_tokens": 9399254.0, "reward": 0.6888071298599243, "reward_std": 0.23255017399787903, "rewards/reward_commands_completeness/mean": 0.13124999403953552, "rewards/reward_commands_completeness/std": 0.050049979239702225, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.028511693701148033, "rewards/reward_diversity/mean": 0.1631820946931839, "rewards/reward_diversity/std": 0.058851227164268494, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.027195284143090248, "rewards/reward_solution_effectiveness/mean": 0.16312499344348907, "rewards/reward_solution_effectiveness/std": 0.07345917075872421, "sampling/importance_sampling_ratio/max": 0.4210710823535919, "sampling/importance_sampling_ratio/mean": 0.07087453454732895, "sampling/importance_sampling_ratio/min": 1.9318231352372095e-06, "sampling/sampling_logp_difference/max": 7.11021089553833, "sampling/sampling_logp_difference/mean": 0.01549974549561739, "step": 142, "step_time": 391.49239008687437 }, { "clip_ratio/high_max": 0.0019736515896511264, "clip_ratio/high_mean": 0.0019736515896511264, "clip_ratio/low_mean": 0.0006567991185875144, "clip_ratio/low_min": 0.0006567991185875144, "clip_ratio/region_mean": 0.002630450704600662, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 1345.0, "completions/mean_terminated_length": 952.0000610351562, "completions/min_length": 640.0, "completions/min_terminated_length": 640.0, "entropy": 0.10732977138832211, "epoch": 2.9183673469387754, "frac_reward_zero_std": 0.0, "grad_norm": 0.04041369631886482, "learning_rate": 7.102040816326531e-06, "loss": -0.0455, "num_tokens": 9439566.0, "reward": 0.6302652955055237, "reward_std": 0.15506823360919952, "rewards/reward_commands_completeness/mean": 0.11499999463558197, "rewards/reward_commands_completeness/std": 0.0458984412252903, "rewards/reward_commands_correctness/mean": 0.0625, "rewards/reward_commands_correctness/std": 0.026457514613866806, "rewards/reward_diversity/mean": 0.15526530146598816, "rewards/reward_diversity/std": 0.055118076503276825, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.02651257812976837, "rewards/reward_solution_effectiveness/mean": 0.13312500715255737, "rewards/reward_solution_effectiveness/std": 0.06660518050193787, "sampling/importance_sampling_ratio/max": 0.8803386688232422, "sampling/importance_sampling_ratio/mean": 0.24861977994441986, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.25880765914917, "sampling/sampling_logp_difference/mean": 0.01356473471969366, "step": 143, "step_time": 253.52072128467262 }, { "clip_ratio/high_max": 0.0014576332469005138, "clip_ratio/high_mean": 0.0014576332469005138, "clip_ratio/low_mean": 0.0006997064992901869, "clip_ratio/low_min": 0.0006997064992901869, "clip_ratio/region_mean": 0.0021573397316387855, "completions/clipped_ratio": 0.0, "completions/max_length": 1260.0, "completions/max_terminated_length": 1260.0, "completions/mean_length": 1042.0, "completions/mean_terminated_length": 1042.0, "completions/min_length": 746.0, "completions/min_terminated_length": 746.0, "entropy": 0.09126830752938986, "epoch": 2.938775510204082, "frac_reward_zero_std": 0.25, "grad_norm": 0.027038637548685074, "learning_rate": 7.081632653061226e-06, "loss": 0.0275, "num_tokens": 9482502.0, "reward": 0.4925266206264496, "reward_std": 0.10360687971115112, "rewards/reward_commands_completeness/mean": 0.0912499949336052, "rewards/reward_commands_completeness/std": 0.060207970440387726, "rewards/reward_commands_correctness/mean": 0.048124998807907104, "rewards/reward_commands_correctness/std": 0.03290770202875137, "rewards/reward_diversity/mean": 0.12127659469842911, "rewards/reward_diversity/std": 0.061022885143756866, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.058750003576278687, "rewards/reward_problem_validity/std": 0.03913651406764984, "rewards/reward_solution_effectiveness/mean": 0.1106249988079071, "rewards/reward_solution_effectiveness/std": 0.0690380334854126, "sampling/importance_sampling_ratio/max": 0.875038743019104, "sampling/importance_sampling_ratio/mean": 0.1310742050409317, "sampling/importance_sampling_ratio/min": 0.0004907698021270335, "sampling/sampling_logp_difference/max": 2.5175466537475586, "sampling/sampling_logp_difference/mean": 0.015256124548614025, "step": 144, "step_time": 133.83653808943927 }, { "clip_ratio/high_max": 0.0009640508287702687, "clip_ratio/high_mean": 0.0009640508287702687, "clip_ratio/low_mean": 0.0011893218106706627, "clip_ratio/low_min": 0.0011893218106706627, "clip_ratio/region_mean": 0.0021533726394409314, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1364.0, "completions/mean_length": 1322.6875, "completions/mean_terminated_length": 1137.800048828125, "completions/min_length": 855.0, "completions/min_terminated_length": 855.0, "entropy": 0.09466969827190042, "epoch": 2.9591836734693877, "frac_reward_zero_std": 0.0, "grad_norm": 0.059409983456134796, "learning_rate": 7.061224489795919e-06, "loss": -0.1377, "num_tokens": 9533577.0, "reward": 0.7678130269050598, "reward_std": 0.07363303005695343, "rewards/reward_commands_completeness/mean": 0.14875000715255737, "rewards/reward_commands_completeness/std": 0.03263434022665024, "rewards/reward_commands_correctness/mean": 0.07437500357627869, "rewards/reward_commands_correctness/std": 0.01263263076543808, "rewards/reward_diversity/mean": 0.1678130030632019, "rewards/reward_diversity/std": 0.03192634880542755, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.18937501311302185, "rewards/reward_solution_effectiveness/std": 0.06727741658687592, "sampling/importance_sampling_ratio/max": 2.366550922393799, "sampling/importance_sampling_ratio/mean": 0.2282046377658844, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.1027116775512695, "sampling/sampling_logp_difference/mean": 0.013109052553772926, "step": 145, "step_time": 307.1017409134656 }, { "clip_ratio/high_max": 0.0026291084941476583, "clip_ratio/high_mean": 0.0026291084941476583, "clip_ratio/low_mean": 0.0007240103441290557, "clip_ratio/low_min": 0.0007240103441290557, "clip_ratio/region_mean": 0.0033531188528286293, "completions/clipped_ratio": 0.0, "completions/max_length": 1550.0, "completions/max_terminated_length": 1550.0, "completions/mean_length": 1137.625, "completions/mean_terminated_length": 1137.625, "completions/min_length": 702.0, "completions/min_terminated_length": 702.0, "entropy": 0.09887119941413403, "epoch": 2.979591836734694, "frac_reward_zero_std": 0.0, "grad_norm": 0.01281419862061739, "learning_rate": 7.0408163265306125e-06, "loss": -0.0257, "num_tokens": 9576291.0, "reward": 0.6256498098373413, "reward_std": 0.15885820984840393, "rewards/reward_commands_completeness/mean": 0.1287499964237213, "rewards/reward_commands_completeness/std": 0.04731103405356407, "rewards/reward_commands_correctness/mean": 0.058125000447034836, "rewards/reward_commands_correctness/std": 0.0242813378572464, "rewards/reward_diversity/mean": 0.12939977645874023, "rewards/reward_diversity/std": 0.10522906482219696, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07562500238418579, "rewards/reward_problem_validity/std": 0.026068823412060738, "rewards/reward_solution_effectiveness/mean": 0.14625000953674316, "rewards/reward_solution_effectiveness/std": 0.06561250239610672, "sampling/importance_sampling_ratio/max": 0.6890290975570679, "sampling/importance_sampling_ratio/mean": 0.12330266833305359, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9194397926330566, "sampling/sampling_logp_difference/mean": 0.016667349264025688, "step": 146, "step_time": 138.27432021312416 }, { "clip_ratio/high_max": 0.0017495794600108638, "clip_ratio/high_mean": 0.0017495794600108638, "clip_ratio/low_mean": 0.0011103039869340137, "clip_ratio/low_min": 0.0011103039869340137, "clip_ratio/region_mean": 0.0028598834469448775, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 2665.0, "completions/mean_length": 1343.375, "completions/mean_terminated_length": 1159.86669921875, "completions/min_length": 791.0, "completions/min_terminated_length": 791.0, "entropy": 0.1200641943141818, "epoch": 3.0, "frac_reward_zero_std": 0.0, "grad_norm": 0.023259304463863373, "learning_rate": 7.020408163265307e-06, "loss": -0.0617, "num_tokens": 9636933.0, "reward": 0.657095193862915, "reward_std": 0.13095298409461975, "rewards/reward_commands_completeness/mean": 0.12125000357627869, "rewards/reward_commands_completeness/std": 0.036124784499406815, "rewards/reward_commands_correctness/mean": 0.06187500059604645, "rewards/reward_commands_correctness/std": 0.0204022079706192, "rewards/reward_diversity/mean": 0.16834516823291779, "rewards/reward_diversity/std": 0.042147647589445114, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08062499761581421, "rewards/reward_problem_validity/std": 0.019482901319861412, "rewards/reward_solution_effectiveness/mean": 0.13124999403953552, "rewards/reward_solution_effectiveness/std": 0.05123475193977356, "sampling/importance_sampling_ratio/max": 1.4137237071990967, "sampling/importance_sampling_ratio/mean": 0.2296113669872284, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.247617721557617, "sampling/sampling_logp_difference/mean": 0.016230998560786247, "step": 147, "step_time": 389.990871315822 }, { "clip_ratio/high_max": 0.0020875236514257267, "clip_ratio/high_mean": 0.0020875236514257267, "clip_ratio/low_mean": 0.0009822493302635849, "clip_ratio/low_min": 0.0009822493302635849, "clip_ratio/region_mean": 0.0030697729816893116, "completions/clipped_ratio": 0.0, "completions/max_length": 1384.0, "completions/max_terminated_length": 1384.0, "completions/mean_length": 1049.3125, "completions/mean_terminated_length": 1049.3125, "completions/min_length": 788.0, "completions/min_terminated_length": 788.0, "entropy": 0.12384253367781639, "epoch": 3.020408163265306, "frac_reward_zero_std": 0.0, "grad_norm": 0.010595868341624737, "learning_rate": 7e-06, "loss": -0.0118, "num_tokens": 9680514.0, "reward": 0.6477862596511841, "reward_std": 0.09582912921905518, "rewards/reward_commands_completeness/mean": 0.11875000596046448, "rewards/reward_commands_completeness/std": 0.03685557842254639, "rewards/reward_commands_correctness/mean": 0.054999999701976776, "rewards/reward_commands_correctness/std": 0.02129162661731243, "rewards/reward_diversity/mean": 0.1784113049507141, "rewards/reward_diversity/std": 0.02879391796886921, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.078125, "rewards/reward_problem_validity/std": 0.01869715005159378, "rewards/reward_solution_effectiveness/mean": 0.1237500011920929, "rewards/reward_solution_effectiveness/std": 0.050049975514411926, "sampling/importance_sampling_ratio/max": 2.6927707195281982, "sampling/importance_sampling_ratio/mean": 0.22617194056510925, "sampling/importance_sampling_ratio/min": 0.00024340552045032382, "sampling/sampling_logp_difference/max": 2.977924346923828, "sampling/sampling_logp_difference/mean": 0.019886566326022148, "step": 148, "step_time": 140.68038208410144 }, { "clip_ratio/high_max": 0.0027560275557334535, "clip_ratio/high_mean": 0.0027560275557334535, "clip_ratio/low_mean": 0.0012599233377841301, "clip_ratio/low_min": 0.0012599233377841301, "clip_ratio/region_mean": 0.004015950929897372, "completions/clipped_ratio": 0.0, "completions/max_length": 1215.0, "completions/max_terminated_length": 1215.0, "completions/mean_length": 1011.125, "completions/mean_terminated_length": 1011.125, "completions/min_length": 674.0, "completions/min_terminated_length": 674.0, "entropy": 0.10718580149114132, "epoch": 3.0408163265306123, "frac_reward_zero_std": 0.0, "grad_norm": 0.07008996605873108, "learning_rate": 6.979591836734695e-06, "loss": 0.1502, "num_tokens": 9720552.0, "reward": 0.7586587071418762, "reward_std": 0.08015425503253937, "rewards/reward_commands_completeness/mean": 0.14249999821186066, "rewards/reward_commands_completeness/std": 0.03568379953503609, "rewards/reward_commands_correctness/mean": 0.07375000417232513, "rewards/reward_commands_correctness/std": 0.00806225836277008, "rewards/reward_diversity/mean": 0.1836586594581604, "rewards/reward_diversity/std": 0.017786186188459396, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.06361014395952225, "sampling/importance_sampling_ratio/max": 1.3801473379135132, "sampling/importance_sampling_ratio/mean": 0.13655737042427063, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.188030958175659, "sampling/sampling_logp_difference/mean": 0.01962905004620552, "step": 149, "step_time": 129.54076802916825 }, { "clip_ratio/high_max": 0.0018864586745621637, "clip_ratio/high_mean": 0.0018864586745621637, "clip_ratio/low_mean": 0.001149663068645168, "clip_ratio/low_min": 0.001149663068645168, "clip_ratio/region_mean": 0.003036121735931374, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1348.0, "completions/mean_length": 1251.875, "completions/mean_terminated_length": 1062.2667236328125, "completions/min_length": 721.0, "completions/min_terminated_length": 721.0, "entropy": 0.12692662933841348, "epoch": 3.061224489795918, "frac_reward_zero_std": 0.0, "grad_norm": 0.02640928514301777, "learning_rate": 6.959183673469388e-06, "loss": -0.0524, "num_tokens": 9762978.0, "reward": 0.6908736228942871, "reward_std": 0.10280480235815048, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.03614784777164459, "rewards/reward_commands_correctness/mean": 0.0637499988079071, "rewards/reward_commands_correctness/std": 0.020289571955800056, "rewards/reward_diversity/mean": 0.1883736550807953, "rewards/reward_diversity/std": 0.020683998242020607, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.13875000178813934, "rewards/reward_solution_effectiveness/std": 0.05123475566506386, "sampling/importance_sampling_ratio/max": 0.8265936374664307, "sampling/importance_sampling_ratio/mean": 0.08290963619947433, "sampling/importance_sampling_ratio/min": 6.337060040095821e-05, "sampling/sampling_logp_difference/max": 7.37949275970459, "sampling/sampling_logp_difference/mean": 0.017663050442934036, "step": 150, "step_time": 263.22055439837277 }, { "clip_ratio/high_max": 0.001616009703866439, "clip_ratio/high_mean": 0.001616009703866439, "clip_ratio/low_mean": 0.0009895510047499556, "clip_ratio/low_min": 0.0009895510047499556, "clip_ratio/region_mean": 0.0026055607086163945, "completions/clipped_ratio": 0.0, "completions/max_length": 2844.0, "completions/max_terminated_length": 2844.0, "completions/mean_length": 1156.875, "completions/mean_terminated_length": 1156.875, "completions/min_length": 868.0, "completions/min_terminated_length": 868.0, "entropy": 0.108652513474226, "epoch": 3.0816326530612246, "frac_reward_zero_std": 0.0, "grad_norm": 0.00751389330253005, "learning_rate": 6.938775510204082e-06, "loss": 0.0137, "num_tokens": 9807008.0, "reward": 0.5612799525260925, "reward_std": 0.10166376829147339, "rewards/reward_commands_completeness/mean": 0.10249999910593033, "rewards/reward_commands_completeness/std": 0.044944409281015396, "rewards/reward_commands_correctness/mean": 0.050624996423721313, "rewards/reward_commands_correctness/std": 0.03151058405637741, "rewards/reward_diversity/mean": 0.15440496802330017, "rewards/reward_diversity/std": 0.06783657521009445, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.06937500089406967, "rewards/reward_problem_validity/std": 0.031930919736623764, "rewards/reward_solution_effectiveness/mean": 0.10312499105930328, "rewards/reward_solution_effectiveness/std": 0.054738011211156845, "sampling/importance_sampling_ratio/max": 0.5260887742042542, "sampling/importance_sampling_ratio/mean": 0.05288383364677429, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.496179580688477, "sampling/sampling_logp_difference/mean": 0.016949743032455444, "step": 151, "step_time": 212.0862814001739 }, { "clip_ratio/high_max": 0.002514217936550267, "clip_ratio/high_mean": 0.002514217936550267, "clip_ratio/low_mean": 0.001004121710138861, "clip_ratio/low_min": 0.001004121710138861, "clip_ratio/region_mean": 0.0035183395812055096, "completions/clipped_ratio": 0.0, "completions/max_length": 1147.0, "completions/max_terminated_length": 1147.0, "completions/mean_length": 1023.1875, "completions/mean_terminated_length": 1023.1875, "completions/min_length": 761.0, "completions/min_terminated_length": 761.0, "entropy": 0.09729345701634884, "epoch": 3.1020408163265305, "frac_reward_zero_std": 0.0, "grad_norm": 0.013679534196853638, "learning_rate": 6.9183673469387755e-06, "loss": 0.0126, "num_tokens": 9848227.0, "reward": 0.778245747089386, "reward_std": 0.0821879431605339, "rewards/reward_commands_completeness/mean": 0.1525000035762787, "rewards/reward_commands_completeness/std": 0.025166118517518044, "rewards/reward_commands_correctness/mean": 0.0768750011920929, "rewards/reward_commands_correctness/std": 0.01447699498385191, "rewards/reward_diversity/mean": 0.16512075066566467, "rewards/reward_diversity/std": 0.02020184136927128, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08875000476837158, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.19499999284744263, "rewards/reward_solution_effectiveness/std": 0.04242640361189842, "sampling/importance_sampling_ratio/max": 0.6466401219367981, "sampling/importance_sampling_ratio/mean": 0.11158531904220581, "sampling/importance_sampling_ratio/min": 0.0009669578284956515, "sampling/sampling_logp_difference/max": 2.993619918823242, "sampling/sampling_logp_difference/mean": 0.018261997029185295, "step": 152, "step_time": 143.39606064930558 }, { "clip_ratio/high_max": 0.0024591088440502062, "clip_ratio/high_mean": 0.0024591088440502062, "clip_ratio/low_mean": 0.0010377625876571983, "clip_ratio/low_min": 0.0010377625876571983, "clip_ratio/region_mean": 0.0034968714462593198, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1218.0, "completions/mean_length": 1191.3125, "completions/mean_terminated_length": 997.666748046875, "completions/min_length": 721.0, "completions/min_terminated_length": 721.0, "entropy": 0.10792609862983227, "epoch": 3.122448979591837, "frac_reward_zero_std": 0.0, "grad_norm": 0.03697845712304115, "learning_rate": 6.8979591836734705e-06, "loss": -0.0928, "num_tokens": 9886996.0, "reward": 0.6220225095748901, "reward_std": 0.17005354166030884, "rewards/reward_commands_completeness/mean": 0.11374999582767487, "rewards/reward_commands_completeness/std": 0.041773200035095215, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.024689743295311928, "rewards/reward_diversity/mean": 0.15952250361442566, "rewards/reward_diversity/std": 0.04165748506784439, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07437500357627869, "rewards/reward_problem_validity/std": 0.025552235543727875, "rewards/reward_solution_effectiveness/mean": 0.1237499937415123, "rewards/reward_solution_effectiveness/std": 0.05463515222072601, "sampling/importance_sampling_ratio/max": 0.9241778254508972, "sampling/importance_sampling_ratio/mean": 0.1347402185201645, "sampling/importance_sampling_ratio/min": 4.6075638238107786e-06, "sampling/sampling_logp_difference/max": 8.31842041015625, "sampling/sampling_logp_difference/mean": 0.018117694184184074, "step": 153, "step_time": 256.27602535299957 }, { "clip_ratio/high_max": 0.0016497200995218009, "clip_ratio/high_mean": 0.0016497200995218009, "clip_ratio/low_mean": 0.0012374745492707007, "clip_ratio/low_min": 0.0012374745492707007, "clip_ratio/region_mean": 0.0028871946269646287, "completions/clipped_ratio": 0.0, "completions/max_length": 1346.0, "completions/max_terminated_length": 1346.0, "completions/mean_length": 1202.25, "completions/mean_terminated_length": 1202.25, "completions/min_length": 807.0, "completions/min_terminated_length": 807.0, "entropy": 0.08742840215563774, "epoch": 3.142857142857143, "frac_reward_zero_std": 0.0, "grad_norm": 0.021196259185671806, "learning_rate": 6.877551020408164e-06, "loss": -0.0078, "num_tokens": 9929216.0, "reward": 0.6266883015632629, "reward_std": 0.10930197685956955, "rewards/reward_commands_completeness/mean": 0.11375000327825546, "rewards/reward_commands_completeness/std": 0.04716990888118744, "rewards/reward_commands_correctness/mean": 0.0625, "rewards/reward_commands_correctness/std": 0.027928480878472328, "rewards/reward_diversity/mean": 0.1585633009672165, "rewards/reward_diversity/std": 0.026140041649341583, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07500000298023224, "rewards/reward_problem_validity/std": 0.025819892063736916, "rewards/reward_solution_effectiveness/mean": 0.12937499582767487, "rewards/reward_solution_effectiveness/std": 0.06361014395952225, "sampling/importance_sampling_ratio/max": 0.5703821182250977, "sampling/importance_sampling_ratio/mean": 0.11209141463041306, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.704590082168579, "sampling/sampling_logp_difference/mean": 0.015529794618487358, "step": 154, "step_time": 139.75645956397057 }, { "clip_ratio/high_max": 0.0012475863841245882, "clip_ratio/high_mean": 0.0012475863841245882, "clip_ratio/low_mean": 0.00104841834945546, "clip_ratio/low_min": 0.00104841834945546, "clip_ratio/region_mean": 0.0022960047135711648, "completions/clipped_ratio": 0.0, "completions/max_length": 3319.0, "completions/max_terminated_length": 3319.0, "completions/mean_length": 1301.3125, "completions/mean_terminated_length": 1301.3125, "completions/min_length": 1043.0, "completions/min_terminated_length": 1043.0, "entropy": 0.0913095735013485, "epoch": 3.163265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.030657237395644188, "learning_rate": 6.857142857142858e-06, "loss": -0.0969, "num_tokens": 9986301.0, "reward": 0.7269445061683655, "reward_std": 0.088139608502388, "rewards/reward_commands_completeness/mean": 0.13624998927116394, "rewards/reward_commands_completeness/std": 0.03117157146334648, "rewards/reward_commands_correctness/mean": 0.06687499582767487, "rewards/reward_commands_correctness/std": 0.01447699498385191, "rewards/reward_diversity/mean": 0.17881950736045837, "rewards/reward_diversity/std": 0.013986669480800629, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500774860382, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.15937501192092896, "rewards/reward_solution_effectiveness/std": 0.053350258618593216, "sampling/importance_sampling_ratio/max": 0.9990645051002502, "sampling/importance_sampling_ratio/mean": 0.17504262924194336, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.576983690261841, "sampling/sampling_logp_difference/mean": 0.014372095465660095, "step": 155, "step_time": 274.8555154018104 }, { "clip_ratio/high_max": 0.0017088044914999045, "clip_ratio/high_mean": 0.0017088044914999045, "clip_ratio/low_mean": 0.0012724708140012808, "clip_ratio/low_min": 0.0012724708140012808, "clip_ratio/region_mean": 0.0029812753637088463, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 1322.0, "completions/mean_terminated_length": 1137.0667724609375, "completions/min_length": 862.0, "completions/min_terminated_length": 862.0, "entropy": 0.1008269414305687, "epoch": 3.183673469387755, "frac_reward_zero_std": 0.0, "grad_norm": 0.039045002311468124, "learning_rate": 6.836734693877551e-06, "loss": -0.0483, "num_tokens": 10053281.0, "reward": 0.5937092900276184, "reward_std": 0.21490681171417236, "rewards/reward_commands_completeness/mean": 0.10624999552965164, "rewards/reward_commands_completeness/std": 0.04937948286533356, "rewards/reward_commands_correctness/mean": 0.06062500178813934, "rewards/reward_commands_correctness/std": 0.027439329773187637, "rewards/reward_diversity/mean": 0.14370927214622498, "rewards/reward_diversity/std": 0.06240123137831688, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07250000536441803, "rewards/reward_problem_validity/std": 0.03130495548248291, "rewards/reward_solution_effectiveness/mean": 0.12937501072883606, "rewards/reward_solution_effectiveness/std": 0.0690380334854126, "sampling/importance_sampling_ratio/max": 0.7995017170906067, "sampling/importance_sampling_ratio/mean": 0.17831267416477203, "sampling/importance_sampling_ratio/min": 6.377002137014642e-06, "sampling/sampling_logp_difference/max": 7.204518795013428, "sampling/sampling_logp_difference/mean": 0.015147611498832703, "step": 156, "step_time": 387.3742290046066 }, { "clip_ratio/high_max": 0.0014563470322173089, "clip_ratio/high_mean": 0.0014563470322173089, "clip_ratio/low_mean": 0.001410683267749846, "clip_ratio/low_min": 0.001410683267749846, "clip_ratio/region_mean": 0.0028670302854152396, "completions/clipped_ratio": 0.0, "completions/max_length": 2619.0, "completions/max_terminated_length": 2619.0, "completions/mean_length": 1150.4375, "completions/mean_terminated_length": 1150.4375, "completions/min_length": 742.0, "completions/min_terminated_length": 742.0, "entropy": 0.1010713716968894, "epoch": 3.204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.028031108900904655, "learning_rate": 6.816326530612245e-06, "loss": -0.0539, "num_tokens": 10094556.0, "reward": 0.5876736640930176, "reward_std": 0.11543097347021103, "rewards/reward_commands_completeness/mean": 0.10625000298023224, "rewards/reward_commands_completeness/std": 0.048836465924978256, "rewards/reward_commands_correctness/mean": 0.0625, "rewards/reward_commands_correctness/std": 0.030000003054738045, "rewards/reward_diversity/mean": 0.14142364263534546, "rewards/reward_diversity/std": 0.06641928851604462, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.06875000149011612, "rewards/reward_problem_validity/std": 0.03159641847014427, "rewards/reward_solution_effectiveness/mean": 0.1274999976158142, "rewards/reward_solution_effectiveness/std": 0.06340347230434418, "sampling/importance_sampling_ratio/max": 1.121285080909729, "sampling/importance_sampling_ratio/mean": 0.13808715343475342, "sampling/importance_sampling_ratio/min": 6.483581273641903e-06, "sampling/sampling_logp_difference/max": 7.494147300720215, "sampling/sampling_logp_difference/mean": 0.01766599901020527, "step": 157, "step_time": 190.1782462373376 }, { "clip_ratio/high_max": 0.002743037897744216, "clip_ratio/high_mean": 0.002743037897744216, "clip_ratio/low_mean": 0.0007319720389205031, "clip_ratio/low_min": 0.0007319720389205031, "clip_ratio/region_mean": 0.0034750099293887615, "completions/clipped_ratio": 0.0, "completions/max_length": 2069.0, "completions/max_terminated_length": 2069.0, "completions/mean_length": 1123.4375, "completions/mean_terminated_length": 1123.4375, "completions/min_length": 733.0, "completions/min_terminated_length": 733.0, "entropy": 0.10405243374407291, "epoch": 3.2244897959183674, "frac_reward_zero_std": 0.0, "grad_norm": 0.0402417816221714, "learning_rate": 6.7959183673469394e-06, "loss": -0.0506, "num_tokens": 10134303.0, "reward": 0.6531339883804321, "reward_std": 0.1745838075876236, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.04704608395695686, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.02468974143266678, "rewards/reward_diversity/mean": 0.1537589281797409, "rewards/reward_diversity/std": 0.04602089151740074, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07750000059604645, "rewards/reward_problem_validity/std": 0.02670830301940441, "rewards/reward_solution_effectiveness/mean": 0.14624999463558197, "rewards/reward_solution_effectiveness/std": 0.07003571093082428, "sampling/importance_sampling_ratio/max": 2.202841281890869, "sampling/importance_sampling_ratio/mean": 0.23489783704280853, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.362853050231934, "sampling/sampling_logp_difference/mean": 0.018992120400071144, "step": 158, "step_time": 163.47528105974197 }, { "clip_ratio/high_max": 0.0018781306134769693, "clip_ratio/high_mean": 0.0018781306134769693, "clip_ratio/low_mean": 0.0007610184693476185, "clip_ratio/low_min": 0.0007610184693476185, "clip_ratio/region_mean": 0.0026391491264803335, "completions/clipped_ratio": 0.0, "completions/max_length": 1366.0, "completions/max_terminated_length": 1366.0, "completions/mean_length": 1072.4375, "completions/mean_terminated_length": 1072.4375, "completions/min_length": 682.0, "completions/min_terminated_length": 682.0, "entropy": 0.11031908076256514, "epoch": 3.2448979591836733, "frac_reward_zero_std": 0.0, "grad_norm": 0.06135181710124016, "learning_rate": 6.7755102040816336e-06, "loss": -0.0535, "num_tokens": 10177790.0, "reward": 0.6783026456832886, "reward_std": 0.1601163148880005, "rewards/reward_commands_completeness/mean": 0.1237499937415123, "rewards/reward_commands_completeness/std": 0.043339744210243225, "rewards/reward_commands_correctness/mean": 0.0625, "rewards/reward_commands_correctness/std": 0.025166116654872894, "rewards/reward_diversity/mean": 0.16955268383026123, "rewards/reward_diversity/std": 0.04096456617116928, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.027195284143090248, "rewards/reward_solution_effectiveness/mean": 0.15562500059604645, "rewards/reward_solution_effectiveness/std": 0.06408002972602844, "sampling/importance_sampling_ratio/max": 1.5987519025802612, "sampling/importance_sampling_ratio/mean": 0.20272785425186157, "sampling/importance_sampling_ratio/min": 0.00022005150094628334, "sampling/sampling_logp_difference/max": 3.276998519897461, "sampling/sampling_logp_difference/mean": 0.01856733113527298, "step": 159, "step_time": 150.9300391227007 }, { "clip_ratio/high_max": 0.001950921279785689, "clip_ratio/high_mean": 0.001950921279785689, "clip_ratio/low_mean": 0.0016017537273000926, "clip_ratio/low_min": 0.0016017537273000926, "clip_ratio/region_mean": 0.0035526750289136544, "completions/clipped_ratio": 0.0, "completions/max_length": 1432.0, "completions/max_terminated_length": 1432.0, "completions/mean_length": 1198.75, "completions/mean_terminated_length": 1198.75, "completions/min_length": 1013.0, "completions/min_terminated_length": 1013.0, "entropy": 0.09411183279007673, "epoch": 3.2653061224489797, "frac_reward_zero_std": 0.0, "grad_norm": 0.04716077819466591, "learning_rate": 6.755102040816327e-06, "loss": -0.1337, "num_tokens": 10222210.0, "reward": 0.5557140111923218, "reward_std": 0.07456067204475403, "rewards/reward_commands_completeness/mean": 0.10624999552965164, "rewards/reward_commands_completeness/std": 0.04716990143060684, "rewards/reward_commands_correctness/mean": 0.05562499910593033, "rewards/reward_commands_correctness/std": 0.025552235543727875, "rewards/reward_diversity/mean": 0.12383908778429031, "rewards/reward_diversity/std": 0.10746260732412338, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07062499970197678, "rewards/reward_problem_validity/std": 0.03043435700237751, "rewards/reward_solution_effectiveness/mean": 0.11812499910593033, "rewards/reward_solution_effectiveness/std": 0.05635823309421539, "sampling/importance_sampling_ratio/max": 1.9965143203735352, "sampling/importance_sampling_ratio/mean": 0.2563055753707886, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6750949621200562, "sampling/sampling_logp_difference/mean": 0.015489418059587479, "step": 160, "step_time": 145.73303684778512 }, { "clip_ratio/high_max": 0.0026391144674562383, "clip_ratio/high_mean": 0.0026391144674562383, "clip_ratio/low_mean": 0.0011011358437826857, "clip_ratio/low_min": 0.0011011358437826857, "clip_ratio/region_mean": 0.003740250234841369, "completions/clipped_ratio": 0.0, "completions/max_length": 1136.0, "completions/max_terminated_length": 1136.0, "completions/mean_length": 939.6875, "completions/mean_terminated_length": 939.6875, "completions/min_length": 677.0, "completions/min_terminated_length": 677.0, "entropy": 0.1266402918845415, "epoch": 3.2857142857142856, "frac_reward_zero_std": 0.0, "grad_norm": 0.02709079347550869, "learning_rate": 6.734693877551021e-06, "loss": -0.0045, "num_tokens": 10276589.0, "reward": 0.6931949257850647, "reward_std": 0.10737545788288116, "rewards/reward_commands_completeness/mean": 0.12250000238418579, "rewards/reward_commands_completeness/std": 0.03642343729734421, "rewards/reward_commands_correctness/mean": 0.06937499344348907, "rewards/reward_commands_correctness/std": 0.02143789827823639, "rewards/reward_diversity/mean": 0.18194490671157837, "rewards/reward_diversity/std": 0.017342649400234222, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.05715111643075943, "sampling/importance_sampling_ratio/max": 0.6477994918823242, "sampling/importance_sampling_ratio/mean": 0.1503894329071045, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.247157573699951, "sampling/sampling_logp_difference/mean": 0.0228677149862051, "step": 161, "step_time": 229.01253251358867 }, { "clip_ratio/high_max": 0.0015405518061015755, "clip_ratio/high_mean": 0.0015405518061015755, "clip_ratio/low_mean": 0.0006856642285129055, "clip_ratio/low_min": 0.0006856642285129055, "clip_ratio/region_mean": 0.002226216034614481, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1409.0, "completions/mean_length": 1378.75, "completions/mean_terminated_length": 1197.60009765625, "completions/min_length": 837.0, "completions/min_terminated_length": 837.0, "entropy": 0.08854001574218273, "epoch": 3.306122448979592, "frac_reward_zero_std": 0.0, "grad_norm": 0.02315008081495762, "learning_rate": 6.714285714285714e-06, "loss": -0.0771, "num_tokens": 10332369.0, "reward": 0.5582437515258789, "reward_std": 0.19884620606899261, "rewards/reward_commands_completeness/mean": 0.09749999642372131, "rewards/reward_commands_completeness/std": 0.0478191077709198, "rewards/reward_commands_correctness/mean": 0.054999999701976776, "rewards/reward_commands_correctness/std": 0.02780887298285961, "rewards/reward_diversity/mean": 0.14449374377727509, "rewards/reward_diversity/std": 0.06343762576580048, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.06937500089406967, "rewards/reward_problem_validity/std": 0.029769953340291977, "rewards/reward_solution_effectiveness/mean": 0.1106249988079071, "rewards/reward_solution_effectiveness/std": 0.06071449816226959, "sampling/importance_sampling_ratio/max": 1.0884720087051392, "sampling/importance_sampling_ratio/mean": 0.13294389843940735, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.193528175354004, "sampling/sampling_logp_difference/mean": 0.0148319648578763, "step": 162, "step_time": 297.2850490361452 }, { "clip_ratio/high_max": 0.002422594970994396, "clip_ratio/high_mean": 0.002422594970994396, "clip_ratio/low_mean": 0.0007121864327928051, "clip_ratio/low_min": 0.0007121864327928051, "clip_ratio/region_mean": 0.003134781392873265, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1459.0, "completions/mean_length": 1456.6875, "completions/mean_terminated_length": 1079.6429443359375, "completions/min_length": 689.0, "completions/min_terminated_length": 689.0, "entropy": 0.10286882892251015, "epoch": 3.326530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.010289882309734821, "learning_rate": 6.693877551020409e-06, "loss": 0.0147, "num_tokens": 10380020.0, "reward": 0.5871776938438416, "reward_std": 0.1586686372756958, "rewards/reward_commands_completeness/mean": 0.10124999284744263, "rewards/reward_commands_completeness/std": 0.040311288088560104, "rewards/reward_commands_correctness/mean": 0.05937499552965164, "rewards/reward_commands_correctness/std": 0.025157837197184563, "rewards/reward_diversity/mean": 0.1609276980161667, "rewards/reward_diversity/std": 0.042417243123054504, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07312500476837158, "rewards/reward_problem_validity/std": 0.024958299472928047, "rewards/reward_solution_effectiveness/mean": 0.10500000417232513, "rewards/reward_solution_effectiveness/std": 0.05253570154309273, "sampling/importance_sampling_ratio/max": 0.23679287731647491, "sampling/importance_sampling_ratio/mean": 0.05037563666701317, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.598575115203857, "sampling/sampling_logp_difference/mean": 0.014392460696399212, "step": 163, "step_time": 268.34935741871595 }, { "clip_ratio/high_max": 0.0016485583437315654, "clip_ratio/high_mean": 0.0016485583437315654, "clip_ratio/low_mean": 0.0008831809464027174, "clip_ratio/low_min": 0.0008831809464027174, "clip_ratio/region_mean": 0.0025317392573924735, "completions/clipped_ratio": 0.0, "completions/max_length": 1263.0, "completions/max_terminated_length": 1263.0, "completions/mean_length": 1085.25, "completions/mean_terminated_length": 1085.25, "completions/min_length": 861.0, "completions/min_terminated_length": 861.0, "entropy": 0.12244276236742735, "epoch": 3.3469387755102042, "frac_reward_zero_std": 0.0, "grad_norm": 0.10483803600072861, "learning_rate": 6.6734693877551025e-06, "loss": 0.1563, "num_tokens": 10426676.0, "reward": 0.6066911816596985, "reward_std": 0.1665652096271515, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.055121079087257385, "rewards/reward_commands_correctness/mean": 0.05687500163912773, "rewards/reward_commands_correctness/std": 0.02960151992738247, "rewards/reward_diversity/mean": 0.16481615602970123, "rewards/reward_diversity/std": 0.05801796913146973, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.06875000149011612, "rewards/reward_problem_validity/std": 0.031171569600701332, "rewards/reward_solution_effectiveness/mean": 0.12000000476837158, "rewards/reward_solution_effectiveness/std": 0.06752777099609375, "sampling/importance_sampling_ratio/max": 2.119562864303589, "sampling/importance_sampling_ratio/mean": 0.25725388526916504, "sampling/importance_sampling_ratio/min": 0.00034788245102390647, "sampling/sampling_logp_difference/max": 2.5627505779266357, "sampling/sampling_logp_difference/mean": 0.021282054483890533, "step": 164, "step_time": 156.36844247952104 }, { "clip_ratio/high_max": 0.0007930876199679915, "clip_ratio/high_mean": 0.0007930876199679915, "clip_ratio/low_mean": 0.0013404605124378577, "clip_ratio/low_min": 0.0013404605124378577, "clip_ratio/region_mean": 0.002133548154233722, "completions/clipped_ratio": 0.0, "completions/max_length": 1246.0, "completions/max_terminated_length": 1246.0, "completions/mean_length": 1103.4375, "completions/mean_terminated_length": 1103.4375, "completions/min_length": 976.0, "completions/min_terminated_length": 976.0, "entropy": 0.09941120818257332, "epoch": 3.36734693877551, "frac_reward_zero_std": 0.25, "grad_norm": 0.08611874282360077, "learning_rate": 6.653061224489797e-06, "loss": -0.2234, "num_tokens": 10472183.0, "reward": 0.5215246677398682, "reward_std": 0.08538417518138885, "rewards/reward_commands_completeness/mean": 0.09874999523162842, "rewards/reward_commands_completeness/std": 0.05289927497506142, "rewards/reward_commands_correctness/mean": 0.04874999821186066, "rewards/reward_commands_correctness/std": 0.02753785438835621, "rewards/reward_diversity/mean": 0.1265246421098709, "rewards/reward_diversity/std": 0.06560441106557846, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06562499701976776, "rewards/reward_problem_validity/std": 0.03346017748117447, "rewards/reward_solution_effectiveness/mean": 0.1068749949336052, "rewards/reward_solution_effectiveness/std": 0.07096654176712036, "sampling/importance_sampling_ratio/max": 2.6552348136901855, "sampling/importance_sampling_ratio/mean": 0.2555970251560211, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.194746255874634, "sampling/sampling_logp_difference/mean": 0.016924699768424034, "step": 165, "step_time": 140.3945412747562 }, { "clip_ratio/high_max": 0.0021280074142850935, "clip_ratio/high_mean": 0.0021280074142850935, "clip_ratio/low_mean": 0.0006991351328906603, "clip_ratio/low_min": 0.0006991351328906603, "clip_ratio/region_mean": 0.002827142547175754, "completions/clipped_ratio": 0.0, "completions/max_length": 1345.0, "completions/max_terminated_length": 1345.0, "completions/mean_length": 1128.0625, "completions/mean_terminated_length": 1128.0625, "completions/min_length": 786.0, "completions/min_terminated_length": 786.0, "entropy": 0.09721376374363899, "epoch": 3.387755102040816, "frac_reward_zero_std": 0.0, "grad_norm": 0.016697602346539497, "learning_rate": 6.63265306122449e-06, "loss": 0.0379, "num_tokens": 10522768.0, "reward": 0.6253312826156616, "reward_std": 0.11128025501966476, "rewards/reward_commands_completeness/mean": 0.11124999821186066, "rewards/reward_commands_completeness/std": 0.037215590476989746, "rewards/reward_commands_correctness/mean": 0.06624999642372131, "rewards/reward_commands_correctness/std": 0.022472208365797997, "rewards/reward_diversity/mean": 0.16033132374286652, "rewards/reward_diversity/std": 0.039415229111909866, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07750000059604645, "rewards/reward_problem_validity/std": 0.018797162920236588, "rewards/reward_solution_effectiveness/mean": 0.11625000834465027, "rewards/reward_solution_effectiveness/std": 0.05239275097846985, "sampling/importance_sampling_ratio/max": 2.179290533065796, "sampling/importance_sampling_ratio/mean": 0.2196475863456726, "sampling/importance_sampling_ratio/min": 0.0019427554216235876, "sampling/sampling_logp_difference/max": 1.9320874214172363, "sampling/sampling_logp_difference/mean": 0.01748763583600521, "step": 166, "step_time": 165.37171362154186 }, { "clip_ratio/high_max": 0.0015277641214197502, "clip_ratio/high_mean": 0.0015277641214197502, "clip_ratio/low_mean": 0.0008043467241805047, "clip_ratio/low_min": 0.0008043467241805047, "clip_ratio/region_mean": 0.002332110845600255, "completions/clipped_ratio": 0.0, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 1124.9375, "completions/mean_terminated_length": 1124.9375, "completions/min_length": 709.0, "completions/min_terminated_length": 709.0, "entropy": 0.09890777710825205, "epoch": 3.4081632653061225, "frac_reward_zero_std": 0.25, "grad_norm": 0.020359572023153305, "learning_rate": 6.612244897959185e-06, "loss": -0.0594, "num_tokens": 10564059.0, "reward": 0.5174346566200256, "reward_std": 0.1456395834684372, "rewards/reward_commands_completeness/mean": 0.09624999761581421, "rewards/reward_commands_completeness/std": 0.061630621552467346, "rewards/reward_commands_correctness/mean": 0.05375000089406967, "rewards/reward_commands_correctness/std": 0.033837851136922836, "rewards/reward_diversity/mean": 0.11930961906909943, "rewards/reward_diversity/std": 0.07078447937965393, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.0612499974668026, "rewards/reward_problem_validity/std": 0.035939764231443405, "rewards/reward_solution_effectiveness/mean": 0.11812499165534973, "rewards/reward_solution_effectiveness/std": 0.08231797814369202, "sampling/importance_sampling_ratio/max": 0.9877572655677795, "sampling/importance_sampling_ratio/mean": 0.12099961936473846, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.980107545852661, "sampling/sampling_logp_difference/mean": 0.017958547919988632, "step": 167, "step_time": 133.00216836854815 }, { "clip_ratio/high_max": 0.002027377558988519, "clip_ratio/high_mean": 0.002027377558988519, "clip_ratio/low_mean": 0.0013992980966577306, "clip_ratio/low_min": 0.0013992980966577306, "clip_ratio/region_mean": 0.00342667568475008, "completions/clipped_ratio": 0.0, "completions/max_length": 1462.0, "completions/max_terminated_length": 1462.0, "completions/mean_length": 1163.5625, "completions/mean_terminated_length": 1163.5625, "completions/min_length": 748.0, "completions/min_terminated_length": 748.0, "entropy": 0.09186290670186281, "epoch": 3.4285714285714284, "frac_reward_zero_std": 0.0, "grad_norm": 0.0054318541660904884, "learning_rate": 6.591836734693878e-06, "loss": -0.0241, "num_tokens": 10606776.0, "reward": 0.6028892993927002, "reward_std": 0.17300724983215332, "rewards/reward_commands_completeness/mean": 0.10999999940395355, "rewards/reward_commands_completeness/std": 0.04898979887366295, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.025223998352885246, "rewards/reward_diversity/mean": 0.1453893482685089, "rewards/reward_diversity/std": 0.05214064568281174, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07312500476837158, "rewards/reward_problem_validity/std": 0.02574716880917549, "rewards/reward_solution_effectiveness/mean": 0.1237499937415123, "rewards/reward_solution_effectiveness/std": 0.058864250779151917, "sampling/importance_sampling_ratio/max": 1.4526443481445312, "sampling/importance_sampling_ratio/mean": 0.10968903452157974, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8207502365112305, "sampling/sampling_logp_difference/mean": 0.0175012219697237, "step": 168, "step_time": 153.9943852685392 }, { "clip_ratio/high_max": 0.0019758076523430645, "clip_ratio/high_mean": 0.0019758076523430645, "clip_ratio/low_mean": 0.0009281277889385819, "clip_ratio/low_min": 0.0009281277889385819, "clip_ratio/region_mean": 0.0029039354267297313, "completions/clipped_ratio": 0.0, "completions/max_length": 1352.0, "completions/max_terminated_length": 1352.0, "completions/mean_length": 1088.375, "completions/mean_terminated_length": 1088.375, "completions/min_length": 776.0, "completions/min_terminated_length": 776.0, "entropy": 0.10126883629709482, "epoch": 3.4489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.015730179846286774, "learning_rate": 6.571428571428572e-06, "loss": -0.0214, "num_tokens": 10654230.0, "reward": 0.5378966927528381, "reward_std": 0.21049652993679047, "rewards/reward_commands_completeness/mean": 0.10000000149011612, "rewards/reward_commands_completeness/std": 0.05656854435801506, "rewards/reward_commands_correctness/mean": 0.05312499776482582, "rewards/reward_commands_correctness/std": 0.03048906847834587, "rewards/reward_diversity/mean": 0.12727166712284088, "rewards/reward_diversity/std": 0.06593968719244003, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06437499821186066, "rewards/reward_problem_validity/std": 0.032653484493494034, "rewards/reward_solution_effectiveness/mean": 0.11812500655651093, "rewards/reward_solution_effectiveness/std": 0.08447632938623428, "sampling/importance_sampling_ratio/max": 0.37090808153152466, "sampling/importance_sampling_ratio/mean": 0.05443674325942993, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3841171264648438, "sampling/sampling_logp_difference/mean": 0.017641546204686165, "step": 169, "step_time": 141.26451500877738 }, { "clip_ratio/high_max": 0.0013493433398252819, "clip_ratio/high_mean": 0.0013493433398252819, "clip_ratio/low_mean": 0.0005912427841394674, "clip_ratio/low_min": 0.0005912427841394674, "clip_ratio/region_mean": 0.001940586127602728, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 1352.3125, "completions/mean_terminated_length": 1169.4000244140625, "completions/min_length": 768.0, "completions/min_terminated_length": 768.0, "entropy": 0.09897319972515106, "epoch": 3.4693877551020407, "frac_reward_zero_std": 0.25, "grad_norm": 0.006879155989736319, "learning_rate": 6.5510204081632656e-06, "loss": -0.0121, "num_tokens": 10702155.0, "reward": 0.7414273619651794, "reward_std": 0.06279744952917099, "rewards/reward_commands_completeness/mean": 0.14375001192092896, "rewards/reward_commands_completeness/std": 0.02753785438835621, "rewards/reward_commands_correctness/mean": 0.07562500238418579, "rewards/reward_commands_correctness/std": 0.01631717011332512, "rewards/reward_diversity/mean": 0.16392731666564941, "rewards/reward_diversity/std": 0.03280438855290413, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.05767364799976349, "sampling/importance_sampling_ratio/max": 0.25712886452674866, "sampling/importance_sampling_ratio/mean": 0.05149443447589874, "sampling/importance_sampling_ratio/min": 4.48175887868274e-05, "sampling/sampling_logp_difference/max": 7.206645488739014, "sampling/sampling_logp_difference/mean": 0.01676766946911812, "step": 170, "step_time": 284.22690217942 }, { "clip_ratio/high_max": 0.001681777648627758, "clip_ratio/high_mean": 0.001681777648627758, "clip_ratio/low_mean": 0.0009489812509855255, "clip_ratio/low_min": 0.0009489812509855255, "clip_ratio/region_mean": 0.002630758928717114, "completions/clipped_ratio": 0.0, "completions/max_length": 1371.0, "completions/max_terminated_length": 1371.0, "completions/mean_length": 1068.875, "completions/mean_terminated_length": 1068.875, "completions/min_length": 665.0, "completions/min_terminated_length": 665.0, "entropy": 0.09988075960427523, "epoch": 3.489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.03517784923315048, "learning_rate": 6.530612244897959e-06, "loss": 0.0076, "num_tokens": 10746981.0, "reward": 0.6296361684799194, "reward_std": 0.14149299263954163, "rewards/reward_commands_completeness/mean": 0.11999999731779099, "rewards/reward_commands_completeness/std": 0.04560701549053192, "rewards/reward_commands_correctness/mean": 0.0624999962747097, "rewards/reward_commands_correctness/std": 0.02408318966627121, "rewards/reward_diversity/mean": 0.15276119112968445, "rewards/reward_diversity/std": 0.05437331646680832, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07562500238418579, "rewards/reward_problem_validity/std": 0.02606882154941559, "rewards/reward_solution_effectiveness/mean": 0.13124999403953552, "rewards/reward_solution_effectiveness/std": 0.059874869883060455, "sampling/importance_sampling_ratio/max": 0.6060723066329956, "sampling/importance_sampling_ratio/mean": 0.17059317231178284, "sampling/importance_sampling_ratio/min": 0.00016325662727467716, "sampling/sampling_logp_difference/max": 7.061387062072754, "sampling/sampling_logp_difference/mean": 0.01831633597612381, "step": 171, "step_time": 136.66465247794986 }, { "clip_ratio/high_max": 0.002284683967445744, "clip_ratio/high_mean": 0.002284683967445744, "clip_ratio/low_mean": 0.0008985647800727747, "clip_ratio/low_min": 0.0008985647800727747, "clip_ratio/region_mean": 0.003183248729328625, "completions/clipped_ratio": 0.0, "completions/max_length": 1334.0, "completions/max_terminated_length": 1334.0, "completions/mean_length": 1161.4375, "completions/mean_terminated_length": 1161.4375, "completions/min_length": 1031.0, "completions/min_terminated_length": 1031.0, "entropy": 0.1049173241481185, "epoch": 3.510204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.03211461380124092, "learning_rate": 6.510204081632654e-06, "loss": -0.0827, "num_tokens": 10795096.0, "reward": 0.7752482891082764, "reward_std": 0.08825305849313736, "rewards/reward_commands_completeness/mean": 0.14875000715255737, "rewards/reward_commands_completeness/std": 0.02526526339352131, "rewards/reward_commands_correctness/mean": 0.06812499463558197, "rewards/reward_commands_correctness/std": 0.01276388205587864, "rewards/reward_diversity/mean": 0.1833733320236206, "rewards/reward_diversity/std": 0.014740710146725178, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.1875, "rewards/reward_solution_effectiveness/std": 0.0594979003071785, "sampling/importance_sampling_ratio/max": 1.2290185689926147, "sampling/importance_sampling_ratio/mean": 0.1107509583234787, "sampling/importance_sampling_ratio/min": 0.00022676207299809903, "sampling/sampling_logp_difference/max": 2.652113437652588, "sampling/sampling_logp_difference/mean": 0.0186904426664114, "step": 172, "step_time": 154.03667660988867 }, { "clip_ratio/high_max": 0.0020887700375169516, "clip_ratio/high_mean": 0.0020887700375169516, "clip_ratio/low_mean": 0.0017767522876965813, "clip_ratio/low_min": 0.0017767522876965813, "clip_ratio/region_mean": 0.0038655222742818296, "completions/clipped_ratio": 0.0, "completions/max_length": 1309.0, "completions/max_terminated_length": 1309.0, "completions/mean_length": 1001.875, "completions/mean_terminated_length": 1001.875, "completions/min_length": 711.0, "completions/min_terminated_length": 711.0, "entropy": 0.10532734729349613, "epoch": 3.5306122448979593, "frac_reward_zero_std": 0.0, "grad_norm": 0.02110794186592102, "learning_rate": 6.489795918367348e-06, "loss": 0.0172, "num_tokens": 10849598.0, "reward": 0.6938669085502625, "reward_std": 0.11005894094705582, "rewards/reward_commands_completeness/mean": 0.13124999403953552, "rewards/reward_commands_completeness/std": 0.030956963077187538, "rewards/reward_commands_correctness/mean": 0.06624999642372131, "rewards/reward_commands_correctness/std": 0.017464250326156616, "rewards/reward_diversity/mean": 0.1582418978214264, "rewards/reward_diversity/std": 0.028985513374209404, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437499403953552, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.1537500023841858, "rewards/reward_solution_effectiveness/std": 0.05886424705386162, "sampling/importance_sampling_ratio/max": 0.5989239811897278, "sampling/importance_sampling_ratio/mean": 0.08499865978956223, "sampling/importance_sampling_ratio/min": 2.969205343106296e-05, "sampling/sampling_logp_difference/max": 3.5024163722991943, "sampling/sampling_logp_difference/mean": 0.019685925915837288, "step": 173, "step_time": 237.2612481676042 }, { "clip_ratio/high_max": 0.0025680382386781275, "clip_ratio/high_mean": 0.0025680382386781275, "clip_ratio/low_mean": 0.0006797800306230783, "clip_ratio/low_min": 0.0006797800306230783, "clip_ratio/region_mean": 0.0032478182401973754, "completions/clipped_ratio": 0.0, "completions/max_length": 1406.0, "completions/max_terminated_length": 1406.0, "completions/mean_length": 1062.375, "completions/mean_terminated_length": 1062.375, "completions/min_length": 843.0, "completions/min_terminated_length": 843.0, "entropy": 0.09096067678183317, "epoch": 3.5510204081632653, "frac_reward_zero_std": 0.25, "grad_norm": 0.09182174503803253, "learning_rate": 6.469387755102041e-06, "loss": 0.1418, "num_tokens": 10894996.0, "reward": 0.47244992852211, "reward_std": 0.20452268421649933, "rewards/reward_commands_completeness/mean": 0.09375, "rewards/reward_commands_completeness/std": 0.0687871128320694, "rewards/reward_commands_correctness/mean": 0.04062499850988388, "rewards/reward_commands_correctness/std": 0.028860297054052353, "rewards/reward_diversity/mean": 0.10932493209838867, "rewards/reward_diversity/std": 0.08224322646856308, "rewards/reward_format/mean": 0.05625000223517418, "rewards/reward_format/std": 0.05123475566506386, "rewards/reward_problem_validity/mean": 0.054375000298023224, "rewards/reward_problem_validity/std": 0.04049177095293999, "rewards/reward_solution_effectiveness/mean": 0.11812499910593033, "rewards/reward_solution_effectiveness/std": 0.08658089488744736, "sampling/importance_sampling_ratio/max": 1.7611305713653564, "sampling/importance_sampling_ratio/mean": 0.18218354880809784, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.074704885482788, "sampling/sampling_logp_difference/mean": 0.016459742560982704, "step": 174, "step_time": 140.85730993188918 }, { "clip_ratio/high_max": 0.002132952446117997, "clip_ratio/high_mean": 0.002132952446117997, "clip_ratio/low_mean": 0.0010112288691743743, "clip_ratio/low_min": 0.0010112288691743743, "clip_ratio/region_mean": 0.003144181304378435, "completions/clipped_ratio": 0.0, "completions/max_length": 1479.0, "completions/max_terminated_length": 1479.0, "completions/mean_length": 1109.0, "completions/mean_terminated_length": 1109.0, "completions/min_length": 749.0, "completions/min_terminated_length": 749.0, "entropy": 0.10325226839631796, "epoch": 3.571428571428571, "frac_reward_zero_std": 0.0, "grad_norm": 0.05933542549610138, "learning_rate": 6.4489795918367345e-06, "loss": 0.1128, "num_tokens": 10936304.0, "reward": 0.6011982560157776, "reward_std": 0.1338203251361847, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.04787135869264603, "rewards/reward_commands_correctness/mean": 0.06499999761581421, "rewards/reward_commands_correctness/std": 0.029439203441143036, "rewards/reward_diversity/mean": 0.14307327568531036, "rewards/reward_diversity/std": 0.06343941390514374, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07375000417232513, "rewards/reward_problem_validity/std": 0.03180671110749245, "rewards/reward_solution_effectiveness/mean": 0.12937501072883606, "rewards/reward_solution_effectiveness/std": 0.05870477482676506, "sampling/importance_sampling_ratio/max": 2.381443977355957, "sampling/importance_sampling_ratio/mean": 0.2671052813529968, "sampling/importance_sampling_ratio/min": 5.979067282169126e-05, "sampling/sampling_logp_difference/max": 2.4729342460632324, "sampling/sampling_logp_difference/mean": 0.019321458414196968, "step": 175, "step_time": 138.56638654321432 }, { "clip_ratio/high_max": 0.0020453419201658107, "clip_ratio/high_mean": 0.0020453419201658107, "clip_ratio/low_mean": 0.0015600913502566982, "clip_ratio/low_min": 0.0015600913502566982, "clip_ratio/region_mean": 0.0036054332449566573, "completions/clipped_ratio": 0.0, "completions/max_length": 1131.0, "completions/max_terminated_length": 1131.0, "completions/mean_length": 991.8125, "completions/mean_terminated_length": 991.8125, "completions/min_length": 688.0, "completions/min_terminated_length": 688.0, "entropy": 0.09740329533815384, "epoch": 3.5918367346938775, "frac_reward_zero_std": 0.0, "grad_norm": 0.06358962506055832, "learning_rate": 6.4285714285714295e-06, "loss": -0.172, "num_tokens": 10974481.0, "reward": 0.5751323699951172, "reward_std": 0.17817792296409607, "rewards/reward_commands_completeness/mean": 0.10749999433755875, "rewards/reward_commands_completeness/std": 0.05000000074505806, "rewards/reward_commands_correctness/mean": 0.05624999850988388, "rewards/reward_commands_correctness/std": 0.03201562166213989, "rewards/reward_diversity/mean": 0.14075739681720734, "rewards/reward_diversity/std": 0.061247628182172775, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.06937499344348907, "rewards/reward_problem_validity/std": 0.029769953340291977, "rewards/reward_solution_effectiveness/mean": 0.12000000476837158, "rewards/reward_solution_effectiveness/std": 0.07509993761777878, "sampling/importance_sampling_ratio/max": 1.791257619857788, "sampling/importance_sampling_ratio/mean": 0.18648496270179749, "sampling/importance_sampling_ratio/min": 4.631620686268434e-05, "sampling/sampling_logp_difference/max": 6.451488971710205, "sampling/sampling_logp_difference/mean": 0.019373387098312378, "step": 176, "step_time": 126.37120875529945 }, { "clip_ratio/high_max": 0.0022119736386230215, "clip_ratio/high_mean": 0.0022119736386230215, "clip_ratio/low_mean": 0.0010816056237672456, "clip_ratio/low_min": 0.0010816056237672456, "clip_ratio/region_mean": 0.0032935792551143095, "completions/clipped_ratio": 0.0, "completions/max_length": 1410.0, "completions/max_terminated_length": 1410.0, "completions/mean_length": 1144.625, "completions/mean_terminated_length": 1144.625, "completions/min_length": 711.0, "completions/min_terminated_length": 711.0, "entropy": 0.09428089810535312, "epoch": 3.612244897959184, "frac_reward_zero_std": 0.0, "grad_norm": 0.04089231789112091, "learning_rate": 6.408163265306124e-06, "loss": -0.0868, "num_tokens": 11016615.0, "reward": 0.7017959356307983, "reward_std": 0.05421756953001022, "rewards/reward_commands_completeness/mean": 0.1287499964237213, "rewards/reward_commands_completeness/std": 0.02918332815170288, "rewards/reward_commands_correctness/mean": 0.07124999910593033, "rewards/reward_commands_correctness/std": 0.009574271738529205, "rewards/reward_diversity/mean": 0.15867093205451965, "rewards/reward_diversity/std": 0.027005713433027267, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.15562500059604645, "rewards/reward_solution_effectiveness/std": 0.04802343249320984, "sampling/importance_sampling_ratio/max": 1.2418617010116577, "sampling/importance_sampling_ratio/mean": 0.16162727773189545, "sampling/importance_sampling_ratio/min": 3.8037622829278916e-08, "sampling/sampling_logp_difference/max": 2.747495174407959, "sampling/sampling_logp_difference/mean": 0.019140686839818954, "step": 177, "step_time": 140.48399739526212 }, { "clip_ratio/high_max": 0.0026260640370310284, "clip_ratio/high_mean": 0.0026260640370310284, "clip_ratio/low_mean": 0.00048562911979388446, "clip_ratio/low_min": 0.00048562911979388446, "clip_ratio/region_mean": 0.0031116931495489553, "completions/clipped_ratio": 0.0, "completions/max_length": 1449.0, "completions/max_terminated_length": 1449.0, "completions/mean_length": 1162.125, "completions/mean_terminated_length": 1162.125, "completions/min_length": 735.0, "completions/min_terminated_length": 735.0, "entropy": 0.10821112338453531, "epoch": 3.63265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.01684364303946495, "learning_rate": 6.387755102040817e-06, "loss": -0.0532, "num_tokens": 11064669.0, "reward": 0.7196499109268188, "reward_std": 0.06265103816986084, "rewards/reward_commands_completeness/mean": 0.13375000655651093, "rewards/reward_commands_completeness/std": 0.027049342170357704, "rewards/reward_commands_correctness/mean": 0.06875000149011612, "rewards/reward_commands_correctness/std": 0.017841899767518044, "rewards/reward_diversity/mean": 0.18089988827705383, "rewards/reward_diversity/std": 0.010466036386787891, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.14812499284744263, "rewards/reward_solution_effectiveness/std": 0.05075677111744881, "sampling/importance_sampling_ratio/max": 0.6398871541023254, "sampling/importance_sampling_ratio/mean": 0.14702880382537842, "sampling/importance_sampling_ratio/min": 3.2774255487311166e-06, "sampling/sampling_logp_difference/max": 1.7408771514892578, "sampling/sampling_logp_difference/mean": 0.020017217844724655, "step": 178, "step_time": 165.65135534852743 }, { "clip_ratio/high_max": 0.0013764597242698073, "clip_ratio/high_mean": 0.0013764597242698073, "clip_ratio/low_mean": 0.0011090045045420993, "clip_ratio/low_min": 0.0011090045045420993, "clip_ratio/region_mean": 0.002485464225173928, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1271.0, "completions/mean_length": 1303.625, "completions/mean_terminated_length": 1117.4666748046875, "completions/min_length": 760.0, "completions/min_terminated_length": 760.0, "entropy": 0.0938743925653398, "epoch": 3.6530612244897958, "frac_reward_zero_std": 0.0, "grad_norm": 0.009106012061238289, "learning_rate": 6.36734693877551e-06, "loss": 0.006, "num_tokens": 11108043.0, "reward": 0.7474556565284729, "reward_std": 0.07447116822004318, "rewards/reward_commands_completeness/mean": 0.14000000059604645, "rewards/reward_commands_completeness/std": 0.02828427404165268, "rewards/reward_commands_correctness/mean": 0.07062500715255737, "rewards/reward_commands_correctness/std": 0.012365948408842087, "rewards/reward_diversity/mean": 0.18058066070079803, "rewards/reward_diversity/std": 0.011743471026420593, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.16874998807907104, "rewards/reward_solution_effectiveness/std": 0.05463515222072601, "sampling/importance_sampling_ratio/max": 1.480325698852539, "sampling/importance_sampling_ratio/mean": 0.197413370013237, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.216714382171631, "sampling/sampling_logp_difference/mean": 0.016208985820412636, "step": 179, "step_time": 262.9326936341822 }, { "clip_ratio/high_max": 0.0017357489123241976, "clip_ratio/high_mean": 0.0017357489123241976, "clip_ratio/low_mean": 0.0008241616742452607, "clip_ratio/low_min": 0.0008241616742452607, "clip_ratio/region_mean": 0.0025599106011213735, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1279.0, "completions/mean_length": 1279.0625, "completions/mean_terminated_length": 1091.2667236328125, "completions/min_length": 664.0, "completions/min_terminated_length": 664.0, "entropy": 0.08210070384666324, "epoch": 3.673469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.07404426485300064, "learning_rate": 6.346938775510204e-06, "loss": -0.1547, "num_tokens": 11149380.0, "reward": 0.7468233108520508, "reward_std": 0.13674448430538177, "rewards/reward_commands_completeness/mean": 0.14375001192092896, "rewards/reward_commands_completeness/std": 0.039475735276937485, "rewards/reward_commands_correctness/mean": 0.078125, "rewards/reward_commands_correctness/std": 0.02257395349442959, "rewards/reward_diversity/mean": 0.1643233746290207, "rewards/reward_diversity/std": 0.0265622828155756, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.020000001415610313, "rewards/reward_solution_effectiveness/mean": 0.18187499046325684, "rewards/reward_solution_effectiveness/std": 0.05946637690067291, "sampling/importance_sampling_ratio/max": 2.194624900817871, "sampling/importance_sampling_ratio/mean": 0.17957235872745514, "sampling/importance_sampling_ratio/min": 0.0001413445279467851, "sampling/sampling_logp_difference/max": 7.155375957489014, "sampling/sampling_logp_difference/mean": 0.015530707314610481, "step": 180, "step_time": 263.4044096227735 }, { "clip_ratio/high_max": 0.002170247447793372, "clip_ratio/high_mean": 0.002170247447793372, "clip_ratio/low_mean": 0.0011831531155621633, "clip_ratio/low_min": 0.0011831531155621633, "clip_ratio/region_mean": 0.0033534005633555353, "completions/clipped_ratio": 0.0, "completions/max_length": 1317.0, "completions/max_terminated_length": 1317.0, "completions/mean_length": 1115.5, "completions/mean_terminated_length": 1115.5, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.10177527368068695, "epoch": 3.693877551020408, "frac_reward_zero_std": 0.0, "grad_norm": 0.014271154999732971, "learning_rate": 6.326530612244899e-06, "loss": -0.002, "num_tokens": 11197920.0, "reward": 0.7160313129425049, "reward_std": 0.0699181780219078, "rewards/reward_commands_completeness/mean": 0.12999999523162842, "rewards/reward_commands_completeness/std": 0.03098386898636818, "rewards/reward_commands_correctness/mean": 0.06937500089406967, "rewards/reward_commands_correctness/std": 0.013400870375335217, "rewards/reward_diversity/mean": 0.17665627598762512, "rewards/reward_diversity/std": 0.022487180307507515, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.1537500023841858, "rewards/reward_solution_effectiveness/std": 0.05239274725317955, "sampling/importance_sampling_ratio/max": 1.4177998304367065, "sampling/importance_sampling_ratio/mean": 0.15116079151630402, "sampling/importance_sampling_ratio/min": 0.00034383998718112707, "sampling/sampling_logp_difference/max": 2.5627505779266357, "sampling/sampling_logp_difference/mean": 0.018653247505426407, "step": 181, "step_time": 161.38740830682218 }, { "clip_ratio/high_max": 0.0013023356441408396, "clip_ratio/high_mean": 0.0013023356441408396, "clip_ratio/low_mean": 0.001685077055299189, "clip_ratio/low_min": 0.001685077055299189, "clip_ratio/region_mean": 0.002987412706715986, "completions/clipped_ratio": 0.0, "completions/max_length": 1223.0, "completions/max_terminated_length": 1223.0, "completions/mean_length": 1064.25, "completions/mean_terminated_length": 1064.25, "completions/min_length": 803.0, "completions/min_terminated_length": 803.0, "entropy": 0.10418496280908585, "epoch": 3.7142857142857144, "frac_reward_zero_std": 0.0, "grad_norm": 0.05024203285574913, "learning_rate": 6.3061224489795925e-06, "loss": -0.1772, "num_tokens": 11268920.0, "reward": 0.5850597620010376, "reward_std": 0.11618730425834656, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.04616997390985489, "rewards/reward_commands_correctness/mean": 0.05312499776482582, "rewards/reward_commands_correctness/std": 0.023300571367144585, "rewards/reward_diversity/mean": 0.15380972623825073, "rewards/reward_diversity/std": 0.06772720068693161, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.06999999284744263, "rewards/reward_problem_validity/std": 0.030110908672213554, "rewards/reward_solution_effectiveness/mean": 0.11812500655651093, "rewards/reward_solution_effectiveness/std": 0.05306835472583771, "sampling/importance_sampling_ratio/max": 1.1020804643630981, "sampling/importance_sampling_ratio/mean": 0.241536945104599, "sampling/importance_sampling_ratio/min": 0.0002021236578002572, "sampling/sampling_logp_difference/max": 5.053809642791748, "sampling/sampling_logp_difference/mean": 0.01900193840265274, "step": 182, "step_time": 218.1421262677759 }, { "clip_ratio/high_max": 0.0017282598710153252, "clip_ratio/high_mean": 0.0017282598710153252, "clip_ratio/low_mean": 0.0007861175836296752, "clip_ratio/low_min": 0.0007861175836296752, "clip_ratio/region_mean": 0.002514377483748831, "completions/clipped_ratio": 0.0, "completions/max_length": 1396.0, "completions/max_terminated_length": 1396.0, "completions/mean_length": 1057.5625, "completions/mean_terminated_length": 1057.5625, "completions/min_length": 633.0, "completions/min_terminated_length": 633.0, "entropy": 0.09634716808795929, "epoch": 3.7346938775510203, "frac_reward_zero_std": 0.0, "grad_norm": 0.0089722303673625, "learning_rate": 6.285714285714286e-06, "loss": -0.0004, "num_tokens": 11305421.0, "reward": 0.6196950078010559, "reward_std": 0.1321144700050354, "rewards/reward_commands_completeness/mean": 0.10624999552965164, "rewards/reward_commands_completeness/std": 0.036308858543634415, "rewards/reward_commands_correctness/mean": 0.06749999523162842, "rewards/reward_commands_correctness/std": 0.02206052467226982, "rewards/reward_diversity/mean": 0.1484449952840805, "rewards/reward_diversity/std": 0.04271009936928749, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.078125, "rewards/reward_problem_validity/std": 0.01869715005159378, "rewards/reward_solution_effectiveness/mean": 0.12562499940395355, "rewards/reward_solution_effectiveness/std": 0.05046038329601288, "sampling/importance_sampling_ratio/max": 0.34866300225257874, "sampling/importance_sampling_ratio/mean": 0.06430734694004059, "sampling/importance_sampling_ratio/min": 0.0004032327269669622, "sampling/sampling_logp_difference/max": 2.4105279445648193, "sampling/sampling_logp_difference/mean": 0.01891353353857994, "step": 183, "step_time": 130.27547342516482 }, { "clip_ratio/high_max": 0.0027432635542936623, "clip_ratio/high_mean": 0.0027432635542936623, "clip_ratio/low_mean": 0.0009454321261728182, "clip_ratio/low_min": 0.0009454321261728182, "clip_ratio/region_mean": 0.0036886956950183958, "completions/clipped_ratio": 0.0, "completions/max_length": 1431.0, "completions/max_terminated_length": 1431.0, "completions/mean_length": 1071.8125, "completions/mean_terminated_length": 1071.8125, "completions/min_length": 720.0, "completions/min_terminated_length": 720.0, "entropy": 0.10225903894752264, "epoch": 3.7551020408163263, "frac_reward_zero_std": 0.0, "grad_norm": 0.15442055463790894, "learning_rate": 6.26530612244898e-06, "loss": 0.2559, "num_tokens": 11350154.0, "reward": 0.5650326609611511, "reward_std": 0.24043738842010498, "rewards/reward_commands_completeness/mean": 0.10374999791383743, "rewards/reward_commands_completeness/std": 0.05426785349845886, "rewards/reward_commands_correctness/mean": 0.05312499776482582, "rewards/reward_commands_correctness/std": 0.029147613793611526, "rewards/reward_diversity/mean": 0.14753267168998718, "rewards/reward_diversity/std": 0.06498432904481888, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06562499701976776, "rewards/reward_problem_validity/std": 0.03346017748117447, "rewards/reward_solution_effectiveness/mean": 0.11999999731779099, "rewards/reward_solution_effectiveness/std": 0.06292853504419327, "sampling/importance_sampling_ratio/max": 2.8138675689697266, "sampling/importance_sampling_ratio/mean": 0.2757929861545563, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9743263721466064, "sampling/sampling_logp_difference/mean": 0.01894170045852661, "step": 184, "step_time": 142.0207443088293 }, { "clip_ratio/high_max": 0.0020460403902688995, "clip_ratio/high_mean": 0.0020460403902688995, "clip_ratio/low_mean": 0.0008215053312596865, "clip_ratio/low_min": 0.0008215053312596865, "clip_ratio/region_mean": 0.002867545685148798, "completions/clipped_ratio": 0.0, "completions/max_length": 1763.0, "completions/max_terminated_length": 1763.0, "completions/mean_length": 1097.3125, "completions/mean_terminated_length": 1097.3125, "completions/min_length": 761.0, "completions/min_terminated_length": 761.0, "entropy": 0.10377032216638327, "epoch": 3.7755102040816326, "frac_reward_zero_std": 0.0, "grad_norm": 0.032326921820640564, "learning_rate": 6.244897959183675e-06, "loss": -0.0516, "num_tokens": 11404611.0, "reward": 0.6101897954940796, "reward_std": 0.1873106062412262, "rewards/reward_commands_completeness/mean": 0.11625000089406967, "rewards/reward_commands_completeness/std": 0.05475703999400139, "rewards/reward_commands_correctness/mean": 0.058750003576278687, "rewards/reward_commands_correctness/std": 0.02825479581952095, "rewards/reward_diversity/mean": 0.1426897644996643, "rewards/reward_diversity/std": 0.06373277306556702, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07062499970197678, "rewards/reward_problem_validity/std": 0.03043435700237751, "rewards/reward_solution_effectiveness/mean": 0.140625, "rewards/reward_solution_effectiveness/std": 0.07646077871322632, "sampling/importance_sampling_ratio/max": 0.8242873549461365, "sampling/importance_sampling_ratio/mean": 0.09466962516307831, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9508728981018066, "sampling/sampling_logp_difference/mean": 0.018386567011475563, "step": 185, "step_time": 170.59160397574306 }, { "clip_ratio/high_max": 0.0021612034179270267, "clip_ratio/high_mean": 0.0021612034179270267, "clip_ratio/low_mean": 0.0005498521059053019, "clip_ratio/low_min": 0.0005498521059053019, "clip_ratio/region_mean": 0.0027110555238323286, "completions/clipped_ratio": 0.0, "completions/max_length": 1302.0, "completions/max_terminated_length": 1302.0, "completions/mean_length": 989.5, "completions/mean_terminated_length": 989.5, "completions/min_length": 673.0, "completions/min_terminated_length": 673.0, "entropy": 0.09436685964465141, "epoch": 3.795918367346939, "frac_reward_zero_std": 0.0, "grad_norm": 0.0582219623029232, "learning_rate": 6.224489795918368e-06, "loss": -0.0076, "num_tokens": 11444775.0, "reward": 0.7232919931411743, "reward_std": 0.10498766601085663, "rewards/reward_commands_completeness/mean": 0.13250000774860382, "rewards/reward_commands_completeness/std": 0.046690475195646286, "rewards/reward_commands_correctness/mean": 0.07062499970197678, "rewards/reward_commands_correctness/std": 0.018427787348628044, "rewards/reward_diversity/mean": 0.1782919466495514, "rewards/reward_diversity/std": 0.013178699649870396, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08312500268220901, "rewards/reward_problem_validity/std": 0.01990603096783161, "rewards/reward_solution_effectiveness/mean": 0.16500000655651093, "rewards/reward_solution_effectiveness/std": 0.0701427161693573, "sampling/importance_sampling_ratio/max": 1.2075049877166748, "sampling/importance_sampling_ratio/mean": 0.19204388558864594, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.971473217010498, "sampling/sampling_logp_difference/mean": 0.016603287309408188, "step": 186, "step_time": 122.95516535639763 }, { "clip_ratio/high_max": 0.002250508936413098, "clip_ratio/high_mean": 0.002250508936413098, "clip_ratio/low_mean": 0.0003421166620682925, "clip_ratio/low_min": 0.0003421166620682925, "clip_ratio/region_mean": 0.0025926256203092635, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1453.0, "completions/mean_length": 1219.125, "completions/mean_terminated_length": 1027.3333740234375, "completions/min_length": 639.0, "completions/min_terminated_length": 639.0, "entropy": 0.10527598578482866, "epoch": 3.816326530612245, "frac_reward_zero_std": 0.0, "grad_norm": 0.05014589801430702, "learning_rate": 6.2040816326530614e-06, "loss": -0.0565, "num_tokens": 11485585.0, "reward": 0.6435601711273193, "reward_std": 0.135589599609375, "rewards/reward_commands_completeness/mean": 0.10999999940395355, "rewards/reward_commands_completeness/std": 0.037237975746393204, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.021281840279698372, "rewards/reward_diversity/mean": 0.1623101830482483, "rewards/reward_diversity/std": 0.04317080229520798, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187499642372131, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.13124999403953552, "rewards/reward_solution_effectiveness/std": 0.05352569743990898, "sampling/importance_sampling_ratio/max": 1.353035807609558, "sampling/importance_sampling_ratio/mean": 0.17410925030708313, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.022036552429199, "sampling/sampling_logp_difference/mean": 0.016451656818389893, "step": 187, "step_time": 258.4290266931057 }, { "clip_ratio/high_max": 0.0018389655160717666, "clip_ratio/high_mean": 0.0018389655160717666, "clip_ratio/low_mean": 0.00046848037891322747, "clip_ratio/low_min": 0.00046848037891322747, "clip_ratio/region_mean": 0.002307445898622973, "completions/clipped_ratio": 0.0, "completions/max_length": 1358.0, "completions/max_terminated_length": 1358.0, "completions/mean_length": 1064.375, "completions/mean_terminated_length": 1064.375, "completions/min_length": 781.0, "completions/min_terminated_length": 781.0, "entropy": 0.1027787821367383, "epoch": 3.836734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.05791821330785751, "learning_rate": 6.1836734693877556e-06, "loss": -0.2366, "num_tokens": 11523971.0, "reward": 0.7009453177452087, "reward_std": 0.14022281765937805, "rewards/reward_commands_completeness/mean": 0.1287499964237213, "rewards/reward_commands_completeness/std": 0.04193248972296715, "rewards/reward_commands_correctness/mean": 0.06812499463558197, "rewards/reward_commands_correctness/std": 0.018337121233344078, "rewards/reward_diversity/mean": 0.17094528675079346, "rewards/reward_diversity/std": 0.043068401515483856, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187499642372131, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.1574999988079071, "rewards/reward_solution_effectiveness/std": 0.06244997680187225, "sampling/importance_sampling_ratio/max": 1.2400484085083008, "sampling/importance_sampling_ratio/mean": 0.29761427640914917, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8348875045776367, "sampling/sampling_logp_difference/mean": 0.017818983644247055, "step": 188, "step_time": 135.88566910102963 }, { "clip_ratio/high_max": 0.002230819227406755, "clip_ratio/high_mean": 0.002230819227406755, "clip_ratio/low_mean": 0.0010966032277792692, "clip_ratio/low_min": 0.0010966032277792692, "clip_ratio/region_mean": 0.003327422426082194, "completions/clipped_ratio": 0.0, "completions/max_length": 1288.0, "completions/max_terminated_length": 1288.0, "completions/mean_length": 1109.625, "completions/mean_terminated_length": 1109.625, "completions/min_length": 808.0, "completions/min_terminated_length": 808.0, "entropy": 0.08898665569722652, "epoch": 3.857142857142857, "frac_reward_zero_std": 0.0, "grad_norm": 0.051607098430395126, "learning_rate": 6.163265306122449e-06, "loss": 0.0552, "num_tokens": 11570417.0, "reward": 0.7027005553245544, "reward_std": 0.04868049547076225, "rewards/reward_commands_completeness/mean": 0.12250000238418579, "rewards/reward_commands_completeness/std": 0.037148356437683105, "rewards/reward_commands_correctness/mean": 0.0768750011920929, "rewards/reward_commands_correctness/std": 0.010781930759549141, "rewards/reward_diversity/mean": 0.17020058631896973, "rewards/reward_diversity/std": 0.020465543493628502, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.005163980647921562, "rewards/reward_solution_effectiveness/mean": 0.14812500774860382, "rewards/reward_solution_effectiveness/std": 0.05946637690067291, "sampling/importance_sampling_ratio/max": 1.9542059898376465, "sampling/importance_sampling_ratio/mean": 0.4080018401145935, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4320015907287598, "sampling/sampling_logp_difference/mean": 0.015732737258076668, "step": 189, "step_time": 173.05429789796472 }, { "clip_ratio/high_max": 0.002453981287544593, "clip_ratio/high_mean": 0.002453981287544593, "clip_ratio/low_mean": 0.0008615755796199664, "clip_ratio/low_min": 0.0008615755796199664, "clip_ratio/region_mean": 0.0033155568671645597, "completions/clipped_ratio": 0.0, "completions/max_length": 1372.0, "completions/max_terminated_length": 1372.0, "completions/mean_length": 968.5, "completions/mean_terminated_length": 968.5, "completions/min_length": 746.0, "completions/min_terminated_length": 746.0, "entropy": 0.10309283714741468, "epoch": 3.877551020408163, "frac_reward_zero_std": 0.0, "grad_norm": 0.1754099577665329, "learning_rate": 6.142857142857144e-06, "loss": 0.2651, "num_tokens": 11608109.0, "reward": 0.5603557229042053, "reward_std": 0.1882983297109604, "rewards/reward_commands_completeness/mean": 0.10124999284744263, "rewards/reward_commands_completeness/std": 0.049244292080402374, "rewards/reward_commands_correctness/mean": 0.05312500149011612, "rewards/reward_commands_correctness/std": 0.02651257812976837, "rewards/reward_diversity/mean": 0.13910570740699768, "rewards/reward_diversity/std": 0.048559945076704025, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07124999910593033, "rewards/reward_problem_validity/std": 0.030740857124328613, "rewards/reward_solution_effectiveness/mean": 0.11437499523162842, "rewards/reward_solution_effectiveness/std": 0.06860211491584778, "sampling/importance_sampling_ratio/max": 2.2695627212524414, "sampling/importance_sampling_ratio/mean": 0.275457501411438, "sampling/importance_sampling_ratio/min": 0.00015175134467426687, "sampling/sampling_logp_difference/max": 3.43996262550354, "sampling/sampling_logp_difference/mean": 0.020076438784599304, "step": 190, "step_time": 138.88180849514902 }, { "clip_ratio/high_max": 0.002650897680723574, "clip_ratio/high_mean": 0.002650897680723574, "clip_ratio/low_mean": 0.0006732417568855453, "clip_ratio/low_min": 0.0006732417568855453, "clip_ratio/region_mean": 0.0033241393975913525, "completions/clipped_ratio": 0.0, "completions/max_length": 1169.0, "completions/max_terminated_length": 1169.0, "completions/mean_length": 950.5, "completions/mean_terminated_length": 950.5, "completions/min_length": 736.0, "completions/min_terminated_length": 736.0, "entropy": 0.11862727627158165, "epoch": 3.8979591836734695, "frac_reward_zero_std": 0.0, "grad_norm": 0.03887297585606575, "learning_rate": 6.122448979591837e-06, "loss": -0.0899, "num_tokens": 11640633.0, "reward": 0.5516880750656128, "reward_std": 0.22185778617858887, "rewards/reward_commands_completeness/mean": 0.0949999988079071, "rewards/reward_commands_completeness/std": 0.05955389887094498, "rewards/reward_commands_correctness/mean": 0.054999999701976776, "rewards/reward_commands_correctness/std": 0.03204163908958435, "rewards/reward_diversity/mean": 0.15543803572654724, "rewards/reward_diversity/std": 0.06785263866186142, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.06312499940395355, "rewards/reward_problem_validity/std": 0.037187591195106506, "rewards/reward_solution_effectiveness/mean": 0.11437499523162842, "rewards/reward_solution_effectiveness/std": 0.07685213536024094, "sampling/importance_sampling_ratio/max": 1.3848060369491577, "sampling/importance_sampling_ratio/mean": 0.20614126324653625, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.249479293823242, "sampling/sampling_logp_difference/mean": 0.02069922536611557, "step": 191, "step_time": 101.34929746389389 }, { "clip_ratio/high_max": 0.001885152239992749, "clip_ratio/high_mean": 0.001885152239992749, "clip_ratio/low_mean": 0.0014281047551776282, "clip_ratio/low_min": 0.0014281047551776282, "clip_ratio/region_mean": 0.003313256995170377, "completions/clipped_ratio": 0.0, "completions/max_length": 1382.0, "completions/max_terminated_length": 1382.0, "completions/mean_length": 1001.625, "completions/mean_terminated_length": 1001.625, "completions/min_length": 681.0, "completions/min_terminated_length": 681.0, "entropy": 0.08460452407598495, "epoch": 3.9183673469387754, "frac_reward_zero_std": 0.0, "grad_norm": 0.04062918201088905, "learning_rate": 6.102040816326531e-06, "loss": -0.0124, "num_tokens": 11682183.0, "reward": 0.7169516086578369, "reward_std": 0.056575048714876175, "rewards/reward_commands_completeness/mean": 0.13624998927116394, "rewards/reward_commands_completeness/std": 0.04272002354264259, "rewards/reward_commands_correctness/mean": 0.07374999672174454, "rewards/reward_commands_correctness/std": 0.011474612168967724, "rewards/reward_diversity/mean": 0.15257664024829865, "rewards/reward_diversity/std": 0.02172457054257393, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.16875000298023224, "rewards/reward_solution_effectiveness/std": 0.08476438373327255, "sampling/importance_sampling_ratio/max": 2.3726980686187744, "sampling/importance_sampling_ratio/mean": 0.3288605809211731, "sampling/importance_sampling_ratio/min": 2.155601032427512e-05, "sampling/sampling_logp_difference/max": 2.8812367916107178, "sampling/sampling_logp_difference/mean": 0.017252324149012566, "step": 192, "step_time": 138.89676713570952 }, { "clip_ratio/high_max": 0.002337424513825681, "clip_ratio/high_mean": 0.002337424513825681, "clip_ratio/low_mean": 0.0007825799184502102, "clip_ratio/low_min": 0.0007825799184502102, "clip_ratio/region_mean": 0.0031200044468278065, "completions/clipped_ratio": 0.0, "completions/max_length": 1344.0, "completions/max_terminated_length": 1344.0, "completions/mean_length": 961.0625, "completions/mean_terminated_length": 961.0625, "completions/min_length": 696.0, "completions/min_terminated_length": 696.0, "entropy": 0.09294410422444344, "epoch": 3.938775510204082, "frac_reward_zero_std": 0.0, "grad_norm": 0.055598851293325424, "learning_rate": 6.0816326530612245e-06, "loss": -0.0886, "num_tokens": 11729008.0, "reward": 0.67756187915802, "reward_std": 0.21249787509441376, "rewards/reward_commands_completeness/mean": 0.1262499988079071, "rewards/reward_commands_completeness/std": 0.05149433761835098, "rewards/reward_commands_correctness/mean": 0.06687499582767487, "rewards/reward_commands_correctness/std": 0.024418232962489128, "rewards/reward_diversity/mean": 0.1606869399547577, "rewards/reward_diversity/std": 0.04059401899576187, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.02651257999241352, "rewards/reward_solution_effectiveness/mean": 0.15937499701976776, "rewards/reward_solution_effectiveness/std": 0.07243099808692932, "sampling/importance_sampling_ratio/max": 2.2663650512695312, "sampling/importance_sampling_ratio/mean": 0.3617284893989563, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1938440799713135, "sampling/sampling_logp_difference/mean": 0.01810400001704693, "step": 193, "step_time": 152.92239927127957 }, { "clip_ratio/high_max": 0.0009108207377721556, "clip_ratio/high_mean": 0.0009108207377721556, "clip_ratio/low_mean": 0.0008116871504171286, "clip_ratio/low_min": 0.0008116871504171286, "clip_ratio/region_mean": 0.001722507891827263, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1451.0, "completions/mean_length": 1192.625, "completions/mean_terminated_length": 999.0667114257812, "completions/min_length": 759.0, "completions/min_terminated_length": 759.0, "entropy": 0.10019599134102464, "epoch": 3.9591836734693877, "frac_reward_zero_std": 0.0, "grad_norm": 0.01824023202061653, "learning_rate": 6.0612244897959195e-06, "loss": 0.0481, "num_tokens": 11771502.0, "reward": 0.6240218877792358, "reward_std": 0.11649617552757263, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.03500000014901161, "rewards/reward_commands_correctness/mean": 0.05874999985098839, "rewards/reward_commands_correctness/std": 0.021252451464533806, "rewards/reward_diversity/mean": 0.17277194559574127, "rewards/reward_diversity/std": 0.01644611731171608, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.019137661904096603, "rewards/reward_solution_effectiveness/mean": 0.1106249988079071, "rewards/reward_solution_effectiveness/std": 0.04864411801099777, "sampling/importance_sampling_ratio/max": 2.38211989402771, "sampling/importance_sampling_ratio/mean": 0.22261814773082733, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0773282051086426, "sampling/sampling_logp_difference/mean": 0.015063375234603882, "step": 194, "step_time": 257.78297601640224 }, { "clip_ratio/high_max": 0.0011177924316143617, "clip_ratio/high_mean": 0.0011177924316143617, "clip_ratio/low_mean": 0.0013808007643092424, "clip_ratio/low_min": 0.0013808007643092424, "clip_ratio/region_mean": 0.002498593181371689, "completions/clipped_ratio": 0.0, "completions/max_length": 1422.0, "completions/max_terminated_length": 1422.0, "completions/mean_length": 1142.625, "completions/mean_terminated_length": 1142.625, "completions/min_length": 836.0, "completions/min_terminated_length": 836.0, "entropy": 0.08282526768743992, "epoch": 3.979591836734694, "frac_reward_zero_std": 0.25, "grad_norm": 0.007112611085176468, "learning_rate": 6.040816326530613e-06, "loss": 0.0228, "num_tokens": 11821940.0, "reward": 0.4260803461074829, "reward_std": 0.10901098698377609, "rewards/reward_commands_completeness/mean": 0.07124999910593033, "rewards/reward_commands_completeness/std": 0.04500000178813934, "rewards/reward_commands_correctness/mean": 0.03999999910593033, "rewards/reward_commands_correctness/std": 0.027568098157644272, "rewards/reward_diversity/mean": 0.12108032405376434, "rewards/reward_diversity/std": 0.08112581074237823, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.054375000298023224, "rewards/reward_problem_validity/std": 0.03558440878987312, "rewards/reward_solution_effectiveness/mean": 0.0768750011920929, "rewards/reward_solution_effectiveness/std": 0.04771006479859352, "sampling/importance_sampling_ratio/max": 0.29754480719566345, "sampling/importance_sampling_ratio/mean": 0.06817731261253357, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9941964149475098, "sampling/sampling_logp_difference/mean": 0.015213966369628906, "step": 195, "step_time": 150.11449288763106 }, { "clip_ratio/high_max": 0.003171245232806541, "clip_ratio/high_mean": 0.003171245232806541, "clip_ratio/low_mean": 0.0006431501606130041, "clip_ratio/low_min": 0.0006431501606130041, "clip_ratio/region_mean": 0.0038143953861435875, "completions/clipped_ratio": 0.0, "completions/max_length": 1316.0, "completions/max_terminated_length": 1316.0, "completions/mean_length": 966.5625, "completions/mean_terminated_length": 966.5625, "completions/min_length": 808.0, "completions/min_terminated_length": 808.0, "entropy": 0.0936196381226182, "epoch": 4.0, "frac_reward_zero_std": 0.0, "grad_norm": 0.04287876561284065, "learning_rate": 6.020408163265307e-06, "loss": 0.0326, "num_tokens": 11862017.0, "reward": 0.6111805438995361, "reward_std": 0.2408646047115326, "rewards/reward_commands_completeness/mean": 0.11749999225139618, "rewards/reward_commands_completeness/std": 0.05409867689013481, "rewards/reward_commands_correctness/mean": 0.05250000208616257, "rewards/reward_commands_correctness/std": 0.02620432712137699, "rewards/reward_diversity/mean": 0.15430548787117004, "rewards/reward_diversity/std": 0.05439180135726929, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07250000536441803, "rewards/reward_problem_validity/std": 0.03130495548248291, "rewards/reward_solution_effectiveness/mean": 0.13312500715255737, "rewards/reward_solution_effectiveness/std": 0.07180703431367874, "sampling/importance_sampling_ratio/max": 1.0284748077392578, "sampling/importance_sampling_ratio/mean": 0.16910234093666077, "sampling/importance_sampling_ratio/min": 0.0011962811695411801, "sampling/sampling_logp_difference/max": 2.3717899322509766, "sampling/sampling_logp_difference/mean": 0.018990805372595787, "step": 196, "step_time": 123.23642376810312 }, { "clip_ratio/high_max": 0.0014190359215717763, "clip_ratio/high_mean": 0.0014190359215717763, "clip_ratio/low_mean": 0.0010322279995307326, "clip_ratio/low_min": 0.0010322279995307326, "clip_ratio/region_mean": 0.0024512639647582546, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1731.0, "completions/mean_length": 1345.875, "completions/mean_terminated_length": 1162.533447265625, "completions/min_length": 744.0, "completions/min_terminated_length": 744.0, "entropy": 0.10085593536496162, "epoch": 4.020408163265306, "frac_reward_zero_std": 0.0, "grad_norm": 0.005420775152742863, "learning_rate": 6e-06, "loss": -0.0096, "num_tokens": 11912699.0, "reward": 0.6873289942741394, "reward_std": 0.14687782526016235, "rewards/reward_commands_completeness/mean": 0.13500000536441803, "rewards/reward_commands_completeness/std": 0.05189733952283859, "rewards/reward_commands_correctness/mean": 0.0637499988079071, "rewards/reward_commands_correctness/std": 0.022173559293150902, "rewards/reward_diversity/mean": 0.1535789668560028, "rewards/reward_diversity/std": 0.05362531915307045, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.027049338445067406, "rewards/reward_solution_effectiveness/mean": 0.16874998807907104, "rewards/reward_solution_effectiveness/std": 0.07256031781435013, "sampling/importance_sampling_ratio/max": 0.14643028378486633, "sampling/importance_sampling_ratio/mean": 0.02682403102517128, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.0244951248168945, "sampling/sampling_logp_difference/mean": 0.016863400116562843, "step": 197, "step_time": 272.5217933766544 }, { "clip_ratio/high_max": 0.0018495015174266882, "clip_ratio/high_mean": 0.0018495015174266882, "clip_ratio/low_mean": 0.0011399724753573537, "clip_ratio/low_min": 0.0011399724753573537, "clip_ratio/region_mean": 0.0029894740364397876, "completions/clipped_ratio": 0.0, "completions/max_length": 1642.0, "completions/max_terminated_length": 1642.0, "completions/mean_length": 982.3125, "completions/mean_terminated_length": 982.3125, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.08684483589604497, "epoch": 4.040816326530612, "frac_reward_zero_std": 0.0, "grad_norm": 0.07806137949228287, "learning_rate": 5.979591836734694e-06, "loss": 0.1852, "num_tokens": 11952276.0, "reward": 0.7281287908554077, "reward_std": 0.12596988677978516, "rewards/reward_commands_completeness/mean": 0.14249999821186066, "rewards/reward_commands_completeness/std": 0.04611579701304436, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.02220172807574272, "rewards/reward_diversity/mean": 0.16500380635261536, "rewards/reward_diversity/std": 0.016725966706871986, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.019989583641290665, "rewards/reward_solution_effectiveness/mean": 0.17812500894069672, "rewards/reward_solution_effectiveness/std": 0.07782191783189774, "sampling/importance_sampling_ratio/max": 2.658215045928955, "sampling/importance_sampling_ratio/mean": 0.3508802056312561, "sampling/importance_sampling_ratio/min": 0.0003145173832308501, "sampling/sampling_logp_difference/max": 1.9586471319198608, "sampling/sampling_logp_difference/mean": 0.017733076587319374, "step": 198, "step_time": 168.1309392247349 }, { "clip_ratio/high_max": 0.0021253429986245465, "clip_ratio/high_mean": 0.0021253429986245465, "clip_ratio/low_mean": 0.0010508657269383548, "clip_ratio/low_min": 0.0010508657269383548, "clip_ratio/region_mean": 0.0031762087091919966, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1504.0, "completions/mean_length": 1289.3125, "completions/mean_terminated_length": 1102.2000732421875, "completions/min_length": 766.0, "completions/min_terminated_length": 766.0, "entropy": 0.08854763256385922, "epoch": 4.061224489795919, "frac_reward_zero_std": 0.0, "grad_norm": 0.029588043689727783, "learning_rate": 5.959183673469388e-06, "loss": 0.079, "num_tokens": 12008977.0, "reward": 0.7644339203834534, "reward_std": 0.04623837396502495, "rewards/reward_commands_completeness/mean": 0.14625000953674316, "rewards/reward_commands_completeness/std": 0.02276693843305111, "rewards/reward_commands_correctness/mean": 0.07187500596046448, "rewards/reward_commands_correctness/std": 0.009810708463191986, "rewards/reward_diversity/mean": 0.17318391799926758, "rewards/reward_diversity/std": 0.018054917454719543, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08937500417232513, "rewards/reward_problem_validity/std": 0.0025000013411045074, "rewards/reward_solution_effectiveness/mean": 0.1837500035762787, "rewards/reward_solution_effectiveness/std": 0.040804408490657806, "sampling/importance_sampling_ratio/max": 2.6465981006622314, "sampling/importance_sampling_ratio/mean": 0.2579687535762787, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.977169990539551, "sampling/sampling_logp_difference/mean": 0.015603412874042988, "step": 199, "step_time": 305.5701991096139 }, { "clip_ratio/high_max": 0.0016094887541839853, "clip_ratio/high_mean": 0.0016094887541839853, "clip_ratio/low_mean": 0.0010262788928230293, "clip_ratio/low_min": 0.0010262788928230293, "clip_ratio/region_mean": 0.0026357676833868027, "completions/clipped_ratio": 0.0, "completions/max_length": 1361.0, "completions/max_terminated_length": 1361.0, "completions/mean_length": 1018.875, "completions/mean_terminated_length": 1018.875, "completions/min_length": 759.0, "completions/min_terminated_length": 759.0, "entropy": 0.08474506856873631, "epoch": 4.081632653061225, "frac_reward_zero_std": 0.0, "grad_norm": 0.10914500057697296, "learning_rate": 5.9387755102040825e-06, "loss": -0.1473, "num_tokens": 12043683.0, "reward": 0.7429919242858887, "reward_std": 0.05921339988708496, "rewards/reward_commands_completeness/mean": 0.14124999940395355, "rewards/reward_commands_completeness/std": 0.024731896817684174, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.014605935662984848, "rewards/reward_diversity/mean": 0.17174193263053894, "rewards/reward_diversity/std": 0.018049588426947594, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.17249999940395355, "rewards/reward_solution_effectiveness/std": 0.053103674203157425, "sampling/importance_sampling_ratio/max": 2.5951619148254395, "sampling/importance_sampling_ratio/mean": 0.28516247868537903, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.667943000793457, "sampling/sampling_logp_difference/mean": 0.017376085743308067, "step": 200, "step_time": 127.2007693015039 }, { "clip_ratio/high_max": 0.0016933983242779505, "clip_ratio/high_mean": 0.0016933983242779505, "clip_ratio/low_mean": 0.0008233178577938816, "clip_ratio/low_min": 0.0008233178577938816, "clip_ratio/region_mean": 0.002516716172976885, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1246.0, "completions/mean_length": 1176.875, "completions/mean_terminated_length": 982.2667236328125, "completions/min_length": 664.0, "completions/min_terminated_length": 664.0, "entropy": 0.08460923796519637, "epoch": 4.1020408163265305, "frac_reward_zero_std": 0.0, "grad_norm": 0.05093446746468544, "learning_rate": 5.918367346938776e-06, "loss": -0.0727, "num_tokens": 12086493.0, "reward": 0.6066921353340149, "reward_std": 0.1760530024766922, "rewards/reward_commands_completeness/mean": 0.10999999940395355, "rewards/reward_commands_completeness/std": 0.0473286397755146, "rewards/reward_commands_correctness/mean": 0.05624999850988388, "rewards/reward_commands_correctness/std": 0.0257875956594944, "rewards/reward_diversity/mean": 0.16294211149215698, "rewards/reward_diversity/std": 0.058234114199876785, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07000000029802322, "rewards/reward_problem_validity/std": 0.026583204045891762, "rewards/reward_solution_effectiveness/mean": 0.11999999731779099, "rewards/reward_solution_effectiveness/std": 0.06387487798929214, "sampling/importance_sampling_ratio/max": 1.6893396377563477, "sampling/importance_sampling_ratio/mean": 0.20797373354434967, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2475547790527344, "sampling/sampling_logp_difference/mean": 0.014662327244877815, "step": 201, "step_time": 261.16120374016464 }, { "clip_ratio/high_max": 0.000915005242859479, "clip_ratio/high_mean": 0.000915005242859479, "clip_ratio/low_mean": 0.0011430845625000075, "clip_ratio/low_min": 0.0011430845625000075, "clip_ratio/region_mean": 0.0020580898053594865, "completions/clipped_ratio": 0.0, "completions/max_length": 1567.0, "completions/max_terminated_length": 1567.0, "completions/mean_length": 1054.5625, "completions/mean_terminated_length": 1054.5625, "completions/min_length": 834.0, "completions/min_terminated_length": 834.0, "entropy": 0.10063335672020912, "epoch": 4.122448979591836, "frac_reward_zero_std": 0.0, "grad_norm": 0.007806889247149229, "learning_rate": 5.89795918367347e-06, "loss": -0.0013, "num_tokens": 12122462.0, "reward": 0.7218716144561768, "reward_std": 0.09755650162696838, "rewards/reward_commands_completeness/mean": 0.13374999165534973, "rewards/reward_commands_completeness/std": 0.033241547644138336, "rewards/reward_commands_correctness/mean": 0.07250000536441803, "rewards/reward_commands_correctness/std": 0.01949359104037285, "rewards/reward_diversity/mean": 0.1631215661764145, "rewards/reward_diversity/std": 0.024929100647568703, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.16687500476837158, "rewards/reward_solution_effectiveness/std": 0.061937469989061356, "sampling/importance_sampling_ratio/max": 0.1908481866121292, "sampling/importance_sampling_ratio/mean": 0.050860028713941574, "sampling/importance_sampling_ratio/min": 1.3711192877963185e-05, "sampling/sampling_logp_difference/max": 5.0950093269348145, "sampling/sampling_logp_difference/mean": 0.020543446764349937, "step": 202, "step_time": 136.65053349547088 }, { "clip_ratio/high_max": 0.0028539422928588465, "clip_ratio/high_mean": 0.0028539422928588465, "clip_ratio/low_mean": 0.000592332195083145, "clip_ratio/low_min": 0.000592332195083145, "clip_ratio/region_mean": 0.0034462744879419915, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1128.0, "completions/mean_length": 1207.1875, "completions/mean_terminated_length": 1014.6000366210938, "completions/min_length": 797.0, "completions/min_terminated_length": 797.0, "entropy": 0.08453904697671533, "epoch": 4.142857142857143, "frac_reward_zero_std": 0.0, "grad_norm": 0.002181750489398837, "learning_rate": 5.877551020408164e-06, "loss": 0.0041, "num_tokens": 12166789.0, "reward": 0.6887893676757812, "reward_std": 0.11656453460454941, "rewards/reward_commands_completeness/mean": 0.13499999046325684, "rewards/reward_commands_completeness/std": 0.028751812875270844, "rewards/reward_commands_correctness/mean": 0.06624999642372131, "rewards/reward_commands_correctness/std": 0.012583058327436447, "rewards/reward_diversity/mean": 0.15066438913345337, "rewards/reward_diversity/std": 0.09615129977464676, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08499999344348907, "rewards/reward_problem_validity/std": 0.005163980647921562, "rewards/reward_solution_effectiveness/mean": 0.15187500417232513, "rewards/reward_solution_effectiveness/std": 0.05418717488646507, "sampling/importance_sampling_ratio/max": 0.09343552589416504, "sampling/importance_sampling_ratio/mean": 0.01765037328004837, "sampling/importance_sampling_ratio/min": 2.736513488343917e-05, "sampling/sampling_logp_difference/max": 7.084932804107666, "sampling/sampling_logp_difference/mean": 0.016480332240462303, "step": 203, "step_time": 271.7467554733157 }, { "clip_ratio/high_max": 0.0009360452531836927, "clip_ratio/high_mean": 0.0009360452531836927, "clip_ratio/low_mean": 0.0021717465788242407, "clip_ratio/low_min": 0.0021717465788242407, "clip_ratio/region_mean": 0.003107791824731976, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 836.3125, "completions/mean_terminated_length": 836.3125, "completions/min_length": 602.0, "completions/min_terminated_length": 602.0, "entropy": 0.0916674779728055, "epoch": 4.163265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.042766962200403214, "learning_rate": 5.857142857142858e-06, "loss": -0.0337, "num_tokens": 12205666.0, "reward": 0.4963979423046112, "reward_std": 0.2805180549621582, "rewards/reward_commands_completeness/mean": 0.09375, "rewards/reward_commands_completeness/std": 0.06520481407642365, "rewards/reward_commands_correctness/mean": 0.051249999552965164, "rewards/reward_commands_correctness/std": 0.03685557469725609, "rewards/reward_diversity/mean": 0.11702293157577515, "rewards/reward_diversity/std": 0.07030390948057175, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.057500001043081284, "rewards/reward_problem_validity/std": 0.0382099486887455, "rewards/reward_solution_effectiveness/mean": 0.11437499523162842, "rewards/reward_solution_effectiveness/std": 0.08571027219295502, "sampling/importance_sampling_ratio/max": 0.7647246718406677, "sampling/importance_sampling_ratio/mean": 0.20788240432739258, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.9992494583129883, "sampling/sampling_logp_difference/mean": 0.01916680485010147, "step": 204, "step_time": 116.40165280736983 }, { "clip_ratio/high_max": 0.0026797797909239307, "clip_ratio/high_mean": 0.0026797797909239307, "clip_ratio/low_mean": 0.0013471215643221512, "clip_ratio/low_min": 0.0013471215643221512, "clip_ratio/region_mean": 0.004026901297038421, "completions/clipped_ratio": 0.0, "completions/max_length": 1236.0, "completions/max_terminated_length": 1236.0, "completions/mean_length": 923.875, "completions/mean_terminated_length": 923.875, "completions/min_length": 693.0, "completions/min_terminated_length": 693.0, "entropy": 0.09095968212932348, "epoch": 4.183673469387755, "frac_reward_zero_std": 0.0, "grad_norm": 0.11500459164381027, "learning_rate": 5.8367346938775515e-06, "loss": 0.3007, "num_tokens": 12247964.0, "reward": 0.7040824294090271, "reward_std": 0.1341506540775299, "rewards/reward_commands_completeness/mean": 0.13749998807907104, "rewards/reward_commands_completeness/std": 0.0443471223115921, "rewards/reward_commands_correctness/mean": 0.06749999523162842, "rewards/reward_commands_correctness/std": 0.020493902266025543, "rewards/reward_diversity/mean": 0.1572074294090271, "rewards/reward_diversity/std": 0.03974871337413788, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.16687500476837158, "rewards/reward_solution_effectiveness/std": 0.07263780385255814, "sampling/importance_sampling_ratio/max": 2.215149164199829, "sampling/importance_sampling_ratio/mean": 0.3671414852142334, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2711434364318848, "sampling/sampling_logp_difference/mean": 0.018635958433151245, "step": 205, "step_time": 123.03550046309829 }, { "clip_ratio/high_max": 0.0014386448892764747, "clip_ratio/high_mean": 0.0014386448892764747, "clip_ratio/low_mean": 0.0013214373284426983, "clip_ratio/low_min": 0.0013214373284426983, "clip_ratio/region_mean": 0.0027600821922533214, "completions/clipped_ratio": 0.0, "completions/max_length": 1405.0, "completions/max_terminated_length": 1405.0, "completions/mean_length": 920.25, "completions/mean_terminated_length": 920.25, "completions/min_length": 658.0, "completions/min_terminated_length": 658.0, "entropy": 0.10911247506737709, "epoch": 4.204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.013716538436710835, "learning_rate": 5.816326530612246e-06, "loss": 0.0136, "num_tokens": 12288820.0, "reward": 0.7173579335212708, "reward_std": 0.09666602313518524, "rewards/reward_commands_completeness/mean": 0.12999999523162842, "rewards/reward_commands_completeness/std": 0.0326598659157753, "rewards/reward_commands_correctness/mean": 0.07625000178813934, "rewards/reward_commands_correctness/std": 0.010246952064335346, "rewards/reward_diversity/mean": 0.1773579716682434, "rewards/reward_diversity/std": 0.017209123820066452, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08375000208616257, "rewards/reward_problem_validity/std": 0.012583059258759022, "rewards/reward_solution_effectiveness/mean": 0.14999999105930328, "rewards/reward_solution_effectiveness/std": 0.06196773424744606, "sampling/importance_sampling_ratio/max": 2.4175336360931396, "sampling/importance_sampling_ratio/mean": 0.1763748973608017, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.826599597930908, "sampling/sampling_logp_difference/mean": 0.021266086027026176, "step": 206, "step_time": 155.8843968566507 }, { "clip_ratio/high_max": 0.0024691528669791296, "clip_ratio/high_mean": 0.0024691528669791296, "clip_ratio/low_mean": 0.0014326778400572948, "clip_ratio/low_min": 0.0014326778400572948, "clip_ratio/region_mean": 0.003901830641552806, "completions/clipped_ratio": 0.0, "completions/max_length": 990.0, "completions/max_terminated_length": 990.0, "completions/mean_length": 782.9375, "completions/mean_terminated_length": 782.9375, "completions/min_length": 637.0, "completions/min_terminated_length": 637.0, "entropy": 0.10350053012371063, "epoch": 4.224489795918367, "frac_reward_zero_std": 0.0, "grad_norm": 0.02298703044652939, "learning_rate": 5.795918367346939e-06, "loss": -0.0121, "num_tokens": 12329859.0, "reward": 0.7050280570983887, "reward_std": 0.09489785134792328, "rewards/reward_commands_completeness/mean": 0.13124999403953552, "rewards/reward_commands_completeness/std": 0.027294687926769257, "rewards/reward_commands_correctness/mean": 0.06499999761581421, "rewards/reward_commands_correctness/std": 0.023380903527140617, "rewards/reward_diversity/mean": 0.16690310835838318, "rewards/reward_diversity/std": 0.020221518352627754, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.15562500059604645, "rewards/reward_solution_effectiveness/std": 0.0516357459127903, "sampling/importance_sampling_ratio/max": 0.629843533039093, "sampling/importance_sampling_ratio/mean": 0.13121843338012695, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.9507157802581787, "sampling/sampling_logp_difference/mean": 0.02119402587413788, "step": 207, "step_time": 143.49711393378675 }, { "clip_ratio/high_max": 0.001741347303322982, "clip_ratio/high_mean": 0.001741347303322982, "clip_ratio/low_mean": 0.0012699662620434538, "clip_ratio/low_min": 0.0012699662620434538, "clip_ratio/region_mean": 0.003011313579918351, "completions/clipped_ratio": 0.0, "completions/max_length": 1132.0, "completions/max_terminated_length": 1132.0, "completions/mean_length": 858.25, "completions/mean_terminated_length": 858.25, "completions/min_length": 717.0, "completions/min_terminated_length": 717.0, "entropy": 0.10394254326820374, "epoch": 4.244897959183674, "frac_reward_zero_std": 0.0, "grad_norm": 0.003205792512744665, "learning_rate": 5.775510204081634e-06, "loss": -0.002, "num_tokens": 12387935.0, "reward": 0.650245189666748, "reward_std": 0.15972504019737244, "rewards/reward_commands_completeness/mean": 0.11749999970197678, "rewards/reward_commands_completeness/std": 0.04374166578054428, "rewards/reward_commands_correctness/mean": 0.07312500476837158, "rewards/reward_commands_correctness/std": 0.020886601880192757, "rewards/reward_diversity/mean": 0.15149521827697754, "rewards/reward_diversity/std": 0.04333963990211487, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.0201556459069252, "rewards/reward_solution_effectiveness/mean": 0.13500000536441803, "rewards/reward_solution_effectiveness/std": 0.07266361266374588, "sampling/importance_sampling_ratio/max": 0.09377942234277725, "sampling/importance_sampling_ratio/mean": 0.016586052253842354, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.993860721588135, "sampling/sampling_logp_difference/mean": 0.022963540628552437, "step": 208, "step_time": 213.71318699605763 }, { "clip_ratio/high_max": 0.002694409660762176, "clip_ratio/high_mean": 0.002694409660762176, "clip_ratio/low_mean": 0.00042648248927434906, "clip_ratio/low_min": 0.00042648248927434906, "clip_ratio/region_mean": 0.003120892113656737, "completions/clipped_ratio": 0.0, "completions/max_length": 1139.0, "completions/max_terminated_length": 1139.0, "completions/mean_length": 860.75, "completions/mean_terminated_length": 860.75, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.10755674820393324, "epoch": 4.26530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.020397208631038666, "learning_rate": 5.755102040816327e-06, "loss": -0.066, "num_tokens": 12429211.0, "reward": 0.5192247033119202, "reward_std": 0.22730973362922668, "rewards/reward_commands_completeness/mean": 0.08875000476837158, "rewards/reward_commands_completeness/std": 0.04674398526549339, "rewards/reward_commands_correctness/mean": 0.04937499761581421, "rewards/reward_commands_correctness/std": 0.028860297054052353, "rewards/reward_diversity/mean": 0.14734968543052673, "rewards/reward_diversity/std": 0.0652310699224472, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06687499582767487, "rewards/reward_problem_validity/std": 0.034199174493551254, "rewards/reward_solution_effectiveness/mean": 0.09187500178813934, "rewards/reward_solution_effectiveness/std": 0.051925431936979294, "sampling/importance_sampling_ratio/max": 1.1368495225906372, "sampling/importance_sampling_ratio/mean": 0.1913164108991623, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.287707567214966, "sampling/sampling_logp_difference/mean": 0.02124919183552265, "step": 209, "step_time": 127.90332173928618 }, { "clip_ratio/high_max": 0.0025515433808322996, "clip_ratio/high_mean": 0.0025515433808322996, "clip_ratio/low_mean": 0.0005815909025841393, "clip_ratio/low_min": 0.0005815909025841393, "clip_ratio/region_mean": 0.003133134297968354, "completions/clipped_ratio": 0.0, "completions/max_length": 1125.0, "completions/max_terminated_length": 1125.0, "completions/mean_length": 847.9375, "completions/mean_terminated_length": 847.9375, "completions/min_length": 646.0, "completions/min_terminated_length": 646.0, "entropy": 0.10937288217246532, "epoch": 4.285714285714286, "frac_reward_zero_std": 0.0, "grad_norm": 0.049621980637311935, "learning_rate": 5.73469387755102e-06, "loss": -0.0797, "num_tokens": 12472602.0, "reward": 0.635785698890686, "reward_std": 0.18000969290733337, "rewards/reward_commands_completeness/mean": 0.12000000476837158, "rewards/reward_commands_completeness/std": 0.04844240844249725, "rewards/reward_commands_correctness/mean": 0.0612499974668026, "rewards/reward_commands_correctness/std": 0.0257875956594944, "rewards/reward_diversity/mean": 0.1595357358455658, "rewards/reward_diversity/std": 0.04235340282320976, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07249999791383743, "rewards/reward_problem_validity/std": 0.027202943339943886, "rewards/reward_solution_effectiveness/mean": 0.13500000536441803, "rewards/reward_solution_effectiveness/std": 0.07099296152591705, "sampling/importance_sampling_ratio/max": 0.7606203556060791, "sampling/importance_sampling_ratio/mean": 0.23954711854457855, "sampling/importance_sampling_ratio/min": 2.9994114356668433e-06, "sampling/sampling_logp_difference/max": 8.67121696472168, "sampling/sampling_logp_difference/mean": 0.022597603499889374, "step": 210, "step_time": 129.0590898487717 }, { "clip_ratio/high_max": 0.0019750803185161203, "clip_ratio/high_mean": 0.0019750803185161203, "clip_ratio/low_mean": 0.0014963017019908875, "clip_ratio/low_min": 0.0014963017019908875, "clip_ratio/region_mean": 0.0034713820641627535, "completions/clipped_ratio": 0.0, "completions/max_length": 1182.0, "completions/max_terminated_length": 1182.0, "completions/mean_length": 823.6875, "completions/mean_terminated_length": 823.6875, "completions/min_length": 622.0, "completions/min_terminated_length": 622.0, "entropy": 0.11373671889305115, "epoch": 4.3061224489795915, "frac_reward_zero_std": 0.0, "grad_norm": 0.03900550678372383, "learning_rate": 5.7142857142857145e-06, "loss": 0.0014, "num_tokens": 12513477.0, "reward": 0.6654390096664429, "reward_std": 0.0655907690525055, "rewards/reward_commands_completeness/mean": 0.11375000327825546, "rewards/reward_commands_completeness/std": 0.030740855261683464, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.01965324394404888, "rewards/reward_diversity/mean": 0.18418899178504944, "rewards/reward_diversity/std": 0.022931808605790138, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08312499523162842, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.11999999731779099, "rewards/reward_solution_effectiveness/std": 0.053665630519390106, "sampling/importance_sampling_ratio/max": 2.1295714378356934, "sampling/importance_sampling_ratio/mean": 0.2859005331993103, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2775096893310547, "sampling/sampling_logp_difference/mean": 0.02200828306376934, "step": 211, "step_time": 144.52069206349552 }, { "clip_ratio/high_max": 0.0016628493467578664, "clip_ratio/high_mean": 0.0016628493467578664, "clip_ratio/low_mean": 0.0006210259452927858, "clip_ratio/low_min": 0.0006210259452927858, "clip_ratio/region_mean": 0.002283875292050652, "completions/clipped_ratio": 0.0, "completions/max_length": 948.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 803.1875, "completions/mean_terminated_length": 803.1875, "completions/min_length": 673.0, "completions/min_terminated_length": 673.0, "entropy": 0.10631932970136404, "epoch": 4.326530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.03629210218787193, "learning_rate": 5.6938775510204095e-06, "loss": -0.0493, "num_tokens": 12553312.0, "reward": 0.5761979222297668, "reward_std": 0.2038668692111969, "rewards/reward_commands_completeness/mean": 0.10625000298023224, "rewards/reward_commands_completeness/std": 0.05965177342295647, "rewards/reward_commands_correctness/mean": 0.054374996572732925, "rewards/reward_commands_correctness/std": 0.031191613525152206, "rewards/reward_diversity/mean": 0.14119791984558105, "rewards/reward_diversity/std": 0.06310629844665527, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06812500208616257, "rewards/reward_problem_validity/std": 0.03487477824091911, "rewards/reward_solution_effectiveness/mean": 0.13124999403953552, "rewards/reward_solution_effectiveness/std": 0.08188407123088837, "sampling/importance_sampling_ratio/max": 1.3047528266906738, "sampling/importance_sampling_ratio/mean": 0.13279229402542114, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.750270366668701, "sampling/sampling_logp_difference/mean": 0.022624546661973, "step": 212, "step_time": 122.60315079800785 }, { "clip_ratio/high_max": 0.00197305268375203, "clip_ratio/high_mean": 0.00197305268375203, "clip_ratio/low_mean": 0.0015718142676632851, "clip_ratio/low_min": 0.0015718142676632851, "clip_ratio/region_mean": 0.0035448669514153153, "completions/clipped_ratio": 0.0, "completions/max_length": 806.0, "completions/max_terminated_length": 806.0, "completions/mean_length": 742.5625, "completions/mean_terminated_length": 742.5625, "completions/min_length": 646.0, "completions/min_terminated_length": 646.0, "entropy": 0.11702447757124901, "epoch": 4.346938775510204, "frac_reward_zero_std": 0.0, "grad_norm": 0.05137217417359352, "learning_rate": 5.673469387755103e-06, "loss": 0.0156, "num_tokens": 12585161.0, "reward": 0.758338451385498, "reward_std": 0.09467947483062744, "rewards/reward_commands_completeness/mean": 0.14249999821186066, "rewards/reward_commands_completeness/std": 0.03172801434993744, "rewards/reward_commands_correctness/mean": 0.07312500476837158, "rewards/reward_commands_correctness/std": 0.015798207372426987, "rewards/reward_diversity/mean": 0.1783384382724762, "rewards/reward_diversity/std": 0.015534374862909317, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.17625001072883606, "rewards/reward_solution_effectiveness/std": 0.06086871027946472, "sampling/importance_sampling_ratio/max": 0.9847621917724609, "sampling/importance_sampling_ratio/mean": 0.19779320061206818, "sampling/importance_sampling_ratio/min": 6.946324333512166e-07, "sampling/sampling_logp_difference/max": 1.969498872756958, "sampling/sampling_logp_difference/mean": 0.024374034255743027, "step": 213, "step_time": 101.97521854937077 }, { "clip_ratio/high_max": 0.002856840757885948, "clip_ratio/high_mean": 0.002856840757885948, "clip_ratio/low_mean": 0.00113815299118869, "clip_ratio/low_min": 0.00113815299118869, "clip_ratio/region_mean": 0.003994993734522723, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 770.125, "completions/mean_terminated_length": 770.125, "completions/min_length": 635.0, "completions/min_terminated_length": 635.0, "entropy": 0.11006729956716299, "epoch": 4.36734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.014436708763241768, "learning_rate": 5.653061224489796e-06, "loss": 0.0003, "num_tokens": 12656691.0, "reward": 0.6115624904632568, "reward_std": 0.1427965611219406, "rewards/reward_commands_completeness/mean": 0.10374999791383743, "rewards/reward_commands_completeness/std": 0.0468863919377327, "rewards/reward_commands_correctness/mean": 0.06749999523162842, "rewards/reward_commands_correctness/std": 0.029552215710282326, "rewards/reward_diversity/mean": 0.16093750298023224, "rewards/reward_diversity/std": 0.04471398517489433, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07375000417232513, "rewards/reward_problem_validity/std": 0.02526525966823101, "rewards/reward_solution_effectiveness/mean": 0.11812499910593033, "rewards/reward_solution_effectiveness/std": 0.07782191783189774, "sampling/importance_sampling_ratio/max": 0.3887225389480591, "sampling/importance_sampling_ratio/mean": 0.05742684751749039, "sampling/importance_sampling_ratio/min": 0.0008755127200856805, "sampling/sampling_logp_difference/max": 4.014948844909668, "sampling/sampling_logp_difference/mean": 0.023180510848760605, "step": 214, "step_time": 220.89723657257855 }, { "clip_ratio/high_max": 0.0013904963852837682, "clip_ratio/high_mean": 0.0013904963852837682, "clip_ratio/low_mean": 0.001484252672526054, "clip_ratio/low_min": 0.001484252672526054, "clip_ratio/region_mean": 0.002874749043257907, "completions/clipped_ratio": 0.0, "completions/max_length": 918.0, "completions/max_terminated_length": 918.0, "completions/mean_length": 759.4375, "completions/mean_terminated_length": 759.4375, "completions/min_length": 629.0, "completions/min_terminated_length": 629.0, "entropy": 0.10817344672977924, "epoch": 4.387755102040816, "frac_reward_zero_std": 0.0, "grad_norm": 0.028396470472216606, "learning_rate": 5.63265306122449e-06, "loss": 0.0243, "num_tokens": 12696834.0, "reward": 0.6833707094192505, "reward_std": 0.15715724229812622, "rewards/reward_commands_completeness/mean": 0.1274999976158142, "rewards/reward_commands_completeness/std": 0.03991658240556717, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.022794371470808983, "rewards/reward_diversity/mean": 0.16837069392204285, "rewards/reward_diversity/std": 0.042329877614974976, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.019137661904096603, "rewards/reward_solution_effectiveness/mean": 0.14999999105930328, "rewards/reward_solution_effectiveness/std": 0.06387487798929214, "sampling/importance_sampling_ratio/max": 0.7939942479133606, "sampling/importance_sampling_ratio/mean": 0.1073598712682724, "sampling/importance_sampling_ratio/min": 2.0694302293122746e-05, "sampling/sampling_logp_difference/max": 7.590192794799805, "sampling/sampling_logp_difference/mean": 0.022781770676374435, "step": 215, "step_time": 135.4583020247519 }, { "clip_ratio/high_max": 0.0018753620606730692, "clip_ratio/high_mean": 0.0018753620606730692, "clip_ratio/low_mean": 0.0012438727571861818, "clip_ratio/low_min": 0.0012438727571861818, "clip_ratio/region_mean": 0.0031192348251352087, "completions/clipped_ratio": 0.0, "completions/max_length": 1147.0, "completions/max_terminated_length": 1147.0, "completions/mean_length": 949.4375, "completions/mean_terminated_length": 949.4375, "completions/min_length": 710.0, "completions/min_terminated_length": 710.0, "entropy": 0.10207764338701963, "epoch": 4.408163265306122, "frac_reward_zero_std": 0.0, "grad_norm": 0.022530194371938705, "learning_rate": 5.6122448979591834e-06, "loss": -0.0501, "num_tokens": 12729845.0, "reward": 0.749775767326355, "reward_std": 0.08545472472906113, "rewards/reward_commands_completeness/mean": 0.14000000059604645, "rewards/reward_commands_completeness/std": 0.031832899898290634, "rewards/reward_commands_correctness/mean": 0.078125, "rewards/reward_commands_correctness/std": 0.01223042793571949, "rewards/reward_diversity/mean": 0.16852577030658722, "rewards/reward_diversity/std": 0.024809617549180984, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.17625001072883606, "rewards/reward_solution_effectiveness/std": 0.06184658408164978, "sampling/importance_sampling_ratio/max": 1.7827738523483276, "sampling/importance_sampling_ratio/mean": 0.19138827919960022, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.712764263153076, "sampling/sampling_logp_difference/mean": 0.0208938829600811, "step": 216, "step_time": 110.55246230587363 }, { "clip_ratio/high_max": 0.0027963741740677506, "clip_ratio/high_mean": 0.0027963741740677506, "clip_ratio/low_mean": 0.0010866018710657954, "clip_ratio/low_min": 0.0010866018710657954, "clip_ratio/region_mean": 0.0038829760742373765, "completions/clipped_ratio": 0.0, "completions/max_length": 1178.0, "completions/max_terminated_length": 1178.0, "completions/mean_length": 844.75, "completions/mean_terminated_length": 844.75, "completions/min_length": 632.0, "completions/min_terminated_length": 632.0, "entropy": 0.11487080343067646, "epoch": 4.428571428571429, "frac_reward_zero_std": 0.0, "grad_norm": 0.010336724109947681, "learning_rate": 5.591836734693878e-06, "loss": 0.0061, "num_tokens": 12774473.0, "reward": 0.6106916069984436, "reward_std": 0.11792200803756714, "rewards/reward_commands_completeness/mean": 0.11374999582767487, "rewards/reward_commands_completeness/std": 0.0430309996008873, "rewards/reward_commands_correctness/mean": 0.0612499974668026, "rewards/reward_commands_correctness/std": 0.026299556717276573, "rewards/reward_diversity/mean": 0.14756658673286438, "rewards/reward_diversity/std": 0.05658821761608124, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07312500476837158, "rewards/reward_problem_validity/std": 0.027500001713633537, "rewards/reward_solution_effectiveness/mean": 0.1274999976158142, "rewards/reward_solution_effectiveness/std": 0.060497935861349106, "sampling/importance_sampling_ratio/max": 1.3633815050125122, "sampling/importance_sampling_ratio/mean": 0.10742300003767014, "sampling/importance_sampling_ratio/min": 1.7489658148406306e-06, "sampling/sampling_logp_difference/max": 7.745074272155762, "sampling/sampling_logp_difference/mean": 0.023813094943761826, "step": 217, "step_time": 148.29209908284247 }, { "clip_ratio/high_max": 0.0023919433369883336, "clip_ratio/high_mean": 0.0023919433369883336, "clip_ratio/low_mean": 0.001353887615550775, "clip_ratio/low_min": 0.001353887615550775, "clip_ratio/region_mean": 0.003745831025298685, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 794.625, "completions/mean_terminated_length": 794.625, "completions/min_length": 642.0, "completions/min_terminated_length": 642.0, "entropy": 0.12134075630456209, "epoch": 4.448979591836735, "frac_reward_zero_std": 0.0, "grad_norm": 0.021282756701111794, "learning_rate": 5.571428571428572e-06, "loss": -0.0444, "num_tokens": 12814083.0, "reward": 0.6887499690055847, "reward_std": 0.1181773915886879, "rewards/reward_commands_completeness/mean": 0.1224999949336052, "rewards/reward_commands_completeness/std": 0.04057914391160011, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.020976178348064423, "rewards/reward_diversity/mean": 0.17624999582767487, "rewards/reward_diversity/std": 0.043524157255887985, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.06408002227544785, "sampling/importance_sampling_ratio/max": 0.9811453223228455, "sampling/importance_sampling_ratio/mean": 0.12903660535812378, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.0111494064331055, "sampling/sampling_logp_difference/mean": 0.02502167969942093, "step": 218, "step_time": 135.49733771011233 }, { "clip_ratio/high_max": 0.002516993263270706, "clip_ratio/high_mean": 0.002516993263270706, "clip_ratio/low_mean": 0.000855165853863582, "clip_ratio/low_min": 0.000855165853863582, "clip_ratio/region_mean": 0.0033721591462381184, "completions/clipped_ratio": 0.0, "completions/max_length": 1011.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 802.0625, "completions/mean_terminated_length": 802.0625, "completions/min_length": 606.0, "completions/min_terminated_length": 606.0, "entropy": 0.12311526760458946, "epoch": 4.469387755102041, "frac_reward_zero_std": 0.0, "grad_norm": 0.022455530241131783, "learning_rate": 5.551020408163266e-06, "loss": -0.0304, "num_tokens": 12845220.0, "reward": 0.6336557865142822, "reward_std": 0.12102064490318298, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.03575378656387329, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.02528998628258705, "rewards/reward_diversity/mean": 0.16803079843521118, "rewards/reward_diversity/std": 0.027501627802848816, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.019137660041451454, "rewards/reward_solution_effectiveness/mean": 0.11812500655651093, "rewards/reward_solution_effectiveness/std": 0.05418717488646507, "sampling/importance_sampling_ratio/max": 0.5121450424194336, "sampling/importance_sampling_ratio/mean": 0.05302570015192032, "sampling/importance_sampling_ratio/min": 1.6624278941890225e-05, "sampling/sampling_logp_difference/max": 3.759316921234131, "sampling/sampling_logp_difference/mean": 0.02492590993642807, "step": 219, "step_time": 118.63934930413961 }, { "clip_ratio/high_max": 0.0026550879701972008, "clip_ratio/high_mean": 0.0026550879701972008, "clip_ratio/low_mean": 0.0016680092885508202, "clip_ratio/low_min": 0.0016680092885508202, "clip_ratio/region_mean": 0.004323097280575894, "completions/clipped_ratio": 0.0, "completions/max_length": 1003.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 757.5625, "completions/mean_terminated_length": 757.5625, "completions/min_length": 563.0, "completions/min_terminated_length": 563.0, "entropy": 0.11732339859008789, "epoch": 4.489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.044399816542863846, "learning_rate": 5.530612244897959e-06, "loss": -0.0818, "num_tokens": 12875897.0, "reward": 0.7453331351280212, "reward_std": 0.10684163868427277, "rewards/reward_commands_completeness/mean": 0.14124999940395355, "rewards/reward_commands_completeness/std": 0.033837851136922836, "rewards/reward_commands_correctness/mean": 0.07312500476837158, "rewards/reward_commands_correctness/std": 0.01922455243766308, "rewards/reward_diversity/mean": 0.16345813870429993, "rewards/reward_diversity/std": 0.032234203070402145, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.18000000715255737, "rewards/reward_solution_effectiveness/std": 0.06752777099609375, "sampling/importance_sampling_ratio/max": 1.1116797924041748, "sampling/importance_sampling_ratio/mean": 0.12763012945652008, "sampling/importance_sampling_ratio/min": 8.471305363855208e-07, "sampling/sampling_logp_difference/max": 6.962869644165039, "sampling/sampling_logp_difference/mean": 0.025017501786351204, "step": 220, "step_time": 109.55591500550508 }, { "clip_ratio/high_max": 0.0017757505484041758, "clip_ratio/high_mean": 0.0017757505484041758, "clip_ratio/low_mean": 0.0007497564802179113, "clip_ratio/low_min": 0.0007497564802179113, "clip_ratio/region_mean": 0.002525507014070172, "completions/clipped_ratio": 0.0, "completions/max_length": 848.0, "completions/max_terminated_length": 848.0, "completions/mean_length": 755.875, "completions/mean_terminated_length": 755.875, "completions/min_length": 677.0, "completions/min_terminated_length": 677.0, "entropy": 0.10586985293775797, "epoch": 4.510204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.0410449355840683, "learning_rate": 5.510204081632653e-06, "loss": -0.0582, "num_tokens": 12915083.0, "reward": 0.754935622215271, "reward_std": 0.12824968993663788, "rewards/reward_commands_completeness/mean": 0.13874998688697815, "rewards/reward_commands_completeness/std": 0.04096747189760208, "rewards/reward_commands_correctness/mean": 0.08249999582767487, "rewards/reward_commands_correctness/std": 0.015275253914296627, "rewards/reward_diversity/mean": 0.17868566513061523, "rewards/reward_diversity/std": 0.02053511142730713, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.16875000298023224, "rewards/reward_solution_effectiveness/std": 0.07965550571680069, "sampling/importance_sampling_ratio/max": 0.9725521802902222, "sampling/importance_sampling_ratio/mean": 0.18372337520122528, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 9.95146369934082, "sampling/sampling_logp_difference/mean": 0.02080170437693596, "step": 221, "step_time": 120.32671659812331 }, { "clip_ratio/high_max": 0.0018306683341506869, "clip_ratio/high_mean": 0.0018306683341506869, "clip_ratio/low_mean": 0.0011448502700659446, "clip_ratio/low_min": 0.0011448502700659446, "clip_ratio/region_mean": 0.002975518611492589, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 730.8125, "completions/mean_terminated_length": 730.8125, "completions/min_length": 623.0, "completions/min_terminated_length": 623.0, "entropy": 0.11632457468658686, "epoch": 4.530612244897959, "frac_reward_zero_std": 0.0, "grad_norm": 0.05211496353149414, "learning_rate": 5.489795918367347e-06, "loss": 0.1473, "num_tokens": 12948296.0, "reward": 0.6106127500534058, "reward_std": 0.1415092945098877, "rewards/reward_commands_completeness/mean": 0.11500000953674316, "rewards/reward_commands_completeness/std": 0.0497996024787426, "rewards/reward_commands_correctness/mean": 0.0625, "rewards/reward_commands_correctness/std": 0.032761771231889725, "rewards/reward_diversity/mean": 0.14248774945735931, "rewards/reward_diversity/std": 0.06381569057703018, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07250000536441803, "rewards/reward_problem_validity/std": 0.03130495548248291, "rewards/reward_solution_effectiveness/mean": 0.1368750035762787, "rewards/reward_solution_effectiveness/std": 0.06569817662239075, "sampling/importance_sampling_ratio/max": 1.3854210376739502, "sampling/importance_sampling_ratio/mean": 0.21224430203437805, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 8.688189506530762, "sampling/sampling_logp_difference/mean": 0.024658508598804474, "step": 222, "step_time": 113.05570154264569 }, { "clip_ratio/high_max": 0.001796923803340178, "clip_ratio/high_mean": 0.001796923803340178, "clip_ratio/low_mean": 0.001251630841579754, "clip_ratio/low_min": 0.001251630841579754, "clip_ratio/region_mean": 0.003048554659471847, "completions/clipped_ratio": 0.0, "completions/max_length": 1010.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 776.5, "completions/mean_terminated_length": 776.5, "completions/min_length": 657.0, "completions/min_terminated_length": 657.0, "entropy": 0.09450289607048035, "epoch": 4.551020408163265, "frac_reward_zero_std": 0.0, "grad_norm": 0.04747253656387329, "learning_rate": 5.4693877551020415e-06, "loss": 0.0144, "num_tokens": 12986680.0, "reward": 0.6745426654815674, "reward_std": 0.18276378512382507, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.06319283694028854, "rewards/reward_commands_correctness/mean": 0.0625, "rewards/reward_commands_correctness/std": 0.0290975384414196, "rewards/reward_diversity/mean": 0.15266762673854828, "rewards/reward_diversity/std": 0.06653447449207306, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07312500476837158, "rewards/reward_problem_validity/std": 0.03156343102455139, "rewards/reward_solution_effectiveness/mean": 0.17249999940395355, "rewards/reward_solution_effectiveness/std": 0.09455158561468124, "sampling/importance_sampling_ratio/max": 2.449131965637207, "sampling/importance_sampling_ratio/mean": 0.24643449485301971, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.690066814422607, "sampling/sampling_logp_difference/mean": 0.0223998986184597, "step": 223, "step_time": 132.2929797936231 }, { "clip_ratio/high_max": 0.002264692753669806, "clip_ratio/high_mean": 0.002264692753669806, "clip_ratio/low_mean": 0.001704702401184477, "clip_ratio/low_min": 0.001704702401184477, "clip_ratio/region_mean": 0.0039693951403023675, "completions/clipped_ratio": 0.0, "completions/max_length": 993.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 817.625, "completions/mean_terminated_length": 817.625, "completions/min_length": 697.0, "completions/min_terminated_length": 697.0, "entropy": 0.11217757780104876, "epoch": 4.571428571428571, "frac_reward_zero_std": 0.0, "grad_norm": 0.015119691379368305, "learning_rate": 5.448979591836735e-06, "loss": -0.025, "num_tokens": 13028446.0, "reward": 0.584160566329956, "reward_std": 0.16400985419750214, "rewards/reward_commands_completeness/mean": 0.10874999314546585, "rewards/reward_commands_completeness/std": 0.0516236387193203, "rewards/reward_commands_correctness/mean": 0.05937499925494194, "rewards/reward_commands_correctness/std": 0.029993055388331413, "rewards/reward_diversity/mean": 0.1435355395078659, "rewards/reward_diversity/std": 0.06749292463064194, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07124999910593033, "rewards/reward_problem_validity/std": 0.030740855261683464, "rewards/reward_solution_effectiveness/mean": 0.11999999731779099, "rewards/reward_solution_effectiveness/std": 0.06663332879543304, "sampling/importance_sampling_ratio/max": 0.26268112659454346, "sampling/importance_sampling_ratio/mean": 0.044652845710515976, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.203342914581299, "sampling/sampling_logp_difference/mean": 0.023913664743304253, "step": 224, "step_time": 121.44010867923498 }, { "clip_ratio/high_max": 0.0016315826651407406, "clip_ratio/high_mean": 0.0016315826651407406, "clip_ratio/low_mean": 0.0007014198636170477, "clip_ratio/low_min": 0.0007014198636170477, "clip_ratio/region_mean": 0.002333002514205873, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 738.3125, "completions/mean_terminated_length": 738.3125, "completions/min_length": 634.0, "completions/min_terminated_length": 634.0, "entropy": 0.09564295411109924, "epoch": 4.591836734693878, "frac_reward_zero_std": 0.25, "grad_norm": 0.014009660109877586, "learning_rate": 5.428571428571429e-06, "loss": 0.0091, "num_tokens": 13065807.0, "reward": 0.5841482877731323, "reward_std": 0.047347329556941986, "rewards/reward_commands_completeness/mean": 0.11374999582767487, "rewards/reward_commands_completeness/std": 0.061413899064064026, "rewards/reward_commands_correctness/mean": 0.0637499988079071, "rewards/reward_commands_correctness/std": 0.037036020308732986, "rewards/reward_diversity/mean": 0.12289828062057495, "rewards/reward_diversity/std": 0.06554631143808365, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06812500208616257, "rewards/reward_problem_validity/std": 0.03487477824091911, "rewards/reward_solution_effectiveness/mean": 0.140625, "rewards/reward_solution_effectiveness/std": 0.0810324028134346, "sampling/importance_sampling_ratio/max": 0.5847676992416382, "sampling/importance_sampling_ratio/mean": 0.06077074259519577, "sampling/importance_sampling_ratio/min": 1.6554823067949087e-09, "sampling/sampling_logp_difference/max": 8.649950981140137, "sampling/sampling_logp_difference/mean": 0.025394251570105553, "step": 225, "step_time": 123.73694661073387 }, { "clip_ratio/high_max": 0.0006759981406503357, "clip_ratio/high_mean": 0.0006759981406503357, "clip_ratio/low_mean": 0.0009662043958087452, "clip_ratio/low_min": 0.0009662043958087452, "clip_ratio/region_mean": 0.0016422025219071656, "completions/clipped_ratio": 0.0, "completions/max_length": 1102.0, "completions/max_terminated_length": 1102.0, "completions/mean_length": 813.9375, "completions/mean_terminated_length": 813.9375, "completions/min_length": 639.0, "completions/min_terminated_length": 639.0, "entropy": 0.09780860133469105, "epoch": 4.612244897959184, "frac_reward_zero_std": 0.0, "grad_norm": 0.09162721037864685, "learning_rate": 5.408163265306123e-06, "loss": -0.122, "num_tokens": 13103334.0, "reward": 0.6396793127059937, "reward_std": 0.07070153951644897, "rewards/reward_commands_completeness/mean": 0.10999999940395355, "rewards/reward_commands_completeness/std": 0.02529822289943695, "rewards/reward_commands_correctness/mean": 0.07249999791383743, "rewards/reward_commands_correctness/std": 0.020493904128670692, "rewards/reward_diversity/mean": 0.16342931985855103, "rewards/reward_diversity/std": 0.0230847354978323, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08312499523162842, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.1106249988079071, "rewards/reward_solution_effectiveness/std": 0.04202678054571152, "sampling/importance_sampling_ratio/max": 1.862479567527771, "sampling/importance_sampling_ratio/mean": 0.27097034454345703, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.557238578796387, "sampling/sampling_logp_difference/mean": 0.02155887521803379, "step": 226, "step_time": 122.20747432298958 }, { "clip_ratio/high_max": 0.0017585401510586962, "clip_ratio/high_mean": 0.0017585401510586962, "clip_ratio/low_mean": 0.0012307494471315295, "clip_ratio/low_min": 0.0012307494471315295, "clip_ratio/region_mean": 0.0029892895836383104, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 771.125, "completions/mean_terminated_length": 771.125, "completions/min_length": 669.0, "completions/min_terminated_length": 669.0, "entropy": 0.1012942511588335, "epoch": 4.63265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.06977149099111557, "learning_rate": 5.387755102040817e-06, "loss": 0.0601, "num_tokens": 13140332.0, "reward": 0.7239953279495239, "reward_std": 0.04785773158073425, "rewards/reward_commands_completeness/mean": 0.13374999165534973, "rewards/reward_commands_completeness/std": 0.02276693843305111, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.010954450815916061, "rewards/reward_diversity/mean": 0.1696203500032425, "rewards/reward_diversity/std": 0.02068803831934929, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.16312500834465027, "rewards/reward_solution_effectiveness/std": 0.04512482509016991, "sampling/importance_sampling_ratio/max": 1.2164064645767212, "sampling/importance_sampling_ratio/mean": 0.17720848321914673, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.002044677734375, "sampling/sampling_logp_difference/mean": 0.02255347929894924, "step": 227, "step_time": 117.6779964119196 }, { "clip_ratio/high_max": 0.002257257787277922, "clip_ratio/high_mean": 0.002257257787277922, "clip_ratio/low_mean": 0.0008637529535917565, "clip_ratio/low_min": 0.0008637529535917565, "clip_ratio/region_mean": 0.0031210107263177633, "completions/clipped_ratio": 0.0, "completions/max_length": 1164.0, "completions/max_terminated_length": 1164.0, "completions/mean_length": 817.125, "completions/mean_terminated_length": 817.125, "completions/min_length": 704.0, "completions/min_terminated_length": 704.0, "entropy": 0.08768667280673981, "epoch": 4.653061224489796, "frac_reward_zero_std": 0.0, "grad_norm": 0.011577533558011055, "learning_rate": 5.36734693877551e-06, "loss": 0.0039, "num_tokens": 13179806.0, "reward": 0.4902757406234741, "reward_std": 0.14979210495948792, "rewards/reward_commands_completeness/mean": 0.09624999761581421, "rewards/reward_commands_completeness/std": 0.059874873608350754, "rewards/reward_commands_correctness/mean": 0.04062499850988388, "rewards/reward_commands_correctness/std": 0.027921020984649658, "rewards/reward_diversity/mean": 0.12527573108673096, "rewards/reward_diversity/std": 0.0655839815735817, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.058125000447034836, "rewards/reward_problem_validity/std": 0.03637192025780678, "rewards/reward_solution_effectiveness/mean": 0.10125000029802322, "rewards/reward_solution_effectiveness/std": 0.07658328860998154, "sampling/importance_sampling_ratio/max": 0.51014244556427, "sampling/importance_sampling_ratio/mean": 0.1280568689107895, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5758936405181885, "sampling/sampling_logp_difference/mean": 0.020245423540472984, "step": 228, "step_time": 102.95065255835652 }, { "clip_ratio/high_max": 0.003975682673626579, "clip_ratio/high_mean": 0.003975682673626579, "clip_ratio/low_mean": 0.0008528281177859753, "clip_ratio/low_min": 0.0008528281177859753, "clip_ratio/region_mean": 0.004828510762308724, "completions/clipped_ratio": 0.0, "completions/max_length": 1078.0, "completions/max_terminated_length": 1078.0, "completions/mean_length": 803.75, "completions/mean_terminated_length": 803.75, "completions/min_length": 654.0, "completions/min_terminated_length": 654.0, "entropy": 0.1241964753717184, "epoch": 4.673469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.07331132143735886, "learning_rate": 5.3469387755102045e-06, "loss": -0.1288, "num_tokens": 13223170.0, "reward": 0.7139235734939575, "reward_std": 0.14193740487098694, "rewards/reward_commands_completeness/mean": 0.13499999046325684, "rewards/reward_commands_completeness/std": 0.043512456119060516, "rewards/reward_commands_correctness/mean": 0.06625000387430191, "rewards/reward_commands_correctness/std": 0.022472206503152847, "rewards/reward_diversity/mean": 0.17517361044883728, "rewards/reward_diversity/std": 0.043203022330999374, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0806250050663948, "rewards/reward_problem_validity/std": 0.019482901319861412, "rewards/reward_solution_effectiveness/mean": 0.16312500834465027, "rewards/reward_solution_effectiveness/std": 0.07180703431367874, "sampling/importance_sampling_ratio/max": 2.7823352813720703, "sampling/importance_sampling_ratio/mean": 0.27279844880104065, "sampling/importance_sampling_ratio/min": 1.0597841537673958e-06, "sampling/sampling_logp_difference/max": 8.674131393432617, "sampling/sampling_logp_difference/mean": 0.02689395658671856, "step": 229, "step_time": 150.78886304609478 }, { "clip_ratio/high_max": 0.0020360252310638316, "clip_ratio/high_mean": 0.0020360252310638316, "clip_ratio/low_mean": 0.0010664491856005043, "clip_ratio/low_min": 0.0010664491856005043, "clip_ratio/region_mean": 0.003102474416664336, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 924.0, "completions/mean_length": 966.875, "completions/mean_terminated_length": 758.2667236328125, "completions/min_length": 646.0, "completions/min_terminated_length": 646.0, "entropy": 0.08381601143628359, "epoch": 4.6938775510204085, "frac_reward_zero_std": 0.0, "grad_norm": 0.01885906048119068, "learning_rate": 5.326530612244898e-06, "loss": -0.042, "num_tokens": 13272240.0, "reward": 0.810104250907898, "reward_std": 0.08131623268127441, "rewards/reward_commands_completeness/mean": 0.16499999165534973, "rewards/reward_commands_completeness/std": 0.02476557157933712, "rewards/reward_commands_correctness/mean": 0.07562500238418579, "rewards/reward_commands_correctness/std": 0.015903355553746223, "rewards/reward_diversity/mean": 0.16885417699813843, "rewards/reward_diversity/std": 0.022331416606903076, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08875000476837158, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.21187502145767212, "rewards/reward_solution_effectiveness/std": 0.06241995468735695, "sampling/importance_sampling_ratio/max": 1.0413868427276611, "sampling/importance_sampling_ratio/mean": 0.1035226359963417, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.850124835968018, "sampling/sampling_logp_difference/mean": 0.018239500001072884, "step": 230, "step_time": 308.36988670378923 }, { "clip_ratio/high_max": 0.0029527682199841365, "clip_ratio/high_mean": 0.0029527682199841365, "clip_ratio/low_mean": 0.0015544692796538584, "clip_ratio/low_min": 0.0015544692796538584, "clip_ratio/region_mean": 0.004507237477810122, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1194.0, "completions/mean_length": 950.625, "completions/mean_terminated_length": 740.933349609375, "completions/min_length": 629.0, "completions/min_terminated_length": 629.0, "entropy": 0.10801581689156592, "epoch": 4.714285714285714, "frac_reward_zero_std": 0.0, "grad_norm": 0.027872459962964058, "learning_rate": 5.306122448979593e-06, "loss": 0.0044, "num_tokens": 13327102.0, "reward": 0.7425857782363892, "reward_std": 0.16124244034290314, "rewards/reward_commands_completeness/mean": 0.14500001072883606, "rewards/reward_commands_completeness/std": 0.04163332283496857, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.018618987873196602, "rewards/reward_diversity/mean": 0.1650857925415039, "rewards/reward_diversity/std": 0.047404855489730835, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08312500268220901, "rewards/reward_problem_validity/std": 0.01990603096783161, "rewards/reward_solution_effectiveness/mean": 0.18562500178813934, "rewards/reward_solution_effectiveness/std": 0.06947121769189835, "sampling/importance_sampling_ratio/max": 0.6982569694519043, "sampling/importance_sampling_ratio/mean": 0.12555451691150665, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.470955848693848, "sampling/sampling_logp_difference/mean": 0.018039492890238762, "step": 231, "step_time": 386.27307445928454 }, { "clip_ratio/high_max": 0.0029313007835298777, "clip_ratio/high_mean": 0.0029313007835298777, "clip_ratio/low_mean": 0.0006976097793085501, "clip_ratio/low_min": 0.0006976097793085501, "clip_ratio/region_mean": 0.0036289105337345973, "completions/clipped_ratio": 0.0, "completions/max_length": 1157.0, "completions/max_terminated_length": 1157.0, "completions/mean_length": 873.6875, "completions/mean_terminated_length": 873.6875, "completions/min_length": 701.0, "completions/min_terminated_length": 701.0, "entropy": 0.0925874700769782, "epoch": 4.73469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.022391531616449356, "learning_rate": 5.285714285714286e-06, "loss": 0.0233, "num_tokens": 13359585.0, "reward": 0.7354786396026611, "reward_std": 0.08534195274114609, "rewards/reward_commands_completeness/mean": 0.13374999165534973, "rewards/reward_commands_completeness/std": 0.02801785245537758, "rewards/reward_commands_correctness/mean": 0.08250000327825546, "rewards/reward_commands_correctness/std": 0.014832398854196072, "rewards/reward_diversity/mean": 0.1642286330461502, "rewards/reward_diversity/std": 0.02572374977171421, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.16874998807907104, "rewards/reward_solution_effectiveness/std": 0.05572252720594406, "sampling/importance_sampling_ratio/max": 0.727293848991394, "sampling/importance_sampling_ratio/mean": 0.12205544859170914, "sampling/importance_sampling_ratio/min": 2.1568943338934332e-05, "sampling/sampling_logp_difference/max": 2.1660258769989014, "sampling/sampling_logp_difference/mean": 0.021504109725356102, "step": 232, "step_time": 134.6660797931254 }, { "clip_ratio/high_max": 0.0020840694196522236, "clip_ratio/high_mean": 0.0020840694196522236, "clip_ratio/low_mean": 0.0019379005971131846, "clip_ratio/low_min": 0.0019379005971131846, "clip_ratio/region_mean": 0.004021970060421154, "completions/clipped_ratio": 0.0, "completions/max_length": 1077.0, "completions/max_terminated_length": 1077.0, "completions/mean_length": 753.1875, "completions/mean_terminated_length": 753.1875, "completions/min_length": 587.0, "completions/min_terminated_length": 587.0, "entropy": 0.10937824659049511, "epoch": 4.755102040816326, "frac_reward_zero_std": 0.0, "grad_norm": 0.00716816633939743, "learning_rate": 5.26530612244898e-06, "loss": 0.0135, "num_tokens": 13395204.0, "reward": 0.624728798866272, "reward_std": 0.05101443827152252, "rewards/reward_commands_completeness/mean": 0.10249999165534973, "rewards/reward_commands_completeness/std": 0.02909754030406475, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.022794371470808983, "rewards/reward_diversity/mean": 0.17285384237766266, "rewards/reward_diversity/std": 0.021565645933151245, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.09937499463558197, "rewards/reward_solution_effectiveness/std": 0.04986231029033661, "sampling/importance_sampling_ratio/max": 0.1887340396642685, "sampling/importance_sampling_ratio/mean": 0.028599033132195473, "sampling/importance_sampling_ratio/min": 4.9422826123191044e-05, "sampling/sampling_logp_difference/max": 3.931929111480713, "sampling/sampling_logp_difference/mean": 0.025394411757588387, "step": 233, "step_time": 139.75919435359538 }, { "clip_ratio/high_max": 0.0009990651451516896, "clip_ratio/high_mean": 0.0009990651451516896, "clip_ratio/low_mean": 0.0004897082835668698, "clip_ratio/low_min": 0.0004897082835668698, "clip_ratio/region_mean": 0.0014887734141666442, "completions/clipped_ratio": 0.0, "completions/max_length": 1717.0, "completions/max_terminated_length": 1717.0, "completions/mean_length": 823.125, "completions/mean_terminated_length": 823.125, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.1074132015928626, "epoch": 4.775510204081632, "frac_reward_zero_std": 0.25, "grad_norm": 0.10789559036493301, "learning_rate": 5.2448979591836735e-06, "loss": 0.0688, "num_tokens": 13442170.0, "reward": 0.6605842113494873, "reward_std": 0.03918295353651047, "rewards/reward_commands_completeness/mean": 0.13375000655651093, "rewards/reward_commands_completeness/std": 0.06994045525789261, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.034970227628946304, "rewards/reward_diversity/mean": 0.13120914995670319, "rewards/reward_diversity/std": 0.06677733361721039, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.07000000029802322, "rewards/reward_problem_validity/std": 0.03577708825469017, "rewards/reward_solution_effectiveness/mean": 0.1875, "rewards/reward_solution_effectiveness/std": 0.10535654425621033, "sampling/importance_sampling_ratio/max": 2.108149528503418, "sampling/importance_sampling_ratio/mean": 0.3320803642272949, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 10.497550964355469, "sampling/sampling_logp_difference/mean": 0.02150198630988598, "step": 234, "step_time": 161.3583866544068 }, { "clip_ratio/high_max": 0.0015950049346429296, "clip_ratio/high_mean": 0.0015950049346429296, "clip_ratio/low_mean": 0.0014314526342786849, "clip_ratio/low_min": 0.0014314526342786849, "clip_ratio/region_mean": 0.0030264575907494873, "completions/clipped_ratio": 0.0, "completions/max_length": 1061.0, "completions/max_terminated_length": 1061.0, "completions/mean_length": 779.25, "completions/mean_terminated_length": 779.25, "completions/min_length": 656.0, "completions/min_terminated_length": 656.0, "entropy": 0.10226458497345448, "epoch": 4.795918367346939, "frac_reward_zero_std": 0.0, "grad_norm": 0.09468071162700653, "learning_rate": 5.2244897959183684e-06, "loss": -0.0082, "num_tokens": 13479002.0, "reward": 0.5920513272285461, "reward_std": 0.26740217208862305, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.06280923634767532, "rewards/reward_commands_correctness/mean": 0.05874999985098839, "rewards/reward_commands_correctness/std": 0.034229617565870285, "rewards/reward_diversity/mean": 0.1408013105392456, "rewards/reward_diversity/std": 0.062889464199543, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06812500208616257, "rewards/reward_problem_validity/std": 0.03487477824091911, "rewards/reward_solution_effectiveness/mean": 0.140625, "rewards/reward_solution_effectiveness/std": 0.09080886840820312, "sampling/importance_sampling_ratio/max": 0.9618623852729797, "sampling/importance_sampling_ratio/mean": 0.14184202253818512, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3240742683410645, "sampling/sampling_logp_difference/mean": 0.023490874096751213, "step": 235, "step_time": 113.67701419815421 }, { "clip_ratio/high_max": 0.0010684117150958627, "clip_ratio/high_mean": 0.0010684117150958627, "clip_ratio/low_mean": 0.001269719134143088, "clip_ratio/low_min": 0.001269719134143088, "clip_ratio/region_mean": 0.0023381308710668236, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1247.0, "completions/mean_length": 1019.375, "completions/mean_terminated_length": 814.2667236328125, "completions/min_length": 625.0, "completions/min_terminated_length": 625.0, "entropy": 0.0910441754385829, "epoch": 4.816326530612245, "frac_reward_zero_std": 0.0, "grad_norm": 0.02489822544157505, "learning_rate": 5.204081632653062e-06, "loss": -0.0411, "num_tokens": 13527616.0, "reward": 0.7073599100112915, "reward_std": 0.07321536540985107, "rewards/reward_commands_completeness/mean": 0.13624998927116394, "rewards/reward_commands_completeness/std": 0.02552776411175728, "rewards/reward_commands_correctness/mean": 0.05999999865889549, "rewards/reward_commands_correctness/std": 0.019999999552965164, "rewards/reward_diversity/mean": 0.17673489451408386, "rewards/reward_diversity/std": 0.014270737767219543, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437499403953552, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.15000000596046448, "rewards/reward_solution_effectiveness/std": 0.06387487798929214, "sampling/importance_sampling_ratio/max": 0.7335032224655151, "sampling/importance_sampling_ratio/mean": 0.11686092615127563, "sampling/importance_sampling_ratio/min": 5.003283263249614e-07, "sampling/sampling_logp_difference/max": 5.55589485168457, "sampling/sampling_logp_difference/mean": 0.017959196120500565, "step": 236, "step_time": 288.345681540668 }, { "clip_ratio/high_max": 0.0017239758744835854, "clip_ratio/high_mean": 0.0017239758744835854, "clip_ratio/low_mean": 0.0020995615486754104, "clip_ratio/low_min": 0.0020995615486754104, "clip_ratio/region_mean": 0.003823537437710911, "completions/clipped_ratio": 0.0, "completions/max_length": 996.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 777.6875, "completions/mean_terminated_length": 777.6875, "completions/min_length": 667.0, "completions/min_terminated_length": 667.0, "entropy": 0.10411571152508259, "epoch": 4.836734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.026100926101207733, "learning_rate": 5.183673469387756e-06, "loss": -0.0446, "num_tokens": 13561395.0, "reward": 0.750368595123291, "reward_std": 0.08096975088119507, "rewards/reward_commands_completeness/mean": 0.14249999821186066, "rewards/reward_commands_completeness/std": 0.02620432898402214, "rewards/reward_commands_correctness/mean": 0.07312500476837158, "rewards/reward_commands_correctness/std": 0.020564937964081764, "rewards/reward_diversity/mean": 0.17536859214305878, "rewards/reward_diversity/std": 0.0175947193056345, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08875000476837158, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.056623756885528564, "sampling/importance_sampling_ratio/max": 0.9906264543533325, "sampling/importance_sampling_ratio/mean": 0.08414028584957123, "sampling/importance_sampling_ratio/min": 1.025537585519487e-06, "sampling/sampling_logp_difference/max": 10.991384506225586, "sampling/sampling_logp_difference/mean": 0.02652890793979168, "step": 237, "step_time": 124.02202024869621 }, { "clip_ratio/high_max": 0.003421306231757626, "clip_ratio/high_mean": 0.003421306231757626, "clip_ratio/low_mean": 0.0017222928581759334, "clip_ratio/low_min": 0.0017222928581759334, "clip_ratio/region_mean": 0.005143599089933559, "completions/clipped_ratio": 0.0, "completions/max_length": 997.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 767.0, "completions/mean_terminated_length": 767.0, "completions/min_length": 608.0, "completions/min_terminated_length": 608.0, "entropy": 0.10155340377241373, "epoch": 4.857142857142857, "frac_reward_zero_std": 0.0, "grad_norm": 0.04133196547627449, "learning_rate": 5.163265306122449e-06, "loss": 0.0609, "num_tokens": 13595167.0, "reward": 0.7498166561126709, "reward_std": 0.06526125222444534, "rewards/reward_commands_completeness/mean": 0.14000000059604645, "rewards/reward_commands_completeness/std": 0.0326598659157753, "rewards/reward_commands_correctness/mean": 0.07562500238418579, "rewards/reward_commands_correctness/std": 0.017876895144581795, "rewards/reward_diversity/mean": 0.1704416573047638, "rewards/reward_diversity/std": 0.02733434922993183, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.17624999582767487, "rewards/reward_solution_effectiveness/std": 0.0717286542057991, "sampling/importance_sampling_ratio/max": 2.044560432434082, "sampling/importance_sampling_ratio/mean": 0.13901551067829132, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2880566120147705, "sampling/sampling_logp_difference/mean": 0.026282455772161484, "step": 238, "step_time": 124.3364196382463 }, { "clip_ratio/high_max": 0.0016563329991186038, "clip_ratio/high_mean": 0.0016563329991186038, "clip_ratio/low_mean": 0.0010881311172852293, "clip_ratio/low_min": 0.0010881311172852293, "clip_ratio/region_mean": 0.002744464058196172, "completions/clipped_ratio": 0.0, "completions/max_length": 1683.0, "completions/max_terminated_length": 1683.0, "completions/mean_length": 806.625, "completions/mean_terminated_length": 806.625, "completions/min_length": 619.0, "completions/min_terminated_length": 619.0, "entropy": 0.09759248606860638, "epoch": 4.877551020408164, "frac_reward_zero_std": 0.0, "grad_norm": 0.018128564581274986, "learning_rate": 5.142857142857142e-06, "loss": -0.0028, "num_tokens": 13631089.0, "reward": 0.6797395944595337, "reward_std": 0.1392129361629486, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.043512456119060516, "rewards/reward_commands_correctness/mean": 0.06812499463558197, "rewards/reward_commands_correctness/std": 0.021669873967766762, "rewards/reward_diversity/mean": 0.1641145795583725, "rewards/reward_diversity/std": 0.040484875440597534, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08250000327825546, "rewards/reward_problem_validity/std": 0.019832635298371315, "rewards/reward_solution_effectiveness/mean": 0.14625000953674316, "rewards/reward_solution_effectiveness/std": 0.06561250239610672, "sampling/importance_sampling_ratio/max": 0.3383900821208954, "sampling/importance_sampling_ratio/mean": 0.05094057694077492, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 15.993284225463867, "sampling/sampling_logp_difference/mean": 0.02740350365638733, "step": 239, "step_time": 150.23365996591747 }, { "clip_ratio/high_max": 0.00219567546446342, "clip_ratio/high_mean": 0.00219567546446342, "clip_ratio/low_mean": 0.0013048934124526568, "clip_ratio/low_min": 0.0013048934124526568, "clip_ratio/region_mean": 0.003500568913295865, "completions/clipped_ratio": 0.0, "completions/max_length": 875.0, "completions/max_terminated_length": 875.0, "completions/mean_length": 721.625, "completions/mean_terminated_length": 721.625, "completions/min_length": 618.0, "completions/min_terminated_length": 618.0, "entropy": 0.09768346790224314, "epoch": 4.8979591836734695, "frac_reward_zero_std": 0.0, "grad_norm": 0.03309145197272301, "learning_rate": 5.122448979591837e-06, "loss": 0.0291, "num_tokens": 13664327.0, "reward": 0.7492014169692993, "reward_std": 0.06710701435804367, "rewards/reward_commands_completeness/mean": 0.14500001072883606, "rewards/reward_commands_completeness/std": 0.02476556971669197, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.018257420510053635, "rewards/reward_diversity/mean": 0.17295140027999878, "rewards/reward_diversity/std": 0.017529187723994255, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.1743749976158142, "rewards/reward_solution_effectiveness/std": 0.05278494209051132, "sampling/importance_sampling_ratio/max": 0.8923509120941162, "sampling/importance_sampling_ratio/mean": 0.14851827919483185, "sampling/importance_sampling_ratio/min": 1.5249361240421422e-05, "sampling/sampling_logp_difference/max": 2.4657177925109863, "sampling/sampling_logp_difference/mean": 0.024944739416241646, "step": 240, "step_time": 109.33227142132819 }, { "clip_ratio/high_max": 0.0013065518141957, "clip_ratio/high_mean": 0.0013065518141957, "clip_ratio/low_mean": 0.00042951533396262676, "clip_ratio/low_min": 0.00042951533396262676, "clip_ratio/region_mean": 0.0017360671481583267, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 754.3125, "completions/mean_terminated_length": 754.3125, "completions/min_length": 651.0, "completions/min_terminated_length": 651.0, "entropy": 0.0902085518464446, "epoch": 4.918367346938775, "frac_reward_zero_std": 0.0, "grad_norm": 0.02153630740940571, "learning_rate": 5.1020408163265315e-06, "loss": 0.0046, "num_tokens": 13698532.0, "reward": 0.7212589979171753, "reward_std": 0.07728058844804764, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.030876098200678825, "rewards/reward_commands_correctness/mean": 0.07562500238418579, "rewards/reward_commands_correctness/std": 0.012093388475477695, "rewards/reward_diversity/mean": 0.17563405632972717, "rewards/reward_diversity/std": 0.0275428369641304, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.15187498927116394, "rewards/reward_solution_effectiveness/std": 0.05418717488646507, "sampling/importance_sampling_ratio/max": 0.30256888270378113, "sampling/importance_sampling_ratio/mean": 0.07064332067966461, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7185511589050293, "sampling/sampling_logp_difference/mean": 0.02304159663617611, "step": 241, "step_time": 107.08872009068727 }, { "clip_ratio/high_max": 0.0026700278103817254, "clip_ratio/high_mean": 0.0026700278103817254, "clip_ratio/low_mean": 0.0012342601403361186, "clip_ratio/low_min": 0.0012342601403361186, "clip_ratio/region_mean": 0.0039042879652697593, "completions/clipped_ratio": 0.0, "completions/max_length": 930.0, "completions/max_terminated_length": 930.0, "completions/mean_length": 770.25, "completions/mean_terminated_length": 770.25, "completions/min_length": 672.0, "completions/min_terminated_length": 672.0, "entropy": 0.10108038503676653, "epoch": 4.938775510204081, "frac_reward_zero_std": 0.0, "grad_norm": 0.019576655700802803, "learning_rate": 5.081632653061225e-06, "loss": -0.0377, "num_tokens": 13752052.0, "reward": 0.6869047284126282, "reward_std": 0.160928875207901, "rewards/reward_commands_completeness/mean": 0.1287499964237213, "rewards/reward_commands_completeness/std": 0.043185651302337646, "rewards/reward_commands_correctness/mean": 0.0612499974668026, "rewards/reward_commands_correctness/std": 0.021563859656453133, "rewards/reward_diversity/mean": 0.17690476775169373, "rewards/reward_diversity/std": 0.0482831746339798, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07999999821186066, "rewards/reward_problem_validity/std": 0.019321836531162262, "rewards/reward_solution_effectiveness/mean": 0.14625000953674316, "rewards/reward_solution_effectiveness/std": 0.07338256388902664, "sampling/importance_sampling_ratio/max": 2.0041964054107666, "sampling/importance_sampling_ratio/mean": 0.1682087481021881, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7999939918518066, "sampling/sampling_logp_difference/mean": 0.024323152378201485, "step": 242, "step_time": 154.94243890978396 }, { "clip_ratio/high_max": 0.0029423505620798096, "clip_ratio/high_mean": 0.0029423505620798096, "clip_ratio/low_mean": 0.0009108953236136585, "clip_ratio/low_min": 0.0009108953236136585, "clip_ratio/region_mean": 0.003853245871141553, "completions/clipped_ratio": 0.0, "completions/max_length": 1073.0, "completions/max_terminated_length": 1073.0, "completions/mean_length": 854.75, "completions/mean_terminated_length": 854.75, "completions/min_length": 728.0, "completions/min_terminated_length": 728.0, "entropy": 0.09371752385050058, "epoch": 4.959183673469388, "frac_reward_zero_std": 0.0, "grad_norm": 0.019346019253134727, "learning_rate": 5.061224489795918e-06, "loss": -0.0133, "num_tokens": 13796692.0, "reward": 0.6742830276489258, "reward_std": 0.11017237603664398, "rewards/reward_commands_completeness/mean": 0.12124999612569809, "rewards/reward_commands_completeness/std": 0.04224925860762596, "rewards/reward_commands_correctness/mean": 0.06875000149011612, "rewards/reward_commands_correctness/std": 0.017841899767518044, "rewards/reward_diversity/mean": 0.17178300023078918, "rewards/reward_diversity/std": 0.04366093501448631, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.1368750035762787, "rewards/reward_solution_effectiveness/std": 0.0599687434732914, "sampling/importance_sampling_ratio/max": 0.5312745571136475, "sampling/importance_sampling_ratio/mean": 0.04651396721601486, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 13.731854438781738, "sampling/sampling_logp_difference/mean": 0.025158878415822983, "step": 243, "step_time": 140.39388618059456 }, { "clip_ratio/high_max": 0.0017361843783874065, "clip_ratio/high_mean": 0.0017361843783874065, "clip_ratio/low_mean": 0.0009839152262429707, "clip_ratio/low_min": 0.0009839152262429707, "clip_ratio/region_mean": 0.00272009962645825, "completions/clipped_ratio": 0.0, "completions/max_length": 807.0, "completions/max_terminated_length": 807.0, "completions/mean_length": 706.5625, "completions/mean_terminated_length": 706.5625, "completions/min_length": 599.0, "completions/min_terminated_length": 599.0, "entropy": 0.08727822732180357, "epoch": 4.979591836734694, "frac_reward_zero_std": 0.0, "grad_norm": 0.009085530415177345, "learning_rate": 5.040816326530613e-06, "loss": 0.0033, "num_tokens": 13835325.0, "reward": 0.6716318726539612, "reward_std": 0.12380920350551605, "rewards/reward_commands_completeness/mean": 0.1274999976158142, "rewards/reward_commands_completeness/std": 0.03924283757805824, "rewards/reward_commands_correctness/mean": 0.057500001043081284, "rewards/reward_commands_correctness/std": 0.02620432712137699, "rewards/reward_diversity/mean": 0.17413194477558136, "rewards/reward_diversity/std": 0.04304269701242447, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07999999821186066, "rewards/reward_problem_validity/std": 0.019321836531162262, "rewards/reward_solution_effectiveness/mean": 0.13875000178813934, "rewards/reward_solution_effectiveness/std": 0.06280923634767532, "sampling/importance_sampling_ratio/max": 0.18333162367343903, "sampling/importance_sampling_ratio/mean": 0.029021665453910828, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 18.231849670410156, "sampling/sampling_logp_difference/mean": 0.02626962959766388, "step": 244, "step_time": 111.67112854123116 }, { "clip_ratio/high_max": 0.0032902389648370445, "clip_ratio/high_mean": 0.0032902389648370445, "clip_ratio/low_mean": 0.0016860233736224473, "clip_ratio/low_min": 0.0016860233736224473, "clip_ratio/region_mean": 0.004976262353011407, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 850.0, "completions/mean_length": 959.5625, "completions/mean_terminated_length": 750.4667358398438, "completions/min_length": 637.0, "completions/min_terminated_length": 637.0, "entropy": 0.10462924931198359, "epoch": 5.0, "frac_reward_zero_std": 0.0, "grad_norm": 0.008470551110804081, "learning_rate": 5.020408163265307e-06, "loss": -0.0198, "num_tokens": 13884374.0, "reward": 0.7547335028648376, "reward_std": 0.08984014391899109, "rewards/reward_commands_completeness/mean": 0.14124999940395355, "rewards/reward_commands_completeness/std": 0.03138471394777298, "rewards/reward_commands_correctness/mean": 0.07187499850988388, "rewards/reward_commands_correctness/std": 0.018337123095989227, "rewards/reward_diversity/mean": 0.18473346531391144, "rewards/reward_diversity/std": 0.018074480816721916, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08624999970197678, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.06546946614980698, "sampling/importance_sampling_ratio/max": 0.6063388586044312, "sampling/importance_sampling_ratio/mean": 0.0532834492623806, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 15.748054504394531, "sampling/sampling_logp_difference/mean": 0.024556012824177742, "step": 245, "step_time": 287.1979053877294 }, { "clip_ratio/high_max": 0.002384846826316789, "clip_ratio/high_mean": 0.002384846826316789, "clip_ratio/low_mean": 0.0010525547404540703, "clip_ratio/low_min": 0.0010525547404540703, "clip_ratio/region_mean": 0.0034374015813227743, "completions/clipped_ratio": 0.0, "completions/max_length": 870.0, "completions/max_terminated_length": 870.0, "completions/mean_length": 748.0, "completions/mean_terminated_length": 748.0, "completions/min_length": 622.0, "completions/min_terminated_length": 622.0, "entropy": 0.0928506562486291, "epoch": 5.020408163265306, "frac_reward_zero_std": 0.0, "grad_norm": 0.04843248799443245, "learning_rate": 5e-06, "loss": -0.1377, "num_tokens": 13929694.0, "reward": 0.684791624546051, "reward_std": 0.16791000962257385, "rewards/reward_commands_completeness/mean": 0.12999999523162842, "rewards/reward_commands_completeness/std": 0.05006662756204605, "rewards/reward_commands_correctness/mean": 0.05812499672174454, "rewards/reward_commands_correctness/std": 0.02509150095283985, "rewards/reward_diversity/mean": 0.17291665077209473, "rewards/reward_diversity/std": 0.045049358159303665, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.027049340307712555, "rewards/reward_solution_effectiveness/mean": 0.1574999988079071, "rewards/reward_solution_effectiveness/std": 0.07389181107282639, "sampling/importance_sampling_ratio/max": 1.9823976755142212, "sampling/importance_sampling_ratio/mean": 0.3100382089614868, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.4029593467712402, "sampling/sampling_logp_difference/mean": 0.022721337154507637, "step": 246, "step_time": 145.71925236284733 }, { "clip_ratio/high_max": 0.0024228485563071445, "clip_ratio/high_mean": 0.0024228485563071445, "clip_ratio/low_mean": 0.0007611911787535064, "clip_ratio/low_min": 0.0007611911787535064, "clip_ratio/region_mean": 0.003184039771440439, "completions/clipped_ratio": 0.0, "completions/max_length": 926.0, "completions/max_terminated_length": 926.0, "completions/mean_length": 717.4375, "completions/mean_terminated_length": 717.4375, "completions/min_length": 602.0, "completions/min_terminated_length": 602.0, "entropy": 0.1125166304409504, "epoch": 5.040816326530612, "frac_reward_zero_std": 0.0, "grad_norm": 0.014760374091565609, "learning_rate": 4.979591836734694e-06, "loss": -0.024, "num_tokens": 13964813.0, "reward": 0.6768749952316284, "reward_std": 0.15931051969528198, "rewards/reward_commands_completeness/mean": 0.12124999612569809, "rewards/reward_commands_completeness/std": 0.04224926233291626, "rewards/reward_commands_correctness/mean": 0.06812499463558197, "rewards/reward_commands_correctness/std": 0.02227666974067688, "rewards/reward_diversity/mean": 0.17374999821186066, "rewards/reward_diversity/std": 0.0430309996008873, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.13875000178813934, "rewards/reward_solution_effectiveness/std": 0.06469158083200455, "sampling/importance_sampling_ratio/max": 0.43540340662002563, "sampling/importance_sampling_ratio/mean": 0.04723338410258293, "sampling/importance_sampling_ratio/min": 3.3782357604650315e-06, "sampling/sampling_logp_difference/max": 3.5626697540283203, "sampling/sampling_logp_difference/mean": 0.028368400409817696, "step": 247, "step_time": 109.9844976067543 }, { "clip_ratio/high_max": 0.0018929462967207655, "clip_ratio/high_mean": 0.0018929462967207655, "clip_ratio/low_mean": 0.0015204187075141817, "clip_ratio/low_min": 0.0015204187075141817, "clip_ratio/region_mean": 0.0034133650333387777, "completions/clipped_ratio": 0.0, "completions/max_length": 1083.0, "completions/max_terminated_length": 1083.0, "completions/mean_length": 786.3125, "completions/mean_terminated_length": 786.3125, "completions/min_length": 705.0, "completions/min_terminated_length": 705.0, "entropy": 0.10889369156211615, "epoch": 5.061224489795919, "frac_reward_zero_std": 0.0, "grad_norm": 0.017423801124095917, "learning_rate": 4.959183673469388e-06, "loss": -0.0298, "num_tokens": 14001666.0, "reward": 0.7247395515441895, "reward_std": 0.047373391687870026, "rewards/reward_commands_completeness/mean": 0.13499999046325684, "rewards/reward_commands_completeness/std": 0.0236643198877573, "rewards/reward_commands_correctness/mean": 0.07124999910593033, "rewards/reward_commands_correctness/std": 0.018574178218841553, "rewards/reward_diversity/mean": 0.18036457896232605, "rewards/reward_diversity/std": 0.02238338068127632, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.1537500023841858, "rewards/reward_solution_effectiveness/std": 0.04759201407432556, "sampling/importance_sampling_ratio/max": 1.297924280166626, "sampling/importance_sampling_ratio/mean": 0.10266848653554916, "sampling/importance_sampling_ratio/min": 3.8356814252571425e-11, "sampling/sampling_logp_difference/max": 15.873064041137695, "sampling/sampling_logp_difference/mean": 0.028379587456583977, "step": 248, "step_time": 133.25715007819235 }, { "clip_ratio/high_max": 0.0025776204565772787, "clip_ratio/high_mean": 0.0025776204565772787, "clip_ratio/low_mean": 0.0017924298736033961, "clip_ratio/low_min": 0.0017924298736033961, "clip_ratio/region_mean": 0.0043700502792489715, "completions/clipped_ratio": 0.0, "completions/max_length": 797.0, "completions/max_terminated_length": 797.0, "completions/mean_length": 713.25, "completions/mean_terminated_length": 713.25, "completions/min_length": 620.0, "completions/min_terminated_length": 620.0, "entropy": 0.09673956409096718, "epoch": 5.081632653061225, "frac_reward_zero_std": 0.0, "grad_norm": 0.0054019843228161335, "learning_rate": 4.938775510204082e-06, "loss": 0.0074, "num_tokens": 14043662.0, "reward": 0.7151378393173218, "reward_std": 0.06468772888183594, "rewards/reward_commands_completeness/mean": 0.13875000178813934, "rewards/reward_commands_completeness/std": 0.02777889184653759, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.018607795238494873, "rewards/reward_diversity/mean": 0.16826286911964417, "rewards/reward_diversity/std": 0.017790867015719414, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.1574999988079071, "rewards/reward_solution_effectiveness/std": 0.049598388373851776, "sampling/importance_sampling_ratio/max": 0.07530795782804489, "sampling/importance_sampling_ratio/mean": 0.007230263203382492, "sampling/importance_sampling_ratio/min": 4.054885767434957e-13, "sampling/sampling_logp_difference/max": 18.174026489257812, "sampling/sampling_logp_difference/mean": 0.03374205529689789, "step": 249, "step_time": 130.16077574901283 }, { "clip_ratio/high_max": 0.0027416081065894105, "clip_ratio/high_mean": 0.0027416081065894105, "clip_ratio/low_mean": 0.0009626510145608336, "clip_ratio/low_min": 0.0009626510145608336, "clip_ratio/region_mean": 0.0037042591284262016, "completions/clipped_ratio": 0.0, "completions/max_length": 886.0, "completions/max_terminated_length": 886.0, "completions/mean_length": 709.125, "completions/mean_terminated_length": 709.125, "completions/min_length": 620.0, "completions/min_terminated_length": 620.0, "entropy": 0.09695312473922968, "epoch": 5.1020408163265305, "frac_reward_zero_std": 0.0, "grad_norm": 0.036570824682712555, "learning_rate": 4.918367346938776e-06, "loss": -0.0857, "num_tokens": 14083508.0, "reward": 0.8013020753860474, "reward_std": 0.07021604478359222, "rewards/reward_commands_completeness/mean": 0.1550000011920929, "rewards/reward_commands_completeness/std": 0.022509261965751648, "rewards/reward_commands_correctness/mean": 0.0806250050663948, "rewards/reward_commands_correctness/std": 0.018062394112348557, "rewards/reward_diversity/mean": 0.17880208790302277, "rewards/reward_diversity/std": 0.019708452746272087, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.19875000417232513, "rewards/reward_solution_effectiveness/std": 0.04883646219968796, "sampling/importance_sampling_ratio/max": 0.6994292140007019, "sampling/importance_sampling_ratio/mean": 0.1115187332034111, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 15.487646102905273, "sampling/sampling_logp_difference/mean": 0.02803461067378521, "step": 250, "step_time": 123.69610901921988 }, { "clip_ratio/high_max": 0.002831749152392149, "clip_ratio/high_mean": 0.002831749152392149, "clip_ratio/low_mean": 0.0016552052620681934, "clip_ratio/low_min": 0.0016552052620681934, "clip_ratio/region_mean": 0.0044869544217363, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 755.9375, "completions/mean_terminated_length": 755.9375, "completions/min_length": 649.0, "completions/min_terminated_length": 649.0, "entropy": 0.09626109525561333, "epoch": 5.122448979591836, "frac_reward_zero_std": 0.0, "grad_norm": 0.00627127755433321, "learning_rate": 4.897959183673469e-06, "loss": -0.0095, "num_tokens": 14121495.0, "reward": 0.7439721822738647, "reward_std": 0.08060699701309204, "rewards/reward_commands_completeness/mean": 0.14374999701976776, "rewards/reward_commands_completeness/std": 0.029410885646939278, "rewards/reward_commands_correctness/mean": 0.07750000059604645, "rewards/reward_commands_correctness/std": 0.01612451672554016, "rewards/reward_diversity/mean": 0.1570972055196762, "rewards/reward_diversity/std": 0.020471949130296707, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.17812499403953552, "rewards/reward_solution_effectiveness/std": 0.05946638062596321, "sampling/importance_sampling_ratio/max": 0.1547713577747345, "sampling/importance_sampling_ratio/mean": 0.029727471992373466, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 18.73583221435547, "sampling/sampling_logp_difference/mean": 0.031712695956230164, "step": 251, "step_time": 127.89374851994216 }, { "clip_ratio/high_max": 0.00269168119848473, "clip_ratio/high_mean": 0.00269168119848473, "clip_ratio/low_mean": 0.0012143204367021099, "clip_ratio/low_min": 0.0012143204367021099, "clip_ratio/region_mean": 0.00390600163518684, "completions/clipped_ratio": 0.0, "completions/max_length": 968.0, "completions/max_terminated_length": 968.0, "completions/mean_length": 760.5625, "completions/mean_terminated_length": 760.5625, "completions/min_length": 634.0, "completions/min_terminated_length": 634.0, "entropy": 0.11528999265283346, "epoch": 5.142857142857143, "frac_reward_zero_std": 0.0, "grad_norm": 0.01765892282128334, "learning_rate": 4.8775510204081635e-06, "loss": 0.0094, "num_tokens": 14160624.0, "reward": 0.5815839171409607, "reward_std": 0.13256126642227173, "rewards/reward_commands_completeness/mean": 0.09999999403953552, "rewards/reward_commands_completeness/std": 0.05006662756204605, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.03004857338964939, "rewards/reward_diversity/mean": 0.15595895051956177, "rewards/reward_diversity/std": 0.0687926709651947, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07250000536441803, "rewards/reward_problem_validity/std": 0.03130495548248291, "rewards/reward_solution_effectiveness/mean": 0.10875000059604645, "rewards/reward_solution_effectiveness/std": 0.06830080598592758, "sampling/importance_sampling_ratio/max": 0.28594228625297546, "sampling/importance_sampling_ratio/mean": 0.06113191694021225, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 17.744670867919922, "sampling/sampling_logp_difference/mean": 0.029232623055577278, "step": 252, "step_time": 123.93285399861634 }, { "clip_ratio/high_max": 0.002332038478925824, "clip_ratio/high_mean": 0.002332038478925824, "clip_ratio/low_mean": 0.0013644620921695605, "clip_ratio/low_min": 0.0013644620921695605, "clip_ratio/region_mean": 0.0036965005565434694, "completions/clipped_ratio": 0.0, "completions/max_length": 996.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 742.8125, "completions/mean_terminated_length": 742.8125, "completions/min_length": 633.0, "completions/min_terminated_length": 633.0, "entropy": 0.09432103112339973, "epoch": 5.163265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.004968675784766674, "learning_rate": 4.857142857142858e-06, "loss": 0.0032, "num_tokens": 14196161.0, "reward": 0.7436649799346924, "reward_std": 0.07273870706558228, "rewards/reward_commands_completeness/mean": 0.13875000178813934, "rewards/reward_commands_completeness/std": 0.03052321821451187, "rewards/reward_commands_correctness/mean": 0.07625000178813934, "rewards/reward_commands_correctness/std": 0.01927866041660309, "rewards/reward_diversity/mean": 0.17366501688957214, "rewards/reward_diversity/std": 0.024268561974167824, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.16875000298023224, "rewards/reward_solution_effectiveness/std": 0.059874873608350754, "sampling/importance_sampling_ratio/max": 0.12485351413488388, "sampling/importance_sampling_ratio/mean": 0.023544656112790108, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9631094932556152, "sampling/sampling_logp_difference/mean": 0.022442057728767395, "step": 253, "step_time": 113.93666947446764 }, { "clip_ratio/high_max": 0.0020193777163513005, "clip_ratio/high_mean": 0.0020193777163513005, "clip_ratio/low_mean": 0.000967332067375537, "clip_ratio/low_min": 0.000967332067375537, "clip_ratio/region_mean": 0.002986709740071092, "completions/clipped_ratio": 0.0, "completions/max_length": 1524.0, "completions/max_terminated_length": 1524.0, "completions/mean_length": 872.75, "completions/mean_terminated_length": 872.75, "completions/min_length": 677.0, "completions/min_terminated_length": 677.0, "entropy": 0.07705172523856163, "epoch": 5.183673469387755, "frac_reward_zero_std": 0.0, "grad_norm": 0.014666557312011719, "learning_rate": 4.836734693877552e-06, "loss": 0.0105, "num_tokens": 14234577.0, "reward": 0.6511189341545105, "reward_std": 0.11401639133691788, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.047046080231666565, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.028744565322995186, "rewards/reward_diversity/mean": 0.13861893117427826, "rewards/reward_diversity/std": 0.05013182386755943, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.02651257812976837, "rewards/reward_solution_effectiveness/mean": 0.1574999988079071, "rewards/reward_solution_effectiveness/std": 0.0730753019452095, "sampling/importance_sampling_ratio/max": 1.187639594078064, "sampling/importance_sampling_ratio/mean": 0.10073068737983704, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 18.839752197265625, "sampling/sampling_logp_difference/mean": 0.024048874154686928, "step": 254, "step_time": 140.18741353601217 }, { "clip_ratio/high_max": 0.0019627770670922473, "clip_ratio/high_mean": 0.0019627770670922473, "clip_ratio/low_mean": 0.0009107803271035664, "clip_ratio/low_min": 0.0009107803271035664, "clip_ratio/region_mean": 0.002873557372367941, "completions/clipped_ratio": 0.0, "completions/max_length": 908.0, "completions/max_terminated_length": 908.0, "completions/mean_length": 755.8125, "completions/mean_terminated_length": 755.8125, "completions/min_length": 673.0, "completions/min_terminated_length": 673.0, "entropy": 0.10380759555846453, "epoch": 5.204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.018271038308739662, "learning_rate": 4.816326530612245e-06, "loss": -0.019, "num_tokens": 14291982.0, "reward": 0.6043872833251953, "reward_std": 0.1418088674545288, "rewards/reward_commands_completeness/mean": 0.10750000178813934, "rewards/reward_commands_completeness/std": 0.03642343729734421, "rewards/reward_commands_correctness/mean": 0.0637499988079071, "rewards/reward_commands_correctness/std": 0.023629080504179, "rewards/reward_diversity/mean": 0.14938725531101227, "rewards/reward_diversity/std": 0.04977220669388771, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07562500238418579, "rewards/reward_problem_validity/std": 0.020966244861483574, "rewards/reward_solution_effectiveness/mean": 0.11437499523162842, "rewards/reward_solution_effectiveness/std": 0.05609144642949104, "sampling/importance_sampling_ratio/max": 1.4641168117523193, "sampling/importance_sampling_ratio/mean": 0.1722572147846222, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.5767056941986084, "sampling/sampling_logp_difference/mean": 0.0246682520955801, "step": 255, "step_time": 206.61784396134317 }, { "clip_ratio/high_max": 0.0013631505498779006, "clip_ratio/high_mean": 0.0013631505498779006, "clip_ratio/low_mean": 0.0009626443425077014, "clip_ratio/low_min": 0.0009626443425077014, "clip_ratio/region_mean": 0.0023257948851096444, "completions/clipped_ratio": 0.0, "completions/max_length": 1118.0, "completions/max_terminated_length": 1118.0, "completions/mean_length": 776.1875, "completions/mean_terminated_length": 776.1875, "completions/min_length": 642.0, "completions/min_terminated_length": 642.0, "entropy": 0.09885053895413876, "epoch": 5.224489795918367, "frac_reward_zero_std": 0.25, "grad_norm": 0.04692806303501129, "learning_rate": 4.795918367346939e-06, "loss": 0.0536, "num_tokens": 14328045.0, "reward": 0.6115601062774658, "reward_std": 0.04910382255911827, "rewards/reward_commands_completeness/mean": 0.11750000715255737, "rewards/reward_commands_completeness/std": 0.06276942789554596, "rewards/reward_commands_correctness/mean": 0.06499999761581421, "rewards/reward_commands_correctness/std": 0.03577708825469017, "rewards/reward_diversity/mean": 0.13218511641025543, "rewards/reward_diversity/std": 0.07230708003044128, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.07000000029802322, "rewards/reward_problem_validity/std": 0.03577708825469017, "rewards/reward_solution_effectiveness/mean": 0.15187498927116394, "rewards/reward_solution_effectiveness/std": 0.08518362790346146, "sampling/importance_sampling_ratio/max": 1.3464349508285522, "sampling/importance_sampling_ratio/mean": 0.10808883607387543, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.452915668487549, "sampling/sampling_logp_difference/mean": 0.028209976851940155, "step": 256, "step_time": 124.33058475330472 }, { "clip_ratio/high_max": 0.0030509289135807194, "clip_ratio/high_mean": 0.0030509289135807194, "clip_ratio/low_mean": 0.0006937323123565875, "clip_ratio/low_min": 0.0006937323123565875, "clip_ratio/region_mean": 0.0037446612550411373, "completions/clipped_ratio": 0.0, "completions/max_length": 912.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 721.0, "completions/mean_terminated_length": 721.0, "completions/min_length": 621.0, "completions/min_terminated_length": 621.0, "entropy": 0.09009670745581388, "epoch": 5.244897959183674, "frac_reward_zero_std": 0.0, "grad_norm": 0.02107309363782406, "learning_rate": 4.775510204081633e-06, "loss": 0.012, "num_tokens": 14370385.0, "reward": 0.6811917424201965, "reward_std": 0.06938278675079346, "rewards/reward_commands_completeness/mean": 0.12124999612569809, "rewards/reward_commands_completeness/std": 0.028722815215587616, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.010781929828226566, "rewards/reward_diversity/mean": 0.17931678891181946, "rewards/reward_diversity/std": 0.012952741235494614, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437499403953552, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.13312499225139618, "rewards/reward_solution_effectiveness/std": 0.058959729969501495, "sampling/importance_sampling_ratio/max": 0.6775665879249573, "sampling/importance_sampling_ratio/mean": 0.11788728833198547, "sampling/importance_sampling_ratio/min": 8.969678901848965e-07, "sampling/sampling_logp_difference/max": 4.688835144042969, "sampling/sampling_logp_difference/mean": 0.02395382896065712, "step": 257, "step_time": 140.47328512743115 }, { "clip_ratio/high_max": 0.002135173126589507, "clip_ratio/high_mean": 0.002135173126589507, "clip_ratio/low_mean": 0.0011879887533723377, "clip_ratio/low_min": 0.0011879887533723377, "clip_ratio/region_mean": 0.003323161887237802, "completions/clipped_ratio": 0.0, "completions/max_length": 817.0, "completions/max_terminated_length": 817.0, "completions/mean_length": 734.25, "completions/mean_terminated_length": 734.25, "completions/min_length": 655.0, "completions/min_terminated_length": 655.0, "entropy": 0.09356306120753288, "epoch": 5.26530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.002564898692071438, "learning_rate": 4.7551020408163265e-06, "loss": 0.0046, "num_tokens": 14404361.0, "reward": 0.752169132232666, "reward_std": 0.06208810955286026, "rewards/reward_commands_completeness/mean": 0.14499999582767487, "rewards/reward_commands_completeness/std": 0.0389871820807457, "rewards/reward_commands_correctness/mean": 0.07187499850988388, "rewards/reward_commands_correctness/std": 0.017969883978366852, "rewards/reward_diversity/mean": 0.17529411613941193, "rewards/reward_diversity/std": 0.023619869723916054, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.1743749976158142, "rewards/reward_solution_effectiveness/std": 0.07201562076807022, "sampling/importance_sampling_ratio/max": 0.03740745410323143, "sampling/importance_sampling_ratio/mean": 0.009890519082546234, "sampling/importance_sampling_ratio/min": 5.314817371981917e-06, "sampling/sampling_logp_difference/max": 4.413317680358887, "sampling/sampling_logp_difference/mean": 0.02564164064824581, "step": 258, "step_time": 109.58840841427445 }, { "clip_ratio/high_max": 0.0022612429020227864, "clip_ratio/high_mean": 0.0022612429020227864, "clip_ratio/low_mean": 0.0007012368878349662, "clip_ratio/low_min": 0.0007012368878349662, "clip_ratio/region_mean": 0.0029624797898577526, "completions/clipped_ratio": 0.0, "completions/max_length": 813.0, "completions/max_terminated_length": 813.0, "completions/mean_length": 720.3125, "completions/mean_terminated_length": 720.3125, "completions/min_length": 672.0, "completions/min_terminated_length": 672.0, "entropy": 0.09998367540538311, "epoch": 5.285714285714286, "frac_reward_zero_std": 0.0, "grad_norm": 0.01830984465777874, "learning_rate": 4.734693877551021e-06, "loss": -0.0321, "num_tokens": 14438166.0, "reward": 0.8029166460037231, "reward_std": 0.07165583968162537, "rewards/reward_commands_completeness/mean": 0.1550000011920929, "rewards/reward_commands_completeness/std": 0.02366432175040245, "rewards/reward_commands_correctness/mean": 0.0768750011920929, "rewards/reward_commands_correctness/std": 0.009464847855269909, "rewards/reward_diversity/mean": 0.17916667461395264, "rewards/reward_diversity/std": 0.027093803510069847, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08937500417232513, "rewards/reward_problem_validity/std": 0.0025000013411045074, "rewards/reward_solution_effectiveness/mean": 0.20250000059604645, "rewards/reward_solution_effectiveness/std": 0.048373546451330185, "sampling/importance_sampling_ratio/max": 0.7531987428665161, "sampling/importance_sampling_ratio/mean": 0.06050153821706772, "sampling/importance_sampling_ratio/min": 4.949616064231499e-11, "sampling/sampling_logp_difference/max": 17.7487850189209, "sampling/sampling_logp_difference/mean": 0.03187485784292221, "step": 259, "step_time": 104.01485963538289 }, { "clip_ratio/high_max": 0.0023384805535897613, "clip_ratio/high_mean": 0.0023384805535897613, "clip_ratio/low_mean": 0.0009935140697052702, "clip_ratio/low_min": 0.0009935140697052702, "clip_ratio/region_mean": 0.0033319946378469467, "completions/clipped_ratio": 0.0, "completions/max_length": 945.0, "completions/max_terminated_length": 945.0, "completions/mean_length": 749.3125, "completions/mean_terminated_length": 749.3125, "completions/min_length": 629.0, "completions/min_terminated_length": 629.0, "entropy": 0.120866390876472, "epoch": 5.3061224489795915, "frac_reward_zero_std": 0.0, "grad_norm": 0.039527274668216705, "learning_rate": 4.714285714285715e-06, "loss": 0.0703, "num_tokens": 14471679.0, "reward": 0.6467341184616089, "reward_std": 0.13548493385314941, "rewards/reward_commands_completeness/mean": 0.11874999850988388, "rewards/reward_commands_completeness/std": 0.04349329322576523, "rewards/reward_commands_correctness/mean": 0.0625, "rewards/reward_commands_correctness/std": 0.02620432712137699, "rewards/reward_diversity/mean": 0.1623590588569641, "rewards/reward_diversity/std": 0.056426092982292175, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.027049340307712555, "rewards/reward_solution_effectiveness/mean": 0.1368750035762787, "rewards/reward_solution_effectiveness/std": 0.05688805133104324, "sampling/importance_sampling_ratio/max": 2.120349168777466, "sampling/importance_sampling_ratio/mean": 0.17773614823818207, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 17.366384506225586, "sampling/sampling_logp_difference/mean": 0.033587999641895294, "step": 260, "step_time": 122.55621318891644 }, { "clip_ratio/high_max": 0.0019525973475538194, "clip_ratio/high_mean": 0.0019525973475538194, "clip_ratio/low_mean": 0.001242925070982892, "clip_ratio/low_min": 0.001242925070982892, "clip_ratio/region_mean": 0.0031955223821569234, "completions/clipped_ratio": 0.0, "completions/max_length": 843.0, "completions/max_terminated_length": 843.0, "completions/mean_length": 727.5, "completions/mean_terminated_length": 727.5, "completions/min_length": 657.0, "completions/min_terminated_length": 657.0, "entropy": 0.08305524103343487, "epoch": 5.326530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.029285458847880363, "learning_rate": 4.693877551020409e-06, "loss": 0.0279, "num_tokens": 14526647.0, "reward": 0.5978180170059204, "reward_std": 0.1089840680360794, "rewards/reward_commands_completeness/mean": 0.11249999701976776, "rewards/reward_commands_completeness/std": 0.05507570877671242, "rewards/reward_commands_correctness/mean": 0.06499999761581421, "rewards/reward_commands_correctness/std": 0.029888683930039406, "rewards/reward_diversity/mean": 0.1384429782629013, "rewards/reward_diversity/std": 0.0614226870238781, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07124999910593033, "rewards/reward_problem_validity/std": 0.030740855261683464, "rewards/reward_solution_effectiveness/mean": 0.12937499582767487, "rewards/reward_solution_effectiveness/std": 0.0748748928308487, "sampling/importance_sampling_ratio/max": 1.08127760887146, "sampling/importance_sampling_ratio/mean": 0.12124594300985336, "sampling/importance_sampling_ratio/min": 0.00015786898438818753, "sampling/sampling_logp_difference/max": 3.6065914630889893, "sampling/sampling_logp_difference/mean": 0.023172719404101372, "step": 261, "step_time": 199.6065175458789 }, { "clip_ratio/high_max": 0.001221591912326403, "clip_ratio/high_mean": 0.001221591912326403, "clip_ratio/low_mean": 0.0020556191957439296, "clip_ratio/low_min": 0.0020556191957439296, "clip_ratio/region_mean": 0.0032772111298982054, "completions/clipped_ratio": 0.0, "completions/max_length": 838.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 717.8125, "completions/mean_terminated_length": 717.8125, "completions/min_length": 602.0, "completions/min_terminated_length": 602.0, "entropy": 0.10513126477599144, "epoch": 5.346938775510204, "frac_reward_zero_std": 0.0, "grad_norm": 0.03852416202425957, "learning_rate": 4.673469387755102e-06, "loss": -0.0668, "num_tokens": 14581896.0, "reward": 0.7368505001068115, "reward_std": 0.09094569087028503, "rewards/reward_commands_completeness/mean": 0.13500000536441803, "rewards/reward_commands_completeness/std": 0.030550507828593254, "rewards/reward_commands_correctness/mean": 0.07187499850988388, "rewards/reward_commands_correctness/std": 0.018337123095989227, "rewards/reward_diversity/mean": 0.1837254762649536, "rewards/reward_diversity/std": 0.014584863558411598, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687499910593033, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.15937499701976776, "rewards/reward_solution_effectiveness/std": 0.06454649567604065, "sampling/importance_sampling_ratio/max": 1.5526342391967773, "sampling/importance_sampling_ratio/mean": 0.1837666779756546, "sampling/importance_sampling_ratio/min": 4.773059725238227e-09, "sampling/sampling_logp_difference/max": 11.62570858001709, "sampling/sampling_logp_difference/mean": 0.029489248991012573, "step": 262, "step_time": 210.4808926898986 }, { "clip_ratio/high_max": 0.0016113861929625273, "clip_ratio/high_mean": 0.0016113861929625273, "clip_ratio/low_mean": 0.0018072241146001033, "clip_ratio/low_min": 0.0018072241146001033, "clip_ratio/region_mean": 0.0034186102711828426, "completions/clipped_ratio": 0.0, "completions/max_length": 893.0, "completions/max_terminated_length": 893.0, "completions/mean_length": 736.5625, "completions/mean_terminated_length": 736.5625, "completions/min_length": 663.0, "completions/min_terminated_length": 663.0, "entropy": 0.10278077516704798, "epoch": 5.36734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.019632840529084206, "learning_rate": 4.653061224489796e-06, "loss": -0.0491, "num_tokens": 14622457.0, "reward": 0.6453645825386047, "reward_std": 0.13644346594810486, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.055377498269081116, "rewards/reward_commands_correctness/mean": 0.061250001192092896, "rewards/reward_commands_correctness/std": 0.02895398810505867, "rewards/reward_diversity/mean": 0.1472395807504654, "rewards/reward_diversity/std": 0.06441813707351685, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07312500476837158, "rewards/reward_problem_validity/std": 0.03156343102455139, "rewards/reward_solution_effectiveness/mean": 0.1574999988079071, "rewards/reward_solution_effectiveness/std": 0.07628893107175827, "sampling/importance_sampling_ratio/max": 0.7229429483413696, "sampling/importance_sampling_ratio/mean": 0.09280559420585632, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 10.965545654296875, "sampling/sampling_logp_difference/mean": 0.027851419523358345, "step": 263, "step_time": 121.91951228305697 }, { "clip_ratio/high_max": 0.0019431073305895552, "clip_ratio/high_mean": 0.0019431073305895552, "clip_ratio/low_mean": 0.0020104402938159183, "clip_ratio/low_min": 0.0020104402938159183, "clip_ratio/region_mean": 0.0039535475661978126, "completions/clipped_ratio": 0.0, "completions/max_length": 966.0, "completions/max_terminated_length": 966.0, "completions/mean_length": 803.5625, "completions/mean_terminated_length": 803.5625, "completions/min_length": 659.0, "completions/min_terminated_length": 659.0, "entropy": 0.0950437793508172, "epoch": 5.387755102040816, "frac_reward_zero_std": 0.0, "grad_norm": 0.04634222015738487, "learning_rate": 4.63265306122449e-06, "loss": 0.074, "num_tokens": 14671782.0, "reward": 0.766230583190918, "reward_std": 0.08255590498447418, "rewards/reward_commands_completeness/mean": 0.15000000596046448, "rewards/reward_commands_completeness/std": 0.0334664024412632, "rewards/reward_commands_correctness/mean": 0.078125, "rewards/reward_commands_correctness/std": 0.01869715005159378, "rewards/reward_diversity/mean": 0.1706055998802185, "rewards/reward_diversity/std": 0.01720181480050087, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.17999999225139618, "rewards/reward_solution_effectiveness/std": 0.06480740755796432, "sampling/importance_sampling_ratio/max": 0.7224483489990234, "sampling/importance_sampling_ratio/mean": 0.10659126192331314, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 17.06169891357422, "sampling/sampling_logp_difference/mean": 0.027385570108890533, "step": 264, "step_time": 151.04796711727977 }, { "clip_ratio/high_max": 0.0018424570371280424, "clip_ratio/high_mean": 0.0018424570371280424, "clip_ratio/low_mean": 0.000771110579080414, "clip_ratio/low_min": 0.000771110579080414, "clip_ratio/region_mean": 0.0026135676671401598, "completions/clipped_ratio": 0.0, "completions/max_length": 862.0, "completions/max_terminated_length": 862.0, "completions/mean_length": 719.0, "completions/mean_terminated_length": 719.0, "completions/min_length": 657.0, "completions/min_terminated_length": 657.0, "entropy": 0.09882883168756962, "epoch": 5.408163265306122, "frac_reward_zero_std": 0.0, "grad_norm": 0.058567266911268234, "learning_rate": 4.612244897959184e-06, "loss": -0.101, "num_tokens": 14707906.0, "reward": 0.6004564762115479, "reward_std": 0.21521711349487305, "rewards/reward_commands_completeness/mean": 0.11499999463558197, "rewards/reward_commands_completeness/std": 0.055856961756944656, "rewards/reward_commands_correctness/mean": 0.05562499910593033, "rewards/reward_commands_correctness/std": 0.027560539543628693, "rewards/reward_diversity/mean": 0.14733149111270905, "rewards/reward_diversity/std": 0.06468940526247025, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07187500596046448, "rewards/reward_problem_validity/std": 0.03103090077638626, "rewards/reward_solution_effectiveness/mean": 0.12937499582767487, "rewards/reward_solution_effectiveness/std": 0.07724150270223618, "sampling/importance_sampling_ratio/max": 1.7957299947738647, "sampling/importance_sampling_ratio/mean": 0.23485925793647766, "sampling/importance_sampling_ratio/min": 1.582198592586792e-06, "sampling/sampling_logp_difference/max": 3.9708027839660645, "sampling/sampling_logp_difference/mean": 0.02676570415496826, "step": 265, "step_time": 108.00691981241107 }, { "clip_ratio/high_max": 0.002923046879004687, "clip_ratio/high_mean": 0.002923046879004687, "clip_ratio/low_mean": 0.0007225174340419471, "clip_ratio/low_min": 0.0007225174340419471, "clip_ratio/region_mean": 0.0036455643421504647, "completions/clipped_ratio": 0.0, "completions/max_length": 1057.0, "completions/max_terminated_length": 1057.0, "completions/mean_length": 804.625, "completions/mean_terminated_length": 804.625, "completions/min_length": 686.0, "completions/min_terminated_length": 686.0, "entropy": 0.12409334070980549, "epoch": 5.428571428571429, "frac_reward_zero_std": 0.0, "grad_norm": 0.011818332597613335, "learning_rate": 4.591836734693878e-06, "loss": -0.012, "num_tokens": 14744412.0, "reward": 0.689693033695221, "reward_std": 0.052725911140441895, "rewards/reward_commands_completeness/mean": 0.11749999970197678, "rewards/reward_commands_completeness/std": 0.027202941477298737, "rewards/reward_commands_correctness/mean": 0.07125000655651093, "rewards/reward_commands_correctness/std": 0.010246952064335346, "rewards/reward_diversity/mean": 0.18281808495521545, "rewards/reward_diversity/std": 0.019263623282313347, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.005163980647921562, "rewards/reward_solution_effectiveness/mean": 0.13312500715255737, "rewards/reward_solution_effectiveness/std": 0.04377499222755432, "sampling/importance_sampling_ratio/max": 0.34282150864601135, "sampling/importance_sampling_ratio/mean": 0.03392709791660309, "sampling/importance_sampling_ratio/min": 5.128386049157996e-10, "sampling/sampling_logp_difference/max": 14.245587348937988, "sampling/sampling_logp_difference/mean": 0.03157814219594002, "step": 266, "step_time": 122.7714088819921 }, { "clip_ratio/high_max": 0.0016914814914343879, "clip_ratio/high_mean": 0.0016914814914343879, "clip_ratio/low_mean": 0.0005099506088299677, "clip_ratio/low_min": 0.0005099506088299677, "clip_ratio/region_mean": 0.002201432071160525, "completions/clipped_ratio": 0.0, "completions/max_length": 1038.0, "completions/max_terminated_length": 1038.0, "completions/mean_length": 756.125, "completions/mean_terminated_length": 756.125, "completions/min_length": 615.0, "completions/min_terminated_length": 615.0, "entropy": 0.09199367742985487, "epoch": 5.448979591836735, "frac_reward_zero_std": 0.25, "grad_norm": 0.008910351432859898, "learning_rate": 4.571428571428572e-06, "loss": 0.016, "num_tokens": 14788434.0, "reward": 0.4698835611343384, "reward_std": 0.11023257672786713, "rewards/reward_commands_completeness/mean": 0.09000000357627869, "rewards/reward_commands_completeness/std": 0.06110101193189621, "rewards/reward_commands_correctness/mean": 0.04874999821186066, "rewards/reward_commands_correctness/std": 0.03403429687023163, "rewards/reward_diversity/mean": 0.10175858438014984, "rewards/reward_diversity/std": 0.07302001118659973, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.058125004172325134, "rewards/reward_problem_validity/std": 0.03868139535188675, "rewards/reward_solution_effectiveness/mean": 0.10875000059604645, "rewards/reward_solution_effectiveness/std": 0.0773627832531929, "sampling/importance_sampling_ratio/max": 0.3102929890155792, "sampling/importance_sampling_ratio/mean": 0.03719572350382805, "sampling/importance_sampling_ratio/min": 9.758840957552195e-11, "sampling/sampling_logp_difference/max": 14.249736785888672, "sampling/sampling_logp_difference/mean": 0.027295202016830444, "step": 267, "step_time": 127.35229461826384 }, { "clip_ratio/high_max": 0.004562132176943123, "clip_ratio/high_mean": 0.004562132176943123, "clip_ratio/low_mean": 0.001457209073123522, "clip_ratio/low_min": 0.001457209073123522, "clip_ratio/region_mean": 0.006019341177307069, "completions/clipped_ratio": 0.0, "completions/max_length": 769.0, "completions/max_terminated_length": 769.0, "completions/mean_length": 687.4375, "completions/mean_terminated_length": 687.4375, "completions/min_length": 630.0, "completions/min_terminated_length": 630.0, "entropy": 0.09899175632745028, "epoch": 5.469387755102041, "frac_reward_zero_std": 0.0, "grad_norm": 0.011948717758059502, "learning_rate": 4.551020408163266e-06, "loss": 0.0093, "num_tokens": 14818537.0, "reward": 0.725477933883667, "reward_std": 0.12019741535186768, "rewards/reward_commands_completeness/mean": 0.13750000298023224, "rewards/reward_commands_completeness/std": 0.043127719312906265, "rewards/reward_commands_correctness/mean": 0.07874999940395355, "rewards/reward_commands_correctness/std": 0.018929695710539818, "rewards/reward_diversity/mean": 0.1692279428243637, "rewards/reward_diversity/std": 0.027263345196843147, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08250000327825546, "rewards/reward_problem_validity/std": 0.014832398854196072, "rewards/reward_solution_effectiveness/mean": 0.1574999988079071, "rewards/reward_solution_effectiveness/std": 0.08865664899349213, "sampling/importance_sampling_ratio/max": 0.36798736453056335, "sampling/importance_sampling_ratio/mean": 0.03266926109790802, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.931476593017578, "sampling/sampling_logp_difference/mean": 0.027751220390200615, "step": 268, "step_time": 99.68554905429482 }, { "clip_ratio/high_max": 0.0029325585928745568, "clip_ratio/high_mean": 0.0029325585928745568, "clip_ratio/low_mean": 0.000980975542915985, "clip_ratio/low_min": 0.000980975542915985, "clip_ratio/region_mean": 0.003913534135790542, "completions/clipped_ratio": 0.0, "completions/max_length": 1109.0, "completions/max_terminated_length": 1109.0, "completions/mean_length": 865.25, "completions/mean_terminated_length": 865.25, "completions/min_length": 714.0, "completions/min_terminated_length": 714.0, "entropy": 0.10799750685691833, "epoch": 5.489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.007325948216021061, "learning_rate": 4.530612244897959e-06, "loss": -0.0179, "num_tokens": 14859545.0, "reward": 0.693270206451416, "reward_std": 0.06623731553554535, "rewards/reward_commands_completeness/mean": 0.1274999976158142, "rewards/reward_commands_completeness/std": 0.0290975384414196, "rewards/reward_commands_correctness/mean": 0.06624999642372131, "rewards/reward_commands_correctness/std": 0.013102162629365921, "rewards/reward_diversity/mean": 0.1763952076435089, "rewards/reward_diversity/std": 0.028263045474886894, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437499403953552, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.13875000178813934, "rewards/reward_solution_effectiveness/std": 0.05123475193977356, "sampling/importance_sampling_ratio/max": 0.21206168830394745, "sampling/importance_sampling_ratio/mean": 0.03438965603709221, "sampling/importance_sampling_ratio/min": 4.059831439917616e-07, "sampling/sampling_logp_difference/max": 3.5029730796813965, "sampling/sampling_logp_difference/mean": 0.027442876249551773, "step": 269, "step_time": 132.72686175629497 }, { "clip_ratio/high_max": 0.0020875581103609875, "clip_ratio/high_mean": 0.0020875581103609875, "clip_ratio/low_mean": 0.001141264394391328, "clip_ratio/low_min": 0.001141264394391328, "clip_ratio/region_mean": 0.0032288225047523156, "completions/clipped_ratio": 0.0, "completions/max_length": 873.0, "completions/max_terminated_length": 873.0, "completions/mean_length": 742.5625, "completions/mean_terminated_length": 742.5625, "completions/min_length": 622.0, "completions/min_terminated_length": 622.0, "entropy": 0.11198491882532835, "epoch": 5.510204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.027969907969236374, "learning_rate": 4.5102040816326535e-06, "loss": -0.0502, "num_tokens": 14895098.0, "reward": 0.7365379333496094, "reward_std": 0.11355136334896088, "rewards/reward_commands_completeness/mean": 0.13875000178813934, "rewards/reward_commands_completeness/std": 0.033837851136922836, "rewards/reward_commands_correctness/mean": 0.07750000059604645, "rewards/reward_commands_correctness/std": 0.022360682487487793, "rewards/reward_diversity/mean": 0.16903798282146454, "rewards/reward_diversity/std": 0.017223643139004707, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.16687500476837158, "rewards/reward_solution_effectiveness/std": 0.0683831125497818, "sampling/importance_sampling_ratio/max": 0.7217108011245728, "sampling/importance_sampling_ratio/mean": 0.08792214095592499, "sampling/importance_sampling_ratio/min": 6.579643923032563e-06, "sampling/sampling_logp_difference/max": 3.0634074211120605, "sampling/sampling_logp_difference/mean": 0.02759404107928276, "step": 270, "step_time": 113.93395608663559 }, { "clip_ratio/high_max": 0.0024531723247491755, "clip_ratio/high_mean": 0.0024531723247491755, "clip_ratio/low_mean": 0.00100368179846555, "clip_ratio/low_min": 0.00100368179846555, "clip_ratio/region_mean": 0.003456854094110895, "completions/clipped_ratio": 0.0, "completions/max_length": 931.0, "completions/max_terminated_length": 931.0, "completions/mean_length": 731.375, "completions/mean_terminated_length": 731.375, "completions/min_length": 671.0, "completions/min_terminated_length": 671.0, "entropy": 0.09480678476393223, "epoch": 5.530612244897959, "frac_reward_zero_std": 0.0, "grad_norm": 0.09245848655700684, "learning_rate": 4.489795918367348e-06, "loss": 0.0662, "num_tokens": 14933164.0, "reward": 0.7894270420074463, "reward_std": 0.07589571177959442, "rewards/reward_commands_completeness/mean": 0.1550000011920929, "rewards/reward_commands_completeness/std": 0.025819892063736916, "rewards/reward_commands_correctness/mean": 0.07750000059604645, "rewards/reward_commands_correctness/std": 0.013416408561170101, "rewards/reward_diversity/mean": 0.16692706942558289, "rewards/reward_diversity/std": 0.024471675977110863, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08937500417232513, "rewards/reward_problem_validity/std": 0.0025000013411045074, "rewards/reward_solution_effectiveness/mean": 0.2006250023841858, "rewards/reward_solution_effectiveness/std": 0.05335025489330292, "sampling/importance_sampling_ratio/max": 2.006387948989868, "sampling/importance_sampling_ratio/mean": 0.28884023427963257, "sampling/importance_sampling_ratio/min": 3.170768104610033e-05, "sampling/sampling_logp_difference/max": 2.8295302391052246, "sampling/sampling_logp_difference/mean": 0.024401048198342323, "step": 271, "step_time": 134.35731339640915 }, { "clip_ratio/high_max": 0.0024683907395228744, "clip_ratio/high_mean": 0.0024683907395228744, "clip_ratio/low_mean": 0.0013316901749931276, "clip_ratio/low_min": 0.0013316901749931276, "clip_ratio/region_mean": 0.003800080914516002, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 761.75, "completions/mean_terminated_length": 761.75, "completions/min_length": 700.0, "completions/min_terminated_length": 700.0, "entropy": 0.09163948986679316, "epoch": 5.551020408163265, "frac_reward_zero_std": 0.0, "grad_norm": 0.16162244975566864, "learning_rate": 4.469387755102041e-06, "loss": 0.0045, "num_tokens": 14970100.0, "reward": 0.6211361885070801, "reward_std": 0.14530405402183533, "rewards/reward_commands_completeness/mean": 0.11750000715255737, "rewards/reward_commands_completeness/std": 0.059721581637859344, "rewards/reward_commands_correctness/mean": 0.06437500566244125, "rewards/reward_commands_correctness/std": 0.0328570157289505, "rewards/reward_diversity/mean": 0.14113619923591614, "rewards/reward_diversity/std": 0.06334442645311356, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07062499970197678, "rewards/reward_problem_validity/std": 0.03043435700237751, "rewards/reward_solution_effectiveness/mean": 0.14625000953674316, "rewards/reward_solution_effectiveness/std": 0.08333666622638702, "sampling/importance_sampling_ratio/max": 2.9100520610809326, "sampling/importance_sampling_ratio/mean": 0.44220322370529175, "sampling/importance_sampling_ratio/min": 6.2432077356788795e-06, "sampling/sampling_logp_difference/max": 4.0324482917785645, "sampling/sampling_logp_difference/mean": 0.024314116686582565, "step": 272, "step_time": 109.26137758418918 }, { "clip_ratio/high_max": 0.001634483429370448, "clip_ratio/high_mean": 0.001634483429370448, "clip_ratio/low_mean": 0.0014547028113156557, "clip_ratio/low_min": 0.0014547028113156557, "clip_ratio/region_mean": 0.003089186327997595, "completions/clipped_ratio": 0.0, "completions/max_length": 886.0, "completions/max_terminated_length": 886.0, "completions/mean_length": 769.25, "completions/mean_terminated_length": 769.25, "completions/min_length": 646.0, "completions/min_terminated_length": 646.0, "entropy": 0.09614263381808996, "epoch": 5.571428571428571, "frac_reward_zero_std": 0.0, "grad_norm": 0.018059248104691505, "learning_rate": 4.448979591836735e-06, "loss": -0.0324, "num_tokens": 15010708.0, "reward": 0.5577604174613953, "reward_std": 0.20194733142852783, "rewards/reward_commands_completeness/mean": 0.09874999523162842, "rewards/reward_commands_completeness/std": 0.05725673586130142, "rewards/reward_commands_correctness/mean": 0.060624998062849045, "rewards/reward_commands_correctness/std": 0.034922532737255096, "rewards/reward_diversity/mean": 0.13088542222976685, "rewards/reward_diversity/std": 0.06936448812484741, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06687499582767487, "rewards/reward_problem_validity/std": 0.034199174493551254, "rewards/reward_solution_effectiveness/mean": 0.12562499940395355, "rewards/reward_solution_effectiveness/std": 0.09179461002349854, "sampling/importance_sampling_ratio/max": 0.4704360365867615, "sampling/importance_sampling_ratio/mean": 0.09843248128890991, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9538865089416504, "sampling/sampling_logp_difference/mean": 0.022830937057733536, "step": 273, "step_time": 117.63558345101774 }, { "clip_ratio/high_max": 0.0036957094707759097, "clip_ratio/high_mean": 0.0036957094707759097, "clip_ratio/low_mean": 0.0014499300450552255, "clip_ratio/low_min": 0.0014499300450552255, "clip_ratio/region_mean": 0.005145639443071559, "completions/clipped_ratio": 0.0, "completions/max_length": 838.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 736.3125, "completions/mean_terminated_length": 736.3125, "completions/min_length": 670.0, "completions/min_terminated_length": 670.0, "entropy": 0.098027303814888, "epoch": 5.591836734693878, "frac_reward_zero_std": 0.0, "grad_norm": 0.005842425394803286, "learning_rate": 4.428571428571429e-06, "loss": -0.0121, "num_tokens": 15050297.0, "reward": 0.8307291269302368, "reward_std": 0.04954755678772926, "rewards/reward_commands_completeness/mean": 0.16749998927116394, "rewards/reward_commands_completeness/std": 0.01437591016292572, "rewards/reward_commands_correctness/mean": 0.078125, "rewards/reward_commands_correctness/std": 0.009810710325837135, "rewards/reward_diversity/mean": 0.17010417580604553, "rewards/reward_diversity/std": 0.022595223039388657, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.09000000357627869, "rewards/reward_problem_validity/std": 0.0, "rewards/reward_solution_effectiveness/mean": 0.22499999403953552, "rewards/reward_solution_effectiveness/std": 0.02683281898498535, "sampling/importance_sampling_ratio/max": 0.1187923476099968, "sampling/importance_sampling_ratio/mean": 0.017424138262867928, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 20.173871994018555, "sampling/sampling_logp_difference/mean": 0.030869584530591965, "step": 274, "step_time": 114.45200804434717 }, { "clip_ratio/high_max": 0.0029040361696388572, "clip_ratio/high_mean": 0.0029040361696388572, "clip_ratio/low_mean": 0.0005876149225514382, "clip_ratio/low_min": 0.0005876149225514382, "clip_ratio/region_mean": 0.0034916510921902955, "completions/clipped_ratio": 0.0, "completions/max_length": 997.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 760.4375, "completions/mean_terminated_length": 760.4375, "completions/min_length": 632.0, "completions/min_terminated_length": 632.0, "entropy": 0.08954156097024679, "epoch": 5.612244897959184, "frac_reward_zero_std": 0.0, "grad_norm": 0.07439351826906204, "learning_rate": 4.408163265306123e-06, "loss": -0.0453, "num_tokens": 15086740.0, "reward": 0.798217236995697, "reward_std": 0.0733303651213646, "rewards/reward_commands_completeness/mean": 0.1574999988079071, "rewards/reward_commands_completeness/std": 0.037859395146369934, "rewards/reward_commands_correctness/mean": 0.07625000178813934, "rewards/reward_commands_correctness/std": 0.010246951133012772, "rewards/reward_diversity/mean": 0.16509225964546204, "rewards/reward_diversity/std": 0.02929648943245411, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.21187499165534973, "rewards/reward_solution_effectiveness/std": 0.0730496421456337, "sampling/importance_sampling_ratio/max": 2.9911115169525146, "sampling/importance_sampling_ratio/mean": 0.2221851944923401, "sampling/importance_sampling_ratio/min": 2.3808111732082082e-11, "sampling/sampling_logp_difference/max": 21.086942672729492, "sampling/sampling_logp_difference/mean": 0.02898119017481804, "step": 275, "step_time": 122.72806461341679 }, { "clip_ratio/high_max": 0.0017660454832366668, "clip_ratio/high_mean": 0.0017660454832366668, "clip_ratio/low_mean": 0.000586276815738529, "clip_ratio/low_min": 0.000586276815738529, "clip_ratio/region_mean": 0.0023523223208030686, "completions/clipped_ratio": 0.0, "completions/max_length": 1091.0, "completions/max_terminated_length": 1091.0, "completions/mean_length": 767.1875, "completions/mean_terminated_length": 767.1875, "completions/min_length": 636.0, "completions/min_terminated_length": 636.0, "entropy": 0.0861095292493701, "epoch": 5.63265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.01980573870241642, "learning_rate": 4.3877551020408165e-06, "loss": -0.0416, "num_tokens": 15122811.0, "reward": 0.7453124523162842, "reward_std": 0.1487872153520584, "rewards/reward_commands_completeness/mean": 0.14249999821186066, "rewards/reward_commands_completeness/std": 0.04374166950583458, "rewards/reward_commands_correctness/mean": 0.07312500476837158, "rewards/reward_commands_correctness/std": 0.02414366975426674, "rewards/reward_diversity/mean": 0.16593749821186066, "rewards/reward_diversity/std": 0.04090286046266556, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.019989583641290665, "rewards/reward_solution_effectiveness/mean": 0.18562500178813934, "rewards/reward_solution_effectiveness/std": 0.06592608988285065, "sampling/importance_sampling_ratio/max": 2.7736332416534424, "sampling/importance_sampling_ratio/mean": 0.20996572077274323, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.829892158508301, "sampling/sampling_logp_difference/mean": 0.023510945960879326, "step": 276, "step_time": 119.07840792275965 }, { "clip_ratio/high_max": 0.0028035232389811426, "clip_ratio/high_mean": 0.0028035232389811426, "clip_ratio/low_mean": 0.000995161957689561, "clip_ratio/low_min": 0.000995161957689561, "clip_ratio/region_mean": 0.003798685225774534, "completions/clipped_ratio": 0.0, "completions/max_length": 1052.0, "completions/max_terminated_length": 1052.0, "completions/mean_length": 766.4375, "completions/mean_terminated_length": 766.4375, "completions/min_length": 613.0, "completions/min_terminated_length": 613.0, "entropy": 0.09306563623249531, "epoch": 5.653061224489796, "frac_reward_zero_std": 0.0, "grad_norm": 0.02264280803501606, "learning_rate": 4.367346938775511e-06, "loss": 0.0105, "num_tokens": 15160498.0, "reward": 0.6535918712615967, "reward_std": 0.12801463901996613, "rewards/reward_commands_completeness/mean": 0.12124999612569809, "rewards/reward_commands_completeness/std": 0.04287578910589218, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.02272113785147667, "rewards/reward_diversity/mean": 0.15671683847904205, "rewards/reward_diversity/std": 0.047205571085214615, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.1368750035762787, "rewards/reward_solution_effectiveness/std": 0.07345917075872421, "sampling/importance_sampling_ratio/max": 0.6195912957191467, "sampling/importance_sampling_ratio/mean": 0.0985308438539505, "sampling/importance_sampling_ratio/min": 1.3288382578445512e-16, "sampling/sampling_logp_difference/max": 19.24970245361328, "sampling/sampling_logp_difference/mean": 0.0295255184173584, "step": 277, "step_time": 116.94342827424407 }, { "clip_ratio/high_max": 0.002873317280318588, "clip_ratio/high_mean": 0.002873317280318588, "clip_ratio/low_mean": 0.0010604984490782954, "clip_ratio/low_min": 0.0010604984490782954, "clip_ratio/region_mean": 0.003933815751224756, "completions/clipped_ratio": 0.0, "completions/max_length": 1290.0, "completions/max_terminated_length": 1290.0, "completions/mean_length": 803.5, "completions/mean_terminated_length": 803.5, "completions/min_length": 619.0, "completions/min_terminated_length": 619.0, "entropy": 0.08796644303947687, "epoch": 5.673469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.038041047751903534, "learning_rate": 4.346938775510205e-06, "loss": -0.0355, "num_tokens": 15193370.0, "reward": 0.7794110178947449, "reward_std": 0.09976033121347427, "rewards/reward_commands_completeness/mean": 0.1550000011920929, "rewards/reward_commands_completeness/std": 0.0268328208476305, "rewards/reward_commands_correctness/mean": 0.07312500476837158, "rewards/reward_commands_correctness/std": 0.01662077195942402, "rewards/reward_diversity/mean": 0.17378602921962738, "rewards/reward_diversity/std": 0.016004573553800583, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.015041609294712543, "rewards/reward_solution_effectiveness/mean": 0.19312499463558197, "rewards/reward_solution_effectiveness/std": 0.06569817662239075, "sampling/importance_sampling_ratio/max": 0.6010211110115051, "sampling/importance_sampling_ratio/mean": 0.0861167311668396, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.25512170791626, "sampling/sampling_logp_difference/mean": 0.025325912982225418, "step": 278, "step_time": 116.31138964928687 }, { "clip_ratio/high_max": 0.002228347701020539, "clip_ratio/high_mean": 0.002228347701020539, "clip_ratio/low_mean": 0.0012784167774952948, "clip_ratio/low_min": 0.0012784167774952948, "clip_ratio/region_mean": 0.003506764449412003, "completions/clipped_ratio": 0.0, "completions/max_length": 891.0, "completions/max_terminated_length": 891.0, "completions/mean_length": 716.6875, "completions/mean_terminated_length": 716.6875, "completions/min_length": 585.0, "completions/min_terminated_length": 585.0, "entropy": 0.10541249718517065, "epoch": 5.6938775510204085, "frac_reward_zero_std": 0.0, "grad_norm": 0.04386361315846443, "learning_rate": 4.326530612244899e-06, "loss": 0.0234, "num_tokens": 15229161.0, "reward": 0.6704687476158142, "reward_std": 0.14664478600025177, "rewards/reward_commands_completeness/mean": 0.11874999850988388, "rewards/reward_commands_completeness/std": 0.0543905645608902, "rewards/reward_commands_correctness/mean": 0.06812499463558197, "rewards/reward_commands_correctness/std": 0.029713915660977364, "rewards/reward_diversity/mean": 0.17421874403953552, "rewards/reward_diversity/std": 0.04294965788722038, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07750000059604645, "rewards/reward_problem_validity/std": 0.02670830301940441, "rewards/reward_solution_effectiveness/mean": 0.1443750113248825, "rewards/reward_solution_effectiveness/std": 0.09500658512115479, "sampling/importance_sampling_ratio/max": 0.5171024799346924, "sampling/importance_sampling_ratio/mean": 0.14674785733222961, "sampling/importance_sampling_ratio/min": 0.00012436941324267536, "sampling/sampling_logp_difference/max": 3.418928384780884, "sampling/sampling_logp_difference/mean": 0.0249137245118618, "step": 279, "step_time": 125.45657096803188 }, { "clip_ratio/high_max": 0.0023913239128887653, "clip_ratio/high_mean": 0.0023913239128887653, "clip_ratio/low_mean": 0.0006997551208769437, "clip_ratio/low_min": 0.0006997551208769437, "clip_ratio/region_mean": 0.0030910790483176243, "completions/clipped_ratio": 0.0, "completions/max_length": 1611.0, "completions/max_terminated_length": 1611.0, "completions/mean_length": 797.6875, "completions/mean_terminated_length": 797.6875, "completions/min_length": 615.0, "completions/min_terminated_length": 615.0, "entropy": 0.09888253919780254, "epoch": 5.714285714285714, "frac_reward_zero_std": 0.0, "grad_norm": 0.04697160795331001, "learning_rate": 4.306122448979592e-06, "loss": 0.0916, "num_tokens": 15266360.0, "reward": 0.7530310750007629, "reward_std": 0.08059919625520706, "rewards/reward_commands_completeness/mean": 0.14250001311302185, "rewards/reward_commands_completeness/std": 0.02909754030406475, "rewards/reward_commands_correctness/mean": 0.07250000536441803, "rewards/reward_commands_correctness/std": 0.01949359104037285, "rewards/reward_diversity/mean": 0.1849060356616974, "rewards/reward_diversity/std": 0.014573507010936737, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.16687500476837158, "rewards/reward_solution_effectiveness/std": 0.06384551525115967, "sampling/importance_sampling_ratio/max": 0.6218666434288025, "sampling/importance_sampling_ratio/mean": 0.09538006782531738, "sampling/importance_sampling_ratio/min": 1.1556678458646275e-07, "sampling/sampling_logp_difference/max": 2.632368326187134, "sampling/sampling_logp_difference/mean": 0.02442891150712967, "step": 280, "step_time": 142.3636731300503 }, { "clip_ratio/high_max": 0.003843338963633869, "clip_ratio/high_mean": 0.003843338963633869, "clip_ratio/low_mean": 0.0008711048430996016, "clip_ratio/low_min": 0.0008711048430996016, "clip_ratio/region_mean": 0.004714443770353682, "completions/clipped_ratio": 0.0, "completions/max_length": 1104.0, "completions/max_terminated_length": 1104.0, "completions/mean_length": 788.3125, "completions/mean_terminated_length": 788.3125, "completions/min_length": 673.0, "completions/min_terminated_length": 673.0, "entropy": 0.08547927532345057, "epoch": 5.73469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.008777671493589878, "learning_rate": 4.2857142857142855e-06, "loss": -0.019, "num_tokens": 15301421.0, "reward": 0.7370237708091736, "reward_std": 0.15604156255722046, "rewards/reward_commands_completeness/mean": 0.14374999701976776, "rewards/reward_commands_completeness/std": 0.04395073279738426, "rewards/reward_commands_correctness/mean": 0.06937499344348907, "rewards/reward_commands_correctness/std": 0.02048373594880104, "rewards/reward_diversity/mean": 0.16264879703521729, "rewards/reward_diversity/std": 0.04111778363585472, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187499642372131, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.18562500178813934, "rewards/reward_solution_effectiveness/std": 0.0728440135717392, "sampling/importance_sampling_ratio/max": 0.24153295159339905, "sampling/importance_sampling_ratio/mean": 0.0530223622918129, "sampling/importance_sampling_ratio/min": 4.089429489484431e-11, "sampling/sampling_logp_difference/max": 15.869691848754883, "sampling/sampling_logp_difference/mean": 0.028181498870253563, "step": 281, "step_time": 124.78982143849134 }, { "clip_ratio/high_max": 0.0021205732264206745, "clip_ratio/high_mean": 0.0021205732264206745, "clip_ratio/low_mean": 0.000668916356517002, "clip_ratio/low_min": 0.000668916356517002, "clip_ratio/region_mean": 0.0027894895829376765, "completions/clipped_ratio": 0.0, "completions/max_length": 818.0, "completions/max_terminated_length": 818.0, "completions/mean_length": 729.25, "completions/mean_terminated_length": 729.25, "completions/min_length": 639.0, "completions/min_terminated_length": 639.0, "entropy": 0.08164293970912695, "epoch": 5.755102040816326, "frac_reward_zero_std": 0.0, "grad_norm": 0.018253561109304428, "learning_rate": 4.2653061224489804e-06, "loss": -0.0106, "num_tokens": 15345809.0, "reward": 0.7681249976158142, "reward_std": 0.08391989767551422, "rewards/reward_commands_completeness/mean": 0.14750000834465027, "rewards/reward_commands_completeness/std": 0.02620432712137699, "rewards/reward_commands_correctness/mean": 0.078125, "rewards/reward_commands_correctness/std": 0.016418995335698128, "rewards/reward_diversity/mean": 0.16749998927116394, "rewards/reward_diversity/std": 0.021204475313425064, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.18937499821186066, "rewards/reward_solution_effectiveness/std": 0.046111274510622025, "sampling/importance_sampling_ratio/max": 0.9251963496208191, "sampling/importance_sampling_ratio/mean": 0.1083516925573349, "sampling/importance_sampling_ratio/min": 3.7121774312254274e-06, "sampling/sampling_logp_difference/max": 3.5906786918640137, "sampling/sampling_logp_difference/mean": 0.02364514023065567, "step": 282, "step_time": 137.48828848265111 }, { "clip_ratio/high_max": 0.0025674808712210506, "clip_ratio/high_mean": 0.0025674808712210506, "clip_ratio/low_mean": 0.0023040613159537315, "clip_ratio/low_min": 0.0023040613159537315, "clip_ratio/region_mean": 0.004871542128967121, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 783.875, "completions/mean_terminated_length": 783.875, "completions/min_length": 651.0, "completions/min_terminated_length": 651.0, "entropy": 0.11176696605980396, "epoch": 5.775510204081632, "frac_reward_zero_std": 0.0, "grad_norm": 0.006295641418546438, "learning_rate": 4.244897959183674e-06, "loss": -0.009, "num_tokens": 15382371.0, "reward": 0.6978124976158142, "reward_std": 0.07495005428791046, "rewards/reward_commands_completeness/mean": 0.11624999344348907, "rewards/reward_commands_completeness/std": 0.04209117218852043, "rewards/reward_commands_correctness/mean": 0.06874999403953552, "rewards/reward_commands_correctness/std": 0.019958291202783585, "rewards/reward_diversity/mean": 0.18781250715255737, "rewards/reward_diversity/std": 0.009845451451838017, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08624999970197678, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.13875000178813934, "rewards/reward_solution_effectiveness/std": 0.07088723033666611, "sampling/importance_sampling_ratio/max": 0.22496670484542847, "sampling/importance_sampling_ratio/mean": 0.02777235396206379, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.845292329788208, "sampling/sampling_logp_difference/mean": 0.028711717575788498, "step": 283, "step_time": 124.86210567317903 }, { "clip_ratio/high_max": 0.0028202018147567287, "clip_ratio/high_mean": 0.0028202018147567287, "clip_ratio/low_mean": 0.0007376614885288291, "clip_ratio/low_min": 0.0007376614885288291, "clip_ratio/region_mean": 0.003557863281457685, "completions/clipped_ratio": 0.0, "completions/max_length": 1168.0, "completions/max_terminated_length": 1168.0, "completions/mean_length": 780.875, "completions/mean_terminated_length": 780.875, "completions/min_length": 634.0, "completions/min_terminated_length": 634.0, "entropy": 0.0824908809736371, "epoch": 5.795918367346939, "frac_reward_zero_std": 0.0, "grad_norm": 0.011844008229672909, "learning_rate": 4.224489795918368e-06, "loss": -0.0052, "num_tokens": 15418733.0, "reward": 0.6590961813926697, "reward_std": 0.12886559963226318, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.04412104934453964, "rewards/reward_commands_correctness/mean": 0.05812499672174454, "rewards/reward_commands_correctness/std": 0.02315707504749298, "rewards/reward_diversity/mean": 0.15909621119499207, "rewards/reward_diversity/std": 0.017026763409376144, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08249999582767487, "rewards/reward_problem_validity/std": 0.019832635298371315, "rewards/reward_solution_effectiveness/mean": 0.140625, "rewards/reward_solution_effectiveness/std": 0.07801442593336105, "sampling/importance_sampling_ratio/max": 0.394781231880188, "sampling/importance_sampling_ratio/mean": 0.06035537272691727, "sampling/importance_sampling_ratio/min": 2.045413838303034e-09, "sampling/sampling_logp_difference/max": 17.214763641357422, "sampling/sampling_logp_difference/mean": 0.02550773322582245, "step": 284, "step_time": 121.05760667845607 }, { "clip_ratio/high_max": 0.001704280570265837, "clip_ratio/high_mean": 0.001704280570265837, "clip_ratio/low_mean": 0.0015103338155313395, "clip_ratio/low_min": 0.0015103338155313395, "clip_ratio/region_mean": 0.0032146143494173884, "completions/clipped_ratio": 0.0, "completions/max_length": 860.0, "completions/max_terminated_length": 860.0, "completions/mean_length": 750.0625, "completions/mean_terminated_length": 750.0625, "completions/min_length": 637.0, "completions/min_terminated_length": 637.0, "entropy": 0.07927345670759678, "epoch": 5.816326530612245, "frac_reward_zero_std": 0.0, "grad_norm": 0.1368827223777771, "learning_rate": 4.204081632653061e-06, "loss": -0.1567, "num_tokens": 15455990.0, "reward": 0.6660181283950806, "reward_std": 0.08369417488574982, "rewards/reward_commands_completeness/mean": 0.11124999821186066, "rewards/reward_commands_completeness/std": 0.045588742941617966, "rewards/reward_commands_correctness/mean": 0.06937500834465027, "rewards/reward_commands_correctness/std": 0.024622147902846336, "rewards/reward_diversity/mean": 0.18289318680763245, "rewards/reward_diversity/std": 0.019330918788909912, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.021563859656453133, "rewards/reward_solution_effectiveness/mean": 0.1237500011920929, "rewards/reward_solution_effectiveness/std": 0.0708872377872467, "sampling/importance_sampling_ratio/max": 2.2673354148864746, "sampling/importance_sampling_ratio/mean": 0.16824790835380554, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.182188987731934, "sampling/sampling_logp_difference/mean": 0.024332966655492783, "step": 285, "step_time": 120.96794236451387 }, { "clip_ratio/high_max": 0.0023335931909969077, "clip_ratio/high_mean": 0.0023335931909969077, "clip_ratio/low_mean": 0.0022395712148863822, "clip_ratio/low_min": 0.0022395712148863822, "clip_ratio/region_mean": 0.004573164420435205, "completions/clipped_ratio": 0.0, "completions/max_length": 1200.0, "completions/max_terminated_length": 1200.0, "completions/mean_length": 865.0625, "completions/mean_terminated_length": 865.0625, "completions/min_length": 662.0, "completions/min_terminated_length": 662.0, "entropy": 0.09164929576218128, "epoch": 5.836734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.03445901721715927, "learning_rate": 4.183673469387755e-06, "loss": 0.0465, "num_tokens": 15492391.0, "reward": 0.7665252685546875, "reward_std": 0.06830492615699768, "rewards/reward_commands_completeness/mean": 0.14249999821186066, "rewards/reward_commands_completeness/std": 0.025166116654872894, "rewards/reward_commands_correctness/mean": 0.07375000417232513, "rewards/reward_commands_correctness/std": 0.01586400717496872, "rewards/reward_diversity/mean": 0.17840030789375305, "rewards/reward_diversity/std": 0.015009699389338493, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.1837500035762787, "rewards/reward_solution_effectiveness/std": 0.04759201779961586, "sampling/importance_sampling_ratio/max": 2.100966215133667, "sampling/importance_sampling_ratio/mean": 0.14897137880325317, "sampling/importance_sampling_ratio/min": 1.0738960298795064e-07, "sampling/sampling_logp_difference/max": 10.000273704528809, "sampling/sampling_logp_difference/mean": 0.02836819365620613, "step": 286, "step_time": 148.30142717249691 }, { "clip_ratio/high_max": 0.002483383475919254, "clip_ratio/high_mean": 0.002483383475919254, "clip_ratio/low_mean": 0.0008613962199888192, "clip_ratio/low_min": 0.0008613962199888192, "clip_ratio/region_mean": 0.0033447797322878614, "completions/clipped_ratio": 0.0, "completions/max_length": 1145.0, "completions/max_terminated_length": 1145.0, "completions/mean_length": 799.9375, "completions/mean_terminated_length": 799.9375, "completions/min_length": 602.0, "completions/min_terminated_length": 602.0, "entropy": 0.09115931950509548, "epoch": 5.857142857142857, "frac_reward_zero_std": 0.0, "grad_norm": 0.0012101808097213507, "learning_rate": 4.163265306122449e-06, "loss": -0.0029, "num_tokens": 15527650.0, "reward": 0.6093530654907227, "reward_std": 0.15214243531227112, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.043185651302337646, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.023655517026782036, "rewards/reward_diversity/mean": 0.15247809886932373, "rewards/reward_diversity/std": 0.04264295846223831, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07750000059604645, "rewards/reward_problem_validity/std": 0.018439089879393578, "rewards/reward_solution_effectiveness/mean": 0.11250000447034836, "rewards/reward_solution_effectiveness/std": 0.05639148876070976, "sampling/importance_sampling_ratio/max": 0.030643319711089134, "sampling/importance_sampling_ratio/mean": 0.008693946525454521, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2230868339538574, "sampling/sampling_logp_difference/mean": 0.02653392218053341, "step": 287, "step_time": 124.34191168099642 }, { "clip_ratio/high_max": 0.00268416698963847, "clip_ratio/high_mean": 0.00268416698963847, "clip_ratio/low_mean": 0.0007346809579757974, "clip_ratio/low_min": 0.0007346809579757974, "clip_ratio/region_mean": 0.003418847976718098, "completions/clipped_ratio": 0.0, "completions/max_length": 872.0, "completions/max_terminated_length": 872.0, "completions/mean_length": 757.5, "completions/mean_terminated_length": 757.5, "completions/min_length": 662.0, "completions/min_terminated_length": 662.0, "entropy": 0.08925999328494072, "epoch": 5.877551020408164, "frac_reward_zero_std": 0.0, "grad_norm": 0.01642237789928913, "learning_rate": 4.1428571428571435e-06, "loss": 0.0093, "num_tokens": 15592842.0, "reward": 0.6826961040496826, "reward_std": 0.1658765971660614, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.04412104934453964, "rewards/reward_commands_correctness/mean": 0.06812500208616257, "rewards/reward_commands_correctness/std": 0.02257395349442959, "rewards/reward_diversity/mean": 0.16707107424736023, "rewards/reward_diversity/std": 0.0434051975607872, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08062499761581421, "rewards/reward_problem_validity/std": 0.019482901319861412, "rewards/reward_solution_effectiveness/mean": 0.14812499284744263, "rewards/reward_solution_effectiveness/std": 0.07704706490039825, "sampling/importance_sampling_ratio/max": 0.8268222808837891, "sampling/importance_sampling_ratio/mean": 0.10968495905399323, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9961256980895996, "sampling/sampling_logp_difference/mean": 0.02593153715133667, "step": 288, "step_time": 205.15486234240234 }, { "clip_ratio/high_max": 0.0026417120825499296, "clip_ratio/high_mean": 0.0026417120825499296, "clip_ratio/low_mean": 0.0014281227486208081, "clip_ratio/low_min": 0.0014281227486208081, "clip_ratio/region_mean": 0.004069834802066907, "completions/clipped_ratio": 0.0, "completions/max_length": 3297.0, "completions/max_terminated_length": 3297.0, "completions/mean_length": 880.9375, "completions/mean_terminated_length": 880.9375, "completions/min_length": 625.0, "completions/min_terminated_length": 625.0, "entropy": 0.09426860231906176, "epoch": 5.8979591836734695, "frac_reward_zero_std": 0.0, "grad_norm": 0.00695775356143713, "learning_rate": 4.122448979591837e-06, "loss": -0.0013, "num_tokens": 15629485.0, "reward": 0.5968167781829834, "reward_std": 0.21309161186218262, "rewards/reward_commands_completeness/mean": 0.10625000298023224, "rewards/reward_commands_completeness/std": 0.05451299995183945, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.02920473739504814, "rewards/reward_diversity/mean": 0.1411917805671692, "rewards/reward_diversity/std": 0.05128110945224762, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07250000536441803, "rewards/reward_problem_validity/std": 0.03130495548248291, "rewards/reward_solution_effectiveness/mean": 0.13124999403953552, "rewards/reward_solution_effectiveness/std": 0.08476438373327255, "sampling/importance_sampling_ratio/max": 0.5914623737335205, "sampling/importance_sampling_ratio/mean": 0.04935751482844353, "sampling/importance_sampling_ratio/min": 1.3234553453145281e-09, "sampling/sampling_logp_difference/max": 4.216054439544678, "sampling/sampling_logp_difference/mean": 0.0251995250582695, "step": 289, "step_time": 231.7502818685025 }, { "clip_ratio/high_max": 0.0021811585465911776, "clip_ratio/high_mean": 0.0021811585465911776, "clip_ratio/low_mean": 0.0014291315674199723, "clip_ratio/low_min": 0.0014291315674199723, "clip_ratio/region_mean": 0.0036102901358390227, "completions/clipped_ratio": 0.0, "completions/max_length": 1011.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 796.3125, "completions/mean_terminated_length": 796.3125, "completions/min_length": 679.0, "completions/min_terminated_length": 679.0, "entropy": 0.07849346986040473, "epoch": 5.918367346938775, "frac_reward_zero_std": 0.0, "grad_norm": 0.024520106613636017, "learning_rate": 4.102040816326531e-06, "loss": -0.0326, "num_tokens": 15671474.0, "reward": 0.6469321846961975, "reward_std": 0.15011322498321533, "rewards/reward_commands_completeness/mean": 0.11249999701976776, "rewards/reward_commands_completeness/std": 0.04250490665435791, "rewards/reward_commands_correctness/mean": 0.07187500596046448, "rewards/reward_commands_correctness/std": 0.019737867638468742, "rewards/reward_diversity/mean": 0.15818218886852264, "rewards/reward_diversity/std": 0.04329465329647064, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.019137661904096603, "rewards/reward_solution_effectiveness/mean": 0.13124999403953552, "rewards/reward_solution_effectiveness/std": 0.06375735253095627, "sampling/importance_sampling_ratio/max": 1.2267487049102783, "sampling/importance_sampling_ratio/mean": 0.10972020775079727, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 16.360794067382812, "sampling/sampling_logp_difference/mean": 0.0255318284034729, "step": 290, "step_time": 132.8022469393909 }, { "clip_ratio/high_max": 0.001921892020618543, "clip_ratio/high_mean": 0.001921892020618543, "clip_ratio/low_mean": 0.0019517305918270722, "clip_ratio/low_min": 0.0019517305918270722, "clip_ratio/region_mean": 0.0038736225978937, "completions/clipped_ratio": 0.0, "completions/max_length": 900.0, "completions/max_terminated_length": 900.0, "completions/mean_length": 744.875, "completions/mean_terminated_length": 744.875, "completions/min_length": 606.0, "completions/min_terminated_length": 606.0, "entropy": 0.08467387221753597, "epoch": 5.938775510204081, "frac_reward_zero_std": 0.0, "grad_norm": 0.07700244337320328, "learning_rate": 4.081632653061225e-06, "loss": 0.1526, "num_tokens": 15709036.0, "reward": 0.7096893191337585, "reward_std": 0.0899796411395073, "rewards/reward_commands_completeness/mean": 0.13124999403953552, "rewards/reward_commands_completeness/std": 0.02825479581952095, "rewards/reward_commands_correctness/mean": 0.06624999642372131, "rewards/reward_commands_correctness/std": 0.017841897904872894, "rewards/reward_diversity/mean": 0.17718932032585144, "rewards/reward_diversity/std": 0.019028495997190475, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.005163980647921562, "rewards/reward_solution_effectiveness/mean": 0.15000000596046448, "rewards/reward_solution_effectiveness/std": 0.06292853504419327, "sampling/importance_sampling_ratio/max": 1.7862787246704102, "sampling/importance_sampling_ratio/mean": 0.285187304019928, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 17.3452091217041, "sampling/sampling_logp_difference/mean": 0.027798928320407867, "step": 291, "step_time": 120.6098185107112 }, { "clip_ratio/high_max": 0.0029213777161203325, "clip_ratio/high_mean": 0.0029213777161203325, "clip_ratio/low_mean": 0.0013578184589277953, "clip_ratio/low_min": 0.0013578184589277953, "clip_ratio/region_mean": 0.004279196145944297, "completions/clipped_ratio": 0.0, "completions/max_length": 900.0, "completions/max_terminated_length": 900.0, "completions/mean_length": 721.625, "completions/mean_terminated_length": 721.625, "completions/min_length": 600.0, "completions/min_terminated_length": 600.0, "entropy": 0.08679882902652025, "epoch": 5.959183673469388, "frac_reward_zero_std": 0.0, "grad_norm": 0.04620588198304176, "learning_rate": 4.061224489795918e-06, "loss": 0.0496, "num_tokens": 15743138.0, "reward": 0.7319270372390747, "reward_std": 0.12030582129955292, "rewards/reward_commands_completeness/mean": 0.13374999165534973, "rewards/reward_commands_completeness/std": 0.03981206193566322, "rewards/reward_commands_correctness/mean": 0.07437500357627869, "rewards/reward_commands_correctness/std": 0.0218993928283453, "rewards/reward_diversity/mean": 0.175677090883255, "rewards/reward_diversity/std": 0.021662861108779907, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.16124999523162842, "rewards/reward_solution_effectiveness/std": 0.07256031781435013, "sampling/importance_sampling_ratio/max": 0.7239569425582886, "sampling/importance_sampling_ratio/mean": 0.11480000615119934, "sampling/importance_sampling_ratio/min": 0.00013560974912252277, "sampling/sampling_logp_difference/max": 2.982761859893799, "sampling/sampling_logp_difference/mean": 0.02553010918200016, "step": 292, "step_time": 106.56112929992378 }, { "clip_ratio/high_max": 0.003500145801808685, "clip_ratio/high_mean": 0.003500145801808685, "clip_ratio/low_mean": 0.002170472464058548, "clip_ratio/low_min": 0.002170472464058548, "clip_ratio/region_mean": 0.005670618265867233, "completions/clipped_ratio": 0.0, "completions/max_length": 844.0, "completions/max_terminated_length": 844.0, "completions/mean_length": 758.5625, "completions/mean_terminated_length": 758.5625, "completions/min_length": 667.0, "completions/min_terminated_length": 667.0, "entropy": 0.10257232282310724, "epoch": 5.979591836734694, "frac_reward_zero_std": 0.0, "grad_norm": 0.0035273507237434387, "learning_rate": 4.040816326530612e-06, "loss": -0.0022, "num_tokens": 15783351.0, "reward": 0.663072943687439, "reward_std": 0.1615298092365265, "rewards/reward_commands_completeness/mean": 0.11874999850988388, "rewards/reward_commands_completeness/std": 0.03827532008290291, "rewards/reward_commands_correctness/mean": 0.07125000655651093, "rewards/reward_commands_correctness/std": 0.024731896817684174, "rewards/reward_diversity/mean": 0.15307292342185974, "rewards/reward_diversity/std": 0.045667603611946106, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187499642372131, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.06592609733343124, "sampling/importance_sampling_ratio/max": 0.1692754179239273, "sampling/importance_sampling_ratio/mean": 0.016754046082496643, "sampling/importance_sampling_ratio/min": 9.51644985036637e-10, "sampling/sampling_logp_difference/max": 15.804229736328125, "sampling/sampling_logp_difference/mean": 0.03364326432347298, "step": 293, "step_time": 108.51412651129067 }, { "clip_ratio/high_max": 0.0023091488401405513, "clip_ratio/high_mean": 0.0023091488401405513, "clip_ratio/low_mean": 0.0006294623963185586, "clip_ratio/low_min": 0.0006294623963185586, "clip_ratio/region_mean": 0.0029386112219071947, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 760.4375, "completions/mean_terminated_length": 760.4375, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.08501783944666386, "epoch": 6.0, "frac_reward_zero_std": 0.0, "grad_norm": 0.01699146442115307, "learning_rate": 4.0204081632653065e-06, "loss": -0.0004, "num_tokens": 15824962.0, "reward": 0.5932812690734863, "reward_std": 0.19229859113693237, "rewards/reward_commands_completeness/mean": 0.09874999523162842, "rewards/reward_commands_completeness/std": 0.04349329695105553, "rewards/reward_commands_correctness/mean": 0.05937499925494194, "rewards/reward_commands_correctness/std": 0.02568235620856285, "rewards/reward_diversity/mean": 0.17265623807907104, "rewards/reward_diversity/std": 0.04388244077563286, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07374999672174454, "rewards/reward_problem_validity/std": 0.02526526153087616, "rewards/reward_solution_effectiveness/mean": 0.10125000029802322, "rewards/reward_solution_effectiveness/std": 0.06830080598592758, "sampling/importance_sampling_ratio/max": 0.34961754083633423, "sampling/importance_sampling_ratio/mean": 0.046117931604385376, "sampling/importance_sampling_ratio/min": 9.339972166344523e-06, "sampling/sampling_logp_difference/max": 3.451010227203369, "sampling/sampling_logp_difference/mean": 0.025298429653048515, "step": 294, "step_time": 114.24744214490056 }, { "clip_ratio/high_max": 0.0030290869617601857, "clip_ratio/high_mean": 0.0030290869617601857, "clip_ratio/low_mean": 0.000715633541403804, "clip_ratio/low_min": 0.000715633541403804, "clip_ratio/region_mean": 0.0037447205031639896, "completions/clipped_ratio": 0.0, "completions/max_length": 1032.0, "completions/max_terminated_length": 1032.0, "completions/mean_length": 704.875, "completions/mean_terminated_length": 704.875, "completions/min_length": 644.0, "completions/min_terminated_length": 644.0, "entropy": 0.09129185136407614, "epoch": 6.020408163265306, "frac_reward_zero_std": 0.0, "grad_norm": 0.02873816527426243, "learning_rate": 4.000000000000001e-06, "loss": -0.0059, "num_tokens": 15860028.0, "reward": 0.7527034878730774, "reward_std": 0.06603483110666275, "rewards/reward_commands_completeness/mean": 0.14500001072883606, "rewards/reward_commands_completeness/std": 0.01712697744369507, "rewards/reward_commands_correctness/mean": 0.07124999910593033, "rewards/reward_commands_correctness/std": 0.01927866041660309, "rewards/reward_diversity/mean": 0.17895351350307465, "rewards/reward_diversity/std": 0.01626928709447384, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.032345786690711975, "sampling/importance_sampling_ratio/max": 0.4684983789920807, "sampling/importance_sampling_ratio/mean": 0.08880814909934998, "sampling/importance_sampling_ratio/min": 4.027458705077591e-14, "sampling/sampling_logp_difference/max": 21.274789810180664, "sampling/sampling_logp_difference/mean": 0.03219754621386528, "step": 295, "step_time": 117.11119874194264 }, { "clip_ratio/high_max": 0.002171352731238585, "clip_ratio/high_mean": 0.002171352731238585, "clip_ratio/low_mean": 0.0014633332611992955, "clip_ratio/low_min": 0.0014633332611992955, "clip_ratio/region_mean": 0.003634685999713838, "completions/clipped_ratio": 0.0, "completions/max_length": 1059.0, "completions/max_terminated_length": 1059.0, "completions/mean_length": 820.5, "completions/mean_terminated_length": 820.5, "completions/min_length": 637.0, "completions/min_terminated_length": 637.0, "entropy": 0.08911711862310767, "epoch": 6.040816326530612, "frac_reward_zero_std": 0.0, "grad_norm": 0.03433939069509506, "learning_rate": 3.979591836734694e-06, "loss": 0.0818, "num_tokens": 15900080.0, "reward": 0.7467947602272034, "reward_std": 0.0636722594499588, "rewards/reward_commands_completeness/mean": 0.14124999940395355, "rewards/reward_commands_completeness/std": 0.02777889184653759, "rewards/reward_commands_correctness/mean": 0.07187500596046448, "rewards/reward_commands_correctness/std": 0.017969883978366852, "rewards/reward_diversity/mean": 0.17929475009441376, "rewards/reward_diversity/std": 0.01810099557042122, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.16687500476837158, "rewards/reward_solution_effectiveness/std": 0.05895973742008209, "sampling/importance_sampling_ratio/max": 0.7254757285118103, "sampling/importance_sampling_ratio/mean": 0.11846482753753662, "sampling/importance_sampling_ratio/min": 2.475650760780379e-12, "sampling/sampling_logp_difference/max": 17.99993896484375, "sampling/sampling_logp_difference/mean": 0.027509508654475212, "step": 296, "step_time": 142.55419136956334 }, { "clip_ratio/high_max": 0.0017172592342831194, "clip_ratio/high_mean": 0.0017172592342831194, "clip_ratio/low_mean": 0.002229212725069374, "clip_ratio/low_min": 0.002229212725069374, "clip_ratio/region_mean": 0.003946471988456324, "completions/clipped_ratio": 0.0, "completions/max_length": 951.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 784.75, "completions/mean_terminated_length": 784.75, "completions/min_length": 674.0, "completions/min_terminated_length": 674.0, "entropy": 0.09276481531560421, "epoch": 6.061224489795919, "frac_reward_zero_std": 0.0, "grad_norm": 0.010342270135879517, "learning_rate": 3.959183673469388e-06, "loss": 0.0022, "num_tokens": 15943096.0, "reward": 0.6140104532241821, "reward_std": 0.14929737150669098, "rewards/reward_commands_completeness/mean": 0.10499999672174454, "rewards/reward_commands_completeness/std": 0.0458984375, "rewards/reward_commands_correctness/mean": 0.06749999523162842, "rewards/reward_commands_correctness/std": 0.02620432712137699, "rewards/reward_diversity/mean": 0.160260409116745, "rewards/reward_diversity/std": 0.056572407484054565, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07562500238418579, "rewards/reward_problem_validity/std": 0.02606882154941559, "rewards/reward_solution_effectiveness/mean": 0.11812499910593033, "rewards/reward_solution_effectiveness/std": 0.07138802856206894, "sampling/importance_sampling_ratio/max": 0.2799050807952881, "sampling/importance_sampling_ratio/mean": 0.042277999222278595, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.7380857467651367, "sampling/sampling_logp_difference/mean": 0.025785015895962715, "step": 297, "step_time": 134.06831474229693 }, { "clip_ratio/high_max": 0.002504758886061609, "clip_ratio/high_mean": 0.002504758886061609, "clip_ratio/low_mean": 0.0010012495040427893, "clip_ratio/low_min": 0.0010012495040427893, "clip_ratio/region_mean": 0.003506008390104398, "completions/clipped_ratio": 0.0, "completions/max_length": 993.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 768.9375, "completions/mean_terminated_length": 768.9375, "completions/min_length": 663.0, "completions/min_terminated_length": 663.0, "entropy": 0.08968405332416296, "epoch": 6.081632653061225, "frac_reward_zero_std": 0.0, "grad_norm": 0.07555828988552094, "learning_rate": 3.938775510204082e-06, "loss": -0.1034, "num_tokens": 15979047.0, "reward": 0.7457433938980103, "reward_std": 0.1264413595199585, "rewards/reward_commands_completeness/mean": 0.14249999821186066, "rewards/reward_commands_completeness/std": 0.044347118586301804, "rewards/reward_commands_correctness/mean": 0.07750000059604645, "rewards/reward_commands_correctness/std": 0.02408318966627121, "rewards/reward_diversity/mean": 0.16511836647987366, "rewards/reward_diversity/std": 0.041962649673223495, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08312500268220901, "rewards/reward_problem_validity/std": 0.01990603096783161, "rewards/reward_solution_effectiveness/mean": 0.1837500035762787, "rewards/reward_solution_effectiveness/std": 0.06652067601680756, "sampling/importance_sampling_ratio/max": 1.5760263204574585, "sampling/importance_sampling_ratio/mean": 0.12226825207471848, "sampling/importance_sampling_ratio/min": 8.456517430754928e-11, "sampling/sampling_logp_difference/max": 15.931427001953125, "sampling/sampling_logp_difference/mean": 0.027453899383544922, "step": 298, "step_time": 113.5768043231219 }, { "clip_ratio/high_max": 0.0029672953533008695, "clip_ratio/high_mean": 0.0029672953533008695, "clip_ratio/low_mean": 0.0010446425803820603, "clip_ratio/low_min": 0.0010446425803820603, "clip_ratio/region_mean": 0.004011937940958887, "completions/clipped_ratio": 0.0, "completions/max_length": 1242.0, "completions/max_terminated_length": 1242.0, "completions/mean_length": 795.9375, "completions/mean_terminated_length": 795.9375, "completions/min_length": 616.0, "completions/min_terminated_length": 616.0, "entropy": 0.08354775700718164, "epoch": 6.1020408163265305, "frac_reward_zero_std": 0.0, "grad_norm": 0.12598608434200287, "learning_rate": 3.9183673469387755e-06, "loss": 0.0616, "num_tokens": 16018802.0, "reward": 0.6916099786758423, "reward_std": 0.08700156211853027, "rewards/reward_commands_completeness/mean": 0.1237499937415123, "rewards/reward_commands_completeness/std": 0.0344238318502903, "rewards/reward_commands_correctness/mean": 0.06875000149011612, "rewards/reward_commands_correctness/std": 0.016683325171470642, "rewards/reward_diversity/mean": 0.17348501086235046, "rewards/reward_diversity/std": 0.027688123285770416, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.13875001668930054, "rewards/reward_solution_effectiveness/std": 0.05463515222072601, "sampling/importance_sampling_ratio/max": 2.286534070968628, "sampling/importance_sampling_ratio/mean": 0.36756184697151184, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 14.624655723571777, "sampling/sampling_logp_difference/mean": 0.028381282463669777, "step": 299, "step_time": 130.04240343347192 }, { "clip_ratio/high_max": 0.0027184506179764867, "clip_ratio/high_mean": 0.0027184506179764867, "clip_ratio/low_mean": 0.0010993191608577035, "clip_ratio/low_min": 0.0010993191608577035, "clip_ratio/region_mean": 0.0038177697715582326, "completions/clipped_ratio": 0.0, "completions/max_length": 1030.0, "completions/max_terminated_length": 1030.0, "completions/mean_length": 790.3125, "completions/mean_terminated_length": 790.3125, "completions/min_length": 649.0, "completions/min_terminated_length": 649.0, "entropy": 0.08237655367702246, "epoch": 6.122448979591836, "frac_reward_zero_std": 0.0, "grad_norm": 0.010550763458013535, "learning_rate": 3.89795918367347e-06, "loss": -0.0109, "num_tokens": 16059535.0, "reward": 0.7025036811828613, "reward_std": 0.09239175915718079, "rewards/reward_commands_completeness/mean": 0.1287499964237213, "rewards/reward_commands_completeness/std": 0.045588746666908264, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.020976178348064423, "rewards/reward_diversity/mean": 0.16937866806983948, "rewards/reward_diversity/std": 0.014734528958797455, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.15937499701976776, "rewards/reward_solution_effectiveness/std": 0.07877975702285767, "sampling/importance_sampling_ratio/max": 1.8082634210586548, "sampling/importance_sampling_ratio/mean": 0.11910338699817657, "sampling/importance_sampling_ratio/min": 2.062981269103023e-14, "sampling/sampling_logp_difference/max": 20.991376876831055, "sampling/sampling_logp_difference/mean": 0.02985842525959015, "step": 300, "step_time": 132.85918582975864 }, { "clip_ratio/high_max": 0.002156238566385582, "clip_ratio/high_mean": 0.002156238566385582, "clip_ratio/low_mean": 0.0012182670616311952, "clip_ratio/low_min": 0.0012182670616311952, "clip_ratio/region_mean": 0.0033745056280167773, "completions/clipped_ratio": 0.0, "completions/max_length": 971.0, "completions/max_terminated_length": 971.0, "completions/mean_length": 768.25, "completions/mean_terminated_length": 768.25, "completions/min_length": 664.0, "completions/min_terminated_length": 664.0, "entropy": 0.07869952823966742, "epoch": 6.142857142857143, "frac_reward_zero_std": 0.0, "grad_norm": 0.019108200445771217, "learning_rate": 3.877551020408164e-06, "loss": -0.0245, "num_tokens": 16099135.0, "reward": 0.6885804533958435, "reward_std": 0.12865762412548065, "rewards/reward_commands_completeness/mean": 0.1287499964237213, "rewards/reward_commands_completeness/std": 0.03862210363149643, "rewards/reward_commands_correctness/mean": 0.06687500327825546, "rewards/reward_commands_correctness/std": 0.021203381940722466, "rewards/reward_diversity/mean": 0.16545547544956207, "rewards/reward_diversity/std": 0.04473461955785751, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.15187500417232513, "rewards/reward_solution_effectiveness/std": 0.06615323573350906, "sampling/importance_sampling_ratio/max": 0.35979804396629333, "sampling/importance_sampling_ratio/mean": 0.0617399699985981, "sampling/importance_sampling_ratio/min": 1.837215131672565e-05, "sampling/sampling_logp_difference/max": 4.122613906860352, "sampling/sampling_logp_difference/mean": 0.024624677374958992, "step": 301, "step_time": 129.11049245856702 }, { "clip_ratio/high_max": 0.0025880070970742963, "clip_ratio/high_mean": 0.0025880070970742963, "clip_ratio/low_mean": 0.0019645543143269606, "clip_ratio/low_min": 0.0019645543143269606, "clip_ratio/region_mean": 0.0045525613240897655, "completions/clipped_ratio": 0.0, "completions/max_length": 1083.0, "completions/max_terminated_length": 1083.0, "completions/mean_length": 807.9375, "completions/mean_terminated_length": 807.9375, "completions/min_length": 695.0, "completions/min_terminated_length": 695.0, "entropy": 0.09293995797634125, "epoch": 6.163265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.00752904312685132, "learning_rate": 3.857142857142858e-06, "loss": -0.0042, "num_tokens": 16134594.0, "reward": 0.6722757816314697, "reward_std": 0.11895851790904999, "rewards/reward_commands_completeness/mean": 0.11999999731779099, "rewards/reward_commands_completeness/std": 0.04258325323462486, "rewards/reward_commands_correctness/mean": 0.06937500089406967, "rewards/reward_commands_correctness/std": 0.017689453437924385, "rewards/reward_diversity/mean": 0.17602579295635223, "rewards/reward_diversity/std": 0.017664264887571335, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07999999821186066, "rewards/reward_problem_validity/std": 0.019321836531162262, "rewards/reward_solution_effectiveness/mean": 0.13312500715255737, "rewards/reward_solution_effectiveness/std": 0.06193746626377106, "sampling/importance_sampling_ratio/max": 0.1741161048412323, "sampling/importance_sampling_ratio/mean": 0.029684029519557953, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.191993713378906, "sampling/sampling_logp_difference/mean": 0.027474170550704002, "step": 302, "step_time": 119.55206113494933 }, { "clip_ratio/high_max": 0.003994434169726446, "clip_ratio/high_mean": 0.003994434169726446, "clip_ratio/low_mean": 0.0009360530384583399, "clip_ratio/low_min": 0.0009360530384583399, "clip_ratio/region_mean": 0.004930487251840532, "completions/clipped_ratio": 0.0, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 791.6875, "completions/mean_terminated_length": 791.6875, "completions/min_length": 676.0, "completions/min_terminated_length": 676.0, "entropy": 0.09140846319496632, "epoch": 6.183673469387755, "frac_reward_zero_std": 0.0, "grad_norm": 0.004159356001764536, "learning_rate": 3.836734693877551e-06, "loss": -0.0057, "num_tokens": 16173737.0, "reward": 0.7263981103897095, "reward_std": 0.17126032710075378, "rewards/reward_commands_completeness/mean": 0.13499999046325684, "rewards/reward_commands_completeness/std": 0.04647580534219742, "rewards/reward_commands_correctness/mean": 0.06687500327825546, "rewards/reward_commands_correctness/std": 0.02151549980044365, "rewards/reward_diversity/mean": 0.17764806747436523, "rewards/reward_diversity/std": 0.01595315895974636, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08249999582767487, "rewards/reward_problem_validity/std": 0.019832635298371315, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.08250000327825546, "sampling/importance_sampling_ratio/max": 0.09254142642021179, "sampling/importance_sampling_ratio/mean": 0.009474297054111958, "sampling/importance_sampling_ratio/min": 2.7164990742956086e-11, "sampling/sampling_logp_difference/max": 19.65082359313965, "sampling/sampling_logp_difference/mean": 0.03304624184966087, "step": 303, "step_time": 141.98457679711282 }, { "clip_ratio/high_max": 0.001433753946912475, "clip_ratio/high_mean": 0.001433753946912475, "clip_ratio/low_mean": 0.001909572005388327, "clip_ratio/low_min": 0.001909572005388327, "clip_ratio/region_mean": 0.0033433259814046323, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 936.0, "completions/mean_length": 974.5, "completions/mean_terminated_length": 766.4000244140625, "completions/min_length": 655.0, "completions/min_terminated_length": 655.0, "entropy": 0.08927878877148032, "epoch": 6.204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.007966185919940472, "learning_rate": 3.816326530612245e-06, "loss": -0.0157, "num_tokens": 16236581.0, "reward": 0.7614068388938904, "reward_std": 0.11241017282009125, "rewards/reward_commands_completeness/mean": 0.14625000953674316, "rewards/reward_commands_completeness/std": 0.030740855261683464, "rewards/reward_commands_correctness/mean": 0.06937500089406967, "rewards/reward_commands_correctness/std": 0.018427787348628044, "rewards/reward_diversity/mean": 0.18203185498714447, "rewards/reward_diversity/std": 0.02365913800895214, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.17812499403953552, "rewards/reward_solution_effectiveness/std": 0.06431367993354797, "sampling/importance_sampling_ratio/max": 0.6590104103088379, "sampling/importance_sampling_ratio/mean": 0.06050703674554825, "sampling/importance_sampling_ratio/min": 1.9806511772912927e-06, "sampling/sampling_logp_difference/max": 7.224592685699463, "sampling/sampling_logp_difference/mean": 0.02310234308242798, "step": 304, "step_time": 390.3962239418179 }, { "clip_ratio/high_max": 0.003392825492483098, "clip_ratio/high_mean": 0.003392825492483098, "clip_ratio/low_mean": 0.000728405881091021, "clip_ratio/low_min": 0.000728405881091021, "clip_ratio/region_mean": 0.004121231388126034, "completions/clipped_ratio": 0.0, "completions/max_length": 903.0, "completions/max_terminated_length": 903.0, "completions/mean_length": 721.0, "completions/mean_terminated_length": 721.0, "completions/min_length": 620.0, "completions/min_terminated_length": 620.0, "entropy": 0.09286261675879359, "epoch": 6.224489795918367, "frac_reward_zero_std": 0.25, "grad_norm": 0.012501486577093601, "learning_rate": 3.795918367346939e-06, "loss": 0.0126, "num_tokens": 16272069.0, "reward": 0.4429057538509369, "reward_std": 0.2255782186985016, "rewards/reward_commands_completeness/mean": 0.07874999940395355, "rewards/reward_commands_completeness/std": 0.061738695949316025, "rewards/reward_commands_correctness/mean": 0.04249999672174454, "rewards/reward_commands_correctness/std": 0.03549648076295853, "rewards/reward_diversity/mean": 0.1160307228565216, "rewards/reward_diversity/std": 0.08021784573793411, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.051249999552965164, "rewards/reward_problem_validity/std": 0.03913651406764984, "rewards/reward_solution_effectiveness/mean": 0.09187500178813934, "rewards/reward_solution_effectiveness/std": 0.08010149747133255, "sampling/importance_sampling_ratio/max": 0.16789653897285461, "sampling/importance_sampling_ratio/mean": 0.02102392539381981, "sampling/importance_sampling_ratio/min": 1.747525146811313e-08, "sampling/sampling_logp_difference/max": 4.339418411254883, "sampling/sampling_logp_difference/mean": 0.03277067467570305, "step": 305, "step_time": 101.56205933354795 }, { "clip_ratio/high_max": 0.003417251951759681, "clip_ratio/high_mean": 0.003417251951759681, "clip_ratio/low_mean": 0.0012150758120696992, "clip_ratio/low_min": 0.0012150758120696992, "clip_ratio/region_mean": 0.004632327792933211, "completions/clipped_ratio": 0.0, "completions/max_length": 852.0, "completions/max_terminated_length": 852.0, "completions/mean_length": 734.8125, "completions/mean_terminated_length": 734.8125, "completions/min_length": 596.0, "completions/min_terminated_length": 596.0, "entropy": 0.09247680474072695, "epoch": 6.244897959183674, "frac_reward_zero_std": 0.0, "grad_norm": 0.03143087774515152, "learning_rate": 3.7755102040816327e-06, "loss": 0.0283, "num_tokens": 16306770.0, "reward": 0.7954687476158142, "reward_std": 0.08949722349643707, "rewards/reward_commands_completeness/mean": 0.1512499898672104, "rewards/reward_commands_completeness/std": 0.034229621291160583, "rewards/reward_commands_correctness/mean": 0.08312500268220901, "rewards/reward_commands_correctness/std": 0.017404502257704735, "rewards/reward_diversity/mean": 0.1704687476158142, "rewards/reward_diversity/std": 0.017554067075252533, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.20250001549720764, "rewards/reward_solution_effectiveness/std": 0.07056912034749985, "sampling/importance_sampling_ratio/max": 0.6547677516937256, "sampling/importance_sampling_ratio/mean": 0.07864490896463394, "sampling/importance_sampling_ratio/min": 2.4348380841913986e-13, "sampling/sampling_logp_difference/max": 21.874982833862305, "sampling/sampling_logp_difference/mean": 0.03312757983803749, "step": 306, "step_time": 139.56485385075212 }, { "clip_ratio/high_max": 0.003088694211328402, "clip_ratio/high_mean": 0.003088694211328402, "clip_ratio/low_mean": 0.0021411893030744977, "clip_ratio/low_min": 0.0021411893030744977, "clip_ratio/region_mean": 0.0052298835144029, "completions/clipped_ratio": 0.0, "completions/max_length": 1071.0, "completions/max_terminated_length": 1071.0, "completions/mean_length": 742.25, "completions/mean_terminated_length": 742.25, "completions/min_length": 680.0, "completions/min_terminated_length": 680.0, "entropy": 0.0845466535538435, "epoch": 6.26530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.04793208837509155, "learning_rate": 3.7551020408163268e-06, "loss": -0.043, "num_tokens": 16349450.0, "reward": 0.7639719247817993, "reward_std": 0.07724674791097641, "rewards/reward_commands_completeness/mean": 0.14874999225139618, "rewards/reward_commands_completeness/std": 0.030083222314715385, "rewards/reward_commands_correctness/mean": 0.07750000059604645, "rewards/reward_commands_correctness/std": 0.018073922023177147, "rewards/reward_diversity/mean": 0.1602219194173813, "rewards/reward_diversity/std": 0.027406461536884308, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.18937499821186066, "rewards/reward_solution_effectiveness/std": 0.05767365172505379, "sampling/importance_sampling_ratio/max": 0.5662639737129211, "sampling/importance_sampling_ratio/mean": 0.048586197197437286, "sampling/importance_sampling_ratio/min": 6.367787315894998e-18, "sampling/sampling_logp_difference/max": 23.498064041137695, "sampling/sampling_logp_difference/mean": 0.03930492699146271, "step": 307, "step_time": 146.8115339949727 }, { "clip_ratio/high_max": 0.002761075273156166, "clip_ratio/high_mean": 0.002761075273156166, "clip_ratio/low_mean": 0.0008582183945691213, "clip_ratio/low_min": 0.0008582183945691213, "clip_ratio/region_mean": 0.003619293653173372, "completions/clipped_ratio": 0.0, "completions/max_length": 950.0, "completions/max_terminated_length": 950.0, "completions/mean_length": 727.6875, "completions/mean_terminated_length": 727.6875, "completions/min_length": 644.0, "completions/min_terminated_length": 644.0, "entropy": 0.07121468940749764, "epoch": 6.285714285714286, "frac_reward_zero_std": 0.0, "grad_norm": 0.2657438814640045, "learning_rate": 3.7346938775510205e-06, "loss": -0.2247, "num_tokens": 16389373.0, "reward": 0.6680712699890137, "reward_std": 0.1596655398607254, "rewards/reward_commands_completeness/mean": 0.1237500011920929, "rewards/reward_commands_completeness/std": 0.05277310311794281, "rewards/reward_commands_correctness/mean": 0.07312499731779099, "rewards/reward_commands_correctness/std": 0.030269624665379524, "rewards/reward_diversity/mean": 0.14244621992111206, "rewards/reward_diversity/std": 0.05034371092915535, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.078125, "rewards/reward_problem_validity/std": 0.026887113228440285, "rewards/reward_solution_effectiveness/mean": 0.16312499344348907, "rewards/reward_solution_effectiveness/std": 0.08047515153884888, "sampling/importance_sampling_ratio/max": 2.635570764541626, "sampling/importance_sampling_ratio/mean": 0.20483121275901794, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.4375293254852295, "sampling/sampling_logp_difference/mean": 0.025095509365200996, "step": 308, "step_time": 130.69938282296062 }, { "clip_ratio/high_max": 0.002619468607008457, "clip_ratio/high_mean": 0.002619468607008457, "clip_ratio/low_mean": 0.0011937623203266412, "clip_ratio/low_min": 0.0011937623203266412, "clip_ratio/region_mean": 0.003813230956438929, "completions/clipped_ratio": 0.0, "completions/max_length": 858.0, "completions/max_terminated_length": 858.0, "completions/mean_length": 707.5, "completions/mean_terminated_length": 707.5, "completions/min_length": 615.0, "completions/min_terminated_length": 615.0, "entropy": 0.0872477050870657, "epoch": 6.3061224489795915, "frac_reward_zero_std": 0.0, "grad_norm": 0.013486603274941444, "learning_rate": 3.7142857142857146e-06, "loss": 0.0176, "num_tokens": 16427685.0, "reward": 0.7513730525970459, "reward_std": 0.07959241420030594, "rewards/reward_commands_completeness/mean": 0.13999998569488525, "rewards/reward_commands_completeness/std": 0.03577708825469017, "rewards/reward_commands_correctness/mean": 0.07999999821186066, "rewards/reward_commands_correctness/std": 0.01632993295788765, "rewards/reward_diversity/mean": 0.17387297749519348, "rewards/reward_diversity/std": 0.014951619319617748, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08875000476837158, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.16875000298023224, "rewards/reward_solution_effectiveness/std": 0.06652067601680756, "sampling/importance_sampling_ratio/max": 0.1507849246263504, "sampling/importance_sampling_ratio/mean": 0.02887118235230446, "sampling/importance_sampling_ratio/min": 5.841771053383127e-05, "sampling/sampling_logp_difference/max": 3.324812889099121, "sampling/sampling_logp_difference/mean": 0.02753637172281742, "step": 309, "step_time": 131.69631147570908 }, { "clip_ratio/high_max": 0.0020008546416647732, "clip_ratio/high_mean": 0.0020008546416647732, "clip_ratio/low_mean": 0.001589604449691251, "clip_ratio/low_min": 0.001589604449691251, "clip_ratio/region_mean": 0.0035904590622521937, "completions/clipped_ratio": 0.0, "completions/max_length": 987.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 746.75, "completions/mean_terminated_length": 746.75, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.08986748568713665, "epoch": 6.326530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.022789742797613144, "learning_rate": 3.6938775510204083e-06, "loss": -0.0059, "num_tokens": 16468717.0, "reward": 0.5972395539283752, "reward_std": 0.11241951584815979, "rewards/reward_commands_completeness/mean": 0.11499999463558197, "rewards/reward_commands_completeness/std": 0.05291502922773361, "rewards/reward_commands_correctness/mean": 0.05562499910593033, "rewards/reward_commands_correctness/std": 0.02682505175471306, "rewards/reward_diversity/mean": 0.1459895819425583, "rewards/reward_diversity/std": 0.06714221090078354, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07000000029802322, "rewards/reward_problem_validity/std": 0.030110908672213554, "rewards/reward_solution_effectiveness/mean": 0.12937501072883606, "rewards/reward_solution_effectiveness/std": 0.0690380334854126, "sampling/importance_sampling_ratio/max": 0.7799925208091736, "sampling/importance_sampling_ratio/mean": 0.10420291125774384, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.95705509185791, "sampling/sampling_logp_difference/mean": 0.030366992577910423, "step": 310, "step_time": 124.24932904914021 }, { "clip_ratio/high_max": 0.0016429067618446425, "clip_ratio/high_mean": 0.0016429067618446425, "clip_ratio/low_mean": 0.001530181267298758, "clip_ratio/low_min": 0.001530181267298758, "clip_ratio/region_mean": 0.0031730880145914853, "completions/clipped_ratio": 0.0, "completions/max_length": 1022.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 743.375, "completions/mean_terminated_length": 743.375, "completions/min_length": 629.0, "completions/min_terminated_length": 629.0, "entropy": 0.08826424181461334, "epoch": 6.346938775510204, "frac_reward_zero_std": 0.0, "grad_norm": 0.036156076937913895, "learning_rate": 3.6734693877551024e-06, "loss": 0.0423, "num_tokens": 16504103.0, "reward": 0.7595086097717285, "reward_std": 0.06077102571725845, "rewards/reward_commands_completeness/mean": 0.14625000953674316, "rewards/reward_commands_completeness/std": 0.03324154391884804, "rewards/reward_commands_correctness/mean": 0.07562500238418579, "rewards/reward_commands_correctness/std": 0.021592823788523674, "rewards/reward_diversity/mean": 0.17075860500335693, "rewards/reward_diversity/std": 0.017198894172906876, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.008164968341588974, "rewards/reward_solution_effectiveness/mean": 0.18187499046325684, "rewards/reward_solution_effectiveness/std": 0.06705408543348312, "sampling/importance_sampling_ratio/max": 0.5838368535041809, "sampling/importance_sampling_ratio/mean": 0.06547404825687408, "sampling/importance_sampling_ratio/min": 5.257227712718304e-06, "sampling/sampling_logp_difference/max": 4.14308500289917, "sampling/sampling_logp_difference/mean": 0.029200999066233635, "step": 311, "step_time": 110.61179056204855 }, { "clip_ratio/high_max": 0.0023485351557610556, "clip_ratio/high_mean": 0.0023485351557610556, "clip_ratio/low_mean": 0.0015354973584180698, "clip_ratio/low_min": 0.0015354973584180698, "clip_ratio/region_mean": 0.0038840325287310407, "completions/clipped_ratio": 0.0, "completions/max_length": 1038.0, "completions/max_terminated_length": 1038.0, "completions/mean_length": 778.1875, "completions/mean_terminated_length": 778.1875, "completions/min_length": 596.0, "completions/min_terminated_length": 596.0, "entropy": 0.09728790447115898, "epoch": 6.36734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.12651999294757843, "learning_rate": 3.653061224489796e-06, "loss": -0.2255, "num_tokens": 16535206.0, "reward": 0.676399827003479, "reward_std": 0.06367968767881393, "rewards/reward_commands_completeness/mean": 0.11749999970197678, "rewards/reward_commands_completeness/std": 0.02816617488861084, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.014591665007174015, "rewards/reward_diversity/mean": 0.18389983475208282, "rewards/reward_diversity/std": 0.016720900312066078, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08312499523162842, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.1274999976158142, "rewards/reward_solution_effectiveness/std": 0.05079369992017746, "sampling/importance_sampling_ratio/max": 1.8108580112457275, "sampling/importance_sampling_ratio/mean": 0.24352100491523743, "sampling/importance_sampling_ratio/min": 1.4724732934692786e-12, "sampling/sampling_logp_difference/max": 14.749913215637207, "sampling/sampling_logp_difference/mean": 0.031295280903577805, "step": 312, "step_time": 109.0486072357744 }, { "clip_ratio/high_max": 0.0024524027976440266, "clip_ratio/high_mean": 0.0024524027976440266, "clip_ratio/low_mean": 0.0009144643263425678, "clip_ratio/low_min": 0.0009144643263425678, "clip_ratio/region_mean": 0.003366867094882764, "completions/clipped_ratio": 0.0, "completions/max_length": 1249.0, "completions/max_terminated_length": 1249.0, "completions/mean_length": 794.25, "completions/mean_terminated_length": 794.25, "completions/min_length": 595.0, "completions/min_terminated_length": 595.0, "entropy": 0.08951525390148163, "epoch": 6.387755102040816, "frac_reward_zero_std": 0.0, "grad_norm": 0.001873848377726972, "learning_rate": 3.6326530612244903e-06, "loss": -0.0049, "num_tokens": 16575090.0, "reward": 0.6602290868759155, "reward_std": 0.09859871119260788, "rewards/reward_commands_completeness/mean": 0.11999999731779099, "rewards/reward_commands_completeness/std": 0.038643673062324524, "rewards/reward_commands_correctness/mean": 0.06687500327825546, "rewards/reward_commands_correctness/std": 0.02548692561686039, "rewards/reward_diversity/mean": 0.1596040427684784, "rewards/reward_diversity/std": 0.027697063982486725, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.021563859656453133, "rewards/reward_solution_effectiveness/mean": 0.13500000536441803, "rewards/reward_solution_effectiveness/std": 0.058991529047489166, "sampling/importance_sampling_ratio/max": 0.07525572925806046, "sampling/importance_sampling_ratio/mean": 0.010512731038033962, "sampling/importance_sampling_ratio/min": 1.880786650552968e-13, "sampling/sampling_logp_difference/max": 21.546106338500977, "sampling/sampling_logp_difference/mean": 0.033533044159412384, "step": 313, "step_time": 138.6647480931133 }, { "clip_ratio/high_max": 0.002008329043746926, "clip_ratio/high_mean": 0.002008329043746926, "clip_ratio/low_mean": 0.0009727680517244153, "clip_ratio/low_min": 0.0009727680517244153, "clip_ratio/region_mean": 0.002981097139127087, "completions/clipped_ratio": 0.0, "completions/max_length": 1022.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 780.375, "completions/mean_terminated_length": 780.375, "completions/min_length": 687.0, "completions/min_terminated_length": 687.0, "entropy": 0.09217225667089224, "epoch": 6.408163265306122, "frac_reward_zero_std": 0.0, "grad_norm": 0.012446639128029346, "learning_rate": 3.612244897959184e-06, "loss": -0.0215, "num_tokens": 16613984.0, "reward": 0.7058854103088379, "reward_std": 0.09954509884119034, "rewards/reward_commands_completeness/mean": 0.11999999731779099, "rewards/reward_commands_completeness/std": 0.03577708825469017, "rewards/reward_commands_correctness/mean": 0.08249999582767487, "rewards/reward_commands_correctness/std": 0.011254630982875824, "rewards/reward_diversity/mean": 0.1783854067325592, "rewards/reward_diversity/std": 0.022266212850809097, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08624999970197678, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.13875000178813934, "rewards/reward_solution_effectiveness/std": 0.06741661578416824, "sampling/importance_sampling_ratio/max": 0.2767435908317566, "sampling/importance_sampling_ratio/mean": 0.033368296921253204, "sampling/importance_sampling_ratio/min": 5.248631623544497e-06, "sampling/sampling_logp_difference/max": 4.013206481933594, "sampling/sampling_logp_difference/mean": 0.02701732888817787, "step": 314, "step_time": 132.73189228773117 }, { "clip_ratio/high_max": 0.002009788091527298, "clip_ratio/high_mean": 0.002009788091527298, "clip_ratio/low_mean": 0.002278823470987845, "clip_ratio/low_min": 0.002278823470987845, "clip_ratio/region_mean": 0.004288611526135355, "completions/clipped_ratio": 0.0, "completions/max_length": 844.0, "completions/max_terminated_length": 844.0, "completions/mean_length": 733.625, "completions/mean_terminated_length": 733.625, "completions/min_length": 677.0, "completions/min_terminated_length": 677.0, "entropy": 0.09619254060089588, "epoch": 6.428571428571429, "frac_reward_zero_std": 0.0, "grad_norm": 0.00882052630186081, "learning_rate": 3.5918367346938777e-06, "loss": -0.0089, "num_tokens": 16654002.0, "reward": 0.7134374976158142, "reward_std": 0.10086274147033691, "rewards/reward_commands_completeness/mean": 0.14374999701976776, "rewards/reward_commands_completeness/std": 0.02217356115579605, "rewards/reward_commands_correctness/mean": 0.07562500238418579, "rewards/reward_commands_correctness/std": 0.015903353691101074, "rewards/reward_diversity/mean": 0.1278124898672104, "rewards/reward_diversity/std": 0.13155758380889893, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.17812499403953552, "rewards/reward_solution_effectiveness/std": 0.04166833311319351, "sampling/importance_sampling_ratio/max": 0.9467154145240784, "sampling/importance_sampling_ratio/mean": 0.07544809579849243, "sampling/importance_sampling_ratio/min": 2.2155248555894014e-13, "sampling/sampling_logp_difference/max": 21.649778366088867, "sampling/sampling_logp_difference/mean": 0.0325380340218544, "step": 315, "step_time": 141.57630347460508 }, { "clip_ratio/high_max": 0.002350327806198038, "clip_ratio/high_mean": 0.002350327806198038, "clip_ratio/low_mean": 0.0008150108551490121, "clip_ratio/low_min": 0.0008150108551490121, "clip_ratio/region_mean": 0.0031653386540710926, "completions/clipped_ratio": 0.0, "completions/max_length": 952.0, "completions/max_terminated_length": 952.0, "completions/mean_length": 769.5625, "completions/mean_terminated_length": 769.5625, "completions/min_length": 638.0, "completions/min_terminated_length": 638.0, "entropy": 0.08429768867790699, "epoch": 6.448979591836735, "frac_reward_zero_std": 0.0, "grad_norm": 0.03347884118556976, "learning_rate": 3.5714285714285718e-06, "loss": -0.0493, "num_tokens": 16691291.0, "reward": 0.7541840076446533, "reward_std": 0.15785135328769684, "rewards/reward_commands_completeness/mean": 0.14875000715255737, "rewards/reward_commands_completeness/std": 0.042563680559396744, "rewards/reward_commands_correctness/mean": 0.07312499731779099, "rewards/reward_commands_correctness/std": 0.02272113785147667, "rewards/reward_diversity/mean": 0.16230902075767517, "rewards/reward_diversity/std": 0.0419846810400486, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.020000001415610313, "rewards/reward_solution_effectiveness/mean": 0.1912499964237213, "rewards/reward_solution_effectiveness/std": 0.06652067601680756, "sampling/importance_sampling_ratio/max": 0.723998486995697, "sampling/importance_sampling_ratio/mean": 0.11998014152050018, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.672760486602783, "sampling/sampling_logp_difference/mean": 0.023699305951595306, "step": 316, "step_time": 110.43848224915564 }, { "clip_ratio/high_max": 0.0016796658019302413, "clip_ratio/high_mean": 0.0016796658019302413, "clip_ratio/low_mean": 0.0018850996275432408, "clip_ratio/low_min": 0.0018850996275432408, "clip_ratio/region_mean": 0.003564765414921567, "completions/clipped_ratio": 0.0, "completions/max_length": 1393.0, "completions/max_terminated_length": 1393.0, "completions/mean_length": 820.875, "completions/mean_terminated_length": 820.875, "completions/min_length": 695.0, "completions/min_terminated_length": 695.0, "entropy": 0.09228093549609184, "epoch": 6.469387755102041, "frac_reward_zero_std": 0.0, "grad_norm": 0.005805172957479954, "learning_rate": 3.5510204081632655e-06, "loss": 0.0001, "num_tokens": 16729337.0, "reward": 0.6434383392333984, "reward_std": 0.11949844658374786, "rewards/reward_commands_completeness/mean": 0.10999999940395355, "rewards/reward_commands_completeness/std": 0.031832896173000336, "rewards/reward_commands_correctness/mean": 0.06625000387430191, "rewards/reward_commands_correctness/std": 0.023345237597823143, "rewards/reward_diversity/mean": 0.17593830823898315, "rewards/reward_diversity/std": 0.015253630466759205, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.019137661904096603, "rewards/reward_solution_effectiveness/mean": 0.11812500655651093, "rewards/reward_solution_effectiveness/std": 0.04833477362990379, "sampling/importance_sampling_ratio/max": 0.5914073586463928, "sampling/importance_sampling_ratio/mean": 0.04419276490807533, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.248839378356934, "sampling/sampling_logp_difference/mean": 0.02804354764521122, "step": 317, "step_time": 130.28399452008307 }, { "clip_ratio/high_max": 0.00251568628300447, "clip_ratio/high_mean": 0.00251568628300447, "clip_ratio/low_mean": 0.0019010494725080207, "clip_ratio/low_min": 0.0019010494725080207, "clip_ratio/region_mean": 0.004416735770064406, "completions/clipped_ratio": 0.0, "completions/max_length": 951.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 759.1875, "completions/mean_terminated_length": 759.1875, "completions/min_length": 631.0, "completions/min_terminated_length": 631.0, "entropy": 0.08235991187393665, "epoch": 6.489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.10067744553089142, "learning_rate": 3.5306122448979596e-06, "loss": 0.1313, "num_tokens": 16780036.0, "reward": 0.7527678608894348, "reward_std": 0.058348044753074646, "rewards/reward_commands_completeness/mean": 0.14374999701976776, "rewards/reward_commands_completeness/std": 0.02941088378429413, "rewards/reward_commands_correctness/mean": 0.07874999940395355, "rewards/reward_commands_correctness/std": 0.01408308744430542, "rewards/reward_diversity/mean": 0.16339285671710968, "rewards/reward_diversity/std": 0.02427174523472786, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08875000476837158, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.17812499403953552, "rewards/reward_solution_effectiveness/std": 0.048334769904613495, "sampling/importance_sampling_ratio/max": 2.2586543560028076, "sampling/importance_sampling_ratio/mean": 0.24150317907333374, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.865506649017334, "sampling/sampling_logp_difference/mean": 0.026693478226661682, "step": 318, "step_time": 211.97283915057778 }, { "clip_ratio/high_max": 0.002364387211855501, "clip_ratio/high_mean": 0.002364387211855501, "clip_ratio/low_mean": 0.0011886313659488223, "clip_ratio/low_min": 0.0011886313659488223, "clip_ratio/region_mean": 0.003553018585080281, "completions/clipped_ratio": 0.0, "completions/max_length": 880.0, "completions/max_terminated_length": 880.0, "completions/mean_length": 740.8125, "completions/mean_terminated_length": 740.8125, "completions/min_length": 653.0, "completions/min_terminated_length": 653.0, "entropy": 0.07858632039278746, "epoch": 6.510204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.05391624942421913, "learning_rate": 3.5102040816326533e-06, "loss": 0.0466, "num_tokens": 16820061.0, "reward": 0.6710764169692993, "reward_std": 0.16461634635925293, "rewards/reward_commands_completeness/mean": 0.12999999523162842, "rewards/reward_commands_completeness/std": 0.0623965859413147, "rewards/reward_commands_correctness/mean": 0.06437499821186066, "rewards/reward_commands_correctness/std": 0.030761178582906723, "rewards/reward_diversity/mean": 0.14857639372348785, "rewards/reward_diversity/std": 0.06460961699485779, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07250000536441803, "rewards/reward_problem_validity/std": 0.03130495548248291, "rewards/reward_solution_effectiveness/mean": 0.1743749976158142, "rewards/reward_solution_effectiveness/std": 0.09244593232870102, "sampling/importance_sampling_ratio/max": 1.2849156856536865, "sampling/importance_sampling_ratio/mean": 0.19597849249839783, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0542187690734863, "sampling/sampling_logp_difference/mean": 0.025614986196160316, "step": 319, "step_time": 122.34517677873373 }, { "clip_ratio/high_max": 0.001400616514729336, "clip_ratio/high_mean": 0.001400616514729336, "clip_ratio/low_mean": 0.0012936740022269078, "clip_ratio/low_min": 0.0012936740022269078, "clip_ratio/region_mean": 0.0026942905242322013, "completions/clipped_ratio": 0.0, "completions/max_length": 997.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 774.375, "completions/mean_terminated_length": 774.375, "completions/min_length": 509.0, "completions/min_terminated_length": 509.0, "entropy": 0.08470458211377263, "epoch": 6.530612244897959, "frac_reward_zero_std": 0.0, "grad_norm": 0.03502129018306732, "learning_rate": 3.4897959183673474e-06, "loss": -0.0962, "num_tokens": 16857943.0, "reward": 0.6248121857643127, "reward_std": 0.26267844438552856, "rewards/reward_commands_completeness/mean": 0.11625000089406967, "rewards/reward_commands_completeness/std": 0.059874869883060455, "rewards/reward_commands_correctness/mean": 0.061250001192092896, "rewards/reward_commands_correctness/std": 0.03180670738220215, "rewards/reward_diversity/mean": 0.14856219291687012, "rewards/reward_diversity/std": 0.05927067622542381, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07124999910593033, "rewards/reward_problem_validity/std": 0.030740855261683464, "rewards/reward_solution_effectiveness/mean": 0.14625000953674316, "rewards/reward_solution_effectiveness/std": 0.0889100655913353, "sampling/importance_sampling_ratio/max": 1.3404428958892822, "sampling/importance_sampling_ratio/mean": 0.13598597049713135, "sampling/importance_sampling_ratio/min": 1.183469294119277e-06, "sampling/sampling_logp_difference/max": 2.801990032196045, "sampling/sampling_logp_difference/mean": 0.026088902726769447, "step": 320, "step_time": 124.11157717742026 }, { "clip_ratio/high_max": 0.0020053765329066664, "clip_ratio/high_mean": 0.0020053765329066664, "clip_ratio/low_mean": 0.001556742296088487, "clip_ratio/low_min": 0.001556742296088487, "clip_ratio/region_mean": 0.0035621188289951533, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 747.5, "completions/mean_terminated_length": 747.5, "completions/min_length": 635.0, "completions/min_terminated_length": 635.0, "entropy": 0.08668118435889482, "epoch": 6.551020408163265, "frac_reward_zero_std": 0.0, "grad_norm": 0.015077825635671616, "learning_rate": 3.469387755102041e-06, "loss": 0.0232, "num_tokens": 16899207.0, "reward": 0.6708333492279053, "reward_std": 0.15544390678405762, "rewards/reward_commands_completeness/mean": 0.11874999850988388, "rewards/reward_commands_completeness/std": 0.04978286102414131, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.026331225410103798, "rewards/reward_diversity/mean": 0.16583332419395447, "rewards/reward_diversity/std": 0.05836666002869606, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.02651257812976837, "rewards/reward_solution_effectiveness/mean": 0.15187500417232513, "rewards/reward_solution_effectiveness/std": 0.07386643439531326, "sampling/importance_sampling_ratio/max": 0.3351738154888153, "sampling/importance_sampling_ratio/mean": 0.0502651110291481, "sampling/importance_sampling_ratio/min": 1.4824754543951713e-05, "sampling/sampling_logp_difference/max": 3.284660577774048, "sampling/sampling_logp_difference/mean": 0.025973422452807426, "step": 321, "step_time": 121.29411921650171 }, { "clip_ratio/high_max": 0.0024866035964805633, "clip_ratio/high_mean": 0.0024866035964805633, "clip_ratio/low_mean": 0.0007374612396233715, "clip_ratio/low_min": 0.0007374612396233715, "clip_ratio/region_mean": 0.0032240648215520196, "completions/clipped_ratio": 0.0, "completions/max_length": 833.0, "completions/max_terminated_length": 833.0, "completions/mean_length": 748.0, "completions/mean_terminated_length": 748.0, "completions/min_length": 688.0, "completions/min_terminated_length": 688.0, "entropy": 0.07992482278496027, "epoch": 6.571428571428571, "frac_reward_zero_std": 0.0, "grad_norm": 0.029519330710172653, "learning_rate": 3.4489795918367353e-06, "loss": 0.0472, "num_tokens": 16941223.0, "reward": 0.6935937404632568, "reward_std": 0.1647789180278778, "rewards/reward_commands_completeness/mean": 0.13375000655651093, "rewards/reward_commands_completeness/std": 0.04485160857439041, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.02394437976181507, "rewards/reward_diversity/mean": 0.14796875417232513, "rewards/reward_diversity/std": 0.04542797431349754, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08125000447034836, "rewards/reward_problem_validity/std": 0.020615531131625175, "rewards/reward_solution_effectiveness/mean": 0.16687500476837158, "rewards/reward_solution_effectiveness/std": 0.07427146285772324, "sampling/importance_sampling_ratio/max": 0.44077518582344055, "sampling/importance_sampling_ratio/mean": 0.07290518283843994, "sampling/importance_sampling_ratio/min": 0.00014565196761395782, "sampling/sampling_logp_difference/max": 3.4240081310272217, "sampling/sampling_logp_difference/mean": 0.02463972195982933, "step": 322, "step_time": 118.19142836332321 }, { "clip_ratio/high_max": 0.003310557469376363, "clip_ratio/high_mean": 0.003310557469376363, "clip_ratio/low_mean": 0.0013124596007401124, "clip_ratio/low_min": 0.0013124596007401124, "clip_ratio/region_mean": 0.004623017041012645, "completions/clipped_ratio": 0.0, "completions/max_length": 975.0, "completions/max_terminated_length": 975.0, "completions/mean_length": 786.8125, "completions/mean_terminated_length": 786.8125, "completions/min_length": 661.0, "completions/min_terminated_length": 661.0, "entropy": 0.08788464870303869, "epoch": 6.591836734693878, "frac_reward_zero_std": 0.0, "grad_norm": 0.07723461091518402, "learning_rate": 3.428571428571429e-06, "loss": -0.1099, "num_tokens": 16976020.0, "reward": 0.7421278953552246, "reward_std": 0.0680057629942894, "rewards/reward_commands_completeness/mean": 0.14374999701976776, "rewards/reward_commands_completeness/std": 0.027537856251001358, "rewards/reward_commands_correctness/mean": 0.07499999552965164, "rewards/reward_commands_correctness/std": 0.018257420510053635, "rewards/reward_diversity/mean": 0.16587789356708527, "rewards/reward_diversity/std": 0.022552773356437683, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.06071449816226959, "sampling/importance_sampling_ratio/max": 2.657968282699585, "sampling/importance_sampling_ratio/mean": 0.3790907859802246, "sampling/importance_sampling_ratio/min": 1.1139223224376771e-11, "sampling/sampling_logp_difference/max": 19.311424255371094, "sampling/sampling_logp_difference/mean": 0.027670659124851227, "step": 323, "step_time": 117.19923908822238 }, { "clip_ratio/high_max": 0.0028709877806250006, "clip_ratio/high_mean": 0.0028709877806250006, "clip_ratio/low_mean": 0.0011626680643530563, "clip_ratio/low_min": 0.0011626680643530563, "clip_ratio/region_mean": 0.004033655830426142, "completions/clipped_ratio": 0.0, "completions/max_length": 849.0, "completions/max_terminated_length": 849.0, "completions/mean_length": 726.875, "completions/mean_terminated_length": 726.875, "completions/min_length": 606.0, "completions/min_terminated_length": 606.0, "entropy": 0.08757084235548973, "epoch": 6.612244897959184, "frac_reward_zero_std": 0.0, "grad_norm": 0.06258460134267807, "learning_rate": 3.4081632653061227e-06, "loss": -0.0294, "num_tokens": 17017730.0, "reward": 0.6811047196388245, "reward_std": 0.10240120440721512, "rewards/reward_commands_completeness/mean": 0.12125000357627869, "rewards/reward_commands_completeness/std": 0.03461695462465286, "rewards/reward_commands_correctness/mean": 0.07124999910593033, "rewards/reward_commands_correctness/std": 0.015438050031661987, "rewards/reward_diversity/mean": 0.1742297112941742, "rewards/reward_diversity/std": 0.02398109622299671, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08312499523162842, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.13124999403953552, "rewards/reward_solution_effectiveness/std": 0.059874869883060455, "sampling/importance_sampling_ratio/max": 0.9177670478820801, "sampling/importance_sampling_ratio/mean": 0.149667426943779, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.4725828170776367, "sampling/sampling_logp_difference/mean": 0.0255071222782135, "step": 324, "step_time": 129.4197769407183 }, { "clip_ratio/high_max": 0.0032175736996578053, "clip_ratio/high_mean": 0.0032175736996578053, "clip_ratio/low_mean": 0.001111109129851684, "clip_ratio/low_min": 0.001111109129851684, "clip_ratio/region_mean": 0.004328682771301828, "completions/clipped_ratio": 0.0, "completions/max_length": 1065.0, "completions/max_terminated_length": 1065.0, "completions/mean_length": 790.6875, "completions/mean_terminated_length": 790.6875, "completions/min_length": 631.0, "completions/min_terminated_length": 631.0, "entropy": 0.09311671834439039, "epoch": 6.63265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.009375525638461113, "learning_rate": 3.3877551020408168e-06, "loss": 0.0072, "num_tokens": 17056653.0, "reward": 0.7599236369132996, "reward_std": 0.10054264962673187, "rewards/reward_commands_completeness/mean": 0.14124999940395355, "rewards/reward_commands_completeness/std": 0.028722815215587616, "rewards/reward_commands_correctness/mean": 0.0793749988079071, "rewards/reward_commands_correctness/std": 0.019482899457216263, "rewards/reward_diversity/mean": 0.1761736273765564, "rewards/reward_diversity/std": 0.019676480442285538, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08875000476837158, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.1743749976158142, "rewards/reward_solution_effectiveness/std": 0.06217917427420616, "sampling/importance_sampling_ratio/max": 0.21707379817962646, "sampling/importance_sampling_ratio/mean": 0.044876135885715485, "sampling/importance_sampling_ratio/min": 2.2207440508736909e-07, "sampling/sampling_logp_difference/max": 3.2050857543945312, "sampling/sampling_logp_difference/mean": 0.024940181523561478, "step": 325, "step_time": 135.68767029047012 }, { "clip_ratio/high_max": 0.003264076658524573, "clip_ratio/high_mean": 0.003264076658524573, "clip_ratio/low_mean": 0.002109103399561718, "clip_ratio/low_min": 0.002109103399561718, "clip_ratio/region_mean": 0.0053731800289824605, "completions/clipped_ratio": 0.0, "completions/max_length": 891.0, "completions/max_terminated_length": 891.0, "completions/mean_length": 698.25, "completions/mean_terminated_length": 698.25, "completions/min_length": 584.0, "completions/min_terminated_length": 584.0, "entropy": 0.09496072120964527, "epoch": 6.653061224489796, "frac_reward_zero_std": 0.0, "grad_norm": 0.023779861629009247, "learning_rate": 3.3673469387755105e-06, "loss": -0.0349, "num_tokens": 17091161.0, "reward": 0.7472671866416931, "reward_std": 0.08745525777339935, "rewards/reward_commands_completeness/mean": 0.13875000178813934, "rewards/reward_commands_completeness/std": 0.03304038196802139, "rewards/reward_commands_correctness/mean": 0.0768750011920929, "rewards/reward_commands_correctness/std": 0.013524670153856277, "rewards/reward_diversity/mean": 0.1735171526670456, "rewards/reward_diversity/std": 0.026755090802907944, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08937500417232513, "rewards/reward_problem_validity/std": 0.0025000013411045074, "rewards/reward_solution_effectiveness/mean": 0.16875000298023224, "rewards/reward_solution_effectiveness/std": 0.06469158083200455, "sampling/importance_sampling_ratio/max": 1.177617073059082, "sampling/importance_sampling_ratio/mean": 0.12954233586788177, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.674060821533203, "sampling/sampling_logp_difference/mean": 0.030692335218191147, "step": 326, "step_time": 106.03084196150303 }, { "clip_ratio/high_max": 0.0021045454050181434, "clip_ratio/high_mean": 0.0021045454050181434, "clip_ratio/low_mean": 0.001417143110302277, "clip_ratio/low_min": 0.001417143110302277, "clip_ratio/region_mean": 0.0035216885153204203, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 755.625, "completions/mean_terminated_length": 755.625, "completions/min_length": 663.0, "completions/min_terminated_length": 663.0, "entropy": 0.09215698670595884, "epoch": 6.673469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.08037713170051575, "learning_rate": 3.3469387755102046e-06, "loss": 0.0738, "num_tokens": 17130127.0, "reward": 0.7562775611877441, "reward_std": 0.06688149273395538, "rewards/reward_commands_completeness/mean": 0.14375001192092896, "rewards/reward_commands_completeness/std": 0.027537856251001358, "rewards/reward_commands_correctness/mean": 0.06874999403953552, "rewards/reward_commands_correctness/std": 0.016278821974992752, "rewards/reward_diversity/mean": 0.18315257132053375, "rewards/reward_diversity/std": 0.011758575215935707, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.1743749976158142, "rewards/reward_solution_effectiveness/std": 0.05819149315357208, "sampling/importance_sampling_ratio/max": 1.7029753923416138, "sampling/importance_sampling_ratio/mean": 0.37169408798217773, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 18.11311149597168, "sampling/sampling_logp_difference/mean": 0.02668137475848198, "step": 327, "step_time": 122.5242517683655 }, { "clip_ratio/high_max": 0.002420360076939687, "clip_ratio/high_mean": 0.002420360076939687, "clip_ratio/low_mean": 0.0005369859136408195, "clip_ratio/low_min": 0.0005369859136408195, "clip_ratio/region_mean": 0.0029573459978564642, "completions/clipped_ratio": 0.0, "completions/max_length": 956.0, "completions/max_terminated_length": 956.0, "completions/mean_length": 784.4375, "completions/mean_terminated_length": 784.4375, "completions/min_length": 709.0, "completions/min_terminated_length": 709.0, "entropy": 0.08795323688536882, "epoch": 6.6938775510204085, "frac_reward_zero_std": 0.0, "grad_norm": 0.057913847267627716, "learning_rate": 3.3265306122448983e-06, "loss": 0.0988, "num_tokens": 17164146.0, "reward": 0.7342647314071655, "reward_std": 0.08881337940692902, "rewards/reward_commands_completeness/mean": 0.13624998927116394, "rewards/reward_commands_completeness/std": 0.03518996387720108, "rewards/reward_commands_correctness/mean": 0.06937500089406967, "rewards/reward_commands_correctness/std": 0.011814538389444351, "rewards/reward_diversity/mean": 0.17051470279693604, "rewards/reward_diversity/std": 0.019101016223430634, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.0663795918226242, "sampling/importance_sampling_ratio/max": 1.1903088092803955, "sampling/importance_sampling_ratio/mean": 0.09571341425180435, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 18.6710205078125, "sampling/sampling_logp_difference/mean": 0.027384374290704727, "step": 328, "step_time": 117.10062811337411 }, { "clip_ratio/high_max": 0.002159796000341885, "clip_ratio/high_mean": 0.002159796000341885, "clip_ratio/low_mean": 0.001494265758083202, "clip_ratio/low_min": 0.001494265758083202, "clip_ratio/region_mean": 0.0036540617584250867, "completions/clipped_ratio": 0.0, "completions/max_length": 812.0, "completions/max_terminated_length": 812.0, "completions/mean_length": 722.4375, "completions/mean_terminated_length": 722.4375, "completions/min_length": 548.0, "completions/min_terminated_length": 548.0, "entropy": 0.09152617119252682, "epoch": 6.714285714285714, "frac_reward_zero_std": 0.0, "grad_norm": 0.027323298156261444, "learning_rate": 3.3061224489795924e-06, "loss": 0.0435, "num_tokens": 17197889.0, "reward": 0.6803125143051147, "reward_std": 0.1628584861755371, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.057735033333301544, "rewards/reward_commands_correctness/mean": 0.07375000417232513, "rewards/reward_commands_correctness/std": 0.029860790818929672, "rewards/reward_diversity/mean": 0.15968748927116394, "rewards/reward_diversity/std": 0.04899062216281891, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.02651257812976837, "rewards/reward_solution_effectiveness/mean": 0.1574999988079071, "rewards/reward_solution_effectiveness/std": 0.10009995102882385, "sampling/importance_sampling_ratio/max": 2.6793107986450195, "sampling/importance_sampling_ratio/mean": 0.24383312463760376, "sampling/importance_sampling_ratio/min": 1.209078800457064e-05, "sampling/sampling_logp_difference/max": 3.32501220703125, "sampling/sampling_logp_difference/mean": 0.025595908984541893, "step": 329, "step_time": 111.17761874198914 }, { "clip_ratio/high_max": 0.0018890102437580936, "clip_ratio/high_mean": 0.0018890102437580936, "clip_ratio/low_mean": 0.0015688725288782734, "clip_ratio/low_min": 0.0015688725288782734, "clip_ratio/region_mean": 0.003457882718066685, "completions/clipped_ratio": 0.0, "completions/max_length": 1527.0, "completions/max_terminated_length": 1527.0, "completions/mean_length": 791.5, "completions/mean_terminated_length": 791.5, "completions/min_length": 650.0, "completions/min_terminated_length": 650.0, "entropy": 0.09030989743769169, "epoch": 6.73469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.005983900278806686, "learning_rate": 3.285714285714286e-06, "loss": -0.013, "num_tokens": 17236889.0, "reward": 0.6160008311271667, "reward_std": 0.18087060749530792, "rewards/reward_commands_completeness/mean": 0.10999999940395355, "rewards/reward_commands_completeness/std": 0.04788876324892044, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.02528998628258705, "rewards/reward_diversity/mean": 0.146625816822052, "rewards/reward_diversity/std": 0.04537675902247429, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.027049340307712555, "rewards/reward_solution_effectiveness/mean": 0.1274999976158142, "rewards/reward_solution_effectiveness/std": 0.07224956899881363, "sampling/importance_sampling_ratio/max": 0.17054983973503113, "sampling/importance_sampling_ratio/mean": 0.023070640861988068, "sampling/importance_sampling_ratio/min": 1.6466137822135352e-06, "sampling/sampling_logp_difference/max": 3.1428797245025635, "sampling/sampling_logp_difference/mean": 0.025364959612488747, "step": 330, "step_time": 162.98687167838216 }, { "clip_ratio/high_max": 0.001386083495162893, "clip_ratio/high_mean": 0.001386083495162893, "clip_ratio/low_mean": 0.0009243082313332707, "clip_ratio/low_min": 0.0009243082313332707, "clip_ratio/region_mean": 0.0023103917264961638, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 729.5, "completions/mean_terminated_length": 729.5, "completions/min_length": 607.0, "completions/min_terminated_length": 607.0, "entropy": 0.0885911206714809, "epoch": 6.755102040816326, "frac_reward_zero_std": 0.25, "grad_norm": 0.09299115836620331, "learning_rate": 3.2653061224489794e-06, "loss": 0.1929, "num_tokens": 17278525.0, "reward": 0.4931770861148834, "reward_std": 0.12704181671142578, "rewards/reward_commands_completeness/mean": 0.0925000011920929, "rewards/reward_commands_completeness/std": 0.064446359872818, "rewards/reward_commands_correctness/mean": 0.054375000298023224, "rewards/reward_commands_correctness/std": 0.03687252849340439, "rewards/reward_diversity/mean": 0.1081770807504654, "rewards/reward_diversity/std": 0.07345710694789886, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.05937500298023224, "rewards/reward_problem_validity/std": 0.039575882256031036, "rewards/reward_solution_effectiveness/mean": 0.11625000089406967, "rewards/reward_solution_effectiveness/std": 0.0889100730419159, "sampling/importance_sampling_ratio/max": 2.3500099182128906, "sampling/importance_sampling_ratio/mean": 0.31313666701316833, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 19.7480411529541, "sampling/sampling_logp_difference/mean": 0.026310360059142113, "step": 331, "step_time": 113.61281251721084 }, { "clip_ratio/high_max": 0.0029658185521839187, "clip_ratio/high_mean": 0.0029658185521839187, "clip_ratio/low_mean": 0.0008708741806913167, "clip_ratio/low_min": 0.0008708741806913167, "clip_ratio/region_mean": 0.00383669271832332, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 917.0, "completions/mean_length": 957.25, "completions/mean_terminated_length": 748.0000610351562, "completions/min_length": 647.0, "completions/min_terminated_length": 647.0, "entropy": 0.09796590264886618, "epoch": 6.775510204081632, "frac_reward_zero_std": 0.0, "grad_norm": 0.06369038671255112, "learning_rate": 3.244897959183674e-06, "loss": 0.088, "num_tokens": 17322905.0, "reward": 0.7835385799407959, "reward_std": 0.13394539058208466, "rewards/reward_commands_completeness/mean": 0.15125000476837158, "rewards/reward_commands_completeness/std": 0.043185651302337646, "rewards/reward_commands_correctness/mean": 0.07437500357627869, "rewards/reward_commands_correctness/std": 0.02528998628258705, "rewards/reward_diversity/mean": 0.16541358828544617, "rewards/reward_diversity/std": 0.01972610503435135, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.020000001415610313, "rewards/reward_solution_effectiveness/mean": 0.21375000476837158, "rewards/reward_solution_effectiveness/std": 0.0708872377872467, "sampling/importance_sampling_ratio/max": 1.7046467065811157, "sampling/importance_sampling_ratio/mean": 0.1520167589187622, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.289227485656738, "sampling/sampling_logp_difference/mean": 0.020914563909173012, "step": 332, "step_time": 272.97552849166095 }, { "clip_ratio/high_max": 0.003105990013864357, "clip_ratio/high_mean": 0.003105990013864357, "clip_ratio/low_mean": 0.0009889561333693564, "clip_ratio/low_min": 0.0009889561333693564, "clip_ratio/region_mean": 0.004094946110853925, "completions/clipped_ratio": 0.0, "completions/max_length": 762.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 694.5625, "completions/mean_terminated_length": 694.5625, "completions/min_length": 613.0, "completions/min_terminated_length": 613.0, "entropy": 0.09428728837519884, "epoch": 6.795918367346939, "frac_reward_zero_std": 0.0, "grad_norm": 0.08521781861782074, "learning_rate": 3.2244897959183672e-06, "loss": 0.087, "num_tokens": 17356822.0, "reward": 0.7016337513923645, "reward_std": 0.1617002785205841, "rewards/reward_commands_completeness/mean": 0.1274999976158142, "rewards/reward_commands_completeness/std": 0.0478191077709198, "rewards/reward_commands_correctness/mean": 0.078125, "rewards/reward_commands_correctness/std": 0.026887111365795135, "rewards/reward_diversity/mean": 0.15913376212120056, "rewards/reward_diversity/std": 0.04267102852463722, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.1612500101327896, "rewards/reward_solution_effectiveness/std": 0.08188407123088837, "sampling/importance_sampling_ratio/max": 1.1443290710449219, "sampling/importance_sampling_ratio/mean": 0.1312551498413086, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2212347984313965, "sampling/sampling_logp_difference/mean": 0.02739867754280567, "step": 333, "step_time": 108.34481780603528 }, { "clip_ratio/high_max": 0.0018402097430225695, "clip_ratio/high_mean": 0.0018402097430225695, "clip_ratio/low_mean": 0.001283374716877006, "clip_ratio/low_min": 0.001283374716877006, "clip_ratio/region_mean": 0.003123584458080586, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 829.0, "completions/mean_length": 946.6875, "completions/mean_terminated_length": 736.7333984375, "completions/min_length": 645.0, "completions/min_terminated_length": 645.0, "entropy": 0.0832192269153893, "epoch": 6.816326530612245, "frac_reward_zero_std": 0.0, "grad_norm": 0.05506173148751259, "learning_rate": 3.204081632653062e-06, "loss": -0.0757, "num_tokens": 17394001.0, "reward": 0.7413133382797241, "reward_std": 0.0737471804022789, "rewards/reward_commands_completeness/mean": 0.13750000298023224, "rewards/reward_commands_completeness/std": 0.03336665406823158, "rewards/reward_commands_correctness/mean": 0.07874999940395355, "rewards/reward_commands_correctness/std": 0.013601472601294518, "rewards/reward_diversity/mean": 0.15943831205368042, "rewards/reward_diversity/std": 0.024315813556313515, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.17812499403953552, "rewards/reward_solution_effectiveness/std": 0.057412974536418915, "sampling/importance_sampling_ratio/max": 1.1035054922103882, "sampling/importance_sampling_ratio/mean": 0.12277565896511078, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.089764595031738, "sampling/sampling_logp_difference/mean": 0.020552923902869225, "step": 334, "step_time": 266.66280581615865 }, { "clip_ratio/high_max": 0.002047093556029722, "clip_ratio/high_mean": 0.002047093556029722, "clip_ratio/low_mean": 0.0008497038506902754, "clip_ratio/low_min": 0.0008497038506902754, "clip_ratio/region_mean": 0.002896797377616167, "completions/clipped_ratio": 0.0, "completions/max_length": 832.0, "completions/max_terminated_length": 832.0, "completions/mean_length": 733.1875, "completions/mean_terminated_length": 733.1875, "completions/min_length": 603.0, "completions/min_terminated_length": 603.0, "entropy": 0.09658186789602041, "epoch": 6.836734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.047864872962236404, "learning_rate": 3.183673469387755e-06, "loss": 0.0794, "num_tokens": 17423112.0, "reward": 0.7640785574913025, "reward_std": 0.07869698852300644, "rewards/reward_commands_completeness/mean": 0.14124999940395355, "rewards/reward_commands_completeness/std": 0.03052321821451187, "rewards/reward_commands_correctness/mean": 0.08249999582767487, "rewards/reward_commands_correctness/std": 0.012909946963191032, "rewards/reward_diversity/mean": 0.17470356822013855, "rewards/reward_diversity/std": 0.01762337237596512, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.17812500894069672, "rewards/reward_solution_effectiveness/std": 0.056358225643634796, "sampling/importance_sampling_ratio/max": 0.8101354241371155, "sampling/importance_sampling_ratio/mean": 0.13637299835681915, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.641782283782959, "sampling/sampling_logp_difference/mean": 0.026064548641443253, "step": 335, "step_time": 96.1847677025944 }, { "clip_ratio/high_max": 0.002512849197955802, "clip_ratio/high_mean": 0.002512849197955802, "clip_ratio/low_mean": 0.0015262482265825383, "clip_ratio/low_min": 0.0015262482265825383, "clip_ratio/region_mean": 0.00403909742453834, "completions/clipped_ratio": 0.0, "completions/max_length": 825.0, "completions/max_terminated_length": 825.0, "completions/mean_length": 737.25, "completions/mean_terminated_length": 737.25, "completions/min_length": 651.0, "completions/min_terminated_length": 651.0, "entropy": 0.08368559926748276, "epoch": 6.857142857142857, "frac_reward_zero_std": 0.0, "grad_norm": 0.08540567755699158, "learning_rate": 3.1632653061224496e-06, "loss": -0.1612, "num_tokens": 17468960.0, "reward": 0.6362226009368896, "reward_std": 0.1510905623435974, "rewards/reward_commands_completeness/mean": 0.11874999850988388, "rewards/reward_commands_completeness/std": 0.04759201779961586, "rewards/reward_commands_correctness/mean": 0.06187500059604645, "rewards/reward_commands_correctness/std": 0.027620946988463402, "rewards/reward_diversity/mean": 0.1512225866317749, "rewards/reward_diversity/std": 0.05552248656749725, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07625000178813934, "rewards/reward_problem_validity/std": 0.026299558579921722, "rewards/reward_solution_effectiveness/mean": 0.140625, "rewards/reward_solution_effectiveness/std": 0.07159783691167831, "sampling/importance_sampling_ratio/max": 1.9189289808273315, "sampling/importance_sampling_ratio/mean": 0.20311790704727173, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 12.931283950805664, "sampling/sampling_logp_difference/mean": 0.02520325407385826, "step": 336, "step_time": 122.68919816054404 }, { "clip_ratio/high_max": 0.0014749014808330685, "clip_ratio/high_mean": 0.0014749014808330685, "clip_ratio/low_mean": 0.00167647999660403, "clip_ratio/low_min": 0.00167647999660403, "clip_ratio/region_mean": 0.003151381475618109, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 836.0, "completions/mean_length": 965.4375, "completions/mean_terminated_length": 756.7333984375, "completions/min_length": 707.0, "completions/min_terminated_length": 707.0, "entropy": 0.10407024808228016, "epoch": 6.877551020408164, "frac_reward_zero_std": 0.0, "grad_norm": 0.021388385444879532, "learning_rate": 3.142857142857143e-06, "loss": 0.0397, "num_tokens": 17508335.0, "reward": 0.71135413646698, "reward_std": 0.05709145963191986, "rewards/reward_commands_completeness/mean": 0.12999999523162842, "rewards/reward_commands_completeness/std": 0.0334664024412632, "rewards/reward_commands_correctness/mean": 0.06937500089406967, "rewards/reward_commands_correctness/std": 0.019482899457216263, "rewards/reward_diversity/mean": 0.17947915196418762, "rewards/reward_diversity/std": 0.013512256555259228, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437499403953552, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.14812500774860382, "rewards/reward_solution_effectiveness/std": 0.06794299185276031, "sampling/importance_sampling_ratio/max": 2.455214262008667, "sampling/importance_sampling_ratio/mean": 0.17607678472995758, "sampling/importance_sampling_ratio/min": 4.2668506949894436e-08, "sampling/sampling_logp_difference/max": 5.54410982131958, "sampling/sampling_logp_difference/mean": 0.021660977974534035, "step": 337, "step_time": 273.40788927301764 }, { "clip_ratio/high_max": 0.0030181388865457848, "clip_ratio/high_mean": 0.0030181388865457848, "clip_ratio/low_mean": 0.0010542368108872324, "clip_ratio/low_min": 0.0010542368108872324, "clip_ratio/region_mean": 0.004072375711984932, "completions/clipped_ratio": 0.0625, "completions/max_length": 2293.0, "completions/max_terminated_length": 1164.0, "completions/mean_length": 914.0, "completions/mean_terminated_length": 822.0667114257812, "completions/min_length": 622.0, "completions/min_terminated_length": 622.0, "entropy": 0.08649077219888568, "epoch": 6.8979591836734695, "frac_reward_zero_std": 0.0, "grad_norm": 0.10296180099248886, "learning_rate": 3.1224489795918374e-06, "loss": 0.0816, "num_tokens": 17564107.0, "reward": 0.6987866163253784, "reward_std": 0.12237219512462616, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.04250490292906761, "rewards/reward_commands_correctness/mean": 0.06875000149011612, "rewards/reward_commands_correctness/std": 0.024731896817684174, "rewards/reward_diversity/mean": 0.15191154181957245, "rewards/reward_diversity/std": 0.04356669262051582, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08312500268220901, "rewards/reward_problem_validity/std": 0.01990603096783161, "rewards/reward_solution_effectiveness/mean": 0.16874998807907104, "rewards/reward_solution_effectiveness/std": 0.06830080598592758, "sampling/importance_sampling_ratio/max": 1.1275218725204468, "sampling/importance_sampling_ratio/mean": 0.15777084231376648, "sampling/importance_sampling_ratio/min": 3.755004172489862e-06, "sampling/sampling_logp_difference/max": 6.321821212768555, "sampling/sampling_logp_difference/mean": 0.023370621725916862, "step": 338, "step_time": 273.260003907606 }, { "clip_ratio/high_max": 0.0020828375782002695, "clip_ratio/high_mean": 0.0020828375782002695, "clip_ratio/low_mean": 0.0019065273299929686, "clip_ratio/low_min": 0.0019065273299929686, "clip_ratio/region_mean": 0.003989364820881747, "completions/clipped_ratio": 0.0, "completions/max_length": 967.0, "completions/max_terminated_length": 967.0, "completions/mean_length": 786.625, "completions/mean_terminated_length": 786.625, "completions/min_length": 649.0, "completions/min_terminated_length": 649.0, "entropy": 0.09210794605314732, "epoch": 6.918367346938775, "frac_reward_zero_std": 0.0, "grad_norm": 0.04266587272286415, "learning_rate": 3.1020408163265307e-06, "loss": -0.0836, "num_tokens": 17603225.0, "reward": 0.6989495754241943, "reward_std": 0.15501706302165985, "rewards/reward_commands_completeness/mean": 0.1300000101327896, "rewards/reward_commands_completeness/std": 0.04066120460629463, "rewards/reward_commands_correctness/mean": 0.07187499850988388, "rewards/reward_commands_correctness/std": 0.022867372259497643, "rewards/reward_diversity/mean": 0.16707460582256317, "rewards/reward_diversity/std": 0.04121100902557373, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.018929695710539818, "rewards/reward_solution_effectiveness/mean": 0.1574999988079071, "rewards/reward_solution_effectiveness/std": 0.0730753093957901, "sampling/importance_sampling_ratio/max": 1.3555543422698975, "sampling/importance_sampling_ratio/mean": 0.09239428490400314, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.064556121826172, "sampling/sampling_logp_difference/mean": 0.027037382125854492, "step": 339, "step_time": 116.05878004617989 }, { "clip_ratio/high_max": 0.0031029805468278937, "clip_ratio/high_mean": 0.0031029805468278937, "clip_ratio/low_mean": 0.0013110756844980642, "clip_ratio/low_min": 0.0013110756844980642, "clip_ratio/region_mean": 0.00441405622405, "completions/clipped_ratio": 0.0, "completions/max_length": 840.0, "completions/max_terminated_length": 840.0, "completions/mean_length": 686.8125, "completions/mean_terminated_length": 686.8125, "completions/min_length": 536.0, "completions/min_terminated_length": 536.0, "entropy": 0.1069959457963705, "epoch": 6.938775510204081, "frac_reward_zero_std": 0.0, "grad_norm": 0.04640711843967438, "learning_rate": 3.0816326530612244e-06, "loss": 0.042, "num_tokens": 17638314.0, "reward": 0.8046875, "reward_std": 0.09346640110015869, "rewards/reward_commands_completeness/mean": 0.14874999225139618, "rewards/reward_commands_completeness/std": 0.03180671110749245, "rewards/reward_commands_correctness/mean": 0.08687499910593033, "rewards/reward_commands_correctness/std": 0.010781930759549141, "rewards/reward_diversity/mean": 0.18843749165534973, "rewards/reward_diversity/std": 0.018693124875426292, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.19312499463558197, "rewards/reward_solution_effectiveness/std": 0.06477846950292587, "sampling/importance_sampling_ratio/max": 0.6607689261436462, "sampling/importance_sampling_ratio/mean": 0.0800071433186531, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.4887585639953613, "sampling/sampling_logp_difference/mean": 0.02967618778347969, "step": 340, "step_time": 105.3149574585259 }, { "clip_ratio/high_max": 0.0013679226249223575, "clip_ratio/high_mean": 0.0013679226249223575, "clip_ratio/low_mean": 0.0014544347286573611, "clip_ratio/low_min": 0.0014544347286573611, "clip_ratio/region_mean": 0.0028223573608556762, "completions/clipped_ratio": 0.0, "completions/max_length": 1446.0, "completions/max_terminated_length": 1446.0, "completions/mean_length": 826.8125, "completions/mean_terminated_length": 826.8125, "completions/min_length": 653.0, "completions/min_terminated_length": 653.0, "entropy": 0.10573560744524002, "epoch": 6.959183673469388, "frac_reward_zero_std": 0.0, "grad_norm": 0.10122062265872955, "learning_rate": 3.0612244897959185e-06, "loss": 0.1631, "num_tokens": 17676655.0, "reward": 0.67520672082901, "reward_std": 0.12562406063079834, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.05744563043117523, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.029147613793611526, "rewards/reward_diversity/mean": 0.15145671367645264, "rewards/reward_diversity/std": 0.06665898859500885, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07437500357627869, "rewards/reward_problem_validity/std": 0.032035138458013535, "rewards/reward_solution_effectiveness/mean": 0.17249999940395355, "rewards/reward_solution_effectiveness/std": 0.07784600555896759, "sampling/importance_sampling_ratio/max": 2.813455104827881, "sampling/importance_sampling_ratio/mean": 0.28410130739212036, "sampling/importance_sampling_ratio/min": 9.49120729926145e-12, "sampling/sampling_logp_difference/max": 18.116350173950195, "sampling/sampling_logp_difference/mean": 0.03067341446876526, "step": 341, "step_time": 133.4973233472556 }, { "clip_ratio/high_max": 0.0030363244732143357, "clip_ratio/high_mean": 0.0030363244732143357, "clip_ratio/low_mean": 0.0014120364576228894, "clip_ratio/low_min": 0.0014120364576228894, "clip_ratio/region_mean": 0.004448360850801691, "completions/clipped_ratio": 0.0, "completions/max_length": 1138.0, "completions/max_terminated_length": 1138.0, "completions/mean_length": 774.0625, "completions/mean_terminated_length": 774.0625, "completions/min_length": 629.0, "completions/min_terminated_length": 629.0, "entropy": 0.09241393255069852, "epoch": 6.979591836734694, "frac_reward_zero_std": 0.0, "grad_norm": 0.08883771300315857, "learning_rate": 3.0408163265306122e-06, "loss": -0.1454, "num_tokens": 17736724.0, "reward": 0.577619194984436, "reward_std": 0.2519264817237854, "rewards/reward_commands_completeness/mean": 0.10500000417232513, "rewards/reward_commands_completeness/std": 0.05033223330974579, "rewards/reward_commands_correctness/mean": 0.054375000298023224, "rewards/reward_commands_correctness/std": 0.030103987082839012, "rewards/reward_diversity/mean": 0.14636918902397156, "rewards/reward_diversity/std": 0.06957211345434189, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07062499970197678, "rewards/reward_problem_validity/std": 0.03043435700237751, "rewards/reward_solution_effectiveness/mean": 0.11999999731779099, "rewards/reward_solution_effectiveness/std": 0.0701427161693573, "sampling/importance_sampling_ratio/max": 2.062683582305908, "sampling/importance_sampling_ratio/mean": 0.20053210854530334, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 15.861818313598633, "sampling/sampling_logp_difference/mean": 0.02758261375129223, "step": 342, "step_time": 206.01094355247915 }, { "clip_ratio/high_max": 0.001916243949381169, "clip_ratio/high_mean": 0.001916243949381169, "clip_ratio/low_mean": 0.0006579498440260068, "clip_ratio/low_min": 0.0006579498440260068, "clip_ratio/region_mean": 0.0025741938006831333, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 760.3125, "completions/mean_terminated_length": 760.3125, "completions/min_length": 625.0, "completions/min_terminated_length": 625.0, "entropy": 0.06761501217260957, "epoch": 7.0, "frac_reward_zero_std": 0.0, "grad_norm": 0.06919360160827637, "learning_rate": 3.0204081632653064e-06, "loss": -0.0927, "num_tokens": 17778273.0, "reward": 0.7790030241012573, "reward_std": 0.087495818734169, "rewards/reward_commands_completeness/mean": 0.15125000476837158, "rewards/reward_commands_completeness/std": 0.03930648788809776, "rewards/reward_commands_correctness/mean": 0.08312500268220901, "rewards/reward_commands_correctness/std": 0.018518010154366493, "rewards/reward_diversity/mean": 0.1558779776096344, "rewards/reward_diversity/std": 0.019225893542170525, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.2006250023841858, "rewards/reward_solution_effectiveness/std": 0.08028854429721832, "sampling/importance_sampling_ratio/max": 2.626270294189453, "sampling/importance_sampling_ratio/mean": 0.22219258546829224, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.529871940612793, "sampling/sampling_logp_difference/mean": 0.02188301272690296, "step": 343, "step_time": 151.06997347809374 }, { "clip_ratio/high_max": 0.0025898029562085867, "clip_ratio/high_mean": 0.0025898029562085867, "clip_ratio/low_mean": 0.001312558822974097, "clip_ratio/low_min": 0.001312558822974097, "clip_ratio/region_mean": 0.003902361757354811, "completions/clipped_ratio": 0.0, "completions/max_length": 825.0, "completions/max_terminated_length": 825.0, "completions/mean_length": 704.1875, "completions/mean_terminated_length": 704.1875, "completions/min_length": 609.0, "completions/min_terminated_length": 609.0, "entropy": 0.09369152039289474, "epoch": 7.020408163265306, "frac_reward_zero_std": 0.0, "grad_norm": 0.009540034458041191, "learning_rate": 3e-06, "loss": -0.0226, "num_tokens": 17812940.0, "reward": 0.6536137461662292, "reward_std": 0.14875838160514832, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.04926121234893799, "rewards/reward_commands_correctness/mean": 0.06687499582767487, "rewards/reward_commands_correctness/std": 0.02651257812976837, "rewards/reward_diversity/mean": 0.15111374855041504, "rewards/reward_diversity/std": 0.053656354546546936, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07500000298023224, "rewards/reward_problem_validity/std": 0.025819890201091766, "rewards/reward_solution_effectiveness/mean": 0.14812499284744263, "rewards/reward_solution_effectiveness/std": 0.07467429339885712, "sampling/importance_sampling_ratio/max": 0.1958303153514862, "sampling/importance_sampling_ratio/mean": 0.04214195907115936, "sampling/importance_sampling_ratio/min": 4.005603386758594e-06, "sampling/sampling_logp_difference/max": 3.0866267681121826, "sampling/sampling_logp_difference/mean": 0.028921552002429962, "step": 344, "step_time": 104.7171855866909 }, { "clip_ratio/high_max": 0.0022758328705094755, "clip_ratio/high_mean": 0.0022758328705094755, "clip_ratio/low_mean": 0.0008103572617983446, "clip_ratio/low_min": 0.0008103572617983446, "clip_ratio/region_mean": 0.00308619013230782, "completions/clipped_ratio": 0.0, "completions/max_length": 922.0, "completions/max_terminated_length": 922.0, "completions/mean_length": 766.6875, "completions/mean_terminated_length": 766.6875, "completions/min_length": 633.0, "completions/min_terminated_length": 633.0, "entropy": 0.0971838803961873, "epoch": 7.040816326530612, "frac_reward_zero_std": 0.0, "grad_norm": 0.040705159306526184, "learning_rate": 2.979591836734694e-06, "loss": -0.0674, "num_tokens": 17845691.0, "reward": 0.6641501784324646, "reward_std": 0.16251064836978912, "rewards/reward_commands_completeness/mean": 0.12250000238418579, "rewards/reward_commands_completeness/std": 0.04892170801758766, "rewards/reward_commands_correctness/mean": 0.06875000149011612, "rewards/reward_commands_correctness/std": 0.02825479581952095, "rewards/reward_diversity/mean": 0.15977516770362854, "rewards/reward_diversity/std": 0.05745889991521835, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07750000059604645, "rewards/reward_problem_validity/std": 0.02670830301940441, "rewards/reward_solution_effectiveness/mean": 0.14812500774860382, "rewards/reward_solution_effectiveness/std": 0.07386643439531326, "sampling/importance_sampling_ratio/max": 2.5068185329437256, "sampling/importance_sampling_ratio/mean": 0.17857547104358673, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.7672677040100098, "sampling/sampling_logp_difference/mean": 0.027731018140912056, "step": 345, "step_time": 102.88608029671013 }, { "clip_ratio/high_max": 0.0017593656375538558, "clip_ratio/high_mean": 0.0017593656375538558, "clip_ratio/low_mean": 0.0002791235328913899, "clip_ratio/low_min": 0.0002791235328913899, "clip_ratio/region_mean": 0.0020384891686262563, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 871.0, "completions/mean_length": 950.75, "completions/mean_terminated_length": 741.0667114257812, "completions/min_length": 662.0, "completions/min_terminated_length": 662.0, "entropy": 0.0713039985857904, "epoch": 7.061224489795919, "frac_reward_zero_std": 0.25, "grad_norm": 0.007101754657924175, "learning_rate": 2.959183673469388e-06, "loss": -0.0022, "num_tokens": 17890303.0, "reward": 0.5470588207244873, "reward_std": 0.1431894600391388, "rewards/reward_commands_completeness/mean": 0.09749999642372131, "rewards/reward_commands_completeness/std": 0.062343139201402664, "rewards/reward_commands_correctness/mean": 0.054999999701976776, "rewards/reward_commands_correctness/std": 0.03386246785521507, "rewards/reward_diversity/mean": 0.137058824300766, "rewards/reward_diversity/std": 0.07070524990558624, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.06312499940395355, "rewards/reward_problem_validity/std": 0.037187591195106506, "rewards/reward_solution_effectiveness/mean": 0.12562499940395355, "rewards/reward_solution_effectiveness/std": 0.0877852514386177, "sampling/importance_sampling_ratio/max": 0.30519378185272217, "sampling/importance_sampling_ratio/mean": 0.06570996344089508, "sampling/importance_sampling_ratio/min": 1.4615720829169732e-06, "sampling/sampling_logp_difference/max": 6.279195308685303, "sampling/sampling_logp_difference/mean": 0.018737668171525, "step": 346, "step_time": 252.79936229623854 }, { "clip_ratio/high_max": 0.004045534893521108, "clip_ratio/high_mean": 0.004045534893521108, "clip_ratio/low_mean": 0.0015980602693161927, "clip_ratio/low_min": 0.0015980602693161927, "clip_ratio/region_mean": 0.005643595097353682, "completions/clipped_ratio": 0.0, "completions/max_length": 2457.0, "completions/max_terminated_length": 2457.0, "completions/mean_length": 868.375, "completions/mean_terminated_length": 868.375, "completions/min_length": 634.0, "completions/min_terminated_length": 634.0, "entropy": 0.09870619233697653, "epoch": 7.081632653061225, "frac_reward_zero_std": 0.0, "grad_norm": 0.05607863888144493, "learning_rate": 2.938775510204082e-06, "loss": -0.2177, "num_tokens": 17926837.0, "reward": 0.7384963631629944, "reward_std": 0.15318801999092102, "rewards/reward_commands_completeness/mean": 0.14499999582767487, "rewards/reward_commands_completeness/std": 0.03966527059674263, "rewards/reward_commands_correctness/mean": 0.07187500596046448, "rewards/reward_commands_correctness/std": 0.024005208164453506, "rewards/reward_diversity/mean": 0.1509963870048523, "rewards/reward_diversity/std": 0.040122680366039276, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08375000208616257, "rewards/reward_problem_validity/std": 0.019958291202783585, "rewards/reward_solution_effectiveness/mean": 0.19312499463558197, "rewards/reward_solution_effectiveness/std": 0.06477847695350647, "sampling/importance_sampling_ratio/max": 1.1741259098052979, "sampling/importance_sampling_ratio/mean": 0.10394683480262756, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 10.42356014251709, "sampling/sampling_logp_difference/mean": 0.027979997918009758, "step": 347, "step_time": 193.6206104271114 }, { "clip_ratio/high_max": 0.001920172173413448, "clip_ratio/high_mean": 0.001920172173413448, "clip_ratio/low_mean": 0.001376707266899757, "clip_ratio/low_min": 0.001376707266899757, "clip_ratio/region_mean": 0.003296879403933417, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 783.3125, "completions/mean_terminated_length": 783.3125, "completions/min_length": 706.0, "completions/min_terminated_length": 706.0, "entropy": 0.10033280868083239, "epoch": 7.1020408163265305, "frac_reward_zero_std": 0.0, "grad_norm": 0.041181568056344986, "learning_rate": 2.9183673469387757e-06, "loss": -0.0963, "num_tokens": 17970394.0, "reward": 0.7319735884666443, "reward_std": 0.057288721203804016, "rewards/reward_commands_completeness/mean": 0.13750000298023224, "rewards/reward_commands_completeness/std": 0.029999999329447746, "rewards/reward_commands_correctness/mean": 0.07375000417232513, "rewards/reward_commands_correctness/std": 0.0145487692207098, "rewards/reward_diversity/mean": 0.1732235997915268, "rewards/reward_diversity/std": 0.015055358409881592, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08624999970197678, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.16124999523162842, "rewards/reward_solution_effectiveness/std": 0.05572252720594406, "sampling/importance_sampling_ratio/max": 1.313941240310669, "sampling/importance_sampling_ratio/mean": 0.16709211468696594, "sampling/importance_sampling_ratio/min": 4.06483650294831e-07, "sampling/sampling_logp_difference/max": 11.249753952026367, "sampling/sampling_logp_difference/mean": 0.026974190026521683, "step": 348, "step_time": 122.69253227487206 }, { "clip_ratio/high_max": 0.0022313603112706915, "clip_ratio/high_mean": 0.0022313603112706915, "clip_ratio/low_mean": 0.0014749936381122097, "clip_ratio/low_min": 0.0014749936381122097, "clip_ratio/region_mean": 0.003706353949382901, "completions/clipped_ratio": 0.0, "completions/max_length": 986.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 772.75, "completions/mean_terminated_length": 772.75, "completions/min_length": 626.0, "completions/min_terminated_length": 626.0, "entropy": 0.08144229836761951, "epoch": 7.122448979591836, "frac_reward_zero_std": 0.0, "grad_norm": 0.010204293765127659, "learning_rate": 2.8979591836734694e-06, "loss": -0.0083, "num_tokens": 18014038.0, "reward": 0.6334191560745239, "reward_std": 0.21582503616809845, "rewards/reward_commands_completeness/mean": 0.12000000476837158, "rewards/reward_commands_completeness/std": 0.05656854435801506, "rewards/reward_commands_correctness/mean": 0.06187500059604645, "rewards/reward_commands_correctness/std": 0.029261751100420952, "rewards/reward_diversity/mean": 0.1402941197156906, "rewards/reward_diversity/std": 0.062301892787218094, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07437500357627869, "rewards/reward_problem_validity/std": 0.032035138458013535, "rewards/reward_solution_effectiveness/mean": 0.15562500059604645, "rewards/reward_solution_effectiveness/std": 0.08286283910274506, "sampling/importance_sampling_ratio/max": 0.42939525842666626, "sampling/importance_sampling_ratio/mean": 0.05053521692752838, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 19.173370361328125, "sampling/sampling_logp_difference/mean": 0.02616208977997303, "step": 349, "step_time": 121.67294057644904 }, { "clip_ratio/high_max": 0.0027499427960719913, "clip_ratio/high_mean": 0.0027499427960719913, "clip_ratio/low_mean": 0.00041213199438061565, "clip_ratio/low_min": 0.00041213199438061565, "clip_ratio/region_mean": 0.003162074790452607, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 3166.0, "completions/mean_length": 1220.0625, "completions/mean_terminated_length": 1028.3333740234375, "completions/min_length": 681.0, "completions/min_terminated_length": 681.0, "entropy": 0.08470012992620468, "epoch": 7.142857142857143, "frac_reward_zero_std": 0.0, "grad_norm": 0.004017538391053677, "learning_rate": 2.8775510204081636e-06, "loss": -0.0055, "num_tokens": 18060067.0, "reward": 0.769853949546814, "reward_std": 0.08191154152154922, "rewards/reward_commands_completeness/mean": 0.1525000035762787, "rewards/reward_commands_completeness/std": 0.021756228059530258, "rewards/reward_commands_correctness/mean": 0.07124999910593033, "rewards/reward_commands_correctness/std": 0.017078252509236336, "rewards/reward_diversity/mean": 0.16485396027565002, "rewards/reward_diversity/std": 0.027373461052775383, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.19312500953674316, "rewards/reward_solution_effectiveness/std": 0.053630683571100235, "sampling/importance_sampling_ratio/max": 0.5333025455474854, "sampling/importance_sampling_ratio/mean": 0.06374113261699677, "sampling/importance_sampling_ratio/min": 1.7180583427034435e-06, "sampling/sampling_logp_difference/max": 6.172033786773682, "sampling/sampling_logp_difference/mean": 0.018949594348669052, "step": 350, "step_time": 265.5482319332659 }, { "clip_ratio/high_max": 0.0028672827320406213, "clip_ratio/high_mean": 0.0028672827320406213, "clip_ratio/low_mean": 0.0011937351955566555, "clip_ratio/low_min": 0.0011937351955566555, "clip_ratio/region_mean": 0.004061017913045362, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 842.0, "completions/mean_length": 956.1875, "completions/mean_terminated_length": 746.86669921875, "completions/min_length": 640.0, "completions/min_terminated_length": 640.0, "entropy": 0.09300491213798523, "epoch": 7.163265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.04270094633102417, "learning_rate": 2.8571428571428573e-06, "loss": 0.0371, "num_tokens": 18095450.0, "reward": 0.7449234127998352, "reward_std": 0.07919324934482574, "rewards/reward_commands_completeness/mean": 0.14374999701976776, "rewards/reward_commands_completeness/std": 0.027537856251001358, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.02476556785404682, "rewards/reward_diversity/mean": 0.17554840445518494, "rewards/reward_diversity/std": 0.01823047548532486, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.16874998807907104, "rewards/reward_solution_effectiveness/std": 0.06280923634767532, "sampling/importance_sampling_ratio/max": 1.2859543561935425, "sampling/importance_sampling_ratio/mean": 0.13643115758895874, "sampling/importance_sampling_ratio/min": 6.190479666656756e-07, "sampling/sampling_logp_difference/max": 6.062254428863525, "sampling/sampling_logp_difference/mean": 0.023076992481946945, "step": 351, "step_time": 249.32925318181515 }, { "clip_ratio/high_max": 0.0022793262396589853, "clip_ratio/high_mean": 0.0022793262396589853, "clip_ratio/low_mean": 0.0006472466338891536, "clip_ratio/low_min": 0.0006472466338891536, "clip_ratio/region_mean": 0.0029265728808240965, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1030.0, "completions/mean_length": 966.0625, "completions/mean_terminated_length": 757.4000244140625, "completions/min_length": 573.0, "completions/min_terminated_length": 573.0, "entropy": 0.08825417375192046, "epoch": 7.183673469387755, "frac_reward_zero_std": 0.0, "grad_norm": 0.013339866884052753, "learning_rate": 2.8367346938775514e-06, "loss": 0.0116, "num_tokens": 18130963.0, "reward": 0.6254860758781433, "reward_std": 0.17303359508514404, "rewards/reward_commands_completeness/mean": 0.11500000208616257, "rewards/reward_commands_completeness/std": 0.048716869205236435, "rewards/reward_commands_correctness/mean": 0.06812500208616257, "rewards/reward_commands_correctness/std": 0.02786126546561718, "rewards/reward_diversity/mean": 0.14173611998558044, "rewards/reward_diversity/std": 0.05028146132826805, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.078125, "rewards/reward_problem_validity/std": 0.026887113228440285, "rewards/reward_solution_effectiveness/mean": 0.13499999046325684, "rewards/reward_solution_effectiveness/std": 0.07266361266374588, "sampling/importance_sampling_ratio/max": 0.20780549943447113, "sampling/importance_sampling_ratio/mean": 0.03473891690373421, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.07578706741333, "sampling/sampling_logp_difference/mean": 0.020439276471734047, "step": 352, "step_time": 256.46098559722304 }, { "clip_ratio/high_max": 0.0018096107742167078, "clip_ratio/high_mean": 0.0018096107742167078, "clip_ratio/low_mean": 0.0006755292270099744, "clip_ratio/low_min": 0.0006755292270099744, "clip_ratio/region_mean": 0.0024851400157785974, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 863.0, "completions/mean_length": 949.0, "completions/mean_terminated_length": 739.2000122070312, "completions/min_length": 609.0, "completions/min_terminated_length": 609.0, "entropy": 0.086367875803262, "epoch": 7.204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.002331208437681198, "learning_rate": 2.816326530612245e-06, "loss": -0.0041, "num_tokens": 18175651.0, "reward": 0.7511029243469238, "reward_std": 0.07067857682704926, "rewards/reward_commands_completeness/mean": 0.14749999344348907, "rewards/reward_commands_completeness/std": 0.021756228059530258, "rewards/reward_commands_correctness/mean": 0.0768750011920929, "rewards/reward_commands_correctness/std": 0.015370427630841732, "rewards/reward_diversity/mean": 0.16235293447971344, "rewards/reward_diversity/std": 0.024727659299969673, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08624999970197678, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.17812499403953552, "rewards/reward_solution_effectiveness/std": 0.04578481987118721, "sampling/importance_sampling_ratio/max": 0.40542110800743103, "sampling/importance_sampling_ratio/mean": 0.03689757362008095, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.20111083984375, "sampling/sampling_logp_difference/mean": 0.020978786051273346, "step": 353, "step_time": 275.3615241423249 }, { "clip_ratio/high_max": 0.0027150867535965517, "clip_ratio/high_mean": 0.0027150867535965517, "clip_ratio/low_mean": 0.0016648902019369416, "clip_ratio/low_min": 0.0016648902019369416, "clip_ratio/region_mean": 0.0043799769337056205, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 872.0, "completions/mean_length": 957.5, "completions/mean_terminated_length": 748.2667236328125, "completions/min_length": 632.0, "completions/min_terminated_length": 632.0, "entropy": 0.09114790987223387, "epoch": 7.224489795918367, "frac_reward_zero_std": 0.0, "grad_norm": 0.023244159296154976, "learning_rate": 2.795918367346939e-06, "loss": -0.0224, "num_tokens": 18216843.0, "reward": 0.7210937738418579, "reward_std": 0.04069104045629501, "rewards/reward_commands_completeness/mean": 0.13124999403953552, "rewards/reward_commands_completeness/std": 0.02825479581952095, "rewards/reward_commands_correctness/mean": 0.06937500089406967, "rewards/reward_commands_correctness/std": 0.020806649699807167, "rewards/reward_diversity/mean": 0.18359375, "rewards/reward_diversity/std": 0.017621850594878197, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.14999999105930328, "rewards/reward_solution_effectiveness/std": 0.05796550586819649, "sampling/importance_sampling_ratio/max": 0.7181809544563293, "sampling/importance_sampling_ratio/mean": 0.08713755011558533, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 17.24998664855957, "sampling/sampling_logp_difference/mean": 0.02290828339755535, "step": 354, "step_time": 299.8895967621356 }, { "clip_ratio/high_max": 0.002780426773824729, "clip_ratio/high_mean": 0.002780426773824729, "clip_ratio/low_mean": 0.000989687308901921, "clip_ratio/low_min": 0.000989687308901921, "clip_ratio/region_mean": 0.0037701141118304804, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 814.0, "completions/mean_length": 923.125, "completions/mean_terminated_length": 711.6000366210938, "completions/min_length": 575.0, "completions/min_terminated_length": 575.0, "entropy": 0.09907444613054395, "epoch": 7.244897959183674, "frac_reward_zero_std": 0.0, "grad_norm": 0.007716229185461998, "learning_rate": 2.775510204081633e-06, "loss": -0.0087, "num_tokens": 18254329.0, "reward": 0.6892524361610413, "reward_std": 0.07836434245109558, "rewards/reward_commands_completeness/mean": 0.12000000476837158, "rewards/reward_commands_completeness/std": 0.029211871325969696, "rewards/reward_commands_correctness/mean": 0.07312499731779099, "rewards/reward_commands_correctness/std": 0.015798209235072136, "rewards/reward_diversity/mean": 0.17737746238708496, "rewards/reward_diversity/std": 0.015117393806576729, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500774860382, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.13312500715255737, "rewards/reward_solution_effectiveness/std": 0.053630683571100235, "sampling/importance_sampling_ratio/max": 0.1769477277994156, "sampling/importance_sampling_ratio/mean": 0.03145075589418411, "sampling/importance_sampling_ratio/min": 2.1128435889750108e-07, "sampling/sampling_logp_difference/max": 5.634073734283447, "sampling/sampling_logp_difference/mean": 0.02337891235947609, "step": 355, "step_time": 253.29385383054614 }, { "clip_ratio/high_max": 0.0017125045487773605, "clip_ratio/high_mean": 0.0017125045487773605, "clip_ratio/low_mean": 0.0021824292634846643, "clip_ratio/low_min": 0.0021824292634846643, "clip_ratio/region_mean": 0.0038949338777456433, "completions/clipped_ratio": 0.0, "completions/max_length": 839.0, "completions/max_terminated_length": 839.0, "completions/mean_length": 725.5625, "completions/mean_terminated_length": 725.5625, "completions/min_length": 619.0, "completions/min_terminated_length": 619.0, "entropy": 0.08623720426112413, "epoch": 7.26530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.029141129925847054, "learning_rate": 2.7551020408163266e-06, "loss": -0.0532, "num_tokens": 18287798.0, "reward": 0.7506250143051147, "reward_std": 0.0652436912059784, "rewards/reward_commands_completeness/mean": 0.14499999582767487, "rewards/reward_commands_completeness/std": 0.03386247158050537, "rewards/reward_commands_correctness/mean": 0.06562500447034836, "rewards/reward_commands_correctness/std": 0.02250000089406967, "rewards/reward_diversity/mean": 0.17249999940395355, "rewards/reward_diversity/std": 0.019757796078920364, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08937500417232513, "rewards/reward_problem_validity/std": 0.0025000013411045074, "rewards/reward_solution_effectiveness/mean": 0.17812499403953552, "rewards/reward_solution_effectiveness/std": 0.07858912646770477, "sampling/importance_sampling_ratio/max": 0.5002302527427673, "sampling/importance_sampling_ratio/mean": 0.0753852128982544, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 19.487646102905273, "sampling/sampling_logp_difference/mean": 0.028891783207654953, "step": 356, "step_time": 105.8131485953927 }, { "clip_ratio/high_max": 0.001818897682824172, "clip_ratio/high_mean": 0.001818897682824172, "clip_ratio/low_mean": 0.0010906528841587715, "clip_ratio/low_min": 0.0010906528841587715, "clip_ratio/region_mean": 0.0029095506033627316, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 893.0, "completions/mean_length": 1060.4375, "completions/mean_terminated_length": 755.5714721679688, "completions/min_length": 635.0, "completions/min_terminated_length": 635.0, "entropy": 0.07811647234484553, "epoch": 7.285714285714286, "frac_reward_zero_std": 0.0, "grad_norm": 0.015395078808069229, "learning_rate": 2.7346938775510207e-06, "loss": -0.0262, "num_tokens": 18355993.0, "reward": 0.7475336790084839, "reward_std": 0.05309321731328964, "rewards/reward_commands_completeness/mean": 0.14124999940395355, "rewards/reward_commands_completeness/std": 0.029636692255735397, "rewards/reward_commands_correctness/mean": 0.06749999523162842, "rewards/reward_commands_correctness/std": 0.013416408561170101, "rewards/reward_diversity/mean": 0.1794086992740631, "rewards/reward_diversity/std": 0.01421035174280405, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.17249999940395355, "rewards/reward_solution_effectiveness/std": 0.06244997680187225, "sampling/importance_sampling_ratio/max": 0.7282242774963379, "sampling/importance_sampling_ratio/mean": 0.10589583218097687, "sampling/importance_sampling_ratio/min": 4.888411181923347e-13, "sampling/sampling_logp_difference/max": 21.31302833557129, "sampling/sampling_logp_difference/mean": 0.019927475601434708, "step": 357, "step_time": 386.29951436445117 }, { "clip_ratio/high_max": 0.002129286716808565, "clip_ratio/high_mean": 0.002129286716808565, "clip_ratio/low_mean": 0.00179919138099649, "clip_ratio/low_min": 0.00179919138099649, "clip_ratio/region_mean": 0.003928478068701224, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 3015.0, "completions/mean_length": 1275.125, "completions/mean_terminated_length": 872.1428833007812, "completions/min_length": 608.0, "completions/min_terminated_length": 608.0, "entropy": 0.07560349209234118, "epoch": 7.3061224489795915, "frac_reward_zero_std": 0.0, "grad_norm": 0.046259041875600815, "learning_rate": 2.7142857142857144e-06, "loss": -0.0949, "num_tokens": 18399691.0, "reward": 0.7242361307144165, "reward_std": 0.1468225121498108, "rewards/reward_commands_completeness/mean": 0.13499999046325684, "rewards/reward_commands_completeness/std": 0.036878179758787155, "rewards/reward_commands_correctness/mean": 0.07625000178813934, "rewards/reward_commands_correctness/std": 0.023345237597823143, "rewards/reward_diversity/mean": 0.16423609852790833, "rewards/reward_diversity/std": 0.039955802261829376, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.019989583641290665, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.05335025489330292, "sampling/importance_sampling_ratio/max": 2.8361315727233887, "sampling/importance_sampling_ratio/mean": 0.20941036939620972, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.910055160522461, "sampling/sampling_logp_difference/mean": 0.016209162771701813, "step": 358, "step_time": 273.70541178807616 }, { "clip_ratio/high_max": 0.0032655670074746013, "clip_ratio/high_mean": 0.0032655670074746013, "clip_ratio/low_mean": 0.0012375373335089535, "clip_ratio/low_min": 0.0012375373335089535, "clip_ratio/region_mean": 0.004503104333707597, "completions/clipped_ratio": 0.0, "completions/max_length": 769.0, "completions/max_terminated_length": 769.0, "completions/mean_length": 718.0, "completions/mean_terminated_length": 718.0, "completions/min_length": 613.0, "completions/min_terminated_length": 613.0, "entropy": 0.1116919107735157, "epoch": 7.326530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.012577002868056297, "learning_rate": 2.6938775510204086e-06, "loss": 0.0011, "num_tokens": 18457987.0, "reward": 0.7360416650772095, "reward_std": 0.11387743055820465, "rewards/reward_commands_completeness/mean": 0.13499999046325684, "rewards/reward_commands_completeness/std": 0.037594325840473175, "rewards/reward_commands_correctness/mean": 0.06812499463558197, "rewards/reward_commands_correctness/std": 0.020726392045617104, "rewards/reward_diversity/mean": 0.19166666269302368, "rewards/reward_diversity/std": 0.008255191147327423, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.019989583641290665, "rewards/reward_solution_effectiveness/mean": 0.16312500834465027, "rewards/reward_solution_effectiveness/std": 0.058959729969501495, "sampling/importance_sampling_ratio/max": 0.16560378670692444, "sampling/importance_sampling_ratio/mean": 0.041546326130628586, "sampling/importance_sampling_ratio/min": 6.368748927343404e-06, "sampling/sampling_logp_difference/max": 2.192018508911133, "sampling/sampling_logp_difference/mean": 0.02909262292087078, "step": 359, "step_time": 202.2416657488793 }, { "clip_ratio/high_max": 0.0027616310253506526, "clip_ratio/high_mean": 0.0027616310253506526, "clip_ratio/low_mean": 0.0015319535305025056, "clip_ratio/low_min": 0.0015319535305025056, "clip_ratio/region_mean": 0.0042935845121974126, "completions/clipped_ratio": 0.0, "completions/max_length": 927.0, "completions/max_terminated_length": 927.0, "completions/mean_length": 725.8125, "completions/mean_terminated_length": 725.8125, "completions/min_length": 589.0, "completions/min_terminated_length": 589.0, "entropy": 0.10229064244776964, "epoch": 7.346938775510204, "frac_reward_zero_std": 0.0, "grad_norm": 0.003386472584679723, "learning_rate": 2.6734693877551023e-06, "loss": -0.0016, "num_tokens": 18500912.0, "reward": 0.7057812213897705, "reward_std": 0.10663825273513794, "rewards/reward_commands_completeness/mean": 0.1274999976158142, "rewards/reward_commands_completeness/std": 0.04494441673159599, "rewards/reward_commands_correctness/mean": 0.07562500238418579, "rewards/reward_commands_correctness/std": 0.021592825651168823, "rewards/reward_diversity/mean": 0.17140623927116394, "rewards/reward_diversity/std": 0.04307800903916359, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07999999821186066, "rewards/reward_problem_validity/std": 0.019321836531162262, "rewards/reward_solution_effectiveness/mean": 0.1574999988079071, "rewards/reward_solution_effectiveness/std": 0.08234075456857681, "sampling/importance_sampling_ratio/max": 0.05990276113152504, "sampling/importance_sampling_ratio/mean": 0.020157964900135994, "sampling/importance_sampling_ratio/min": 1.2586748425746919e-06, "sampling/sampling_logp_difference/max": 3.30530047416687, "sampling/sampling_logp_difference/mean": 0.030193418264389038, "step": 360, "step_time": 135.15757500566542 }, { "clip_ratio/high_max": 0.0036423251149244606, "clip_ratio/high_mean": 0.0036423251149244606, "clip_ratio/low_mean": 0.0007372155960183591, "clip_ratio/low_min": 0.0007372155960183591, "clip_ratio/region_mean": 0.00437954071094282, "completions/clipped_ratio": 0.0, "completions/max_length": 1030.0, "completions/max_terminated_length": 1030.0, "completions/mean_length": 866.9375, "completions/mean_terminated_length": 866.9375, "completions/min_length": 702.0, "completions/min_terminated_length": 702.0, "entropy": 0.08878780296072364, "epoch": 7.36734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.004960460588335991, "learning_rate": 2.6530612244897964e-06, "loss": -0.0066, "num_tokens": 18535015.0, "reward": 0.7303171157836914, "reward_std": 0.11685595661401749, "rewards/reward_commands_completeness/mean": 0.13874998688697815, "rewards/reward_commands_completeness/std": 0.0403112918138504, "rewards/reward_commands_correctness/mean": 0.06937500089406967, "rewards/reward_commands_correctness/std": 0.0201556459069252, "rewards/reward_diversity/mean": 0.16344211995601654, "rewards/reward_diversity/std": 0.04365392029285431, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08312499523162842, "rewards/reward_problem_validity/std": 0.01990603096783161, "rewards/reward_solution_effectiveness/mean": 0.18187499046325684, "rewards/reward_solution_effectiveness/std": 0.06794299185276031, "sampling/importance_sampling_ratio/max": 0.1060020849108696, "sampling/importance_sampling_ratio/mean": 0.008034744299948215, "sampling/importance_sampling_ratio/min": 3.671888097397935e-13, "sampling/sampling_logp_difference/max": 18.173946380615234, "sampling/sampling_logp_difference/mean": 0.033649247139692307, "step": 361, "step_time": 118.96129309013486 }, { "clip_ratio/high_max": 0.002769599675957579, "clip_ratio/high_mean": 0.002769599675957579, "clip_ratio/low_mean": 0.0007679955851926934, "clip_ratio/low_min": 0.0007679955851926934, "clip_ratio/region_mean": 0.003537595275702188, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 928.8125, "completions/mean_terminated_length": 717.6666870117188, "completions/min_length": 638.0, "completions/min_terminated_length": 638.0, "entropy": 0.09412513114511967, "epoch": 7.387755102040816, "frac_reward_zero_std": 0.0, "grad_norm": 0.017428234219551086, "learning_rate": 2.63265306122449e-06, "loss": -0.0303, "num_tokens": 18578604.0, "reward": 0.7539174556732178, "reward_std": 0.07235035300254822, "rewards/reward_commands_completeness/mean": 0.13750000298023224, "rewards/reward_commands_completeness/std": 0.030000003054738045, "rewards/reward_commands_correctness/mean": 0.08187499642372131, "rewards/reward_commands_correctness/std": 0.019737867638468742, "rewards/reward_diversity/mean": 0.1751674860715866, "rewards/reward_diversity/std": 0.01745370589196682, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.0047871386632323265, "rewards/reward_solution_effectiveness/mean": 0.17250001430511475, "rewards/reward_solution_effectiveness/std": 0.060497935861349106, "sampling/importance_sampling_ratio/max": 0.6341201066970825, "sampling/importance_sampling_ratio/mean": 0.042614784091711044, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.987644195556641, "sampling/sampling_logp_difference/mean": 0.022164154797792435, "step": 362, "step_time": 298.2614269964397 }, { "clip_ratio/high_max": 0.0032535872742300853, "clip_ratio/high_mean": 0.0032535872742300853, "clip_ratio/low_mean": 0.001396705411025323, "clip_ratio/low_min": 0.001396705411025323, "clip_ratio/region_mean": 0.004650292685255408, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 806.0, "completions/mean_length": 899.8125, "completions/mean_terminated_length": 686.7333984375, "completions/min_length": 579.0, "completions/min_terminated_length": 579.0, "entropy": 0.09624654427170753, "epoch": 7.408163265306122, "frac_reward_zero_std": 0.0, "grad_norm": 0.054792940616607666, "learning_rate": 2.6122448979591842e-06, "loss": -0.0713, "num_tokens": 18613433.0, "reward": 0.6866421699523926, "reward_std": 0.13657741248607635, "rewards/reward_commands_completeness/mean": 0.1237500011920929, "rewards/reward_commands_completeness/std": 0.04455333948135376, "rewards/reward_commands_correctness/mean": 0.06812500208616257, "rewards/reward_commands_correctness/std": 0.01939716562628746, "rewards/reward_diversity/mean": 0.17414215207099915, "rewards/reward_diversity/std": 0.04304029047489166, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08250000327825546, "rewards/reward_problem_validity/std": 0.019832637161016464, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.07201562076807022, "sampling/importance_sampling_ratio/max": 1.8728857040405273, "sampling/importance_sampling_ratio/mean": 0.12833984196186066, "sampling/importance_sampling_ratio/min": 1.560944838274736e-06, "sampling/sampling_logp_difference/max": 5.321249008178711, "sampling/sampling_logp_difference/mean": 0.024311231449246407, "step": 363, "step_time": 258.2423755656928 }, { "clip_ratio/high_max": 0.0021891445794608444, "clip_ratio/high_mean": 0.0021891445794608444, "clip_ratio/low_mean": 0.0016750302747823298, "clip_ratio/low_min": 0.0016750302747823298, "clip_ratio/region_mean": 0.0038641748833470047, "completions/clipped_ratio": 0.0, "completions/max_length": 867.0, "completions/max_terminated_length": 867.0, "completions/mean_length": 724.1875, "completions/mean_terminated_length": 724.1875, "completions/min_length": 640.0, "completions/min_terminated_length": 640.0, "entropy": 0.08010216522961855, "epoch": 7.428571428571429, "frac_reward_zero_std": 0.0, "grad_norm": 0.040052350610494614, "learning_rate": 2.591836734693878e-06, "loss": -0.033, "num_tokens": 18645516.0, "reward": 0.7789880633354187, "reward_std": 0.06775350123643875, "rewards/reward_commands_completeness/mean": 0.15125000476837158, "rewards/reward_commands_completeness/std": 0.023057900369167328, "rewards/reward_commands_correctness/mean": 0.07750000059604645, "rewards/reward_commands_correctness/std": 0.017320509999990463, "rewards/reward_diversity/mean": 0.17148809134960175, "rewards/reward_diversity/std": 0.019238675013184547, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08937500417232513, "rewards/reward_problem_validity/std": 0.0025000013411045074, "rewards/reward_solution_effectiveness/mean": 0.18937501311302185, "rewards/reward_solution_effectiveness/std": 0.05105144530534744, "sampling/importance_sampling_ratio/max": 0.7809209823608398, "sampling/importance_sampling_ratio/mean": 0.07527413964271545, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 18.21194839477539, "sampling/sampling_logp_difference/mean": 0.024894770234823227, "step": 364, "step_time": 113.30998464673758 }, { "clip_ratio/high_max": 0.0018836176313925534, "clip_ratio/high_mean": 0.0018836176313925534, "clip_ratio/low_mean": 0.001741853862768039, "clip_ratio/low_min": 0.001741853862768039, "clip_ratio/region_mean": 0.0036254714941605926, "completions/clipped_ratio": 0.0, "completions/max_length": 1122.0, "completions/max_terminated_length": 1122.0, "completions/mean_length": 803.0, "completions/mean_terminated_length": 803.0, "completions/min_length": 672.0, "completions/min_terminated_length": 672.0, "entropy": 0.09564095921814442, "epoch": 7.448979591836735, "frac_reward_zero_std": 0.0, "grad_norm": 0.025485169142484665, "learning_rate": 2.571428571428571e-06, "loss": -0.0441, "num_tokens": 18682760.0, "reward": 0.759595513343811, "reward_std": 0.05425434559583664, "rewards/reward_commands_completeness/mean": 0.14000000059604645, "rewards/reward_commands_completeness/std": 0.03098387084901333, "rewards/reward_commands_correctness/mean": 0.07500000298023224, "rewards/reward_commands_correctness/std": 0.013662601821124554, "rewards/reward_diversity/mean": 0.17334555089473724, "rewards/reward_diversity/std": 0.014265070669353008, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.18562500178813934, "rewards/reward_solution_effectiveness/std": 0.05819149687886238, "sampling/importance_sampling_ratio/max": 1.1924890279769897, "sampling/importance_sampling_ratio/mean": 0.09671057760715485, "sampling/importance_sampling_ratio/min": 2.068532438670445e-07, "sampling/sampling_logp_difference/max": 3.6445817947387695, "sampling/sampling_logp_difference/mean": 0.027134913951158524, "step": 365, "step_time": 137.0457143187523 }, { "clip_ratio/high_max": 0.002973620328702964, "clip_ratio/high_mean": 0.002973620328702964, "clip_ratio/low_mean": 0.0013512396399164572, "clip_ratio/low_min": 0.0013512396399164572, "clip_ratio/region_mean": 0.004324859939515591, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 846.0, "completions/mean_length": 979.5, "completions/mean_terminated_length": 771.7333984375, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.1104377843439579, "epoch": 7.469387755102041, "frac_reward_zero_std": 0.0, "grad_norm": 0.08026380091905594, "learning_rate": 2.5510204081632657e-06, "loss": -0.1694, "num_tokens": 18723004.0, "reward": 0.7292242050170898, "reward_std": 0.07395477592945099, "rewards/reward_commands_completeness/mean": 0.12999999523162842, "rewards/reward_commands_completeness/std": 0.03651484102010727, "rewards/reward_commands_correctness/mean": 0.07437499612569809, "rewards/reward_commands_correctness/std": 0.017876895144581795, "rewards/reward_diversity/mean": 0.18609923124313354, "rewards/reward_diversity/std": 0.017130674794316292, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.005163980647921562, "rewards/reward_solution_effectiveness/mean": 0.1537500023841858, "rewards/reward_solution_effectiveness/std": 0.07003570348024368, "sampling/importance_sampling_ratio/max": 2.1447296142578125, "sampling/importance_sampling_ratio/mean": 0.22148063778877258, "sampling/importance_sampling_ratio/min": 6.841837651982985e-10, "sampling/sampling_logp_difference/max": 5.573967933654785, "sampling/sampling_logp_difference/mean": 0.02383190393447876, "step": 366, "step_time": 276.8509143013507 }, { "clip_ratio/high_max": 0.00230578712944407, "clip_ratio/high_mean": 0.00230578712944407, "clip_ratio/low_mean": 0.002321960469998885, "clip_ratio/low_min": 0.002321960469998885, "clip_ratio/region_mean": 0.004627747664926574, "completions/clipped_ratio": 0.0, "completions/max_length": 934.0, "completions/max_terminated_length": 934.0, "completions/mean_length": 776.4375, "completions/mean_terminated_length": 776.4375, "completions/min_length": 635.0, "completions/min_terminated_length": 635.0, "entropy": 0.0998386237770319, "epoch": 7.489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.007538960315287113, "learning_rate": 2.530612244897959e-06, "loss": -0.0205, "num_tokens": 18762663.0, "reward": 0.714538037776947, "reward_std": 0.09155020117759705, "rewards/reward_commands_completeness/mean": 0.1262499988079071, "rewards/reward_commands_completeness/std": 0.04047633334994316, "rewards/reward_commands_correctness/mean": 0.07062499970197678, "rewards/reward_commands_correctness/std": 0.016520190984010696, "rewards/reward_diversity/mean": 0.19078804552555084, "rewards/reward_diversity/std": 0.010382414795458317, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.14250001311302185, "rewards/reward_solution_effectiveness/std": 0.0714142844080925, "sampling/importance_sampling_ratio/max": 0.14546364545822144, "sampling/importance_sampling_ratio/mean": 0.030181461945176125, "sampling/importance_sampling_ratio/min": 3.2439631922898116e-06, "sampling/sampling_logp_difference/max": 3.6566076278686523, "sampling/sampling_logp_difference/mean": 0.028211669996380806, "step": 367, "step_time": 124.5349212884903 }, { "clip_ratio/high_max": 0.003809617119259201, "clip_ratio/high_mean": 0.003809617119259201, "clip_ratio/low_mean": 0.0008910027099773288, "clip_ratio/low_min": 0.0008910027099773288, "clip_ratio/region_mean": 0.004700619800132699, "completions/clipped_ratio": 0.0, "completions/max_length": 995.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 782.5625, "completions/mean_terminated_length": 782.5625, "completions/min_length": 649.0, "completions/min_terminated_length": 649.0, "entropy": 0.10712667927145958, "epoch": 7.510204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.024683428928256035, "learning_rate": 2.5102040816326536e-06, "loss": -0.0602, "num_tokens": 18812840.0, "reward": 0.7163480520248413, "reward_std": 0.17224471271038055, "rewards/reward_commands_completeness/mean": 0.13625000417232513, "rewards/reward_commands_completeness/std": 0.04272002354264259, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.02250000089406967, "rewards/reward_diversity/mean": 0.1763480305671692, "rewards/reward_diversity/std": 0.04471224173903465, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.16312499344348907, "rewards/reward_solution_effectiveness/std": 0.07180703431367874, "sampling/importance_sampling_ratio/max": 0.8744932413101196, "sampling/importance_sampling_ratio/mean": 0.10729032754898071, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.316556930541992, "sampling/sampling_logp_difference/mean": 0.02793123386800289, "step": 368, "step_time": 155.55907196179032 }, { "clip_ratio/high_max": 0.0020137338069616817, "clip_ratio/high_mean": 0.0020137338069616817, "clip_ratio/low_mean": 0.00111778148857411, "clip_ratio/low_min": 0.00111778148857411, "clip_ratio/region_mean": 0.0031315153173636645, "completions/clipped_ratio": 0.0, "completions/max_length": 1021.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 767.9375, "completions/mean_terminated_length": 767.9375, "completions/min_length": 625.0, "completions/min_terminated_length": 625.0, "entropy": 0.09050503000617027, "epoch": 7.530612244897959, "frac_reward_zero_std": 0.0, "grad_norm": 0.08479863405227661, "learning_rate": 2.489795918367347e-06, "loss": -0.2019, "num_tokens": 18850887.0, "reward": 0.7622569799423218, "reward_std": 0.15009397268295288, "rewards/reward_commands_completeness/mean": 0.14374999701976776, "rewards/reward_commands_completeness/std": 0.040145572274923325, "rewards/reward_commands_correctness/mean": 0.07562500238418579, "rewards/reward_commands_correctness/std": 0.023935679346323013, "rewards/reward_diversity/mean": 0.17975693941116333, "rewards/reward_diversity/std": 0.014278564602136612, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.1875, "rewards/reward_solution_effectiveness/std": 0.07389181107282639, "sampling/importance_sampling_ratio/max": 2.4803788661956787, "sampling/importance_sampling_ratio/mean": 0.2798113524913788, "sampling/importance_sampling_ratio/min": 5.112437695409255e-10, "sampling/sampling_logp_difference/max": 15.623775482177734, "sampling/sampling_logp_difference/mean": 0.027288567274808884, "step": 369, "step_time": 137.32288685254753 }, { "clip_ratio/high_max": 0.0032360500626964495, "clip_ratio/high_mean": 0.0032360500626964495, "clip_ratio/low_mean": 0.0006552792110596783, "clip_ratio/low_min": 0.0006552792110596783, "clip_ratio/region_mean": 0.003891329324687831, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 935.0, "completions/mean_length": 995.125, "completions/mean_terminated_length": 788.4000244140625, "completions/min_length": 669.0, "completions/min_terminated_length": 669.0, "entropy": 0.08570791594684124, "epoch": 7.551020408163265, "frac_reward_zero_std": 0.0, "grad_norm": 0.01920890435576439, "learning_rate": 2.469387755102041e-06, "loss": -0.0217, "num_tokens": 18913565.0, "reward": 0.7440625429153442, "reward_std": 0.14566393196582794, "rewards/reward_commands_completeness/mean": 0.14249998331069946, "rewards/reward_commands_completeness/std": 0.04187282919883728, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.02160247042775154, "rewards/reward_diversity/mean": 0.16906249523162842, "rewards/reward_diversity/std": 0.042119622230529785, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08312500268220901, "rewards/reward_problem_validity/std": 0.01990603096783161, "rewards/reward_solution_effectiveness/mean": 0.18562500178813934, "rewards/reward_solution_effectiveness/std": 0.06217917427420616, "sampling/importance_sampling_ratio/max": 0.6876504421234131, "sampling/importance_sampling_ratio/mean": 0.06306039541959763, "sampling/importance_sampling_ratio/min": 7.411140541080385e-05, "sampling/sampling_logp_difference/max": 2.3960719108581543, "sampling/sampling_logp_difference/mean": 0.02038518711924553, "step": 370, "step_time": 388.4596825912595 }, { "clip_ratio/high_max": 0.001721432534395717, "clip_ratio/high_mean": 0.001721432534395717, "clip_ratio/low_mean": 0.000597492340602912, "clip_ratio/low_min": 0.000597492340602912, "clip_ratio/region_mean": 0.0023189248458947986, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1286.0, "completions/mean_length": 991.375, "completions/mean_terminated_length": 784.4000244140625, "completions/min_length": 628.0, "completions/min_terminated_length": 628.0, "entropy": 0.07412581518292427, "epoch": 7.571428571428571, "frac_reward_zero_std": 0.0, "grad_norm": 0.010279865004122257, "learning_rate": 2.4489795918367347e-06, "loss": -0.015, "num_tokens": 18959535.0, "reward": 0.7461857795715332, "reward_std": 0.08590222895145416, "rewards/reward_commands_completeness/mean": 0.14250001311302185, "rewards/reward_commands_completeness/std": 0.03492850065231323, "rewards/reward_commands_correctness/mean": 0.07625000178813934, "rewards/reward_commands_correctness/std": 0.018211718648672104, "rewards/reward_diversity/mean": 0.15993580222129822, "rewards/reward_diversity/std": 0.028559250757098198, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.18187502026557922, "rewards/reward_solution_effectiveness/std": 0.0730496421456337, "sampling/importance_sampling_ratio/max": 0.8225593566894531, "sampling/importance_sampling_ratio/mean": 0.07711614668369293, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.226767539978027, "sampling/sampling_logp_difference/mean": 0.01970679685473442, "step": 371, "step_time": 282.9589295666665 }, { "clip_ratio/high_max": 0.002164043893571943, "clip_ratio/high_mean": 0.002164043893571943, "clip_ratio/low_mean": 0.0015574541030218825, "clip_ratio/low_min": 0.0015574541030218825, "clip_ratio/region_mean": 0.0037214980548014864, "completions/clipped_ratio": 0.0, "completions/max_length": 1041.0, "completions/max_terminated_length": 1041.0, "completions/mean_length": 751.5625, "completions/mean_terminated_length": 751.5625, "completions/min_length": 610.0, "completions/min_terminated_length": 610.0, "entropy": 0.09704652521759272, "epoch": 7.591836734693878, "frac_reward_zero_std": 0.0, "grad_norm": 0.01818113960325718, "learning_rate": 2.428571428571429e-06, "loss": -0.0454, "num_tokens": 18999468.0, "reward": 0.7816179394721985, "reward_std": 0.06408318877220154, "rewards/reward_commands_completeness/mean": 0.14750000834465027, "rewards/reward_commands_completeness/std": 0.030876100063323975, "rewards/reward_commands_correctness/mean": 0.078125, "rewards/reward_commands_correctness/std": 0.01276388205587864, "rewards/reward_diversity/mean": 0.17661792039871216, "rewards/reward_diversity/std": 0.01832813210785389, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.1912499964237213, "rewards/reward_solution_effectiveness/std": 0.06375735998153687, "sampling/importance_sampling_ratio/max": 0.7615633606910706, "sampling/importance_sampling_ratio/mean": 0.07994315773248672, "sampling/importance_sampling_ratio/min": 4.684322266257368e-05, "sampling/sampling_logp_difference/max": 3.2713515758514404, "sampling/sampling_logp_difference/mean": 0.02600639872252941, "step": 372, "step_time": 138.18785087578 }, { "clip_ratio/high_max": 0.002618158279801719, "clip_ratio/high_mean": 0.002618158279801719, "clip_ratio/low_mean": 0.001204982261697296, "clip_ratio/low_min": 0.001204982261697296, "clip_ratio/region_mean": 0.0038231405269470997, "completions/clipped_ratio": 0.0, "completions/max_length": 868.0, "completions/max_terminated_length": 868.0, "completions/mean_length": 749.75, "completions/mean_terminated_length": 749.75, "completions/min_length": 673.0, "completions/min_terminated_length": 673.0, "entropy": 0.0915583735331893, "epoch": 7.612244897959184, "frac_reward_zero_std": 0.0, "grad_norm": 0.04376206547021866, "learning_rate": 2.4081632653061225e-06, "loss": 0.0818, "num_tokens": 19045800.0, "reward": 0.7229963541030884, "reward_std": 0.06332787871360779, "rewards/reward_commands_completeness/mean": 0.13124999403953552, "rewards/reward_commands_completeness/std": 0.03575379028916359, "rewards/reward_commands_correctness/mean": 0.07249999791383743, "rewards/reward_commands_correctness/std": 0.020493902266025543, "rewards/reward_diversity/mean": 0.17237132787704468, "rewards/reward_diversity/std": 0.020163092762231827, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.16124999523162842, "rewards/reward_solution_effectiveness/std": 0.06652067601680756, "sampling/importance_sampling_ratio/max": 1.9183552265167236, "sampling/importance_sampling_ratio/mean": 0.14872705936431885, "sampling/importance_sampling_ratio/min": 9.556043778502499e-07, "sampling/sampling_logp_difference/max": 9.439857482910156, "sampling/sampling_logp_difference/mean": 0.026425892487168312, "step": 373, "step_time": 154.29334705509245 }, { "clip_ratio/high_max": 0.0017057987861335278, "clip_ratio/high_mean": 0.0017057987861335278, "clip_ratio/low_mean": 0.0018610417610034347, "clip_ratio/low_min": 0.0018610417610034347, "clip_ratio/region_mean": 0.003566840576240793, "completions/clipped_ratio": 0.0, "completions/max_length": 881.0, "completions/max_terminated_length": 881.0, "completions/mean_length": 756.0, "completions/mean_terminated_length": 756.0, "completions/min_length": 679.0, "completions/min_terminated_length": 679.0, "entropy": 0.09453888423740864, "epoch": 7.63265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.010790609754621983, "learning_rate": 2.3877551020408166e-06, "loss": -0.0238, "num_tokens": 19076908.0, "reward": 0.7618566155433655, "reward_std": 0.10427466779947281, "rewards/reward_commands_completeness/mean": 0.14624999463558197, "rewards/reward_commands_completeness/std": 0.03913651406764984, "rewards/reward_commands_correctness/mean": 0.0768750011920929, "rewards/reward_commands_correctness/std": 0.0230126790702343, "rewards/reward_diversity/mean": 0.1756066232919693, "rewards/reward_diversity/std": 0.011340923607349396, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08375000208616257, "rewards/reward_problem_validity/std": 0.019958291202783585, "rewards/reward_solution_effectiveness/mean": 0.18562500178813934, "rewards/reward_solution_effectiveness/std": 0.056091442704200745, "sampling/importance_sampling_ratio/max": 0.8290479779243469, "sampling/importance_sampling_ratio/mean": 0.07268498837947845, "sampling/importance_sampling_ratio/min": 4.3685458028797086e-11, "sampling/sampling_logp_difference/max": 17.6229305267334, "sampling/sampling_logp_difference/mean": 0.028504101559519768, "step": 374, "step_time": 103.59769313782454 }, { "clip_ratio/high_max": 0.0011848046196973883, "clip_ratio/high_mean": 0.0011848046196973883, "clip_ratio/low_mean": 0.001926851189637091, "clip_ratio/low_min": 0.001926851189637091, "clip_ratio/region_mean": 0.003111655780230649, "completions/clipped_ratio": 0.0, "completions/max_length": 787.0, "completions/max_terminated_length": 787.0, "completions/mean_length": 715.5, "completions/mean_terminated_length": 715.5, "completions/min_length": 602.0, "completions/min_terminated_length": 602.0, "entropy": 0.1008687848225236, "epoch": 7.653061224489796, "frac_reward_zero_std": 0.0, "grad_norm": 0.016727205365896225, "learning_rate": 2.3673469387755103e-06, "loss": -0.0011, "num_tokens": 19116692.0, "reward": 0.5969791412353516, "reward_std": 0.13044321537017822, "rewards/reward_commands_completeness/mean": 0.10874999314546585, "rewards/reward_commands_completeness/std": 0.05162363499403, "rewards/reward_commands_correctness/mean": 0.05812499672174454, "rewards/reward_commands_correctness/std": 0.026638008654117584, "rewards/reward_diversity/mean": 0.14760416746139526, "rewards/reward_diversity/std": 0.06511662155389786, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07187500596046448, "rewards/reward_problem_validity/std": 0.03103090077638626, "rewards/reward_solution_effectiveness/mean": 0.12937499582767487, "rewards/reward_solution_effectiveness/std": 0.07325468957424164, "sampling/importance_sampling_ratio/max": 0.2786717116832733, "sampling/importance_sampling_ratio/mean": 0.0376841276884079, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2241106033325195, "sampling/sampling_logp_difference/mean": 0.025454336777329445, "step": 375, "step_time": 116.26588817313313 }, { "clip_ratio/high_max": 0.0038049464637879282, "clip_ratio/high_mean": 0.0038049464637879282, "clip_ratio/low_mean": 0.0007702297370997258, "clip_ratio/low_min": 0.0007702297370997258, "clip_ratio/region_mean": 0.004575176200887654, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 792.0, "completions/mean_length": 1148.125, "completions/mean_terminated_length": 727.0000610351562, "completions/min_length": 653.0, "completions/min_terminated_length": 653.0, "entropy": 0.08659524330869317, "epoch": 7.673469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.06219640001654625, "learning_rate": 2.3469387755102044e-06, "loss": 0.0681, "num_tokens": 19161282.0, "reward": 0.7044791579246521, "reward_std": 0.09517397731542587, "rewards/reward_commands_completeness/mean": 0.13374999165534973, "rewards/reward_commands_completeness/std": 0.0460253581404686, "rewards/reward_commands_correctness/mean": 0.06687499582767487, "rewards/reward_commands_correctness/std": 0.0230126790702343, "rewards/reward_diversity/mean": 0.16322916746139526, "rewards/reward_diversity/std": 0.044941194355487823, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07999999821186066, "rewards/reward_problem_validity/std": 0.01932183839380741, "rewards/reward_solution_effectiveness/mean": 0.16687500476837158, "rewards/reward_solution_effectiveness/std": 0.06925496459007263, "sampling/importance_sampling_ratio/max": 1.1946157217025757, "sampling/importance_sampling_ratio/mean": 0.09919554740190506, "sampling/importance_sampling_ratio/min": 0.0001692012883722782, "sampling/sampling_logp_difference/max": 5.557572364807129, "sampling/sampling_logp_difference/mean": 0.01789020746946335, "step": 376, "step_time": 277.6168970838189 }, { "clip_ratio/high_max": 0.0027512859669514, "clip_ratio/high_mean": 0.0027512859669514, "clip_ratio/low_mean": 0.0011505747606861405, "clip_ratio/low_min": 0.0011505747606861405, "clip_ratio/region_mean": 0.0039018607058096677, "completions/clipped_ratio": 0.0, "completions/max_length": 882.0, "completions/max_terminated_length": 882.0, "completions/mean_length": 750.875, "completions/mean_terminated_length": 750.875, "completions/min_length": 615.0, "completions/min_terminated_length": 615.0, "entropy": 0.0903065437451005, "epoch": 7.6938775510204085, "frac_reward_zero_std": 0.0, "grad_norm": 0.029308335855603218, "learning_rate": 2.326530612244898e-06, "loss": -0.0457, "num_tokens": 19204568.0, "reward": 0.6653993129730225, "reward_std": 0.20314277708530426, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.04816638305783272, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.02606882154941559, "rewards/reward_diversity/mean": 0.16289930045604706, "rewards/reward_diversity/std": 0.05711307376623154, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07625000178813934, "rewards/reward_problem_validity/std": 0.026299558579921722, "rewards/reward_solution_effectiveness/mean": 0.14812499284744263, "rewards/reward_solution_effectiveness/std": 0.06794299185276031, "sampling/importance_sampling_ratio/max": 0.8683831095695496, "sampling/importance_sampling_ratio/mean": 0.09483170509338379, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.77910041809082, "sampling/sampling_logp_difference/mean": 0.026575526222586632, "step": 377, "step_time": 128.63946434669197 }, { "clip_ratio/high_max": 0.0029284022020874545, "clip_ratio/high_mean": 0.0029284022020874545, "clip_ratio/low_mean": 0.0012605422452907078, "clip_ratio/low_min": 0.0012605422452907078, "clip_ratio/region_mean": 0.004188944512861781, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 836.0, "completions/mean_length": 951.1875, "completions/mean_terminated_length": 741.5333862304688, "completions/min_length": 665.0, "completions/min_terminated_length": 665.0, "entropy": 0.08267838461324573, "epoch": 7.714285714285714, "frac_reward_zero_std": 0.0, "grad_norm": 0.00492048216983676, "learning_rate": 2.306122448979592e-06, "loss": -0.0075, "num_tokens": 19242691.0, "reward": 0.7622855305671692, "reward_std": 0.09717118740081787, "rewards/reward_commands_completeness/mean": 0.14374999701976776, "rewards/reward_commands_completeness/std": 0.02753785438835621, "rewards/reward_commands_correctness/mean": 0.07124999910593033, "rewards/reward_commands_correctness/std": 0.021252453327178955, "rewards/reward_diversity/mean": 0.1747855395078659, "rewards/reward_diversity/std": 0.01834998093545437, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08875000476837158, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.1837500035762787, "rewards/reward_solution_effectiveness/std": 0.06280923634767532, "sampling/importance_sampling_ratio/max": 0.23389890789985657, "sampling/importance_sampling_ratio/mean": 0.025907672941684723, "sampling/importance_sampling_ratio/min": 1.0602233487588819e-05, "sampling/sampling_logp_difference/max": 5.548621654510498, "sampling/sampling_logp_difference/mean": 0.019978078082203865, "step": 378, "step_time": 263.38554656505585 }, { "clip_ratio/high_max": 0.0031948429968906567, "clip_ratio/high_mean": 0.0031948429968906567, "clip_ratio/low_mean": 0.0014140490602585487, "clip_ratio/low_min": 0.0014140490602585487, "clip_ratio/region_mean": 0.004608892078977078, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 877.0, "completions/mean_length": 960.125, "completions/mean_terminated_length": 751.0667114257812, "completions/min_length": 631.0, "completions/min_terminated_length": 631.0, "entropy": 0.09664122248068452, "epoch": 7.73469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.026501866057515144, "learning_rate": 2.285714285714286e-06, "loss": 0.0065, "num_tokens": 19286701.0, "reward": 0.6402021646499634, "reward_std": 0.13486900925636292, "rewards/reward_commands_completeness/mean": 0.11374999582767487, "rewards/reward_commands_completeness/std": 0.04047633334994316, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.02548692561686039, "rewards/reward_diversity/mean": 0.17645220458507538, "rewards/reward_diversity/std": 0.04390418156981468, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.02151549980044365, "rewards/reward_solution_effectiveness/mean": 0.11625000089406967, "rewards/reward_solution_effectiveness/std": 0.06652067601680756, "sampling/importance_sampling_ratio/max": 0.6705601215362549, "sampling/importance_sampling_ratio/mean": 0.09784354269504547, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.803741455078125, "sampling/sampling_logp_difference/mean": 0.02063409797847271, "step": 379, "step_time": 274.30237629264593 }, { "clip_ratio/high_max": 0.001889840241346974, "clip_ratio/high_mean": 0.001889840241346974, "clip_ratio/low_mean": 0.002410514614894055, "clip_ratio/low_min": 0.002410514614894055, "clip_ratio/region_mean": 0.004300354834413156, "completions/clipped_ratio": 0.0, "completions/max_length": 1022.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 761.625, "completions/mean_terminated_length": 761.625, "completions/min_length": 620.0, "completions/min_terminated_length": 620.0, "entropy": 0.10382113419473171, "epoch": 7.755102040816326, "frac_reward_zero_std": 0.0, "grad_norm": 0.007031077519059181, "learning_rate": 2.2653061224489797e-06, "loss": 0.0079, "num_tokens": 19325611.0, "reward": 0.7395710945129395, "reward_std": 0.12973669171333313, "rewards/reward_commands_completeness/mean": 0.14249999821186066, "rewards/reward_commands_completeness/std": 0.037859391421079636, "rewards/reward_commands_correctness/mean": 0.07625000178813934, "rewards/reward_commands_correctness/std": 0.02500000223517418, "rewards/reward_diversity/mean": 0.16269607841968536, "rewards/reward_diversity/std": 0.04278995469212532, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.019989583641290665, "rewards/reward_solution_effectiveness/mean": 0.18000000715255737, "rewards/reward_solution_effectiveness/std": 0.05899152532219887, "sampling/importance_sampling_ratio/max": 0.15363943576812744, "sampling/importance_sampling_ratio/mean": 0.03804568946361542, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.7807323932647705, "sampling/sampling_logp_difference/mean": 0.025273608043789864, "step": 380, "step_time": 132.39175215922296 }, { "clip_ratio/high_max": 0.002757219655904919, "clip_ratio/high_mean": 0.002757219655904919, "clip_ratio/low_mean": 0.0006894640100654215, "clip_ratio/low_min": 0.0006894640100654215, "clip_ratio/region_mean": 0.0034466836805222556, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 797.0, "completions/mean_length": 897.375, "completions/mean_terminated_length": 684.1333618164062, "completions/min_length": 617.0, "completions/min_terminated_length": 617.0, "entropy": 0.0828881892375648, "epoch": 7.775510204081632, "frac_reward_zero_std": 0.0, "grad_norm": 0.03877919912338257, "learning_rate": 2.244897959183674e-06, "loss": -0.0777, "num_tokens": 19369373.0, "reward": 0.7905729413032532, "reward_std": 0.09233224391937256, "rewards/reward_commands_completeness/mean": 0.15000000596046448, "rewards/reward_commands_completeness/std": 0.03651484102010727, "rewards/reward_commands_correctness/mean": 0.08374999463558197, "rewards/reward_commands_correctness/std": 0.01586400717496872, "rewards/reward_diversity/mean": 0.16869792342185974, "rewards/reward_diversity/std": 0.02833818458020687, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.2006250023841858, "rewards/reward_solution_effectiveness/std": 0.07325468957424164, "sampling/importance_sampling_ratio/max": 1.2650535106658936, "sampling/importance_sampling_ratio/mean": 0.11875972151756287, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.570486068725586, "sampling/sampling_logp_difference/mean": 0.020011156797409058, "step": 381, "step_time": 278.8944112248719 }, { "clip_ratio/high_max": 0.002440647833282128, "clip_ratio/high_mean": 0.002440647833282128, "clip_ratio/low_mean": 0.0010648899115039967, "clip_ratio/low_min": 0.0010648899115039967, "clip_ratio/region_mean": 0.003505537722958252, "completions/clipped_ratio": 0.0, "completions/max_length": 837.0, "completions/max_terminated_length": 837.0, "completions/mean_length": 686.1875, "completions/mean_terminated_length": 686.1875, "completions/min_length": 602.0, "completions/min_terminated_length": 602.0, "entropy": 0.08778166864067316, "epoch": 7.795918367346939, "frac_reward_zero_std": 0.0, "grad_norm": 0.015855250880122185, "learning_rate": 2.2244897959183675e-06, "loss": -0.0025, "num_tokens": 19405012.0, "reward": 0.7929626703262329, "reward_std": 0.07549013942480087, "rewards/reward_commands_completeness/mean": 0.1525000035762787, "rewards/reward_commands_completeness/std": 0.032557643949985504, "rewards/reward_commands_correctness/mean": 0.08562500029802322, "rewards/reward_commands_correctness/std": 0.017500000074505806, "rewards/reward_diversity/mean": 0.16233761608600616, "rewards/reward_diversity/std": 0.0245205070823431, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.2043749988079071, "rewards/reward_solution_effectiveness/std": 0.06947122514247894, "sampling/importance_sampling_ratio/max": 0.6770644783973694, "sampling/importance_sampling_ratio/mean": 0.1014411672949791, "sampling/importance_sampling_ratio/min": 0.00011360784992575645, "sampling/sampling_logp_difference/max": 3.5254080295562744, "sampling/sampling_logp_difference/mean": 0.022437764331698418, "step": 382, "step_time": 121.18938753195107 }, { "clip_ratio/high_max": 0.0019700828488566913, "clip_ratio/high_mean": 0.0019700828488566913, "clip_ratio/low_mean": 0.0012324705967330374, "clip_ratio/low_min": 0.0012324705967330374, "clip_ratio/region_mean": 0.003202553416485898, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 795.0, "completions/mean_length": 939.625, "completions/mean_terminated_length": 729.2000122070312, "completions/min_length": 656.0, "completions/min_terminated_length": 656.0, "entropy": 0.09465849585831165, "epoch": 7.816326530612245, "frac_reward_zero_std": 0.0, "grad_norm": 0.005054810084402561, "learning_rate": 2.2040816326530616e-06, "loss": 0.0074, "num_tokens": 19448314.0, "reward": 0.8128137588500977, "reward_std": 0.037849754095077515, "rewards/reward_commands_completeness/mean": 0.1574999988079071, "rewards/reward_commands_completeness/std": 0.01914854533970356, "rewards/reward_commands_correctness/mean": 0.0768750011920929, "rewards/reward_commands_correctness/std": 0.01302241813391447, "rewards/reward_diversity/mean": 0.18531382083892822, "rewards/reward_diversity/std": 0.014027845114469528, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08875000476837158, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.2043749988079071, "rewards/reward_solution_effectiveness/std": 0.03669127821922302, "sampling/importance_sampling_ratio/max": 0.10187028348445892, "sampling/importance_sampling_ratio/mean": 0.014969603158533573, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 19.249420166015625, "sampling/sampling_logp_difference/mean": 0.022643981501460075, "step": 383, "step_time": 290.26467374525964 }, { "clip_ratio/high_max": 0.001923912946949713, "clip_ratio/high_mean": 0.001923912946949713, "clip_ratio/low_mean": 0.001621378809431917, "clip_ratio/low_min": 0.001621378809431917, "clip_ratio/region_mean": 0.003545291743648704, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 769.0, "completions/mean_length": 912.25, "completions/mean_terminated_length": 700.0000610351562, "completions/min_length": 596.0, "completions/min_terminated_length": 596.0, "entropy": 0.10273813083767891, "epoch": 7.836734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.0706053227186203, "learning_rate": 2.1836734693877553e-06, "loss": 0.0531, "num_tokens": 19483482.0, "reward": 0.7322396039962769, "reward_std": 0.05837945640087128, "rewards/reward_commands_completeness/mean": 0.12999999523162842, "rewards/reward_commands_completeness/std": 0.03098386898636818, "rewards/reward_commands_correctness/mean": 0.078125, "rewards/reward_commands_correctness/std": 0.01327591948211193, "rewards/reward_diversity/mean": 0.1847395896911621, "rewards/reward_diversity/std": 0.016582341864705086, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.1537500023841858, "rewards/reward_solution_effectiveness/std": 0.05239275097846985, "sampling/importance_sampling_ratio/max": 0.9614602327346802, "sampling/importance_sampling_ratio/mean": 0.18992017209529877, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.783847332000732, "sampling/sampling_logp_difference/mean": 0.021060993894934654, "step": 384, "step_time": 262.12706941738725 }, { "clip_ratio/high_max": 0.002814347404637374, "clip_ratio/high_mean": 0.002814347404637374, "clip_ratio/low_mean": 0.0017693450863589533, "clip_ratio/low_min": 0.0017693450863589533, "clip_ratio/region_mean": 0.004583692556479946, "completions/clipped_ratio": 0.0, "completions/max_length": 870.0, "completions/max_terminated_length": 870.0, "completions/mean_length": 756.25, "completions/mean_terminated_length": 756.25, "completions/min_length": 657.0, "completions/min_terminated_length": 657.0, "entropy": 0.08875551400706172, "epoch": 7.857142857142857, "frac_reward_zero_std": 0.0, "grad_norm": 0.018587645143270493, "learning_rate": 2.1632653061224495e-06, "loss": -0.0567, "num_tokens": 19521678.0, "reward": 0.7495710849761963, "reward_std": 0.07421247661113739, "rewards/reward_commands_completeness/mean": 0.14249999821186066, "rewards/reward_commands_completeness/std": 0.037148356437683105, "rewards/reward_commands_correctness/mean": 0.07499999552965164, "rewards/reward_commands_correctness/std": 0.01632993295788765, "rewards/reward_diversity/mean": 0.16582107543945312, "rewards/reward_diversity/std": 0.020667044445872307, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08624999970197678, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.18000000715255737, "rewards/reward_solution_effectiveness/std": 0.07668115943670273, "sampling/importance_sampling_ratio/max": 2.023444890975952, "sampling/importance_sampling_ratio/mean": 0.23007437586784363, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6406736373901367, "sampling/sampling_logp_difference/mean": 0.02380371280014515, "step": 385, "step_time": 116.26633999869227 }, { "clip_ratio/high_max": 0.001991226206882857, "clip_ratio/high_mean": 0.001991226206882857, "clip_ratio/low_mean": 0.001217470555275213, "clip_ratio/low_min": 0.001217470555275213, "clip_ratio/region_mean": 0.003208696754882112, "completions/clipped_ratio": 0.0, "completions/max_length": 956.0, "completions/max_terminated_length": 956.0, "completions/mean_length": 777.4375, "completions/mean_terminated_length": 777.4375, "completions/min_length": 639.0, "completions/min_terminated_length": 639.0, "entropy": 0.0928023336455226, "epoch": 7.877551020408164, "frac_reward_zero_std": 0.0, "grad_norm": 0.010812962427735329, "learning_rate": 2.1428571428571427e-06, "loss": -0.0208, "num_tokens": 19569809.0, "reward": 0.6736315488815308, "reward_std": 0.14140698313713074, "rewards/reward_commands_completeness/mean": 0.13124999403953552, "rewards/reward_commands_completeness/std": 0.04787135869264603, "rewards/reward_commands_correctness/mean": 0.06499999761581421, "rewards/reward_commands_correctness/std": 0.02756810002028942, "rewards/reward_diversity/mean": 0.15050652623176575, "rewards/reward_diversity/std": 0.05622756853699684, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.078125, "rewards/reward_problem_validity/std": 0.026887113228440285, "rewards/reward_solution_effectiveness/mean": 0.16124999523162842, "rewards/reward_solution_effectiveness/std": 0.06375734508037567, "sampling/importance_sampling_ratio/max": 0.36850640177726746, "sampling/importance_sampling_ratio/mean": 0.058931902050971985, "sampling/importance_sampling_ratio/min": 1.6679687178111635e-05, "sampling/sampling_logp_difference/max": 6.875977039337158, "sampling/sampling_logp_difference/mean": 0.023571757599711418, "step": 386, "step_time": 144.4976090360433 }, { "clip_ratio/high_max": 0.0017240285669686273, "clip_ratio/high_mean": 0.0017240285669686273, "clip_ratio/low_mean": 0.0025851609316305257, "clip_ratio/low_min": 0.0025851609316305257, "clip_ratio/region_mean": 0.004309189513151068, "completions/clipped_ratio": 0.0, "completions/max_length": 823.0, "completions/max_terminated_length": 823.0, "completions/mean_length": 735.9375, "completions/mean_terminated_length": 735.9375, "completions/min_length": 649.0, "completions/min_terminated_length": 649.0, "entropy": 0.10169129725545645, "epoch": 7.8979591836734695, "frac_reward_zero_std": 0.0, "grad_norm": 0.03912101686000824, "learning_rate": 2.122448979591837e-06, "loss": -0.0872, "num_tokens": 19609376.0, "reward": 0.5850398540496826, "reward_std": 0.150003582239151, "rewards/reward_commands_completeness/mean": 0.10499999672174454, "rewards/reward_commands_completeness/std": 0.048716865479946136, "rewards/reward_commands_correctness/mean": 0.05249999836087227, "rewards/reward_commands_correctness/std": 0.025948667898774147, "rewards/reward_diversity/mean": 0.1556648313999176, "rewards/reward_diversity/std": 0.06849393248558044, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07250000536441803, "rewards/reward_problem_validity/std": 0.03130495548248291, "rewards/reward_solution_effectiveness/mean": 0.11812500655651093, "rewards/reward_solution_effectiveness/std": 0.06523995101451874, "sampling/importance_sampling_ratio/max": 1.2846795320510864, "sampling/importance_sampling_ratio/mean": 0.19470611214637756, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.72328519821167, "sampling/sampling_logp_difference/mean": 0.02520930767059326, "step": 387, "step_time": 126.52137777768075 }, { "clip_ratio/high_max": 0.002161068798159249, "clip_ratio/high_mean": 0.002161068798159249, "clip_ratio/low_mean": 0.0011762406429625116, "clip_ratio/low_min": 0.0011762406429625116, "clip_ratio/region_mean": 0.0033373094629496336, "completions/clipped_ratio": 0.0, "completions/max_length": 845.0, "completions/max_terminated_length": 845.0, "completions/mean_length": 759.375, "completions/mean_terminated_length": 759.375, "completions/min_length": 648.0, "completions/min_terminated_length": 648.0, "entropy": 0.1133577972650528, "epoch": 7.918367346938775, "frac_reward_zero_std": 0.0, "grad_norm": 0.08919859677553177, "learning_rate": 2.1020408163265306e-06, "loss": -0.1233, "num_tokens": 19639914.0, "reward": 0.6845189332962036, "reward_std": 0.0765983909368515, "rewards/reward_commands_completeness/mean": 0.11624999344348907, "rewards/reward_commands_completeness/std": 0.02446085214614868, "rewards/reward_commands_correctness/mean": 0.07062499970197678, "rewards/reward_commands_correctness/std": 0.014361408539116383, "rewards/reward_diversity/mean": 0.1895189881324768, "rewards/reward_diversity/std": 0.011858063749969006, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.008139412850141525, "rewards/reward_solution_effectiveness/mean": 0.1237500011920929, "rewards/reward_solution_effectiveness/std": 0.04500000178813934, "sampling/importance_sampling_ratio/max": 2.9398555755615234, "sampling/importance_sampling_ratio/mean": 0.22968097031116486, "sampling/importance_sampling_ratio/min": 1.9886110749212094e-05, "sampling/sampling_logp_difference/max": 2.9894371032714844, "sampling/sampling_logp_difference/mean": 0.027824455872178078, "step": 388, "step_time": 106.67320966161788 }, { "clip_ratio/high_max": 0.002129364322172478, "clip_ratio/high_mean": 0.002129364322172478, "clip_ratio/low_mean": 0.0016794017137726769, "clip_ratio/low_min": 0.0016794017137726769, "clip_ratio/region_mean": 0.003808766065048985, "completions/clipped_ratio": 0.0, "completions/max_length": 1105.0, "completions/max_terminated_length": 1105.0, "completions/mean_length": 755.375, "completions/mean_terminated_length": 755.375, "completions/min_length": 653.0, "completions/min_terminated_length": 653.0, "entropy": 0.10403596609830856, "epoch": 7.938775510204081, "frac_reward_zero_std": 0.0, "grad_norm": 0.06001855805516243, "learning_rate": 2.0816326530612247e-06, "loss": -0.0792, "num_tokens": 19674732.0, "reward": 0.7399930953979492, "reward_std": 0.07631979882717133, "rewards/reward_commands_completeness/mean": 0.13999998569488525, "rewards/reward_commands_completeness/std": 0.026331225410103798, "rewards/reward_commands_correctness/mean": 0.0768750011920929, "rewards/reward_commands_correctness/std": 0.016214706003665924, "rewards/reward_diversity/mean": 0.1606181263923645, "rewards/reward_diversity/std": 0.029337607324123383, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.1743749976158142, "rewards/reward_solution_effectiveness/std": 0.042734645307064056, "sampling/importance_sampling_ratio/max": 1.749634861946106, "sampling/importance_sampling_ratio/mean": 0.1305815875530243, "sampling/importance_sampling_ratio/min": 5.280751183556198e-11, "sampling/sampling_logp_difference/max": 20.15090560913086, "sampling/sampling_logp_difference/mean": 0.028859296813607216, "step": 389, "step_time": 124.72855177335441 }, { "clip_ratio/high_max": 0.002729011161136441, "clip_ratio/high_mean": 0.002729011161136441, "clip_ratio/low_mean": 0.001385101189953275, "clip_ratio/low_min": 0.001385101189953275, "clip_ratio/region_mean": 0.00411411227833014, "completions/clipped_ratio": 0.0, "completions/max_length": 794.0, "completions/max_terminated_length": 794.0, "completions/mean_length": 715.0625, "completions/mean_terminated_length": 715.0625, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.10708046983927488, "epoch": 7.959183673469388, "frac_reward_zero_std": 0.0, "grad_norm": 0.041925206780433655, "learning_rate": 2.0612244897959184e-06, "loss": 0.0132, "num_tokens": 19705289.0, "reward": 0.7881771326065063, "reward_std": 0.1665920615196228, "rewards/reward_commands_completeness/mean": 0.1525000035762787, "rewards/reward_commands_completeness/std": 0.046690475195646286, "rewards/reward_commands_correctness/mean": 0.08124999701976776, "rewards/reward_commands_correctness/std": 0.02500000037252903, "rewards/reward_diversity/mean": 0.16880208253860474, "rewards/reward_diversity/std": 0.045219600200653076, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08375000208616257, "rewards/reward_problem_validity/std": 0.019958291202783585, "rewards/reward_solution_effectiveness/mean": 0.20812499523162842, "rewards/reward_solution_effectiveness/std": 0.083043672144413, "sampling/importance_sampling_ratio/max": 1.2785570621490479, "sampling/importance_sampling_ratio/mean": 0.17772819101810455, "sampling/importance_sampling_ratio/min": 0.00024514502729289234, "sampling/sampling_logp_difference/max": 2.7067410945892334, "sampling/sampling_logp_difference/mean": 0.026849521324038506, "step": 390, "step_time": 96.54471090994775 }, { "clip_ratio/high_max": 0.002837010979419574, "clip_ratio/high_mean": 0.002837010979419574, "clip_ratio/low_mean": 0.0012299075824557804, "clip_ratio/low_min": 0.0012299075824557804, "clip_ratio/region_mean": 0.004066918583703227, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 750.5625, "completions/mean_terminated_length": 750.5625, "completions/min_length": 674.0, "completions/min_terminated_length": 674.0, "entropy": 0.10497986245900393, "epoch": 7.979591836734694, "frac_reward_zero_std": 0.0, "grad_norm": 0.015457040630280972, "learning_rate": 2.0408163265306125e-06, "loss": 0.0165, "num_tokens": 19754570.0, "reward": 0.6953584551811218, "reward_std": 0.14020130038261414, "rewards/reward_commands_completeness/mean": 0.1237500011920929, "rewards/reward_commands_completeness/std": 0.041452787816524506, "rewards/reward_commands_correctness/mean": 0.07437500357627869, "rewards/reward_commands_correctness/std": 0.025024987757205963, "rewards/reward_diversity/mean": 0.1747334599494934, "rewards/reward_diversity/std": 0.04585989937186241, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08249999582767487, "rewards/reward_problem_validity/std": 0.019832635298371315, "rewards/reward_solution_effectiveness/mean": 0.14625000953674316, "rewards/reward_solution_effectiveness/std": 0.06280923634767532, "sampling/importance_sampling_ratio/max": 0.3306352496147156, "sampling/importance_sampling_ratio/mean": 0.07221272587776184, "sampling/importance_sampling_ratio/min": 1.1206996363365661e-14, "sampling/sampling_logp_difference/max": 20.623065948486328, "sampling/sampling_logp_difference/mean": 0.029206642881035805, "step": 391, "step_time": 210.21809154003859 }, { "clip_ratio/high_max": 0.0018672125588636845, "clip_ratio/high_mean": 0.0018672125588636845, "clip_ratio/low_mean": 0.0015552334680251079, "clip_ratio/low_min": 0.0015552334680251079, "clip_ratio/region_mean": 0.003422445997784962, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 844.0, "completions/mean_length": 934.4375, "completions/mean_terminated_length": 723.6666870117188, "completions/min_length": 618.0, "completions/min_terminated_length": 618.0, "entropy": 0.11037856061011553, "epoch": 8.0, "frac_reward_zero_std": 0.0, "grad_norm": 0.0015426134923473, "learning_rate": 2.020408163265306e-06, "loss": -0.0026, "num_tokens": 19793285.0, "reward": 0.785824179649353, "reward_std": 0.044380560517311096, "rewards/reward_commands_completeness/mean": 0.15000000596046448, "rewards/reward_commands_completeness/std": 0.024221204221248627, "rewards/reward_commands_correctness/mean": 0.07750000059604645, "rewards/reward_commands_correctness/std": 0.019832633435726166, "rewards/reward_diversity/mean": 0.17894914746284485, "rewards/reward_diversity/std": 0.02428184263408184, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.1912499964237213, "rewards/reward_solution_effectiveness/std": 0.046314142644405365, "sampling/importance_sampling_ratio/max": 0.03319227322936058, "sampling/importance_sampling_ratio/mean": 0.004834193270653486, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 19.931684494018555, "sampling/sampling_logp_difference/mean": 0.02553451433777809, "step": 392, "step_time": 263.57859957404435 }, { "clip_ratio/high_max": 0.002234152561868541, "clip_ratio/high_mean": 0.002234152561868541, "clip_ratio/low_mean": 0.0016349963989341632, "clip_ratio/low_min": 0.0016349963989341632, "clip_ratio/region_mean": 0.0038691489316988736, "completions/clipped_ratio": 0.0, "completions/max_length": 965.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 736.3125, "completions/mean_terminated_length": 736.3125, "completions/min_length": 619.0, "completions/min_terminated_length": 619.0, "entropy": 0.09465653263032436, "epoch": 8.020408163265307, "frac_reward_zero_std": 0.0, "grad_norm": 0.02667171321809292, "learning_rate": 2.0000000000000003e-06, "loss": -0.0656, "num_tokens": 19848510.0, "reward": 0.6136090755462646, "reward_std": 0.13636845350265503, "rewards/reward_commands_completeness/mean": 0.11249999701976776, "rewards/reward_commands_completeness/std": 0.043127719312906265, "rewards/reward_commands_correctness/mean": 0.061250001192092896, "rewards/reward_commands_correctness/std": 0.027294689789414406, "rewards/reward_diversity/mean": 0.14735907316207886, "rewards/reward_diversity/std": 0.05125198885798454, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07562500238418579, "rewards/reward_problem_validity/std": 0.02606882154941559, "rewards/reward_solution_effectiveness/mean": 0.12937501072883606, "rewards/reward_solution_effectiveness/std": 0.06265980005264282, "sampling/importance_sampling_ratio/max": 0.5905424952507019, "sampling/importance_sampling_ratio/mean": 0.07618701457977295, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 16.05685043334961, "sampling/sampling_logp_difference/mean": 0.0274641253054142, "step": 393, "step_time": 209.81752302870154 }, { "clip_ratio/high_max": 0.002322913489479106, "clip_ratio/high_mean": 0.002322913489479106, "clip_ratio/low_mean": 0.0009275590782635845, "clip_ratio/low_min": 0.0009275590782635845, "clip_ratio/region_mean": 0.003250472553190775, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1079.0, "completions/mean_length": 991.625, "completions/mean_terminated_length": 784.6666870117188, "completions/min_length": 623.0, "completions/min_terminated_length": 623.0, "entropy": 0.09668098110705614, "epoch": 8.040816326530612, "frac_reward_zero_std": 0.0, "grad_norm": 0.053799714893102646, "learning_rate": 1.979591836734694e-06, "loss": -0.0615, "num_tokens": 19884776.0, "reward": 0.7644895315170288, "reward_std": 0.09631270170211792, "rewards/reward_commands_completeness/mean": 0.14624999463558197, "rewards/reward_commands_completeness/std": 0.02895398996770382, "rewards/reward_commands_correctness/mean": 0.07249999791383743, "rewards/reward_commands_correctness/std": 0.018797164782881737, "rewards/reward_diversity/mean": 0.17386452853679657, "rewards/reward_diversity/std": 0.017259465530514717, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.18562500178813934, "rewards/reward_solution_effectiveness/std": 0.06217917799949646, "sampling/importance_sampling_ratio/max": 1.199235200881958, "sampling/importance_sampling_ratio/mean": 0.09512430429458618, "sampling/importance_sampling_ratio/min": 3.168623052829389e-08, "sampling/sampling_logp_difference/max": 5.349507808685303, "sampling/sampling_logp_difference/mean": 0.021184222772717476, "step": 394, "step_time": 253.1124576665461 }, { "clip_ratio/high_max": 0.0019389697263250127, "clip_ratio/high_mean": 0.0019389697263250127, "clip_ratio/low_mean": 0.0016089858545456082, "clip_ratio/low_min": 0.0016089858545456082, "clip_ratio/region_mean": 0.0035479555372148752, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 782.0, "completions/mean_length": 913.0625, "completions/mean_terminated_length": 700.86669921875, "completions/min_length": 571.0, "completions/min_terminated_length": 571.0, "entropy": 0.10515253990888596, "epoch": 8.061224489795919, "frac_reward_zero_std": 0.0, "grad_norm": 0.046893008053302765, "learning_rate": 1.9591836734693877e-06, "loss": -0.0722, "num_tokens": 19917993.0, "reward": 0.7055544853210449, "reward_std": 0.05614283308386803, "rewards/reward_commands_completeness/mean": 0.1262499988079071, "rewards/reward_commands_completeness/std": 0.02801785245537758, "rewards/reward_commands_correctness/mean": 0.07062499970197678, "rewards/reward_commands_correctness/std": 0.019822128117084503, "rewards/reward_diversity/mean": 0.1911795288324356, "rewards/reward_diversity/std": 0.01186780259013176, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08249999582767487, "rewards/reward_problem_validity/std": 0.01238278578966856, "rewards/reward_solution_effectiveness/mean": 0.13500000536441803, "rewards/reward_solution_effectiveness/std": 0.06196773797273636, "sampling/importance_sampling_ratio/max": 2.151460647583008, "sampling/importance_sampling_ratio/mean": 0.2201620638370514, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.877444267272949, "sampling/sampling_logp_difference/mean": 0.02087363600730896, "step": 395, "step_time": 254.75672884471714 }, { "clip_ratio/high_max": 0.003222169616492465, "clip_ratio/high_mean": 0.003222169616492465, "clip_ratio/low_mean": 0.0009954276465578005, "clip_ratio/low_min": 0.0009954276465578005, "clip_ratio/region_mean": 0.00421759724849835, "completions/clipped_ratio": 0.0, "completions/max_length": 844.0, "completions/max_terminated_length": 844.0, "completions/mean_length": 737.375, "completions/mean_terminated_length": 737.375, "completions/min_length": 638.0, "completions/min_terminated_length": 638.0, "entropy": 0.08837772440165281, "epoch": 8.081632653061224, "frac_reward_zero_std": 0.0, "grad_norm": 0.10750206559896469, "learning_rate": 1.938775510204082e-06, "loss": -0.2759, "num_tokens": 19955067.0, "reward": 0.6485416889190674, "reward_std": 0.22484493255615234, "rewards/reward_commands_completeness/mean": 0.12250000238418579, "rewards/reward_commands_completeness/std": 0.05603570491075516, "rewards/reward_commands_correctness/mean": 0.06812499463558197, "rewards/reward_commands_correctness/std": 0.031245002523064613, "rewards/reward_diversity/mean": 0.14416667819023132, "rewards/reward_diversity/std": 0.06454900652170181, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07312500476837158, "rewards/reward_problem_validity/std": 0.03156343102455139, "rewards/reward_solution_effectiveness/mean": 0.15937499701976776, "rewards/reward_solution_effectiveness/std": 0.08250000327825546, "sampling/importance_sampling_ratio/max": 2.688046455383301, "sampling/importance_sampling_ratio/mean": 0.2685626745223999, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2048282623291016, "sampling/sampling_logp_difference/mean": 0.02450370043516159, "step": 396, "step_time": 125.05294954776764 }, { "clip_ratio/high_max": 0.003052034415304661, "clip_ratio/high_mean": 0.003052034415304661, "clip_ratio/low_mean": 0.0019134160538669676, "clip_ratio/low_min": 0.0019134160538669676, "clip_ratio/region_mean": 0.004965450454619713, "completions/clipped_ratio": 0.0, "completions/max_length": 998.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 775.875, "completions/mean_terminated_length": 775.875, "completions/min_length": 686.0, "completions/min_terminated_length": 686.0, "entropy": 0.09442322980612516, "epoch": 8.10204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.06557086855173111, "learning_rate": 1.9183673469387756e-06, "loss": 0.0894, "num_tokens": 19987917.0, "reward": 0.73124098777771, "reward_std": 0.140663743019104, "rewards/reward_commands_completeness/mean": 0.13624998927116394, "rewards/reward_commands_completeness/std": 0.040145572274923325, "rewards/reward_commands_correctness/mean": 0.0768750011920929, "rewards/reward_commands_correctness/std": 0.021203381940722466, "rewards/reward_diversity/mean": 0.16499103605747223, "rewards/reward_diversity/std": 0.04109139367938042, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08312499523162842, "rewards/reward_problem_validity/std": 0.01990603096783161, "rewards/reward_solution_effectiveness/mean": 0.17624999582767487, "rewards/reward_solution_effectiveness/std": 0.06561250239610672, "sampling/importance_sampling_ratio/max": 1.3307210206985474, "sampling/importance_sampling_ratio/mean": 0.18566739559173584, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 14.997932434082031, "sampling/sampling_logp_difference/mean": 0.028494184836745262, "step": 397, "step_time": 115.9647927545011 }, { "clip_ratio/high_max": 0.0032011389193939976, "clip_ratio/high_mean": 0.0032011389193939976, "clip_ratio/low_mean": 0.0009499347070232034, "clip_ratio/low_min": 0.0009499347070232034, "clip_ratio/region_mean": 0.004151073626417201, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 744.4375, "completions/mean_terminated_length": 744.4375, "completions/min_length": 615.0, "completions/min_terminated_length": 615.0, "entropy": 0.09921184927225113, "epoch": 8.122448979591837, "frac_reward_zero_std": 0.0, "grad_norm": 0.016098635271191597, "learning_rate": 1.8979591836734695e-06, "loss": -0.0084, "num_tokens": 20023208.0, "reward": 0.7140434384346008, "reward_std": 0.20019438862800598, "rewards/reward_commands_completeness/mean": 0.13750000298023224, "rewards/reward_commands_completeness/std": 0.05208967253565788, "rewards/reward_commands_correctness/mean": 0.07375000417232513, "rewards/reward_commands_correctness/std": 0.029636690393090248, "rewards/reward_diversity/mean": 0.1596684753894806, "rewards/reward_diversity/std": 0.04527045041322708, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07750000059604645, "rewards/reward_problem_validity/std": 0.02670830301940441, "rewards/reward_solution_effectiveness/mean": 0.17812499403953552, "rewards/reward_solution_effectiveness/std": 0.07626434415578842, "sampling/importance_sampling_ratio/max": 0.33739152550697327, "sampling/importance_sampling_ratio/mean": 0.09249933063983917, "sampling/importance_sampling_ratio/min": 0.00011773083679145202, "sampling/sampling_logp_difference/max": 2.3884191513061523, "sampling/sampling_logp_difference/mean": 0.0249567199498415, "step": 398, "step_time": 110.12998922169209 }, { "clip_ratio/high_max": 0.0021458534101839177, "clip_ratio/high_mean": 0.0021458534101839177, "clip_ratio/low_mean": 0.0009934524932759814, "clip_ratio/low_min": 0.0009934524932759814, "clip_ratio/region_mean": 0.003139305903459899, "completions/clipped_ratio": 0.0, "completions/max_length": 1605.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 785.9375, "completions/mean_terminated_length": 785.9375, "completions/min_length": 607.0, "completions/min_terminated_length": 607.0, "entropy": 0.0926872044801712, "epoch": 8.142857142857142, "frac_reward_zero_std": 0.0, "grad_norm": 0.016367128118872643, "learning_rate": 1.8775510204081634e-06, "loss": -0.0113, "num_tokens": 20060763.0, "reward": 0.5799009799957275, "reward_std": 0.22239750623703003, "rewards/reward_commands_completeness/mean": 0.10625000298023224, "rewards/reward_commands_completeness/std": 0.055482737720012665, "rewards/reward_commands_correctness/mean": 0.05812499672174454, "rewards/reward_commands_correctness/std": 0.030815312638878822, "rewards/reward_diversity/mean": 0.1392759382724762, "rewards/reward_diversity/std": 0.06156575307250023, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07124999910593033, "rewards/reward_problem_validity/std": 0.030740855261683464, "rewards/reward_solution_effectiveness/mean": 0.1237500011920929, "rewards/reward_solution_effectiveness/std": 0.08405355364084244, "sampling/importance_sampling_ratio/max": 1.4183084964752197, "sampling/importance_sampling_ratio/mean": 0.16126272082328796, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.672590255737305, "sampling/sampling_logp_difference/mean": 0.022719182074069977, "step": 399, "step_time": 136.45036859065294 }, { "clip_ratio/high_max": 0.001860740696429275, "clip_ratio/high_mean": 0.001860740696429275, "clip_ratio/low_mean": 0.0015911192604107782, "clip_ratio/low_min": 0.0015911192604107782, "clip_ratio/region_mean": 0.003451859942288138, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 797.0, "completions/mean_length": 923.0625, "completions/mean_terminated_length": 711.5333862304688, "completions/min_length": 644.0, "completions/min_terminated_length": 644.0, "entropy": 0.08724122494459152, "epoch": 8.16326530612245, "frac_reward_zero_std": 0.0, "grad_norm": 0.023416990414261818, "learning_rate": 1.8571428571428573e-06, "loss": -0.0489, "num_tokens": 20097288.0, "reward": 0.7649418115615845, "reward_std": 0.09823839366436005, "rewards/reward_commands_completeness/mean": 0.14499999582767487, "rewards/reward_commands_completeness/std": 0.036878179758787155, "rewards/reward_commands_correctness/mean": 0.07624999433755875, "rewards/reward_commands_correctness/std": 0.017078252509236336, "rewards/reward_diversity/mean": 0.1761917918920517, "rewards/reward_diversity/std": 0.01769929565489292, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.18000000715255737, "rewards/reward_solution_effectiveness/std": 0.06841052323579788, "sampling/importance_sampling_ratio/max": 0.7083312273025513, "sampling/importance_sampling_ratio/mean": 0.09162335097789764, "sampling/importance_sampling_ratio/min": 2.4327947989455367e-10, "sampling/sampling_logp_difference/max": 18.938005447387695, "sampling/sampling_logp_difference/mean": 0.02074158936738968, "step": 400, "step_time": 254.8052302710712 }, { "clip_ratio/high_max": 0.0011834013348561712, "clip_ratio/high_mean": 0.0011834013348561712, "clip_ratio/low_mean": 0.0019528809789335355, "clip_ratio/low_min": 0.0019528809789335355, "clip_ratio/region_mean": 0.0031362823137897067, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 887.0, "completions/mean_length": 963.25, "completions/mean_terminated_length": 754.4000244140625, "completions/min_length": 646.0, "completions/min_terminated_length": 646.0, "entropy": 0.0880681062117219, "epoch": 8.183673469387756, "frac_reward_zero_std": 0.0, "grad_norm": 0.016249334439635277, "learning_rate": 1.8367346938775512e-06, "loss": -0.0145, "num_tokens": 20137456.0, "reward": 0.7107230424880981, "reward_std": 0.1523810625076294, "rewards/reward_commands_completeness/mean": 0.13500000536441803, "rewards/reward_commands_completeness/std": 0.05085928738117218, "rewards/reward_commands_correctness/mean": 0.07625000178813934, "rewards/reward_commands_correctness/std": 0.029636690393090248, "rewards/reward_diversity/mean": 0.15072304010391235, "rewards/reward_diversity/std": 0.056007251143455505, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.027195284143090248, "rewards/reward_solution_effectiveness/mean": 0.18187500536441803, "rewards/reward_solution_effectiveness/std": 0.07858912646770477, "sampling/importance_sampling_ratio/max": 0.7785878777503967, "sampling/importance_sampling_ratio/mean": 0.109616719186306, "sampling/importance_sampling_ratio/min": 1.1115033149249314e-14, "sampling/sampling_logp_difference/max": 16.748064041137695, "sampling/sampling_logp_difference/mean": 0.020334681496024132, "step": 401, "step_time": 270.0248341243714 }, { "clip_ratio/high_max": 0.0024259369893115945, "clip_ratio/high_mean": 0.0024259369893115945, "clip_ratio/low_mean": 0.0006276443600654602, "clip_ratio/low_min": 0.0006276443600654602, "clip_ratio/region_mean": 0.0030535813493770547, "completions/clipped_ratio": 0.0, "completions/max_length": 1092.0, "completions/max_terminated_length": 1092.0, "completions/mean_length": 773.25, "completions/mean_terminated_length": 773.25, "completions/min_length": 645.0, "completions/min_terminated_length": 645.0, "entropy": 0.1016376088373363, "epoch": 8.204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.11602271348237991, "learning_rate": 1.8163265306122451e-06, "loss": 0.0339, "num_tokens": 20185340.0, "reward": 0.8257970809936523, "reward_std": 0.05155642703175545, "rewards/reward_commands_completeness/mean": 0.16124999523162842, "rewards/reward_commands_completeness/std": 0.021252455189824104, "rewards/reward_commands_correctness/mean": 0.08249999582767487, "rewards/reward_commands_correctness/std": 0.014832398854196072, "rewards/reward_diversity/mean": 0.17517203092575073, "rewards/reward_diversity/std": 0.023438602685928345, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08937500417232513, "rewards/reward_problem_validity/std": 0.0025000013411045074, "rewards/reward_solution_effectiveness/mean": 0.2175000011920929, "rewards/reward_solution_effectiveness/std": 0.049598392099142075, "sampling/importance_sampling_ratio/max": 2.5627593994140625, "sampling/importance_sampling_ratio/mean": 0.4401116371154785, "sampling/importance_sampling_ratio/min": 1.5610235326590782e-10, "sampling/sampling_logp_difference/max": 19.61394691467285, "sampling/sampling_logp_difference/mean": 0.02634485810995102, "step": 402, "step_time": 160.61476992443204 }, { "clip_ratio/high_max": 0.0008869396551745012, "clip_ratio/high_mean": 0.0008869396551745012, "clip_ratio/low_mean": 0.0010455610463395715, "clip_ratio/low_min": 0.0010455610463395715, "clip_ratio/region_mean": 0.0019325006869621575, "completions/clipped_ratio": 0.0, "completions/max_length": 1600.0, "completions/max_terminated_length": 1600.0, "completions/mean_length": 821.6875, "completions/mean_terminated_length": 821.6875, "completions/min_length": 627.0, "completions/min_terminated_length": 627.0, "entropy": 0.08779572695493698, "epoch": 8.224489795918368, "frac_reward_zero_std": 0.0, "grad_norm": 0.019763238728046417, "learning_rate": 1.7959183673469388e-06, "loss": -0.0412, "num_tokens": 20228623.0, "reward": 0.6208333373069763, "reward_std": 0.1543668806552887, "rewards/reward_commands_completeness/mean": 0.10750000178813934, "rewards/reward_commands_completeness/std": 0.04837355017662048, "rewards/reward_commands_correctness/mean": 0.06624999642372131, "rewards/reward_commands_correctness/std": 0.02500000037252903, "rewards/reward_diversity/mean": 0.16083332896232605, "rewards/reward_diversity/std": 0.055982805788517, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.02651257812976837, "rewards/reward_solution_effectiveness/mean": 0.12187500298023224, "rewards/reward_solution_effectiveness/std": 0.07386643439531326, "sampling/importance_sampling_ratio/max": 1.480290174484253, "sampling/importance_sampling_ratio/mean": 0.17921042442321777, "sampling/importance_sampling_ratio/min": 0.0006236835033632815, "sampling/sampling_logp_difference/max": 2.703273296356201, "sampling/sampling_logp_difference/mean": 0.02036520093679428, "step": 403, "step_time": 166.91007558628917 }, { "clip_ratio/high_max": 0.0020800577112822793, "clip_ratio/high_mean": 0.0020800577112822793, "clip_ratio/low_mean": 0.001787067980330903, "clip_ratio/low_min": 0.001787067980330903, "clip_ratio/region_mean": 0.003867125720717013, "completions/clipped_ratio": 0.0, "completions/max_length": 1070.0, "completions/max_terminated_length": 1070.0, "completions/mean_length": 772.125, "completions/mean_terminated_length": 772.125, "completions/min_length": 625.0, "completions/min_terminated_length": 625.0, "entropy": 0.11026289220899343, "epoch": 8.244897959183673, "frac_reward_zero_std": 0.0, "grad_norm": 0.018783804029226303, "learning_rate": 1.7755102040816327e-06, "loss": -0.049, "num_tokens": 20257237.0, "reward": 0.7242536544799805, "reward_std": 0.062502421438694, "rewards/reward_commands_completeness/mean": 0.12999999523162842, "rewards/reward_commands_completeness/std": 0.03932768478989601, "rewards/reward_commands_correctness/mean": 0.07750000059604645, "rewards/reward_commands_correctness/std": 0.01064581423997879, "rewards/reward_diversity/mean": 0.17987869679927826, "rewards/reward_diversity/std": 0.01655118353664875, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.15000000596046448, "rewards/reward_solution_effectiveness/std": 0.07266360521316528, "sampling/importance_sampling_ratio/max": 1.8391698598861694, "sampling/importance_sampling_ratio/mean": 0.15341228246688843, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.879322052001953, "sampling/sampling_logp_difference/mean": 0.02556890808045864, "step": 404, "step_time": 108.68609856255352 }, { "clip_ratio/high_max": 0.0013074050511932, "clip_ratio/high_mean": 0.0013074050511932, "clip_ratio/low_mean": 0.001200166945636738, "clip_ratio/low_min": 0.001200166945636738, "clip_ratio/region_mean": 0.0025075719895539805, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 839.0, "completions/mean_length": 1135.25, "completions/mean_terminated_length": 712.2857666015625, "completions/min_length": 625.0, "completions/min_terminated_length": 625.0, "entropy": 0.07656510220840573, "epoch": 8.26530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.01783185452222824, "learning_rate": 1.7551020408163267e-06, "loss": -0.0254, "num_tokens": 20299909.0, "reward": 0.7890380024909973, "reward_std": 0.09285090863704681, "rewards/reward_commands_completeness/mean": 0.1537500023841858, "rewards/reward_commands_completeness/std": 0.02276693843305111, "rewards/reward_commands_correctness/mean": 0.07874999940395355, "rewards/reward_commands_correctness/std": 0.01927866041660309, "rewards/reward_diversity/mean": 0.16903799772262573, "rewards/reward_diversity/std": 0.027787117287516594, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08875000476837158, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.19874998927116394, "rewards/reward_solution_effectiveness/std": 0.05572253093123436, "sampling/importance_sampling_ratio/max": 0.45007655024528503, "sampling/importance_sampling_ratio/mean": 0.05440057814121246, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 16.363018035888672, "sampling/sampling_logp_difference/mean": 0.016643308103084564, "step": 405, "step_time": 287.305628888309 }, { "clip_ratio/high_max": 0.001998778789129574, "clip_ratio/high_mean": 0.001998778789129574, "clip_ratio/low_mean": 0.0020194883400108665, "clip_ratio/low_min": 0.0020194883400108665, "clip_ratio/region_mean": 0.004018267180072144, "completions/clipped_ratio": 0.0, "completions/max_length": 895.0, "completions/max_terminated_length": 895.0, "completions/mean_length": 736.3125, "completions/mean_terminated_length": 736.3125, "completions/min_length": 579.0, "completions/min_terminated_length": 579.0, "entropy": 0.09381629433482885, "epoch": 8.285714285714286, "frac_reward_zero_std": 0.0, "grad_norm": 0.006111864000558853, "learning_rate": 1.7346938775510206e-06, "loss": -0.0076, "num_tokens": 20339750.0, "reward": 0.7495925426483154, "reward_std": 0.1653246432542801, "rewards/reward_commands_completeness/mean": 0.14249999821186066, "rewards/reward_commands_completeness/std": 0.03924283757805824, "rewards/reward_commands_correctness/mean": 0.08125000447034836, "rewards/reward_commands_correctness/std": 0.02276693843305111, "rewards/reward_diversity/mean": 0.1614675223827362, "rewards/reward_diversity/std": 0.04389118775725365, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.020000001415610313, "rewards/reward_solution_effectiveness/mean": 0.18562498688697815, "rewards/reward_solution_effectiveness/std": 0.06860212236642838, "sampling/importance_sampling_ratio/max": 0.08243050426244736, "sampling/importance_sampling_ratio/mean": 0.01664399355649948, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 16.24998664855957, "sampling/sampling_logp_difference/mean": 0.027060674503445625, "step": 406, "step_time": 127.46204641647637 }, { "clip_ratio/high_max": 0.0015719187795184553, "clip_ratio/high_mean": 0.0015719187795184553, "clip_ratio/low_mean": 0.0019869433890562505, "clip_ratio/low_min": 0.0019869433890562505, "clip_ratio/region_mean": 0.0035588621394708753, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 794.75, "completions/mean_terminated_length": 794.75, "completions/min_length": 624.0, "completions/min_terminated_length": 624.0, "entropy": 0.0994364283978939, "epoch": 8.306122448979592, "frac_reward_zero_std": 0.0, "grad_norm": 0.02804296649992466, "learning_rate": 1.7142857142857145e-06, "loss": 0.0405, "num_tokens": 20381362.0, "reward": 0.6639368534088135, "reward_std": 0.07520319521427155, "rewards/reward_commands_completeness/mean": 0.11874999850988388, "rewards/reward_commands_completeness/std": 0.033837851136922836, "rewards/reward_commands_correctness/mean": 0.06624999642372131, "rewards/reward_commands_correctness/std": 0.01927866041660309, "rewards/reward_diversity/mean": 0.1714368760585785, "rewards/reward_diversity/std": 0.024974163621664047, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500774860382, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.12187500298023224, "rewards/reward_solution_effectiveness/std": 0.060466933995485306, "sampling/importance_sampling_ratio/max": 0.5825209617614746, "sampling/importance_sampling_ratio/mean": 0.05516102910041809, "sampling/importance_sampling_ratio/min": 0.00010425619984744117, "sampling/sampling_logp_difference/max": 4.925294876098633, "sampling/sampling_logp_difference/mean": 0.02306384965777397, "step": 407, "step_time": 163.30360864847898 }, { "clip_ratio/high_max": 0.002256375999422744, "clip_ratio/high_mean": 0.002256375999422744, "clip_ratio/low_mean": 0.0014663608890259638, "clip_ratio/low_min": 0.0014663608890259638, "clip_ratio/region_mean": 0.003722736975760199, "completions/clipped_ratio": 0.0, "completions/max_length": 1023.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 762.9375, "completions/mean_terminated_length": 762.9375, "completions/min_length": 657.0, "completions/min_terminated_length": 657.0, "entropy": 0.09823432099074125, "epoch": 8.326530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.009913893416523933, "learning_rate": 1.6938775510204084e-06, "loss": -0.0222, "num_tokens": 20416637.0, "reward": 0.7329167127609253, "reward_std": 0.11951577663421631, "rewards/reward_commands_completeness/mean": 0.13375000655651093, "rewards/reward_commands_completeness/std": 0.04364630952477455, "rewards/reward_commands_correctness/mean": 0.08249999582767487, "rewards/reward_commands_correctness/std": 0.01570562645792961, "rewards/reward_diversity/mean": 0.16979166865348816, "rewards/reward_diversity/std": 0.019831467419862747, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500774860382, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.16124999523162842, "rewards/reward_solution_effectiveness/std": 0.08261356502771378, "sampling/importance_sampling_ratio/max": 0.40149354934692383, "sampling/importance_sampling_ratio/mean": 0.06260636448860168, "sampling/importance_sampling_ratio/min": 1.1906765394087415e-05, "sampling/sampling_logp_difference/max": 3.4092345237731934, "sampling/sampling_logp_difference/mean": 0.025347745046019554, "step": 408, "step_time": 117.89358588680625 }, { "clip_ratio/high_max": 0.0028554187301779166, "clip_ratio/high_mean": 0.0028554187301779166, "clip_ratio/low_mean": 0.0013007294255658053, "clip_ratio/low_min": 0.0013007294255658053, "clip_ratio/region_mean": 0.004156148206675425, "completions/clipped_ratio": 0.0, "completions/max_length": 893.0, "completions/max_terminated_length": 893.0, "completions/mean_length": 730.625, "completions/mean_terminated_length": 730.625, "completions/min_length": 624.0, "completions/min_terminated_length": 624.0, "entropy": 0.09322003833949566, "epoch": 8.346938775510203, "frac_reward_zero_std": 0.0, "grad_norm": 0.06406934559345245, "learning_rate": 1.6734693877551023e-06, "loss": 0.1096, "num_tokens": 20459659.0, "reward": 0.7986979484558105, "reward_std": 0.06307715177536011, "rewards/reward_commands_completeness/mean": 0.1587499976158142, "rewards/reward_commands_completeness/std": 0.02578759752213955, "rewards/reward_commands_correctness/mean": 0.08250000327825546, "rewards/reward_commands_correctness/std": 0.01570562645792961, "rewards/reward_diversity/mean": 0.15432292222976685, "rewards/reward_diversity/std": 0.02410474605858326, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08937500417232513, "rewards/reward_problem_validity/std": 0.0025000013411045074, "rewards/reward_solution_effectiveness/mean": 0.21375000476837158, "rewards/reward_solution_effectiveness/std": 0.05123475566506386, "sampling/importance_sampling_ratio/max": 1.4991687536239624, "sampling/importance_sampling_ratio/mean": 0.15726616978645325, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.762234926223755, "sampling/sampling_logp_difference/mean": 0.022056998685002327, "step": 409, "step_time": 155.16280849650502 }, { "clip_ratio/high_max": 0.0019217113440390676, "clip_ratio/high_mean": 0.0019217113440390676, "clip_ratio/low_mean": 0.0004462083561520558, "clip_ratio/low_min": 0.0004462083561520558, "clip_ratio/region_mean": 0.002367919671087293, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 952.0, "completions/mean_length": 944.875, "completions/mean_terminated_length": 734.800048828125, "completions/min_length": 633.0, "completions/min_terminated_length": 633.0, "entropy": 0.09184915898367763, "epoch": 8.36734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.014370178803801537, "learning_rate": 1.6530612244897962e-06, "loss": 0.0029, "num_tokens": 20502337.0, "reward": 0.718267560005188, "reward_std": 0.12003706395626068, "rewards/reward_commands_completeness/mean": 0.12999999523162842, "rewards/reward_commands_completeness/std": 0.04258325323462486, "rewards/reward_commands_correctness/mean": 0.06812499463558197, "rewards/reward_commands_correctness/std": 0.021360009908676147, "rewards/reward_diversity/mean": 0.17889255285263062, "rewards/reward_diversity/std": 0.015173899941146374, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08249999582767487, "rewards/reward_problem_validity/std": 0.019832635298371315, "rewards/reward_solution_effectiveness/mean": 0.16500000655651093, "rewards/reward_solution_effectiveness/std": 0.07509993761777878, "sampling/importance_sampling_ratio/max": 0.4586326479911804, "sampling/importance_sampling_ratio/mean": 0.12152665108442307, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.236084938049316, "sampling/sampling_logp_difference/mean": 0.0189240463078022, "step": 410, "step_time": 263.4167956318706 }, { "clip_ratio/high_max": 0.0032460306538268924, "clip_ratio/high_mean": 0.0032460306538268924, "clip_ratio/low_mean": 0.0008873581828083843, "clip_ratio/low_min": 0.0008873581828083843, "clip_ratio/region_mean": 0.004133388836635277, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 1006.9375, "completions/mean_terminated_length": 801.0000610351562, "completions/min_length": 606.0, "completions/min_terminated_length": 606.0, "entropy": 0.10328027186915278, "epoch": 8.387755102040817, "frac_reward_zero_std": 0.0, "grad_norm": 0.009275101125240326, "learning_rate": 1.6326530612244897e-06, "loss": 0.0109, "num_tokens": 20551948.0, "reward": 0.5682291984558105, "reward_std": 0.2626327872276306, "rewards/reward_commands_completeness/mean": 0.10374999791383743, "rewards/reward_commands_completeness/std": 0.056199051439762115, "rewards/reward_commands_correctness/mean": 0.04874999821186066, "rewards/reward_commands_correctness/std": 0.02604483626782894, "rewards/reward_diversity/mean": 0.15197916328907013, "rewards/reward_diversity/std": 0.06746527552604675, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06687499582767487, "rewards/reward_problem_validity/std": 0.034199174493551254, "rewards/reward_solution_effectiveness/mean": 0.12187500298023224, "rewards/reward_solution_effectiveness/std": 0.07704706490039825, "sampling/importance_sampling_ratio/max": 0.1722683608531952, "sampling/importance_sampling_ratio/mean": 0.022809255868196487, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.538362503051758, "sampling/sampling_logp_difference/mean": 0.021365055814385414, "step": 411, "step_time": 293.06353323720396 }, { "clip_ratio/high_max": 0.003165709160384722, "clip_ratio/high_mean": 0.003165709160384722, "clip_ratio/low_mean": 0.0007939644347061403, "clip_ratio/low_min": 0.0007939644347061403, "clip_ratio/region_mean": 0.003959673587814905, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 953.0, "completions/mean_length": 971.25, "completions/mean_terminated_length": 762.933349609375, "completions/min_length": 633.0, "completions/min_terminated_length": 633.0, "entropy": 0.09013841487467289, "epoch": 8.408163265306122, "frac_reward_zero_std": 0.0, "grad_norm": 0.0077002402395009995, "learning_rate": 1.6122448979591836e-06, "loss": -0.0006, "num_tokens": 20592564.0, "reward": 0.7140451669692993, "reward_std": 0.15568532049655914, "rewards/reward_commands_completeness/mean": 0.13375000655651093, "rewards/reward_commands_completeness/std": 0.04485161229968071, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.029432127252221107, "rewards/reward_diversity/mean": 0.16904513537883759, "rewards/reward_diversity/std": 0.04252982512116432, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.07567199319601059, "sampling/importance_sampling_ratio/max": 0.8905856013298035, "sampling/importance_sampling_ratio/mean": 0.09674694389104843, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.932476997375488, "sampling/sampling_logp_difference/mean": 0.018084527924656868, "step": 412, "step_time": 269.0075912103057 }, { "clip_ratio/high_max": 0.001974480546778068, "clip_ratio/high_mean": 0.001974480546778068, "clip_ratio/low_mean": 0.0004688372719101608, "clip_ratio/low_min": 0.0004688372719101608, "clip_ratio/region_mean": 0.002443317833240144, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 867.0, "completions/mean_length": 1174.3125, "completions/mean_terminated_length": 756.9285888671875, "completions/min_length": 663.0, "completions/min_terminated_length": 663.0, "entropy": 0.08144257264211774, "epoch": 8.428571428571429, "frac_reward_zero_std": 0.0, "grad_norm": 0.005767181981354952, "learning_rate": 1.5918367346938775e-06, "loss": -0.012, "num_tokens": 20639957.0, "reward": 0.7192426919937134, "reward_std": 0.16337114572525024, "rewards/reward_commands_completeness/mean": 0.13624998927116394, "rewards/reward_commands_completeness/std": 0.04209117218852043, "rewards/reward_commands_correctness/mean": 0.06687499582767487, "rewards/reward_commands_correctness/std": 0.01956825703382492, "rewards/reward_diversity/mean": 0.16736763715744019, "rewards/reward_diversity/std": 0.042367953807115555, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08249999582767487, "rewards/reward_problem_validity/std": 0.019832635298371315, "rewards/reward_solution_effectiveness/mean": 0.17250001430511475, "rewards/reward_solution_effectiveness/std": 0.0714142844080925, "sampling/importance_sampling_ratio/max": 0.41986602544784546, "sampling/importance_sampling_ratio/mean": 0.05462852120399475, "sampling/importance_sampling_ratio/min": 2.9757165975752287e-07, "sampling/sampling_logp_difference/max": 14.74979305267334, "sampling/sampling_logp_difference/mean": 0.017322691157460213, "step": 413, "step_time": 270.4690263196826 }, { "clip_ratio/high_max": 0.0014385040776687674, "clip_ratio/high_mean": 0.0014385040776687674, "clip_ratio/low_mean": 0.0010311639489373192, "clip_ratio/low_min": 0.0010311639489373192, "clip_ratio/region_mean": 0.0024696680120541714, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 843.0, "completions/mean_length": 1174.8125, "completions/mean_terminated_length": 757.5000610351562, "completions/min_length": 681.0, "completions/min_terminated_length": 681.0, "entropy": 0.08473193878307939, "epoch": 8.448979591836734, "frac_reward_zero_std": 0.25, "grad_norm": 0.02658432349562645, "learning_rate": 1.5714285714285714e-06, "loss": 0.0927, "num_tokens": 20690618.0, "reward": 0.6214582920074463, "reward_std": 0.0614987388253212, "rewards/reward_commands_completeness/mean": 0.11874999850988388, "rewards/reward_commands_completeness/std": 0.06302116066217422, "rewards/reward_commands_correctness/mean": 0.059999994933605194, "rewards/reward_commands_correctness/std": 0.03224903345108032, "rewards/reward_diversity/mean": 0.1458333283662796, "rewards/reward_diversity/std": 0.07604580372571945, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06812500208616257, "rewards/reward_problem_validity/std": 0.03487477824091911, "rewards/reward_solution_effectiveness/mean": 0.1537500023841858, "rewards/reward_solution_effectiveness/std": 0.08823265135288239, "sampling/importance_sampling_ratio/max": 0.9847172498703003, "sampling/importance_sampling_ratio/mean": 0.14492008090019226, "sampling/importance_sampling_ratio/min": 2.9276902751007583e-06, "sampling/sampling_logp_difference/max": 5.4148054122924805, "sampling/sampling_logp_difference/mean": 0.015725011005997658, "step": 414, "step_time": 277.10050407610834 }, { "clip_ratio/high_max": 0.002454106739605777, "clip_ratio/high_mean": 0.002454106739605777, "clip_ratio/low_mean": 0.002119611846865155, "clip_ratio/low_min": 0.002119611846865155, "clip_ratio/region_mean": 0.004573718586470932, "completions/clipped_ratio": 0.0, "completions/max_length": 777.0, "completions/max_terminated_length": 777.0, "completions/mean_length": 711.0625, "completions/mean_terminated_length": 711.0625, "completions/min_length": 612.0, "completions/min_terminated_length": 612.0, "entropy": 0.1050789374858141, "epoch": 8.46938775510204, "frac_reward_zero_std": 0.0, "grad_norm": 0.043120212852954865, "learning_rate": 1.5510204081632654e-06, "loss": 0.0862, "num_tokens": 20739463.0, "reward": 0.6846691370010376, "reward_std": 0.13658857345581055, "rewards/reward_commands_completeness/mean": 0.1224999949336052, "rewards/reward_commands_completeness/std": 0.043127719312906265, "rewards/reward_commands_correctness/mean": 0.07249999791383743, "rewards/reward_commands_correctness/std": 0.022656861692667007, "rewards/reward_diversity/mean": 0.1677941232919693, "rewards/reward_diversity/std": 0.04457880184054375, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07999999821186066, "rewards/reward_problem_validity/std": 0.019321836531162262, "rewards/reward_solution_effectiveness/mean": 0.14812499284744263, "rewards/reward_solution_effectiveness/std": 0.07054253667593002, "sampling/importance_sampling_ratio/max": 0.6906630396842957, "sampling/importance_sampling_ratio/mean": 0.06245064362883568, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0440149307250977, "sampling/sampling_logp_difference/mean": 0.025293318554759026, "step": 415, "step_time": 207.37275670096278 }, { "clip_ratio/high_max": 0.002601197105832398, "clip_ratio/high_mean": 0.002601197105832398, "clip_ratio/low_mean": 0.0020892086904495955, "clip_ratio/low_min": 0.0020892086904495955, "clip_ratio/region_mean": 0.004690405912697315, "completions/clipped_ratio": 0.0, "completions/max_length": 881.0, "completions/max_terminated_length": 881.0, "completions/mean_length": 750.0, "completions/mean_terminated_length": 750.0, "completions/min_length": 647.0, "completions/min_terminated_length": 647.0, "entropy": 0.10369480028748512, "epoch": 8.489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.020066112279891968, "learning_rate": 1.5306122448979593e-06, "loss": -0.0413, "num_tokens": 20776315.0, "reward": 0.7677274942398071, "reward_std": 0.10001999139785767, "rewards/reward_commands_completeness/mean": 0.14124999940395355, "rewards/reward_commands_completeness/std": 0.029636690393090248, "rewards/reward_commands_correctness/mean": 0.07500000298023224, "rewards/reward_commands_correctness/std": 0.019321836531162262, "rewards/reward_diversity/mean": 0.18522752821445465, "rewards/reward_diversity/std": 0.014549768529832363, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.012632631696760654, "rewards/reward_solution_effectiveness/mean": 0.18187499046325684, "rewards/reward_solution_effectiveness/std": 0.06705408543348312, "sampling/importance_sampling_ratio/max": 0.47039854526519775, "sampling/importance_sampling_ratio/mean": 0.07365117967128754, "sampling/importance_sampling_ratio/min": 2.697845744546734e-13, "sampling/sampling_logp_difference/max": 20.24327850341797, "sampling/sampling_logp_difference/mean": 0.029481440782546997, "step": 416, "step_time": 117.04545244574547 }, { "clip_ratio/high_max": 0.0025378772224939894, "clip_ratio/high_mean": 0.0025378772224939894, "clip_ratio/low_mean": 0.000826838409921038, "clip_ratio/low_min": 0.000826838409921038, "clip_ratio/region_mean": 0.003364715630596038, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 971.0, "completions/mean_length": 1179.75, "completions/mean_terminated_length": 763.1428833007812, "completions/min_length": 660.0, "completions/min_terminated_length": 660.0, "entropy": 0.06767496978864074, "epoch": 8.510204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.051081106066703796, "learning_rate": 1.5102040816326532e-06, "loss": 0.0821, "num_tokens": 20822571.0, "reward": 0.7552344799041748, "reward_std": 0.12832169234752655, "rewards/reward_commands_completeness/mean": 0.14874999225139618, "rewards/reward_commands_completeness/std": 0.03997916728258133, "rewards/reward_commands_correctness/mean": 0.07437500357627869, "rewards/reward_commands_correctness/std": 0.022794373333454132, "rewards/reward_diversity/mean": 0.156484454870224, "rewards/reward_diversity/std": 0.040854789316654205, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.020000001415610313, "rewards/reward_solution_effectiveness/mean": 0.19687500596046448, "rewards/reward_solution_effectiveness/std": 0.06289873272180557, "sampling/importance_sampling_ratio/max": 1.1159900426864624, "sampling/importance_sampling_ratio/mean": 0.14380748569965363, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 13.929727554321289, "sampling/sampling_logp_difference/mean": 0.014629878103733063, "step": 417, "step_time": 280.30624510906637 }, { "clip_ratio/high_max": 0.002157885523047298, "clip_ratio/high_mean": 0.002157885523047298, "clip_ratio/low_mean": 0.0015213780425256118, "clip_ratio/low_min": 0.0015213780425256118, "clip_ratio/region_mean": 0.0036792635801248252, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 959.0, "completions/mean_length": 1018.375, "completions/mean_terminated_length": 813.2000122070312, "completions/min_length": 688.0, "completions/min_terminated_length": 688.0, "entropy": 0.08840299351140857, "epoch": 8.53061224489796, "frac_reward_zero_std": 0.0, "grad_norm": 0.015538561157882214, "learning_rate": 1.489795918367347e-06, "loss": 0.0071, "num_tokens": 20879937.0, "reward": 0.7379934191703796, "reward_std": 0.0826212614774704, "rewards/reward_commands_completeness/mean": 0.13624998927116394, "rewards/reward_commands_completeness/std": 0.03201562166213989, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.012093386612832546, "rewards/reward_diversity/mean": 0.17924341559410095, "rewards/reward_diversity/std": 0.01836923323571682, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.16875000298023224, "rewards/reward_solution_effectiveness/std": 0.05572252720594406, "sampling/importance_sampling_ratio/max": 0.4005916118621826, "sampling/importance_sampling_ratio/mean": 0.0691891685128212, "sampling/importance_sampling_ratio/min": 0.0001896808680612594, "sampling/sampling_logp_difference/max": 5.683904647827148, "sampling/sampling_logp_difference/mean": 0.01808175817131996, "step": 418, "step_time": 301.5095568113029 }, { "clip_ratio/high_max": 0.0009386791716678999, "clip_ratio/high_mean": 0.0009386791716678999, "clip_ratio/low_mean": 0.0018971394529216923, "clip_ratio/low_min": 0.0018971394529216923, "clip_ratio/region_mean": 0.002835818610037677, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1028.0, "completions/mean_length": 1015.6875, "completions/mean_terminated_length": 810.3333740234375, "completions/min_length": 660.0, "completions/min_terminated_length": 660.0, "entropy": 0.09545716689899564, "epoch": 8.551020408163264, "frac_reward_zero_std": 0.0, "grad_norm": 0.10625610500574112, "learning_rate": 1.469387755102041e-06, "loss": -0.3071, "num_tokens": 20926544.0, "reward": 0.7400152683258057, "reward_std": 0.05880137160420418, "rewards/reward_commands_completeness/mean": 0.14124999940395355, "rewards/reward_commands_completeness/std": 0.033837854862213135, "rewards/reward_commands_correctness/mean": 0.06750000268220901, "rewards/reward_commands_correctness/std": 0.022060522809624672, "rewards/reward_diversity/mean": 0.17876532673835754, "rewards/reward_diversity/std": 0.018533676862716675, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500029802322, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.16687500476837158, "rewards/reward_solution_effectiveness/std": 0.07263780385255814, "sampling/importance_sampling_ratio/max": 2.354804515838623, "sampling/importance_sampling_ratio/mean": 0.3978261351585388, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 17.87277603149414, "sampling/sampling_logp_difference/mean": 0.02121604233980179, "step": 419, "step_time": 293.9800994787365 }, { "clip_ratio/high_max": 0.0031982634536689147, "clip_ratio/high_mean": 0.0031982634536689147, "clip_ratio/low_mean": 0.0006900070875417441, "clip_ratio/low_min": 0.0006900070875417441, "clip_ratio/region_mean": 0.003888270555762574, "completions/clipped_ratio": 0.0, "completions/max_length": 840.0, "completions/max_terminated_length": 840.0, "completions/mean_length": 740.25, "completions/mean_terminated_length": 740.25, "completions/min_length": 627.0, "completions/min_terminated_length": 627.0, "entropy": 0.09625833481550217, "epoch": 8.571428571428571, "frac_reward_zero_std": 0.0, "grad_norm": 0.010389424860477448, "learning_rate": 1.4489795918367347e-06, "loss": -0.0161, "num_tokens": 20961752.0, "reward": 0.7309045791625977, "reward_std": 0.10042861104011536, "rewards/reward_commands_completeness/mean": 0.13749998807907104, "rewards/reward_commands_completeness/std": 0.03415650501847267, "rewards/reward_commands_correctness/mean": 0.07125000655651093, "rewards/reward_commands_correctness/std": 0.014548770152032375, "rewards/reward_diversity/mean": 0.1702796071767807, "rewards/reward_diversity/std": 0.01445984747260809, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08875000476837158, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.16312500834465027, "rewards/reward_solution_effectiveness/std": 0.06289872527122498, "sampling/importance_sampling_ratio/max": 0.5999763607978821, "sampling/importance_sampling_ratio/mean": 0.09187997877597809, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9251880645751953, "sampling/sampling_logp_difference/mean": 0.02371709607541561, "step": 420, "step_time": 126.6902174167335 }, { "clip_ratio/high_max": 0.0019593562465161085, "clip_ratio/high_mean": 0.0019593562465161085, "clip_ratio/low_mean": 0.0012665255489991978, "clip_ratio/low_min": 0.0012665255489991978, "clip_ratio/region_mean": 0.0032258817955153063, "completions/clipped_ratio": 0.0, "completions/max_length": 2191.0, "completions/max_terminated_length": 2191.0, "completions/mean_length": 871.3125, "completions/mean_terminated_length": 871.3125, "completions/min_length": 670.0, "completions/min_terminated_length": 670.0, "entropy": 0.10975820291787386, "epoch": 8.591836734693878, "frac_reward_zero_std": 0.0, "grad_norm": 0.015450048260390759, "learning_rate": 1.4285714285714286e-06, "loss": 0.011, "num_tokens": 21000157.0, "reward": 0.7140971422195435, "reward_std": 0.09192641824483871, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.0290975384414196, "rewards/reward_commands_correctness/mean": 0.06812500208616257, "rewards/reward_commands_correctness/std": 0.021669872105121613, "rewards/reward_diversity/mean": 0.16847223043441772, "rewards/reward_diversity/std": 0.022373102605342865, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08374999463558197, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.1612500101327896, "rewards/reward_solution_effectiveness/std": 0.06086871027946472, "sampling/importance_sampling_ratio/max": 0.34076523780822754, "sampling/importance_sampling_ratio/mean": 0.044498343020677567, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.2288312911987305, "sampling/sampling_logp_difference/mean": 0.022937631234526634, "step": 421, "step_time": 177.96579953283072 }, { "clip_ratio/high_max": 0.002309555871761404, "clip_ratio/high_mean": 0.002309555871761404, "clip_ratio/low_mean": 0.0011152396764373407, "clip_ratio/low_min": 0.0011152396764373407, "clip_ratio/region_mean": 0.00342479556275066, "completions/clipped_ratio": 0.0, "completions/max_length": 937.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 764.25, "completions/mean_terminated_length": 764.25, "completions/min_length": 620.0, "completions/min_terminated_length": 620.0, "entropy": 0.10080202575773001, "epoch": 8.612244897959183, "frac_reward_zero_std": 0.0, "grad_norm": 0.024832533672451973, "learning_rate": 1.4081632653061225e-06, "loss": -0.0181, "num_tokens": 21039513.0, "reward": 0.7023821473121643, "reward_std": 0.08243609219789505, "rewards/reward_commands_completeness/mean": 0.125, "rewards/reward_commands_completeness/std": 0.033065591007471085, "rewards/reward_commands_correctness/mean": 0.06937499344348907, "rewards/reward_commands_correctness/std": 0.020806651562452316, "rewards/reward_diversity/mean": 0.17675712704658508, "rewards/reward_diversity/std": 0.019372450187802315, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.056091442704200745, "sampling/importance_sampling_ratio/max": 0.5480059385299683, "sampling/importance_sampling_ratio/mean": 0.08298276364803314, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.8755252361297607, "sampling/sampling_logp_difference/mean": 0.021772390231490135, "step": 422, "step_time": 132.29823834449053 }, { "clip_ratio/high_max": 0.002190031977079343, "clip_ratio/high_mean": 0.002190031977079343, "clip_ratio/low_mean": 0.0010966812988044694, "clip_ratio/low_min": 0.0010966812988044694, "clip_ratio/region_mean": 0.0032867132904357277, "completions/clipped_ratio": 0.0, "completions/max_length": 981.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 804.8125, "completions/mean_terminated_length": 804.8125, "completions/min_length": 660.0, "completions/min_terminated_length": 660.0, "entropy": 0.12206480652093887, "epoch": 8.63265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.03438318520784378, "learning_rate": 1.3877551020408165e-06, "loss": -0.0469, "num_tokens": 21079594.0, "reward": 0.5227022171020508, "reward_std": 0.30611491203308105, "rewards/reward_commands_completeness/mean": 0.10000000149011612, "rewards/reward_commands_completeness/std": 0.0669328048825264, "rewards/reward_commands_correctness/mean": 0.04999999701976776, "rewards/reward_commands_correctness/std": 0.03405877575278282, "rewards/reward_diversity/mean": 0.12207720428705215, "rewards/reward_diversity/std": 0.08414749801158905, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.058750003576278687, "rewards/reward_problem_validity/std": 0.03913651779294014, "rewards/reward_solution_effectiveness/mean": 0.12937499582767487, "rewards/reward_solution_effectiveness/std": 0.09146721661090851, "sampling/importance_sampling_ratio/max": 1.8296833038330078, "sampling/importance_sampling_ratio/mean": 0.1470947414636612, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 16.274150848388672, "sampling/sampling_logp_difference/mean": 0.027557088062167168, "step": 423, "step_time": 115.56934751011431 }, { "clip_ratio/high_max": 0.0027338006111676805, "clip_ratio/high_mean": 0.0027338006111676805, "clip_ratio/low_mean": 0.0011297048404230736, "clip_ratio/low_min": 0.0011297048404230736, "clip_ratio/region_mean": 0.0038635054952464998, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 748.625, "completions/mean_terminated_length": 748.625, "completions/min_length": 648.0, "completions/min_terminated_length": 648.0, "entropy": 0.09315296541899443, "epoch": 8.653061224489797, "frac_reward_zero_std": 0.0, "grad_norm": 0.07140696793794632, "learning_rate": 1.3673469387755104e-06, "loss": -0.091, "num_tokens": 21115924.0, "reward": 0.7404528856277466, "reward_std": 0.053923122584819794, "rewards/reward_commands_completeness/mean": 0.13499999046325684, "rewards/reward_commands_completeness/std": 0.0225092563778162, "rewards/reward_commands_correctness/mean": 0.07562500238418579, "rewards/reward_commands_correctness/std": 0.0072743846103549, "rewards/reward_diversity/mean": 0.17170289158821106, "rewards/reward_diversity/std": 0.01366808544844389, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08937500417232513, "rewards/reward_problem_validity/std": 0.0025000013411045074, "rewards/reward_solution_effectiveness/mean": 0.16875000298023224, "rewards/reward_solution_effectiveness/std": 0.044999998062849045, "sampling/importance_sampling_ratio/max": 1.63173246383667, "sampling/importance_sampling_ratio/mean": 0.18555563688278198, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 17.369760513305664, "sampling/sampling_logp_difference/mean": 0.024730414152145386, "step": 424, "step_time": 121.05954643711448 }, { "clip_ratio/high_max": 0.002554263061028905, "clip_ratio/high_mean": 0.002554263061028905, "clip_ratio/low_mean": 0.001992236386286095, "clip_ratio/low_min": 0.001992236386286095, "clip_ratio/region_mean": 0.004546499403659254, "completions/clipped_ratio": 0.0, "completions/max_length": 954.0, "completions/max_terminated_length": 954.0, "completions/mean_length": 785.3125, "completions/mean_terminated_length": 785.3125, "completions/min_length": 572.0, "completions/min_terminated_length": 572.0, "entropy": 0.10347798559814692, "epoch": 8.673469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.003219934180378914, "learning_rate": 1.3469387755102043e-06, "loss": -0.0021, "num_tokens": 21154265.0, "reward": 0.622978150844574, "reward_std": 0.16646930575370789, "rewards/reward_commands_completeness/mean": 0.10999999940395355, "rewards/reward_commands_completeness/std": 0.0505964457988739, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.03156343102455139, "rewards/reward_diversity/mean": 0.16360318660736084, "rewards/reward_diversity/std": 0.0566280335187912, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.0768750011920929, "rewards/reward_problem_validity/std": 0.02651257812976837, "rewards/reward_solution_effectiveness/mean": 0.12187500298023224, "rewards/reward_solution_effectiveness/std": 0.07934891432523727, "sampling/importance_sampling_ratio/max": 0.37934744358062744, "sampling/importance_sampling_ratio/mean": 0.03721495717763901, "sampling/importance_sampling_ratio/min": 2.293906618433539e-06, "sampling/sampling_logp_difference/max": 3.173718214035034, "sampling/sampling_logp_difference/mean": 0.025220317766070366, "step": 425, "step_time": 133.9534895736724 }, { "clip_ratio/high_max": 0.002320630199392326, "clip_ratio/high_mean": 0.002320630199392326, "clip_ratio/low_mean": 0.001022872282192111, "clip_ratio/low_min": 0.001022872282192111, "clip_ratio/region_mean": 0.003343502481584437, "completions/clipped_ratio": 0.0, "completions/max_length": 931.0, "completions/max_terminated_length": 931.0, "completions/mean_length": 745.75, "completions/mean_terminated_length": 745.75, "completions/min_length": 642.0, "completions/min_terminated_length": 642.0, "entropy": 0.11105980444699526, "epoch": 8.693877551020408, "frac_reward_zero_std": 0.0, "grad_norm": 0.011479797773063183, "learning_rate": 1.3265306122448982e-06, "loss": -0.0039, "num_tokens": 21188449.0, "reward": 0.6860631704330444, "reward_std": 0.04585318639874458, "rewards/reward_commands_completeness/mean": 0.12124999612569809, "rewards/reward_commands_completeness/std": 0.02680174447596073, "rewards/reward_commands_correctness/mean": 0.060624998062849045, "rewards/reward_commands_correctness/std": 0.019822128117084503, "rewards/reward_diversity/mean": 0.18793810904026031, "rewards/reward_diversity/std": 0.01766408421099186, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.12937499582767487, "rewards/reward_solution_effectiveness/std": 0.04739462211728096, "sampling/importance_sampling_ratio/max": 0.3969372808933258, "sampling/importance_sampling_ratio/mean": 0.0528382807970047, "sampling/importance_sampling_ratio/min": 3.526380470026659e-17, "sampling/sampling_logp_difference/max": 20.548580169677734, "sampling/sampling_logp_difference/mean": 0.029538504779338837, "step": 426, "step_time": 115.87039827741683 }, { "clip_ratio/high_max": 0.002098351571476087, "clip_ratio/high_mean": 0.002098351571476087, "clip_ratio/low_mean": 0.001168063965451438, "clip_ratio/low_min": 0.001168063965451438, "clip_ratio/region_mean": 0.0032664155296515673, "completions/clipped_ratio": 0.0, "completions/max_length": 937.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 795.25, "completions/mean_terminated_length": 795.25, "completions/min_length": 668.0, "completions/min_terminated_length": 668.0, "entropy": 0.10714566148817539, "epoch": 8.714285714285714, "frac_reward_zero_std": 0.0, "grad_norm": 0.057344820350408554, "learning_rate": 1.3061224489795921e-06, "loss": 0.0142, "num_tokens": 21237133.0, "reward": 0.7359914779663086, "reward_std": 0.07238438725471497, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.03991658240556717, "rewards/reward_commands_correctness/mean": 0.08062499761581421, "rewards/reward_commands_correctness/std": 0.010626227594912052, "rewards/reward_diversity/mean": 0.17474153637886047, "rewards/reward_diversity/std": 0.018207920715212822, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08312500268220901, "rewards/reward_problem_validity/std": 0.008732127025723457, "rewards/reward_solution_effectiveness/mean": 0.16499999165534973, "rewards/reward_solution_effectiveness/std": 0.08342661708593369, "sampling/importance_sampling_ratio/max": 1.0731109380722046, "sampling/importance_sampling_ratio/mean": 0.18610382080078125, "sampling/importance_sampling_ratio/min": 1.0056372047984041e-05, "sampling/sampling_logp_difference/max": 2.869243621826172, "sampling/sampling_logp_difference/mean": 0.024894893169403076, "step": 427, "step_time": 145.54117694869637 }, { "clip_ratio/high_max": 0.0019551540026441216, "clip_ratio/high_mean": 0.0019551540026441216, "clip_ratio/low_mean": 0.0014927358242857736, "clip_ratio/low_min": 0.0014927358242857736, "clip_ratio/region_mean": 0.0034478898414818104, "completions/clipped_ratio": 0.0, "completions/max_length": 1387.0, "completions/max_terminated_length": 1387.0, "completions/mean_length": 781.625, "completions/mean_terminated_length": 781.625, "completions/min_length": 630.0, "completions/min_terminated_length": 630.0, "entropy": 0.09620509762316942, "epoch": 8.73469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.022369297221302986, "learning_rate": 1.2857142857142856e-06, "loss": -0.0283, "num_tokens": 21275203.0, "reward": 0.6708149313926697, "reward_std": 0.15586939454078674, "rewards/reward_commands_completeness/mean": 0.12250000238418579, "rewards/reward_commands_completeness/std": 0.04187282547354698, "rewards/reward_commands_correctness/mean": 0.06312499940395355, "rewards/reward_commands_correctness/std": 0.02651257812976837, "rewards/reward_diversity/mean": 0.17268994450569153, "rewards/reward_diversity/std": 0.019238291308283806, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.1368750035762787, "rewards/reward_solution_effectiveness/std": 0.06569816917181015, "sampling/importance_sampling_ratio/max": 0.5175623893737793, "sampling/importance_sampling_ratio/mean": 0.0996965691447258, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.439915657043457, "sampling/sampling_logp_difference/mean": 0.024238569661974907, "step": 428, "step_time": 129.90957279503345 }, { "clip_ratio/high_max": 0.002147055813111365, "clip_ratio/high_mean": 0.002147055813111365, "clip_ratio/low_mean": 0.0013713526423089206, "clip_ratio/low_min": 0.0013713526423089206, "clip_ratio/region_mean": 0.0035184084699722007, "completions/clipped_ratio": 0.0, "completions/max_length": 1163.0, "completions/max_terminated_length": 1163.0, "completions/mean_length": 786.4375, "completions/mean_terminated_length": 786.4375, "completions/min_length": 610.0, "completions/min_terminated_length": 610.0, "entropy": 0.11528732161968946, "epoch": 8.755102040816327, "frac_reward_zero_std": 0.0, "grad_norm": 0.08901073038578033, "learning_rate": 1.2653061224489795e-06, "loss": -0.1126, "num_tokens": 21307714.0, "reward": 0.6148179173469543, "reward_std": 0.16776365041732788, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.045588742941617966, "rewards/reward_commands_correctness/mean": 0.06062500178813934, "rewards/reward_commands_correctness/std": 0.026700502261519432, "rewards/reward_diversity/mean": 0.16044293344020844, "rewards/reward_diversity/std": 0.06041407212615013, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07562500238418579, "rewards/reward_problem_validity/std": 0.02606882154941559, "rewards/reward_solution_effectiveness/mean": 0.12187500298023224, "rewards/reward_solution_effectiveness/std": 0.06337389349937439, "sampling/importance_sampling_ratio/max": 2.924203395843506, "sampling/importance_sampling_ratio/mean": 0.2212933450937271, "sampling/importance_sampling_ratio/min": 1.7552025383338332e-05, "sampling/sampling_logp_difference/max": 4.354211807250977, "sampling/sampling_logp_difference/mean": 0.027754275128245354, "step": 429, "step_time": 117.22477511502802 }, { "clip_ratio/high_max": 0.0023671995804761536, "clip_ratio/high_mean": 0.0023671995804761536, "clip_ratio/low_mean": 0.0015964923368301243, "clip_ratio/low_min": 0.0015964923368301243, "clip_ratio/region_mean": 0.00396369188092649, "completions/clipped_ratio": 0.0, "completions/max_length": 843.0, "completions/max_terminated_length": 843.0, "completions/mean_length": 746.375, "completions/mean_terminated_length": 746.375, "completions/min_length": 651.0, "completions/min_terminated_length": 651.0, "entropy": 0.10602437611669302, "epoch": 8.775510204081632, "frac_reward_zero_std": 0.0, "grad_norm": 0.06301624327898026, "learning_rate": 1.2448979591836734e-06, "loss": -0.062, "num_tokens": 21345076.0, "reward": 0.6993749737739563, "reward_std": 0.13613003492355347, "rewards/reward_commands_completeness/mean": 0.1274999976158142, "rewards/reward_commands_completeness/std": 0.03991657868027687, "rewards/reward_commands_correctness/mean": 0.06937500089406967, "rewards/reward_commands_correctness/std": 0.026449641212821007, "rewards/reward_diversity/mean": 0.16562500596046448, "rewards/reward_diversity/std": 0.034545328468084335, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08375000208616257, "rewards/reward_problem_validity/std": 0.019958291202783585, "rewards/reward_solution_effectiveness/mean": 0.15937501192092896, "rewards/reward_solution_effectiveness/std": 0.05767364799976349, "sampling/importance_sampling_ratio/max": 1.7244067192077637, "sampling/importance_sampling_ratio/mean": 0.22857147455215454, "sampling/importance_sampling_ratio/min": 1.3461030068384616e-09, "sampling/sampling_logp_difference/max": 16.123062133789062, "sampling/sampling_logp_difference/mean": 0.02782776579260826, "step": 430, "step_time": 136.77577488310635 }, { "clip_ratio/high_max": 0.0029556072840932757, "clip_ratio/high_mean": 0.0029556072840932757, "clip_ratio/low_mean": 0.001106752737541683, "clip_ratio/low_min": 0.001106752737541683, "clip_ratio/region_mean": 0.004062360021634959, "completions/clipped_ratio": 0.0, "completions/max_length": 890.0, "completions/max_terminated_length": 890.0, "completions/mean_length": 721.1875, "completions/mean_terminated_length": 721.1875, "completions/min_length": 659.0, "completions/min_terminated_length": 659.0, "entropy": 0.11545737273991108, "epoch": 8.795918367346939, "frac_reward_zero_std": 0.0, "grad_norm": 0.16923145949840546, "learning_rate": 1.2244897959183673e-06, "loss": 0.2202, "num_tokens": 21379891.0, "reward": 0.7533149719238281, "reward_std": 0.09802432358264923, "rewards/reward_commands_completeness/mean": 0.13625000417232513, "rewards/reward_commands_completeness/std": 0.03879433497786522, "rewards/reward_commands_correctness/mean": 0.07312499731779099, "rewards/reward_commands_correctness/std": 0.01922455057501793, "rewards/reward_diversity/mean": 0.18456494808197021, "rewards/reward_diversity/std": 0.01422814466059208, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08875000476837158, "rewards/reward_problem_validity/std": 0.0034156523179262877, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.07159783691167831, "sampling/importance_sampling_ratio/max": 2.857597827911377, "sampling/importance_sampling_ratio/mean": 0.3980486989021301, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.563666820526123, "sampling/sampling_logp_difference/mean": 0.025465738028287888, "step": 431, "step_time": 104.3933524955064 }, { "clip_ratio/high_max": 0.0015936127019813284, "clip_ratio/high_mean": 0.0015936127019813284, "clip_ratio/low_mean": 0.0014404951907636132, "clip_ratio/low_min": 0.0014404951907636132, "clip_ratio/region_mean": 0.0030341078527271748, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1073.0, "completions/mean_length": 989.0625, "completions/mean_terminated_length": 781.933349609375, "completions/min_length": 688.0, "completions/min_terminated_length": 688.0, "entropy": 0.09067516215145588, "epoch": 8.816326530612244, "frac_reward_zero_std": 0.0, "grad_norm": 0.025530340149998665, "learning_rate": 1.2040816326530612e-06, "loss": 0.0677, "num_tokens": 21421024.0, "reward": 0.669055163860321, "reward_std": 0.06642869859933853, "rewards/reward_commands_completeness/mean": 0.11500000208616257, "rewards/reward_commands_completeness/std": 0.030550505965948105, "rewards/reward_commands_correctness/mean": 0.06687500327825546, "rewards/reward_commands_correctness/std": 0.02151549980044365, "rewards/reward_diversity/mean": 0.17155514657497406, "rewards/reward_diversity/std": 0.01685892418026924, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08437500149011612, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.13124999403953552, "rewards/reward_solution_effectiveness/std": 0.059874869883060455, "sampling/importance_sampling_ratio/max": 1.3619714975357056, "sampling/importance_sampling_ratio/mean": 0.14674630761146545, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.690466403961182, "sampling/sampling_logp_difference/mean": 0.01992359198629856, "step": 432, "step_time": 287.1433847527951 }, { "clip_ratio/high_max": 0.0024294600589200854, "clip_ratio/high_mean": 0.0024294600589200854, "clip_ratio/low_mean": 0.0009251469055016059, "clip_ratio/low_min": 0.0009251469055016059, "clip_ratio/region_mean": 0.0033546069462317973, "completions/clipped_ratio": 0.0, "completions/max_length": 1842.0, "completions/max_terminated_length": 1842.0, "completions/mean_length": 836.75, "completions/mean_terminated_length": 836.75, "completions/min_length": 667.0, "completions/min_terminated_length": 667.0, "entropy": 0.0965965585783124, "epoch": 8.83673469387755, "frac_reward_zero_std": 0.0, "grad_norm": 0.04531364515423775, "learning_rate": 1.1836734693877552e-06, "loss": 0.0568, "num_tokens": 21457596.0, "reward": 0.6402717232704163, "reward_std": 0.06971703469753265, "rewards/reward_commands_completeness/mean": 0.11124999821186066, "rewards/reward_commands_completeness/std": 0.02305789850652218, "rewards/reward_commands_correctness/mean": 0.06499999761581421, "rewards/reward_commands_correctness/std": 0.010327955707907677, "rewards/reward_diversity/mean": 0.16589674353599548, "rewards/reward_diversity/std": 0.018727218732237816, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500774860382, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.11250000447034836, "rewards/reward_solution_effectiveness/std": 0.03549648076295853, "sampling/importance_sampling_ratio/max": 0.5779037475585938, "sampling/importance_sampling_ratio/mean": 0.13285663723945618, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.8691630363464355, "sampling/sampling_logp_difference/mean": 0.02091819979250431, "step": 433, "step_time": 160.6860660240054 }, { "clip_ratio/high_max": 0.001736104786687065, "clip_ratio/high_mean": 0.001736104786687065, "clip_ratio/low_mean": 0.0014535723748849705, "clip_ratio/low_min": 0.0014535723748849705, "clip_ratio/region_mean": 0.003189677168847993, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 772.0, "completions/mean_length": 921.5625, "completions/mean_terminated_length": 709.933349609375, "completions/min_length": 633.0, "completions/min_terminated_length": 633.0, "entropy": 0.09081587265245616, "epoch": 8.857142857142858, "frac_reward_zero_std": 0.0, "grad_norm": 0.002666102722287178, "learning_rate": 1.163265306122449e-06, "loss": -0.0005, "num_tokens": 21496141.0, "reward": 0.7159374952316284, "reward_std": 0.06263577938079834, "rewards/reward_commands_completeness/mean": 0.13749998807907104, "rewards/reward_commands_completeness/std": 0.03172801434993744, "rewards/reward_commands_correctness/mean": 0.06749999523162842, "rewards/reward_commands_correctness/std": 0.013416408561170101, "rewards/reward_diversity/mean": 0.1653124988079071, "rewards/reward_diversity/std": 0.021293528378009796, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.15937499701976776, "rewards/reward_solution_effectiveness/std": 0.05767365172505379, "sampling/importance_sampling_ratio/max": 0.5809204578399658, "sampling/importance_sampling_ratio/mean": 0.05400995537638664, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.399260520935059, "sampling/sampling_logp_difference/mean": 0.019987935200333595, "step": 434, "step_time": 280.50008512474597 }, { "clip_ratio/high_max": 0.002589017283753492, "clip_ratio/high_mean": 0.002589017283753492, "clip_ratio/low_mean": 0.0005994971070322208, "clip_ratio/low_min": 0.0005994971070322208, "clip_ratio/region_mean": 0.0031885143907857127, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 894.0, "completions/mean_length": 956.75, "completions/mean_terminated_length": 747.4667358398438, "completions/min_length": 600.0, "completions/min_terminated_length": 600.0, "entropy": 0.09245162131264806, "epoch": 8.877551020408163, "frac_reward_zero_std": 0.0, "grad_norm": 0.003742811968550086, "learning_rate": 1.142857142857143e-06, "loss": -0.0099, "num_tokens": 21570957.0, "reward": 0.6625000238418579, "reward_std": 0.2056632936000824, "rewards/reward_commands_completeness/mean": 0.1237499937415123, "rewards/reward_commands_completeness/std": 0.056199051439762115, "rewards/reward_commands_correctness/mean": 0.06937500089406967, "rewards/reward_commands_correctness/std": 0.03395462781190872, "rewards/reward_diversity/mean": 0.15187498927116394, "rewards/reward_diversity/std": 0.05668586492538452, "rewards/reward_format/mean": 0.08125000447034836, "rewards/reward_format/std": 0.0403112918138504, "rewards/reward_problem_validity/mean": 0.07500000298023224, "rewards/reward_problem_validity/std": 0.03224903345108032, "rewards/reward_solution_effectiveness/mean": 0.16124999523162842, "rewards/reward_solution_effectiveness/std": 0.07658328860998154, "sampling/importance_sampling_ratio/max": 0.4102711081504822, "sampling/importance_sampling_ratio/mean": 0.04488569125533104, "sampling/importance_sampling_ratio/min": 2.660908648977056e-05, "sampling/sampling_logp_difference/max": 5.243115425109863, "sampling/sampling_logp_difference/mean": 0.018929850310087204, "step": 435, "step_time": 378.7464231867343 }, { "clip_ratio/high_max": 0.0015149118207773427, "clip_ratio/high_mean": 0.0015149118207773427, "clip_ratio/low_mean": 0.000739258925023023, "clip_ratio/low_min": 0.000739258925023023, "clip_ratio/region_mean": 0.0022541707585332915, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1105.0, "completions/mean_length": 1230.125, "completions/mean_terminated_length": 820.7142944335938, "completions/min_length": 716.0, "completions/min_terminated_length": 716.0, "entropy": 0.09385509882122278, "epoch": 8.89795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.03306989744305611, "learning_rate": 1.122448979591837e-06, "loss": -0.0209, "num_tokens": 21616255.0, "reward": 0.7125015258789062, "reward_std": 0.056207895278930664, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.032557643949985504, "rewards/reward_commands_correctness/mean": 0.06499999761581421, "rewards/reward_commands_correctness/std": 0.020000001415610313, "rewards/reward_diversity/mean": 0.18000152707099915, "rewards/reward_diversity/std": 0.01247334573417902, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08312499523162842, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.15187500417232513, "rewards/reward_solution_effectiveness/std": 0.06337389349937439, "sampling/importance_sampling_ratio/max": 1.1904178857803345, "sampling/importance_sampling_ratio/mean": 0.22781912982463837, "sampling/importance_sampling_ratio/min": 0.0005937464884482324, "sampling/sampling_logp_difference/max": 5.607219696044922, "sampling/sampling_logp_difference/mean": 0.01608051359653473, "step": 436, "step_time": 279.0187246929854 }, { "clip_ratio/high_max": 0.0022395748528651893, "clip_ratio/high_mean": 0.0022395748528651893, "clip_ratio/low_mean": 0.0007997808497748338, "clip_ratio/low_min": 0.0007997808497748338, "clip_ratio/region_mean": 0.0030393557171919383, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 855.0, "completions/mean_length": 1168.4375, "completions/mean_terminated_length": 750.2142944335938, "completions/min_length": 654.0, "completions/min_terminated_length": 654.0, "entropy": 0.06743052648380399, "epoch": 8.918367346938776, "frac_reward_zero_std": 0.0, "grad_norm": 0.003979156259447336, "learning_rate": 1.1020408163265308e-06, "loss": 0.0071, "num_tokens": 21657902.0, "reward": 0.7935937643051147, "reward_std": 0.07917508482933044, "rewards/reward_commands_completeness/mean": 0.1550000011920929, "rewards/reward_commands_completeness/std": 0.03224903345108032, "rewards/reward_commands_correctness/mean": 0.08374999463558197, "rewards/reward_commands_correctness/std": 0.018929695710539818, "rewards/reward_diversity/mean": 0.1667187511920929, "rewards/reward_diversity/std": 0.03480164706707001, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.2006250023841858, "rewards/reward_solution_effectiveness/std": 0.0699017196893692, "sampling/importance_sampling_ratio/max": 0.16164924204349518, "sampling/importance_sampling_ratio/mean": 0.03620715066790581, "sampling/importance_sampling_ratio/min": 1.7025915440171957e-05, "sampling/sampling_logp_difference/max": 6.1010284423828125, "sampling/sampling_logp_difference/mean": 0.014312723651528358, "step": 437, "step_time": 278.27272963523865 }, { "clip_ratio/high_max": 0.0030759206565562636, "clip_ratio/high_mean": 0.0030759206565562636, "clip_ratio/low_mean": 0.0013488573240465485, "clip_ratio/low_min": 0.0013488573240465485, "clip_ratio/region_mean": 0.00442477798787877, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1364.0, "completions/mean_length": 1061.25, "completions/mean_terminated_length": 858.933349609375, "completions/min_length": 705.0, "completions/min_terminated_length": 705.0, "entropy": 0.12479407014325261, "epoch": 8.938775510204081, "frac_reward_zero_std": 0.0, "grad_norm": 0.015275135636329651, "learning_rate": 1.0816326530612247e-06, "loss": -0.0137, "num_tokens": 21702662.0, "reward": 0.7384636402130127, "reward_std": 0.07802146673202515, "rewards/reward_commands_completeness/mean": 0.13500000536441803, "rewards/reward_commands_completeness/std": 0.03224903345108032, "rewards/reward_commands_correctness/mean": 0.06937499344348907, "rewards/reward_commands_correctness/std": 0.009979145601391792, "rewards/reward_diversity/mean": 0.18408861756324768, "rewards/reward_diversity/std": 0.017586268484592438, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.16312499344348907, "rewards/reward_solution_effectiveness/std": 0.05249999463558197, "sampling/importance_sampling_ratio/max": 0.6680004596710205, "sampling/importance_sampling_ratio/mean": 0.09200222790241241, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 9.23634147644043, "sampling/sampling_logp_difference/mean": 0.021603994071483612, "step": 438, "step_time": 302.539324330166 }, { "clip_ratio/high_max": 0.0020413850870681927, "clip_ratio/high_mean": 0.0020413850870681927, "clip_ratio/low_mean": 0.001038786820572568, "clip_ratio/low_min": 0.001038786820572568, "clip_ratio/region_mean": 0.0030801719149167184, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 824.0625, "completions/mean_terminated_length": 824.0625, "completions/min_length": 603.0, "completions/min_terminated_length": 603.0, "entropy": 0.1178899509832263, "epoch": 8.959183673469388, "frac_reward_zero_std": 0.0, "grad_norm": 0.04645416513085365, "learning_rate": 1.0612244897959184e-06, "loss": -0.0874, "num_tokens": 21733203.0, "reward": 0.7459919452667236, "reward_std": 0.07332257926464081, "rewards/reward_commands_completeness/mean": 0.13749998807907104, "rewards/reward_commands_completeness/std": 0.03415650501847267, "rewards/reward_commands_correctness/mean": 0.07874999940395355, "rewards/reward_commands_correctness/std": 0.018211718648672104, "rewards/reward_diversity/mean": 0.17661693692207336, "rewards/reward_diversity/std": 0.02030724659562111, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.013102165423333645, "rewards/reward_solution_effectiveness/mean": 0.16687500476837158, "rewards/reward_solution_effectiveness/std": 0.06569816917181015, "sampling/importance_sampling_ratio/max": 2.736140012741089, "sampling/importance_sampling_ratio/mean": 0.2518633008003235, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 16.999963760375977, "sampling/sampling_logp_difference/mean": 0.02714143693447113, "step": 439, "step_time": 119.77412211894989 }, { "clip_ratio/high_max": 0.00205041203298606, "clip_ratio/high_mean": 0.00205041203298606, "clip_ratio/low_mean": 0.0017432282111258246, "clip_ratio/low_min": 0.0017432282111258246, "clip_ratio/region_mean": 0.0037936401859042235, "completions/clipped_ratio": 0.0, "completions/max_length": 1040.0, "completions/max_terminated_length": 1040.0, "completions/mean_length": 789.625, "completions/mean_terminated_length": 789.625, "completions/min_length": 599.0, "completions/min_terminated_length": 599.0, "entropy": 0.09325657319277525, "epoch": 8.979591836734693, "frac_reward_zero_std": 0.0, "grad_norm": 0.03302445262670517, "learning_rate": 1.0408163265306123e-06, "loss": 0.0553, "num_tokens": 21776893.0, "reward": 0.7157502174377441, "reward_std": 0.1164984405040741, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.0472581572830677, "rewards/reward_commands_correctness/mean": 0.07562500238418579, "rewards/reward_commands_correctness/std": 0.024486390873789787, "rewards/reward_diversity/mean": 0.1670001745223999, "rewards/reward_diversity/std": 0.043253518640995026, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.16500000655651093, "rewards/reward_solution_effectiveness/std": 0.08694826811552048, "sampling/importance_sampling_ratio/max": 0.4957256019115448, "sampling/importance_sampling_ratio/mean": 0.1445075273513794, "sampling/importance_sampling_ratio/min": 1.2033937309752218e-05, "sampling/sampling_logp_difference/max": 3.289987564086914, "sampling/sampling_logp_difference/mean": 0.02397795021533966, "step": 440, "step_time": 143.66625700891018 }, { "clip_ratio/high_max": 0.002247750453534536, "clip_ratio/high_mean": 0.002247750453534536, "clip_ratio/low_mean": 0.0007921919896034524, "clip_ratio/low_min": 0.0007921919896034524, "clip_ratio/region_mean": 0.003039942472241819, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 873.0, "completions/mean_length": 1175.1875, "completions/mean_terminated_length": 757.9285888671875, "completions/min_length": 646.0, "completions/min_terminated_length": 646.0, "entropy": 0.07802598038688302, "epoch": 9.0, "frac_reward_zero_std": 0.0, "grad_norm": 0.031044507399201393, "learning_rate": 1.0204081632653063e-06, "loss": 0.0617, "num_tokens": 21828868.0, "reward": 0.6859007477760315, "reward_std": 0.1756579577922821, "rewards/reward_commands_completeness/mean": 0.1274999976158142, "rewards/reward_commands_completeness/std": 0.043127719312906265, "rewards/reward_commands_correctness/mean": 0.06187500059604645, "rewards/reward_commands_correctness/std": 0.0242813378572464, "rewards/reward_diversity/mean": 0.16965073347091675, "rewards/reward_diversity/std": 0.04187532886862755, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08125000447034836, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.15187498927116394, "rewards/reward_solution_effectiveness/std": 0.07547350227832794, "sampling/importance_sampling_ratio/max": 0.6232930421829224, "sampling/importance_sampling_ratio/mean": 0.11385183781385422, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.320809364318848, "sampling/sampling_logp_difference/mean": 0.017031598836183548, "step": 441, "step_time": 301.78986358083785 }, { "clip_ratio/high_max": 0.0028495113292592578, "clip_ratio/high_mean": 0.0028495113292592578, "clip_ratio/low_mean": 0.001225423184223473, "clip_ratio/low_min": 0.001225423184223473, "clip_ratio/region_mean": 0.0040749345207586884, "completions/clipped_ratio": 0.0, "completions/max_length": 983.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 811.6875, "completions/mean_terminated_length": 811.6875, "completions/min_length": 663.0, "completions/min_terminated_length": 663.0, "entropy": 0.10865582432597876, "epoch": 9.020408163265307, "frac_reward_zero_std": 0.0, "grad_norm": 0.02633552998304367, "learning_rate": 1.0000000000000002e-06, "loss": -0.0364, "num_tokens": 21877579.0, "reward": 0.5285842418670654, "reward_std": 0.2080157846212387, "rewards/reward_commands_completeness/mean": 0.08875000476837158, "rewards/reward_commands_completeness/std": 0.057951126247644424, "rewards/reward_commands_correctness/mean": 0.054999999701976776, "rewards/reward_commands_correctness/std": 0.03425395488739014, "rewards/reward_diversity/mean": 0.13545924425125122, "rewards/reward_diversity/std": 0.07296442240476608, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06187500059604645, "rewards/reward_problem_validity/std": 0.0378098338842392, "rewards/reward_solution_effectiveness/mean": 0.11249999701976776, "rewards/reward_solution_effectiveness/std": 0.07861297577619553, "sampling/importance_sampling_ratio/max": 0.9241535067558289, "sampling/importance_sampling_ratio/mean": 0.17775289714336395, "sampling/importance_sampling_ratio/min": 0.00014770431153010577, "sampling/sampling_logp_difference/max": 3.3395533561706543, "sampling/sampling_logp_difference/mean": 0.024685028940439224, "step": 442, "step_time": 142.50936095789075 }, { "clip_ratio/high_max": 0.0022831659007351846, "clip_ratio/high_mean": 0.0022831659007351846, "clip_ratio/low_mean": 0.0008230261955759488, "clip_ratio/low_min": 0.0008230261955759488, "clip_ratio/region_mean": 0.0031061921326909214, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 1519.0, "completions/mean_length": 1178.5625, "completions/mean_terminated_length": 761.7857666015625, "completions/min_length": 636.0, "completions/min_terminated_length": 636.0, "entropy": 0.0847571468912065, "epoch": 9.040816326530612, "frac_reward_zero_std": 0.0, "grad_norm": 0.05142473429441452, "learning_rate": 9.795918367346939e-07, "loss": -0.098, "num_tokens": 21917208.0, "reward": 0.6578237414360046, "reward_std": 0.1639532744884491, "rewards/reward_commands_completeness/mean": 0.11625000089406967, "rewards/reward_commands_completeness/std": 0.03879432752728462, "rewards/reward_commands_correctness/mean": 0.07562500238418579, "rewards/reward_commands_correctness/std": 0.02682505175471306, "rewards/reward_diversity/mean": 0.15407373011112213, "rewards/reward_diversity/std": 0.04221506789326668, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.1368750035762787, "rewards/reward_solution_effectiveness/std": 0.06750000268220901, "sampling/importance_sampling_ratio/max": 1.6244032382965088, "sampling/importance_sampling_ratio/mean": 0.23352330923080444, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.805202484130859, "sampling/sampling_logp_difference/mean": 0.01578972302377224, "step": 443, "step_time": 255.5785269420594 }, { "clip_ratio/high_max": 0.002532849699491635, "clip_ratio/high_mean": 0.002532849699491635, "clip_ratio/low_mean": 0.001191036331874784, "clip_ratio/low_min": 0.001191036331874784, "clip_ratio/region_mean": 0.003723886053194292, "completions/clipped_ratio": 0.0, "completions/max_length": 935.0, "completions/max_terminated_length": 935.0, "completions/mean_length": 752.9375, "completions/mean_terminated_length": 752.9375, "completions/min_length": 620.0, "completions/min_terminated_length": 620.0, "entropy": 0.09963917639106512, "epoch": 9.061224489795919, "frac_reward_zero_std": 0.0, "grad_norm": 0.0293466467410326, "learning_rate": 9.591836734693878e-07, "loss": 0.0029, "num_tokens": 21955799.0, "reward": 0.7385355234146118, "reward_std": 0.05013730749487877, "rewards/reward_commands_completeness/mean": 0.13875000178813934, "rewards/reward_commands_completeness/std": 0.02777889184653759, "rewards/reward_commands_correctness/mean": 0.07437500357627869, "rewards/reward_commands_correctness/std": 0.010307765565812588, "rewards/reward_diversity/mean": 0.17541053891181946, "rewards/reward_diversity/std": 0.019255071878433228, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08500000089406967, "rewards/reward_problem_validity/std": 0.005163980647921562, "rewards/reward_solution_effectiveness/mean": 0.16499999165534973, "rewards/reward_solution_effectiveness/std": 0.053665630519390106, "sampling/importance_sampling_ratio/max": 0.7925037741661072, "sampling/importance_sampling_ratio/mean": 0.16103217005729675, "sampling/importance_sampling_ratio/min": 1.4973145986973968e-11, "sampling/sampling_logp_difference/max": 16.613107681274414, "sampling/sampling_logp_difference/mean": 0.026654860004782677, "step": 444, "step_time": 128.04429041594267 }, { "clip_ratio/high_max": 0.0014105195441516116, "clip_ratio/high_mean": 0.0014105195441516116, "clip_ratio/low_mean": 0.0018986323339049704, "clip_ratio/low_min": 0.0018986323339049704, "clip_ratio/region_mean": 0.0033091518926084973, "completions/clipped_ratio": 0.0, "completions/max_length": 1413.0, "completions/max_terminated_length": 1413.0, "completions/mean_length": 811.25, "completions/mean_terminated_length": 811.25, "completions/min_length": 532.0, "completions/min_terminated_length": 532.0, "entropy": 0.10456107091158628, "epoch": 9.081632653061224, "frac_reward_zero_std": 0.0, "grad_norm": 0.008873412385582924, "learning_rate": 9.387755102040817e-07, "loss": -0.0025, "num_tokens": 21993783.0, "reward": 0.4920138716697693, "reward_std": 0.2130347490310669, "rewards/reward_commands_completeness/mean": 0.08374999463558197, "rewards/reward_commands_completeness/std": 0.050711605697870255, "rewards/reward_commands_correctness/mean": 0.05374999716877937, "rewards/reward_commands_correctness/std": 0.03263433650135994, "rewards/reward_diversity/mean": 0.12888887524604797, "rewards/reward_diversity/std": 0.069324791431427, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.0612499974668026, "rewards/reward_problem_validity/std": 0.035939764231443405, "rewards/reward_solution_effectiveness/mean": 0.09562500566244125, "rewards/reward_solution_effectiveness/std": 0.06500961631536484, "sampling/importance_sampling_ratio/max": 0.18718278408050537, "sampling/importance_sampling_ratio/mean": 0.033754974603652954, "sampling/importance_sampling_ratio/min": 3.1382226097775856e-06, "sampling/sampling_logp_difference/max": 3.2164244651794434, "sampling/sampling_logp_difference/mean": 0.02482610009610653, "step": 445, "step_time": 141.46472429856658 }, { "clip_ratio/high_max": 0.0018769037706078961, "clip_ratio/high_mean": 0.0018769037706078961, "clip_ratio/low_mean": 0.0012180699122836813, "clip_ratio/low_min": 0.0012180699122836813, "clip_ratio/region_mean": 0.0030949736828915775, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1038.0, "completions/mean_length": 1003.9375, "completions/mean_terminated_length": 797.800048828125, "completions/min_length": 609.0, "completions/min_terminated_length": 609.0, "entropy": 0.0990963401272893, "epoch": 9.10204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.005143088288605213, "learning_rate": 9.183673469387756e-07, "loss": 0.0073, "num_tokens": 22035642.0, "reward": 0.5982520580291748, "reward_std": 0.21400782465934753, "rewards/reward_commands_completeness/mean": 0.11124999821186066, "rewards/reward_commands_completeness/std": 0.06238322705030441, "rewards/reward_commands_correctness/mean": 0.05874999985098839, "rewards/reward_commands_correctness/std": 0.03324154019355774, "rewards/reward_diversity/mean": 0.1388770043849945, "rewards/reward_diversity/std": 0.072355717420578, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.07000000029802322, "rewards/reward_problem_validity/std": 0.03577708825469017, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.08500735461711884, "sampling/importance_sampling_ratio/max": 0.21869000792503357, "sampling/importance_sampling_ratio/mean": 0.034601204097270966, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 16.249963760375977, "sampling/sampling_logp_difference/mean": 0.02277468517422676, "step": 446, "step_time": 263.15786540322006 }, { "clip_ratio/high_max": 0.002946107961179223, "clip_ratio/high_mean": 0.002946107961179223, "clip_ratio/low_mean": 0.0011295749209239148, "clip_ratio/low_min": 0.0011295749209239148, "clip_ratio/region_mean": 0.004075682852999307, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 2794.0, "completions/mean_length": 1046.8125, "completions/mean_terminated_length": 843.5333862304688, "completions/min_length": 578.0, "completions/min_terminated_length": 578.0, "entropy": 0.10594210308045149, "epoch": 9.122448979591837, "frac_reward_zero_std": 0.0, "grad_norm": 0.014699100516736507, "learning_rate": 8.979591836734694e-07, "loss": 0.0114, "num_tokens": 22071371.0, "reward": 0.7260353565216064, "reward_std": 0.08028295636177063, "rewards/reward_commands_completeness/mean": 0.1262499988079071, "rewards/reward_commands_completeness/std": 0.03630886226892471, "rewards/reward_commands_correctness/mean": 0.08124999701976776, "rewards/reward_commands_correctness/std": 0.01147461123764515, "rewards/reward_diversity/mean": 0.17478537559509277, "rewards/reward_diversity/std": 0.014828371815383434, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.15562498569488525, "rewards/reward_solution_effectiveness/std": 0.06408002227544785, "sampling/importance_sampling_ratio/max": 0.24304920434951782, "sampling/importance_sampling_ratio/mean": 0.06683811545372009, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 18.622467041015625, "sampling/sampling_logp_difference/mean": 0.021286491304636, "step": 447, "step_time": 267.5308559201658 }, { "clip_ratio/high_max": 0.003936394467018545, "clip_ratio/high_mean": 0.003936394467018545, "clip_ratio/low_mean": 0.0008019410815904848, "clip_ratio/low_min": 0.0008019410815904848, "clip_ratio/region_mean": 0.004738335555884987, "completions/clipped_ratio": 0.0, "completions/max_length": 996.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 783.5625, "completions/mean_terminated_length": 783.5625, "completions/min_length": 656.0, "completions/min_terminated_length": 656.0, "entropy": 0.1032112818211317, "epoch": 9.142857142857142, "frac_reward_zero_std": 0.0, "grad_norm": 0.027032846584916115, "learning_rate": 8.775510204081633e-07, "loss": -0.0726, "num_tokens": 22101924.0, "reward": 0.7076747417449951, "reward_std": 0.16941776871681213, "rewards/reward_commands_completeness/mean": 0.13374999165534973, "rewards/reward_commands_completeness/std": 0.04425306245684624, "rewards/reward_commands_correctness/mean": 0.07187499850988388, "rewards/reward_commands_correctness/std": 0.022867372259497643, "rewards/reward_diversity/mean": 0.16329969465732574, "rewards/reward_diversity/std": 0.041077837347984314, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187499642372131, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.16312499344348907, "rewards/reward_solution_effectiveness/std": 0.0683831125497818, "sampling/importance_sampling_ratio/max": 1.0285521745681763, "sampling/importance_sampling_ratio/mean": 0.17186297476291656, "sampling/importance_sampling_ratio/min": 3.1228572083819017e-07, "sampling/sampling_logp_difference/max": 11.968563079833984, "sampling/sampling_logp_difference/mean": 0.027418002486228943, "step": 448, "step_time": 118.64390867203474 }, { "clip_ratio/high_max": 0.0010389996095909737, "clip_ratio/high_mean": 0.0010389996095909737, "clip_ratio/low_mean": 0.0008986730026663281, "clip_ratio/low_min": 0.0008986730026663281, "clip_ratio/region_mean": 0.0019376726049813442, "completions/clipped_ratio": 0.0, "completions/max_length": 1030.0, "completions/max_terminated_length": 1030.0, "completions/mean_length": 850.3125, "completions/mean_terminated_length": 850.3125, "completions/min_length": 726.0, "completions/min_terminated_length": 726.0, "entropy": 0.11852216720581055, "epoch": 9.16326530612245, "frac_reward_zero_std": 0.0, "grad_norm": 0.007311915513128042, "learning_rate": 8.571428571428572e-07, "loss": 0.001, "num_tokens": 22139177.0, "reward": 0.6539233922958374, "reward_std": 0.13212701678276062, "rewards/reward_commands_completeness/mean": 0.11124999821186066, "rewards/reward_commands_completeness/std": 0.03862210363149643, "rewards/reward_commands_correctness/mean": 0.06499999761581421, "rewards/reward_commands_correctness/std": 0.02529822289943695, "rewards/reward_diversity/mean": 0.17829841375350952, "rewards/reward_diversity/std": 0.04424242302775383, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.1237500011920929, "rewards/reward_solution_effectiveness/std": 0.06469158083200455, "sampling/importance_sampling_ratio/max": 0.18856777250766754, "sampling/importance_sampling_ratio/mean": 0.04388090595602989, "sampling/importance_sampling_ratio/min": 3.945768912672065e-05, "sampling/sampling_logp_difference/max": 2.686622381210327, "sampling/sampling_logp_difference/mean": 0.025655275210738182, "step": 449, "step_time": 117.64225514419377 }, { "clip_ratio/high_max": 0.0022295126400422305, "clip_ratio/high_mean": 0.0022295126400422305, "clip_ratio/low_mean": 0.0013694234658032656, "clip_ratio/low_min": 0.0013694234658032656, "clip_ratio/region_mean": 0.00359893603308592, "completions/clipped_ratio": 0.0, "completions/max_length": 2849.0, "completions/max_terminated_length": 2849.0, "completions/mean_length": 913.875, "completions/mean_terminated_length": 913.875, "completions/min_length": 630.0, "completions/min_terminated_length": 630.0, "entropy": 0.10427532717585564, "epoch": 9.183673469387756, "frac_reward_zero_std": 0.0, "grad_norm": 0.01849873550236225, "learning_rate": 8.367346938775512e-07, "loss": -0.0415, "num_tokens": 22176235.0, "reward": 0.78187495470047, "reward_std": 0.0730326771736145, "rewards/reward_commands_completeness/mean": 0.1512499898672104, "rewards/reward_commands_completeness/std": 0.023057900369167328, "rewards/reward_commands_correctness/mean": 0.07500000298023224, "rewards/reward_commands_correctness/std": 0.020655911415815353, "rewards/reward_diversity/mean": 0.17624999582767487, "rewards/reward_diversity/std": 0.021169513463974, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.09000000357627869, "rewards/reward_problem_validity/std": 0.0, "rewards/reward_solution_effectiveness/mean": 0.18937499821186066, "rewards/reward_solution_effectiveness/std": 0.04986230656504631, "sampling/importance_sampling_ratio/max": 0.4342193901538849, "sampling/importance_sampling_ratio/mean": 0.058872684836387634, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 19.67405891418457, "sampling/sampling_logp_difference/mean": 0.02688351459801197, "step": 450, "step_time": 204.13655443117023 }, { "clip_ratio/high_max": 0.0033565569028723985, "clip_ratio/high_mean": 0.0033565569028723985, "clip_ratio/low_mean": 0.0018636302447703201, "clip_ratio/low_min": 0.0018636302447703201, "clip_ratio/region_mean": 0.00522018717310857, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 793.0, "completions/mean_length": 915.375, "completions/mean_terminated_length": 703.3333740234375, "completions/min_length": 642.0, "completions/min_terminated_length": 642.0, "entropy": 0.11188037134706974, "epoch": 9.204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.006762199103832245, "learning_rate": 8.163265306122449e-07, "loss": -0.0134, "num_tokens": 22227089.0, "reward": 0.6470190286636353, "reward_std": 0.1502024382352829, "rewards/reward_commands_completeness/mean": 0.11249999701976776, "rewards/reward_commands_completeness/std": 0.04187282547354698, "rewards/reward_commands_correctness/mean": 0.06499999761581421, "rewards/reward_commands_correctness/std": 0.025033313781023026, "rewards/reward_diversity/mean": 0.16701897978782654, "rewards/reward_diversity/std": 0.04303021728992462, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08312500268220901, "rewards/reward_problem_validity/std": 0.01990603096783161, "rewards/reward_solution_effectiveness/mean": 0.12562501430511475, "rewards/reward_solution_effectiveness/std": 0.0631367564201355, "sampling/importance_sampling_ratio/max": 0.2643636465072632, "sampling/importance_sampling_ratio/mean": 0.05142601579427719, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.477161884307861, "sampling/sampling_logp_difference/mean": 0.02272229827940464, "step": 451, "step_time": 385.4219755381346 }, { "clip_ratio/high_max": 0.0021209264232311398, "clip_ratio/high_mean": 0.0021209264232311398, "clip_ratio/low_mean": 0.0024678559420863166, "clip_ratio/low_min": 0.0024678559420863166, "clip_ratio/region_mean": 0.004588782365317456, "completions/clipped_ratio": 0.0, "completions/max_length": 921.0, "completions/max_terminated_length": 921.0, "completions/mean_length": 771.0, "completions/mean_terminated_length": 771.0, "completions/min_length": 638.0, "completions/min_terminated_length": 638.0, "entropy": 0.08999055996537209, "epoch": 9.224489795918368, "frac_reward_zero_std": 0.0, "grad_norm": 0.09923937916755676, "learning_rate": 7.959183673469388e-07, "loss": 0.0525, "num_tokens": 22269701.0, "reward": 0.7111458778381348, "reward_std": 0.15191957354545593, "rewards/reward_commands_completeness/mean": 0.13875000178813934, "rewards/reward_commands_completeness/std": 0.04096747189760208, "rewards/reward_commands_correctness/mean": 0.06875000149011612, "rewards/reward_commands_correctness/std": 0.023057900369167328, "rewards/reward_diversity/mean": 0.17239582538604736, "rewards/reward_diversity/std": 0.045998066663742065, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187499642372131, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.15562501549720764, "rewards/reward_solution_effectiveness/std": 0.06592609733343124, "sampling/importance_sampling_ratio/max": 2.25935697555542, "sampling/importance_sampling_ratio/mean": 0.3653048276901245, "sampling/importance_sampling_ratio/min": 0.00019639130914583802, "sampling/sampling_logp_difference/max": 3.0440731048583984, "sampling/sampling_logp_difference/mean": 0.02192925475537777, "step": 452, "step_time": 137.04035883955657 }, { "clip_ratio/high_max": 0.0020934851199854165, "clip_ratio/high_mean": 0.0020934851199854165, "clip_ratio/low_mean": 0.000891781415703008, "clip_ratio/low_min": 0.000891781415703008, "clip_ratio/region_mean": 0.0029852665393264033, "completions/clipped_ratio": 0.0, "completions/max_length": 2969.0, "completions/max_terminated_length": 2969.0, "completions/mean_length": 922.25, "completions/mean_terminated_length": 922.25, "completions/min_length": 662.0, "completions/min_terminated_length": 662.0, "entropy": 0.10007980279624462, "epoch": 9.244897959183673, "frac_reward_zero_std": 0.0, "grad_norm": 0.10435076802968979, "learning_rate": 7.755102040816327e-07, "loss": -0.1061, "num_tokens": 22311297.0, "reward": 0.7391605377197266, "reward_std": 0.15398122370243073, "rewards/reward_commands_completeness/mean": 0.14375001192092896, "rewards/reward_commands_completeness/std": 0.05327601730823517, "rewards/reward_commands_correctness/mean": 0.06875000149011612, "rewards/reward_commands_correctness/std": 0.028489766642451286, "rewards/reward_diversity/mean": 0.1747855544090271, "rewards/reward_diversity/std": 0.019573215395212173, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07874999940395355, "rewards/reward_problem_validity/std": 0.027049340307712555, "rewards/reward_solution_effectiveness/mean": 0.18562500178813934, "rewards/reward_solution_effectiveness/std": 0.08429858088493347, "sampling/importance_sampling_ratio/max": 2.5815255641937256, "sampling/importance_sampling_ratio/mean": 0.5074887871742249, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 15.624411582946777, "sampling/sampling_logp_difference/mean": 0.020544521510601044, "step": 453, "step_time": 221.68141817487776 }, { "clip_ratio/high_max": 0.002134458190994337, "clip_ratio/high_mean": 0.002134458190994337, "clip_ratio/low_mean": 0.0016379833832615986, "clip_ratio/low_min": 0.0016379833832615986, "clip_ratio/region_mean": 0.0037724415597040206, "completions/clipped_ratio": 0.0625, "completions/max_length": 2293.0, "completions/max_terminated_length": 900.0, "completions/mean_length": 828.3125, "completions/mean_terminated_length": 730.6666870117188, "completions/min_length": 622.0, "completions/min_terminated_length": 622.0, "entropy": 0.09153787512332201, "epoch": 9.26530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.0185824204236269, "learning_rate": 7.551020408163266e-07, "loss": -0.0101, "num_tokens": 22365818.0, "reward": 0.672447919845581, "reward_std": 0.1435500979423523, "rewards/reward_commands_completeness/mean": 0.1237499937415123, "rewards/reward_commands_completeness/std": 0.040804412215948105, "rewards/reward_commands_correctness/mean": 0.06750000268220901, "rewards/reward_commands_correctness/std": 0.022656861692667007, "rewards/reward_diversity/mean": 0.16307291388511658, "rewards/reward_diversity/std": 0.0437413714826107, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.07999999821186066, "rewards/reward_problem_validity/std": 0.019321836531162262, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.06683001667261124, "sampling/importance_sampling_ratio/max": 0.5554821491241455, "sampling/importance_sampling_ratio/mean": 0.07106082886457443, "sampling/importance_sampling_ratio/min": 7.174771599238738e-05, "sampling/sampling_logp_difference/max": 5.174931526184082, "sampling/sampling_logp_difference/mean": 0.02177399955689907, "step": 454, "step_time": 285.31294919364154 }, { "clip_ratio/high_max": 0.0019698893884196877, "clip_ratio/high_mean": 0.0019698893884196877, "clip_ratio/low_mean": 0.0015236589824780822, "clip_ratio/low_min": 0.0015236589824780822, "clip_ratio/region_mean": 0.00349354837089777, "completions/clipped_ratio": 0.0, "completions/max_length": 2830.0, "completions/max_terminated_length": 2830.0, "completions/mean_length": 885.875, "completions/mean_terminated_length": 885.875, "completions/min_length": 666.0, "completions/min_terminated_length": 666.0, "entropy": 0.0902435164898634, "epoch": 9.285714285714286, "frac_reward_zero_std": 0.0, "grad_norm": 0.05148046463727951, "learning_rate": 7.346938775510205e-07, "loss": -0.1146, "num_tokens": 22403644.0, "reward": 0.8110345005989075, "reward_std": 0.08230000734329224, "rewards/reward_commands_completeness/mean": 0.1599999964237213, "rewards/reward_commands_completeness/std": 0.034253958612680435, "rewards/reward_commands_correctness/mean": 0.08312500268220901, "rewards/reward_commands_correctness/std": 0.01302241813391447, "rewards/reward_diversity/mean": 0.16665951907634735, "rewards/reward_diversity/std": 0.024458592757582664, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.21375000476837158, "rewards/reward_solution_effectiveness/std": 0.07658329606056213, "sampling/importance_sampling_ratio/max": 0.8766595125198364, "sampling/importance_sampling_ratio/mean": 0.11704722046852112, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 12.999082565307617, "sampling/sampling_logp_difference/mean": 0.02285221964120865, "step": 455, "step_time": 223.81214103475213 }, { "clip_ratio/high_max": 0.0017568604234838858, "clip_ratio/high_mean": 0.0017568604234838858, "clip_ratio/low_mean": 0.0005918638198636472, "clip_ratio/low_min": 0.0005918638198636472, "clip_ratio/region_mean": 0.002348724243347533, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 833.0, "completions/mean_length": 940.625, "completions/mean_terminated_length": 730.2667236328125, "completions/min_length": 635.0, "completions/min_terminated_length": 635.0, "entropy": 0.09479045635089278, "epoch": 9.306122448979592, "frac_reward_zero_std": 0.0, "grad_norm": 0.011455360800027847, "learning_rate": 7.142857142857143e-07, "loss": -0.0153, "num_tokens": 22449050.0, "reward": 0.70723956823349, "reward_std": 0.16338221728801727, "rewards/reward_commands_completeness/mean": 0.13375000655651093, "rewards/reward_commands_completeness/std": 0.041773200035095215, "rewards/reward_commands_correctness/mean": 0.0625, "rewards/reward_commands_correctness/std": 0.02620432712137699, "rewards/reward_diversity/mean": 0.17473956942558289, "rewards/reward_diversity/std": 0.04445318132638931, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.1612500101327896, "rewards/reward_solution_effectiveness/std": 0.07419569045305252, "sampling/importance_sampling_ratio/max": 0.6495106816291809, "sampling/importance_sampling_ratio/mean": 0.08727486431598663, "sampling/importance_sampling_ratio/min": 5.049170431448147e-05, "sampling/sampling_logp_difference/max": 2.6140449047088623, "sampling/sampling_logp_difference/mean": 0.018011141568422318, "step": 456, "step_time": 305.02741977386177 }, { "clip_ratio/high_max": 0.0015593187708873302, "clip_ratio/high_mean": 0.0015593187708873302, "clip_ratio/low_mean": 0.001496624987339601, "clip_ratio/low_min": 0.001496624987339601, "clip_ratio/region_mean": 0.0030559437145711854, "completions/clipped_ratio": 0.0, "completions/max_length": 1046.0, "completions/max_terminated_length": 1046.0, "completions/mean_length": 808.8125, "completions/mean_terminated_length": 808.8125, "completions/min_length": 686.0, "completions/min_terminated_length": 686.0, "entropy": 0.09388390695676208, "epoch": 9.326530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.13036958873271942, "learning_rate": 6.938775510204082e-07, "loss": -0.0608, "num_tokens": 22488679.0, "reward": 0.7499885559082031, "reward_std": 0.0922466367483139, "rewards/reward_commands_completeness/mean": 0.14124999940395355, "rewards/reward_commands_completeness/std": 0.038965802639722824, "rewards/reward_commands_correctness/mean": 0.06937500089406967, "rewards/reward_commands_correctness/std": 0.015261608175933361, "rewards/reward_diversity/mean": 0.17873859405517578, "rewards/reward_diversity/std": 0.021892916411161423, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.1743749976158142, "rewards/reward_solution_effectiveness/std": 0.07117759436368942, "sampling/importance_sampling_ratio/max": 1.4023264646530151, "sampling/importance_sampling_ratio/mean": 0.1686231940984726, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4599807262420654, "sampling/sampling_logp_difference/mean": 0.024317938834428787, "step": 457, "step_time": 135.33992583490908 }, { "clip_ratio/high_max": 0.0025461883396928897, "clip_ratio/high_mean": 0.0025461883396928897, "clip_ratio/low_mean": 0.0010151659589610063, "clip_ratio/low_min": 0.0010151659589610063, "clip_ratio/region_mean": 0.0035613543059298536, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1084.0, "completions/mean_length": 1047.125, "completions/mean_terminated_length": 843.86669921875, "completions/min_length": 734.0, "completions/min_terminated_length": 734.0, "entropy": 0.09738777624443173, "epoch": 9.346938775510203, "frac_reward_zero_std": 0.0, "grad_norm": 0.020556477829813957, "learning_rate": 6.734693877551021e-07, "loss": -0.054, "num_tokens": 22538369.0, "reward": 0.6806251406669617, "reward_std": 0.08749618381261826, "rewards/reward_commands_completeness/mean": 0.11749999970197678, "rewards/reward_commands_completeness/std": 0.027202943339943886, "rewards/reward_commands_correctness/mean": 0.07250000536441803, "rewards/reward_commands_correctness/std": 0.019832633435726166, "rewards/reward_diversity/mean": 0.17187514901161194, "rewards/reward_diversity/std": 0.015977703034877777, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08187499642372131, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.1368750035762787, "rewards/reward_solution_effectiveness/std": 0.05250000208616257, "sampling/importance_sampling_ratio/max": 0.8675920963287354, "sampling/importance_sampling_ratio/mean": 0.15794923901557922, "sampling/importance_sampling_ratio/min": 1.0985723615519749e-11, "sampling/sampling_logp_difference/max": 15.548311233520508, "sampling/sampling_logp_difference/mean": 0.019446173682808876, "step": 458, "step_time": 297.4190748408437 }, { "clip_ratio/high_max": 0.0025020976027008146, "clip_ratio/high_mean": 0.0025020976027008146, "clip_ratio/low_mean": 0.0007079208444338292, "clip_ratio/low_min": 0.0007079208444338292, "clip_ratio/region_mean": 0.0032100184325827286, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 775.625, "completions/mean_terminated_length": 775.625, "completions/min_length": 620.0, "completions/min_terminated_length": 620.0, "entropy": 0.08816062100231647, "epoch": 9.36734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.02734045684337616, "learning_rate": 6.530612244897961e-07, "loss": -0.039, "num_tokens": 22577987.0, "reward": 0.7432886958122253, "reward_std": 0.14122815430164337, "rewards/reward_commands_completeness/mean": 0.14749999344348907, "rewards/reward_commands_completeness/std": 0.037859391421079636, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.020976178348064423, "rewards/reward_diversity/mean": 0.16141369938850403, "rewards/reward_diversity/std": 0.025507912039756775, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08250000327825546, "rewards/reward_problem_validity/std": 0.012382784858345985, "rewards/reward_solution_effectiveness/mean": 0.18187500536441803, "rewards/reward_solution_effectiveness/std": 0.08084707707166672, "sampling/importance_sampling_ratio/max": 0.5078179836273193, "sampling/importance_sampling_ratio/mean": 0.08298061788082123, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2214207649230957, "sampling/sampling_logp_difference/mean": 0.024342073127627373, "step": 459, "step_time": 135.2339091449976 }, { "clip_ratio/high_max": 0.002698584590689279, "clip_ratio/high_mean": 0.002698584590689279, "clip_ratio/low_mean": 0.0006859094864921644, "clip_ratio/low_min": 0.0006859094864921644, "clip_ratio/region_mean": 0.0033844941208371893, "completions/clipped_ratio": 0.0, "completions/max_length": 1379.0, "completions/max_terminated_length": 1379.0, "completions/mean_length": 793.625, "completions/mean_terminated_length": 793.625, "completions/min_length": 638.0, "completions/min_terminated_length": 638.0, "entropy": 0.08415229246020317, "epoch": 9.387755102040817, "frac_reward_zero_std": 0.0, "grad_norm": 0.01811559870839119, "learning_rate": 6.326530612244898e-07, "loss": -0.0488, "num_tokens": 22626625.0, "reward": 0.6690475940704346, "reward_std": 0.19661767780780792, "rewards/reward_commands_completeness/mean": 0.1287499964237213, "rewards/reward_commands_completeness/std": 0.04787135869264603, "rewards/reward_commands_correctness/mean": 0.061250001192092896, "rewards/reward_commands_correctness/std": 0.0257875956594944, "rewards/reward_diversity/mean": 0.150297611951828, "rewards/reward_diversity/std": 0.03747788444161415, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.0793749988079071, "rewards/reward_problem_validity/std": 0.02379601076245308, "rewards/reward_solution_effectiveness/mean": 0.15562500059604645, "rewards/reward_solution_effectiveness/std": 0.07839801907539368, "sampling/importance_sampling_ratio/max": 0.734679639339447, "sampling/importance_sampling_ratio/mean": 0.09833225607872009, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.65739107131958, "sampling/sampling_logp_difference/mean": 0.021386289969086647, "step": 460, "step_time": 173.56222548894584 }, { "clip_ratio/high_max": 0.0011496684310259297, "clip_ratio/high_mean": 0.0011496684310259297, "clip_ratio/low_mean": 0.0008989287671283819, "clip_ratio/low_min": 0.0008989287671283819, "clip_ratio/region_mean": 0.0020485971981543116, "completions/clipped_ratio": 0.0, "completions/max_length": 1066.0, "completions/max_terminated_length": 1066.0, "completions/mean_length": 774.5625, "completions/mean_terminated_length": 774.5625, "completions/min_length": 642.0, "completions/min_terminated_length": 642.0, "entropy": 0.09521850012242794, "epoch": 9.408163265306122, "frac_reward_zero_std": 0.0, "grad_norm": 0.045282796025276184, "learning_rate": 6.122448979591837e-07, "loss": -0.0674, "num_tokens": 22661394.0, "reward": 0.7699201107025146, "reward_std": 0.08216683566570282, "rewards/reward_commands_completeness/mean": 0.14875000715255737, "rewards/reward_commands_completeness/std": 0.03344149515032768, "rewards/reward_commands_correctness/mean": 0.07437500357627869, "rewards/reward_commands_correctness/std": 0.023084988817572594, "rewards/reward_diversity/mean": 0.1699201464653015, "rewards/reward_diversity/std": 0.020266445353627205, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.18937499821186066, "rewards/reward_solution_effectiveness/std": 0.0699017196893692, "sampling/importance_sampling_ratio/max": 1.215774416923523, "sampling/importance_sampling_ratio/mean": 0.11403883993625641, "sampling/importance_sampling_ratio/min": 5.6758100823450786e-12, "sampling/sampling_logp_difference/max": 17.806623458862305, "sampling/sampling_logp_difference/mean": 0.026076333597302437, "step": 461, "step_time": 128.5305767841637 }, { "clip_ratio/high_max": 0.0009285110427299514, "clip_ratio/high_mean": 0.0009285110427299514, "clip_ratio/low_mean": 0.0011057313604396768, "clip_ratio/low_min": 0.0011057313604396768, "clip_ratio/region_mean": 0.002034242424997501, "completions/clipped_ratio": 0.0, "completions/max_length": 813.0, "completions/max_terminated_length": 813.0, "completions/mean_length": 730.875, "completions/mean_terminated_length": 730.875, "completions/min_length": 671.0, "completions/min_terminated_length": 671.0, "entropy": 0.10502135008573532, "epoch": 9.428571428571429, "frac_reward_zero_std": 0.0, "grad_norm": 0.18494996428489685, "learning_rate": 5.918367346938776e-07, "loss": -0.1289, "num_tokens": 22693064.0, "reward": 0.7903921604156494, "reward_std": 0.09314776957035065, "rewards/reward_commands_completeness/mean": 0.14999999105930328, "rewards/reward_commands_completeness/std": 0.035023804754018784, "rewards/reward_commands_correctness/mean": 0.0793749988079071, "rewards/reward_commands_correctness/std": 0.019822128117084503, "rewards/reward_diversity/mean": 0.17789216339588165, "rewards/reward_diversity/std": 0.025484316051006317, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.19499999284744263, "rewards/reward_solution_effectiveness/std": 0.06928203999996185, "sampling/importance_sampling_ratio/max": 2.3780477046966553, "sampling/importance_sampling_ratio/mean": 0.5028574466705322, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 17.116052627563477, "sampling/sampling_logp_difference/mean": 0.025828668847680092, "step": 462, "step_time": 98.55686357803643 }, { "clip_ratio/high_max": 0.002646284265210852, "clip_ratio/high_mean": 0.002646284265210852, "clip_ratio/low_mean": 0.0005479894753079861, "clip_ratio/low_min": 0.0005479894753079861, "clip_ratio/region_mean": 0.003194273725966923, "completions/clipped_ratio": 0.0, "completions/max_length": 1298.0, "completions/max_terminated_length": 1298.0, "completions/mean_length": 786.8125, "completions/mean_terminated_length": 786.8125, "completions/min_length": 658.0, "completions/min_terminated_length": 658.0, "entropy": 0.09947673603892326, "epoch": 9.448979591836734, "frac_reward_zero_std": 0.0, "grad_norm": 0.03907610476016998, "learning_rate": 5.714285714285715e-07, "loss": -0.1063, "num_tokens": 22730373.0, "reward": 0.8211999535560608, "reward_std": 0.07351594418287277, "rewards/reward_commands_completeness/mean": 0.1599999964237213, "rewards/reward_commands_completeness/std": 0.02190890721976757, "rewards/reward_commands_correctness/mean": 0.0793749988079071, "rewards/reward_commands_correctness/std": 0.010626226663589478, "rewards/reward_diversity/mean": 0.1805749535560608, "rewards/reward_diversity/std": 0.02170000970363617, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08937500417232513, "rewards/reward_problem_validity/std": 0.0025000013411045074, "rewards/reward_solution_effectiveness/mean": 0.21187499165534973, "rewards/reward_solution_effectiveness/std": 0.04020261391997337, "sampling/importance_sampling_ratio/max": 1.4766862392425537, "sampling/importance_sampling_ratio/mean": 0.19559916853904724, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 17.498043060302734, "sampling/sampling_logp_difference/mean": 0.025417622178792953, "step": 463, "step_time": 160.78295081853867 }, { "clip_ratio/high_max": 0.002505403961549746, "clip_ratio/high_mean": 0.002505403961549746, "clip_ratio/low_mean": 0.0009535771278024185, "clip_ratio/low_min": 0.0009535771278024185, "clip_ratio/region_mean": 0.0034589811039040796, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1038.0, "completions/mean_length": 1010.25, "completions/mean_terminated_length": 804.5333862304688, "completions/min_length": 584.0, "completions/min_terminated_length": 584.0, "entropy": 0.090147087816149, "epoch": 9.46938775510204, "frac_reward_zero_std": 0.0, "grad_norm": 0.03682594746351242, "learning_rate": 5.510204081632654e-07, "loss": 0.0646, "num_tokens": 22775297.0, "reward": 0.6417379379272461, "reward_std": 0.13524916768074036, "rewards/reward_commands_completeness/mean": 0.1237500011920929, "rewards/reward_commands_completeness/std": 0.061630621552467346, "rewards/reward_commands_correctness/mean": 0.06187500059604645, "rewards/reward_commands_correctness/std": 0.03208712488412857, "rewards/reward_diversity/mean": 0.14923793077468872, "rewards/reward_diversity/std": 0.06660357862710953, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07187500596046448, "rewards/reward_problem_validity/std": 0.03103090077638626, "rewards/reward_solution_effectiveness/mean": 0.1537500023841858, "rewards/reward_solution_effectiveness/std": 0.09542012959718704, "sampling/importance_sampling_ratio/max": 1.2026742696762085, "sampling/importance_sampling_ratio/mean": 0.12465672940015793, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 14.839491844177246, "sampling/sampling_logp_difference/mean": 0.019686253741383553, "step": 464, "step_time": 276.5509714689106 }, { "clip_ratio/high_max": 0.002129415064700879, "clip_ratio/high_mean": 0.002129415064700879, "clip_ratio/low_mean": 0.0016545804683119059, "clip_ratio/low_min": 0.0016545804683119059, "clip_ratio/region_mean": 0.00378399551846087, "completions/clipped_ratio": 0.0, "completions/max_length": 948.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 733.6875, "completions/mean_terminated_length": 733.6875, "completions/min_length": 614.0, "completions/min_terminated_length": 614.0, "entropy": 0.10045939683914185, "epoch": 9.489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.011658224277198315, "learning_rate": 5.306122448979592e-07, "loss": -0.0188, "num_tokens": 22816992.0, "reward": 0.7615624666213989, "reward_std": 0.07427583634853363, "rewards/reward_commands_completeness/mean": 0.13874998688697815, "rewards/reward_commands_completeness/std": 0.03052321821451187, "rewards/reward_commands_correctness/mean": 0.0793749988079071, "rewards/reward_commands_correctness/std": 0.01569235697388649, "rewards/reward_diversity/mean": 0.1834374964237213, "rewards/reward_diversity/std": 0.014465399086475372, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08562500774860382, "rewards/reward_problem_validity/std": 0.00512347836047411, "rewards/reward_solution_effectiveness/mean": 0.1743749976158142, "rewards/reward_solution_effectiveness/std": 0.05819149687886238, "sampling/importance_sampling_ratio/max": 0.4478585124015808, "sampling/importance_sampling_ratio/mean": 0.09441883862018585, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.758437156677246, "sampling/sampling_logp_difference/mean": 0.025973092764616013, "step": 465, "step_time": 151.23977380804718 }, { "clip_ratio/high_max": 0.0023084114800440148, "clip_ratio/high_mean": 0.0023084114800440148, "clip_ratio/low_mean": 0.0021022819710196927, "clip_ratio/low_min": 0.0021022819710196927, "clip_ratio/region_mean": 0.0044106933928560466, "completions/clipped_ratio": 0.0, "completions/max_length": 885.0, "completions/max_terminated_length": 885.0, "completions/mean_length": 754.4375, "completions/mean_terminated_length": 754.4375, "completions/min_length": 695.0, "completions/min_terminated_length": 695.0, "entropy": 0.10413684509694576, "epoch": 9.510204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.019928421825170517, "learning_rate": 5.102040816326531e-07, "loss": -0.0429, "num_tokens": 22849631.0, "reward": 0.6777880191802979, "reward_std": 0.10644005239009857, "rewards/reward_commands_completeness/mean": 0.1224999949336052, "rewards/reward_commands_completeness/std": 0.03642343729734421, "rewards/reward_commands_correctness/mean": 0.07375000417232513, "rewards/reward_commands_correctness/std": 0.020615529268980026, "rewards/reward_diversity/mean": 0.15716299414634705, "rewards/reward_diversity/std": 0.02703590877354145, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08187499642372131, "rewards/reward_problem_validity/std": 0.012230428867042065, "rewards/reward_solution_effectiveness/mean": 0.14249999821186066, "rewards/reward_solution_effectiveness/std": 0.0714142844080925, "sampling/importance_sampling_ratio/max": 0.43855729699134827, "sampling/importance_sampling_ratio/mean": 0.0670451670885086, "sampling/importance_sampling_ratio/min": 2.9779296255583176e-06, "sampling/sampling_logp_difference/max": 2.952859878540039, "sampling/sampling_logp_difference/mean": 0.025164803490042686, "step": 466, "step_time": 113.08476561680436 }, { "clip_ratio/high_max": 0.002387456908763852, "clip_ratio/high_mean": 0.002387456908763852, "clip_ratio/low_mean": 0.0014347681717481464, "clip_ratio/low_min": 0.0014347681717481464, "clip_ratio/region_mean": 0.003822225087787956, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 740.5625, "completions/mean_terminated_length": 740.5625, "completions/min_length": 598.0, "completions/min_terminated_length": 598.0, "entropy": 0.10041836276650429, "epoch": 9.53061224489796, "frac_reward_zero_std": 0.0, "grad_norm": 0.03351103141903877, "learning_rate": 4.897959183673469e-07, "loss": 0.0467, "num_tokens": 22878848.0, "reward": 0.7230207920074463, "reward_std": 0.15246796607971191, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.044347118586301804, "rewards/reward_commands_correctness/mean": 0.07062499970197678, "rewards/reward_commands_correctness/std": 0.026196375489234924, "rewards/reward_diversity/mean": 0.17739582061767578, "rewards/reward_diversity/std": 0.020504912361502647, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08187500387430191, "rewards/reward_problem_validity/std": 0.019737867638468742, "rewards/reward_solution_effectiveness/mean": 0.16687500476837158, "rewards/reward_solution_effectiveness/std": 0.07665670663118362, "sampling/importance_sampling_ratio/max": 0.3535289466381073, "sampling/importance_sampling_ratio/mean": 0.0707271620631218, "sampling/importance_sampling_ratio/min": 0.00021607351663988084, "sampling/sampling_logp_difference/max": 3.9894981384277344, "sampling/sampling_logp_difference/mean": 0.02528947964310646, "step": 467, "step_time": 97.2449074499309 }, { "clip_ratio/high_max": 0.0022093912957643624, "clip_ratio/high_mean": 0.0022093912957643624, "clip_ratio/low_mean": 0.0005649337981594726, "clip_ratio/low_min": 0.0005649337981594726, "clip_ratio/region_mean": 0.002774325093923835, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 844.0, "completions/mean_length": 1145.375, "completions/mean_terminated_length": 723.857177734375, "completions/min_length": 648.0, "completions/min_terminated_length": 648.0, "entropy": 0.08654334163293242, "epoch": 9.551020408163264, "frac_reward_zero_std": 0.0, "grad_norm": 0.054496001452207565, "learning_rate": 4.6938775510204085e-07, "loss": 0.0761, "num_tokens": 22936070.0, "reward": 0.6998437643051147, "reward_std": 0.07919885963201523, "rewards/reward_commands_completeness/mean": 0.1274999976158142, "rewards/reward_commands_completeness/std": 0.037148356437683105, "rewards/reward_commands_correctness/mean": 0.06812499463558197, "rewards/reward_commands_correctness/std": 0.017969883978366852, "rewards/reward_diversity/mean": 0.17484374344348907, "rewards/reward_diversity/std": 0.02381879836320877, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08312499523162842, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.14624999463558197, "rewards/reward_solution_effectiveness/std": 0.07256032526493073, "sampling/importance_sampling_ratio/max": 1.5197609663009644, "sampling/importance_sampling_ratio/mean": 0.16722989082336426, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.700717926025391, "sampling/sampling_logp_difference/mean": 0.01478979829698801, "step": 468, "step_time": 308.5483252108097 }, { "clip_ratio/high_max": 0.0020606438047252595, "clip_ratio/high_mean": 0.0020606438047252595, "clip_ratio/low_mean": 0.0015944674669299275, "clip_ratio/low_min": 0.0015944674669299275, "clip_ratio/region_mean": 0.0036551112425513566, "completions/clipped_ratio": 0.0, "completions/max_length": 932.0, "completions/max_terminated_length": 932.0, "completions/mean_length": 762.5, "completions/mean_terminated_length": 762.5, "completions/min_length": 675.0, "completions/min_terminated_length": 675.0, "entropy": 0.09210528805851936, "epoch": 9.571428571428571, "frac_reward_zero_std": 0.0, "grad_norm": 0.15532183647155762, "learning_rate": 4.489795918367347e-07, "loss": -0.3052, "num_tokens": 22979506.0, "reward": 0.5557291507720947, "reward_std": 0.26884812116622925, "rewards/reward_commands_completeness/mean": 0.10875000059604645, "rewards/reward_commands_completeness/std": 0.07228416204452515, "rewards/reward_commands_correctness/mean": 0.054999999701976776, "rewards/reward_commands_correctness/std": 0.037237975746393204, "rewards/reward_diversity/mean": 0.125104159116745, "rewards/reward_diversity/std": 0.07426983118057251, "rewards/reward_format/mean": 0.0625, "rewards/reward_format/std": 0.05000000074505806, "rewards/reward_problem_validity/mean": 0.06000000238418579, "rewards/reward_problem_validity/std": 0.04000000283122063, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.09373500198125839, "sampling/importance_sampling_ratio/max": 2.4665699005126953, "sampling/importance_sampling_ratio/mean": 0.27282917499542236, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 13.749638557434082, "sampling/sampling_logp_difference/mean": 0.023423152044415474, "step": 469, "step_time": 120.93293284624815 }, { "clip_ratio/high_max": 0.0028055703442078084, "clip_ratio/high_mean": 0.0028055703442078084, "clip_ratio/low_mean": 0.0005628809667541645, "clip_ratio/low_min": 0.0005628809667541645, "clip_ratio/region_mean": 0.0033684512891341, "completions/clipped_ratio": 0.0, "completions/max_length": 1173.0, "completions/max_terminated_length": 1173.0, "completions/mean_length": 767.0, "completions/mean_terminated_length": 767.0, "completions/min_length": 544.0, "completions/min_terminated_length": 544.0, "entropy": 0.11029389686882496, "epoch": 9.591836734693878, "frac_reward_zero_std": 0.0, "grad_norm": 0.014110194519162178, "learning_rate": 4.285714285714286e-07, "loss": -0.0326, "num_tokens": 23009050.0, "reward": 0.7816702127456665, "reward_std": 0.08939172327518463, "rewards/reward_commands_completeness/mean": 0.14500001072883606, "rewards/reward_commands_completeness/std": 0.029664797708392143, "rewards/reward_commands_correctness/mean": 0.08312500268220901, "rewards/reward_commands_correctness/std": 0.016214707866311073, "rewards/reward_diversity/mean": 0.17604517936706543, "rewards/reward_diversity/std": 0.020364802330732346, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.18937499821186066, "rewards/reward_solution_effectiveness/std": 0.06169481202960014, "sampling/importance_sampling_ratio/max": 1.1292980909347534, "sampling/importance_sampling_ratio/mean": 0.09209266304969788, "sampling/importance_sampling_ratio/min": 1.7240068747592652e-11, "sampling/sampling_logp_difference/max": 16.99881935119629, "sampling/sampling_logp_difference/mean": 0.028575971722602844, "step": 470, "step_time": 126.77960970811546 }, { "clip_ratio/high_max": 0.0030092048618826084, "clip_ratio/high_mean": 0.0030092048618826084, "clip_ratio/low_mean": 0.0010381656247773208, "clip_ratio/low_min": 0.0010381656247773208, "clip_ratio/region_mean": 0.004047370457556099, "completions/clipped_ratio": 0.0, "completions/max_length": 882.0, "completions/max_terminated_length": 882.0, "completions/mean_length": 763.75, "completions/mean_terminated_length": 763.75, "completions/min_length": 688.0, "completions/min_terminated_length": 688.0, "entropy": 0.1083463141694665, "epoch": 9.612244897959183, "frac_reward_zero_std": 0.0, "grad_norm": 0.013644423335790634, "learning_rate": 4.0816326530612243e-07, "loss": 0.0175, "num_tokens": 23064530.0, "reward": 0.6757107973098755, "reward_std": 0.1365063637495041, "rewards/reward_commands_completeness/mean": 0.1287499964237213, "rewards/reward_commands_completeness/std": 0.03862210363149643, "rewards/reward_commands_correctness/mean": 0.060624998062849045, "rewards/reward_commands_correctness/std": 0.025940638035535812, "rewards/reward_diversity/mean": 0.17258578538894653, "rewards/reward_diversity/std": 0.044178884476423264, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08125000447034836, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.13875000178813934, "rewards/reward_solution_effectiveness/std": 0.06280923634767532, "sampling/importance_sampling_ratio/max": 0.536504328250885, "sampling/importance_sampling_ratio/mean": 0.0656682699918747, "sampling/importance_sampling_ratio/min": 1.574997259012889e-05, "sampling/sampling_logp_difference/max": 3.1538712978363037, "sampling/sampling_logp_difference/mean": 0.02428159862756729, "step": 471, "step_time": 209.47379294410348 }, { "clip_ratio/high_max": 0.002290994583745487, "clip_ratio/high_mean": 0.002290994583745487, "clip_ratio/low_mean": 0.0009560768885421567, "clip_ratio/low_min": 0.0009560768885421567, "clip_ratio/region_mean": 0.0032470714795636013, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 731.4375, "completions/mean_terminated_length": 731.4375, "completions/min_length": 627.0, "completions/min_terminated_length": 627.0, "entropy": 0.0977513063699007, "epoch": 9.63265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.010516464710235596, "learning_rate": 3.8775510204081634e-07, "loss": -0.0102, "num_tokens": 23102417.0, "reward": 0.7430698275566101, "reward_std": 0.0818139910697937, "rewards/reward_commands_completeness/mean": 0.14750000834465027, "rewards/reward_commands_completeness/std": 0.03642343729734421, "rewards/reward_commands_correctness/mean": 0.07312500476837158, "rewards/reward_commands_correctness/std": 0.018518010154366493, "rewards/reward_diversity/mean": 0.1530698537826538, "rewards/reward_diversity/std": 0.0327531173825264, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.18187499046325684, "rewards/reward_solution_effectiveness/std": 0.07934891432523727, "sampling/importance_sampling_ratio/max": 0.5252264738082886, "sampling/importance_sampling_ratio/mean": 0.05945760756731033, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.279916763305664, "sampling/sampling_logp_difference/mean": 0.024125874042510986, "step": 472, "step_time": 115.66093474626541 }, { "clip_ratio/high_max": 0.002404100203420967, "clip_ratio/high_mean": 0.002404100203420967, "clip_ratio/low_mean": 0.001207485307531897, "clip_ratio/low_min": 0.001207485307531897, "clip_ratio/region_mean": 0.0036115855327807367, "completions/clipped_ratio": 0.0, "completions/max_length": 822.0, "completions/max_terminated_length": 822.0, "completions/mean_length": 722.1875, "completions/mean_terminated_length": 722.1875, "completions/min_length": 617.0, "completions/min_terminated_length": 617.0, "entropy": 0.09368776064366102, "epoch": 9.653061224489797, "frac_reward_zero_std": 0.0, "grad_norm": 0.0066702584736049175, "learning_rate": 3.6734693877551025e-07, "loss": 0.0063, "num_tokens": 23137624.0, "reward": 0.7473713159561157, "reward_std": 0.08975356817245483, "rewards/reward_commands_completeness/mean": 0.14000000059604645, "rewards/reward_commands_completeness/std": 0.03098386898636818, "rewards/reward_commands_correctness/mean": 0.0768750011920929, "rewards/reward_commands_correctness/std": 0.01662077195942402, "rewards/reward_diversity/mean": 0.1654963195323944, "rewards/reward_diversity/std": 0.029779843986034393, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.17812499403953552, "rewards/reward_solution_effectiveness/std": 0.05741297826170921, "sampling/importance_sampling_ratio/max": 0.4302048683166504, "sampling/importance_sampling_ratio/mean": 0.053179457783699036, "sampling/importance_sampling_ratio/min": 4.365568599951075e-07, "sampling/sampling_logp_difference/max": 2.7966480255126953, "sampling/sampling_logp_difference/mean": 0.02397041767835617, "step": 473, "step_time": 121.62348346039653 }, { "clip_ratio/high_max": 0.0023339207691606134, "clip_ratio/high_mean": 0.0023339207691606134, "clip_ratio/low_mean": 0.000881683939951472, "clip_ratio/low_min": 0.000881683939951472, "clip_ratio/region_mean": 0.00321560469456017, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 795.25, "completions/mean_terminated_length": 795.25, "completions/min_length": 628.0, "completions/min_terminated_length": 628.0, "entropy": 0.09436355624347925, "epoch": 9.673469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.01388089545071125, "learning_rate": 3.469387755102041e-07, "loss": -0.0196, "num_tokens": 23177216.0, "reward": 0.7145925760269165, "reward_std": 0.12161460518836975, "rewards/reward_commands_completeness/mean": 0.14000000059604645, "rewards/reward_commands_completeness/std": 0.054650407284498215, "rewards/reward_commands_correctness/mean": 0.07249999791383743, "rewards/reward_commands_correctness/std": 0.027446920052170753, "rewards/reward_diversity/mean": 0.1470925211906433, "rewards/reward_diversity/std": 0.05660131573677063, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.078125, "rewards/reward_problem_validity/std": 0.026887113228440285, "rewards/reward_solution_effectiveness/mean": 0.18937501311302185, "rewards/reward_solution_effectiveness/std": 0.08465371280908585, "sampling/importance_sampling_ratio/max": 0.4632129967212677, "sampling/importance_sampling_ratio/mean": 0.04368731752038002, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 17.299013137817383, "sampling/sampling_logp_difference/mean": 0.025699922814965248, "step": 474, "step_time": 116.57955537550151 }, { "clip_ratio/high_max": 0.0023996356103452854, "clip_ratio/high_mean": 0.0023996356103452854, "clip_ratio/low_mean": 0.001122326219046954, "clip_ratio/low_min": 0.001122326219046954, "clip_ratio/region_mean": 0.0035219618293922395, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 848.6875, "completions/mean_terminated_length": 848.6875, "completions/min_length": 669.0, "completions/min_terminated_length": 669.0, "entropy": 0.09897684585303068, "epoch": 9.693877551020408, "frac_reward_zero_std": 0.0, "grad_norm": 0.05111366882920265, "learning_rate": 3.2653061224489803e-07, "loss": 0.0513, "num_tokens": 23215715.0, "reward": 0.7432024478912354, "reward_std": 0.1113542839884758, "rewards/reward_commands_completeness/mean": 0.13749998807907104, "rewards/reward_commands_completeness/std": 0.04250490665435791, "rewards/reward_commands_correctness/mean": 0.07437500357627869, "rewards/reward_commands_correctness/std": 0.02421260066330433, "rewards/reward_diversity/mean": 0.17257742583751678, "rewards/reward_diversity/std": 0.021084103733301163, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.1706250011920929, "rewards/reward_solution_effectiveness/std": 0.07646077871322632, "sampling/importance_sampling_ratio/max": 1.7344887256622314, "sampling/importance_sampling_ratio/mean": 0.1449568122625351, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 12.994004249572754, "sampling/sampling_logp_difference/mean": 0.02437179535627365, "step": 475, "step_time": 134.66543434932828 }, { "clip_ratio/high_max": 0.002430152781016659, "clip_ratio/high_mean": 0.002430152781016659, "clip_ratio/low_mean": 0.001066914337570779, "clip_ratio/low_min": 0.001066914337570779, "clip_ratio/region_mean": 0.0034970671258633956, "completions/clipped_ratio": 0.0, "completions/max_length": 933.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 759.8125, "completions/mean_terminated_length": 759.8125, "completions/min_length": 591.0, "completions/min_terminated_length": 591.0, "entropy": 0.09034902136772871, "epoch": 9.714285714285714, "frac_reward_zero_std": 0.0, "grad_norm": 0.12724396586418152, "learning_rate": 3.0612244897959183e-07, "loss": -0.1507, "num_tokens": 23267204.0, "reward": 0.6829503774642944, "reward_std": 0.15504713356494904, "rewards/reward_commands_completeness/mean": 0.12125000357627869, "rewards/reward_commands_completeness/std": 0.04161330312490463, "rewards/reward_commands_correctness/mean": 0.07124999910593033, "rewards/reward_commands_correctness/std": 0.024731896817684174, "rewards/reward_diversity/mean": 0.1667003631591797, "rewards/reward_diversity/std": 0.043511006981134415, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08375000208616257, "rewards/reward_problem_validity/std": 0.019958291202783585, "rewards/reward_solution_effectiveness/mean": 0.14625000953674316, "rewards/reward_solution_effectiveness/std": 0.06917369365692139, "sampling/importance_sampling_ratio/max": 2.327787160873413, "sampling/importance_sampling_ratio/mean": 0.4359661936759949, "sampling/importance_sampling_ratio/min": 0.001467563328333199, "sampling/sampling_logp_difference/max": 2.2604475021362305, "sampling/sampling_logp_difference/mean": 0.022245531901717186, "step": 476, "step_time": 196.5964713525027 }, { "clip_ratio/high_max": 0.0017353116272715852, "clip_ratio/high_mean": 0.0017353116272715852, "clip_ratio/low_mean": 0.0025144438550341874, "clip_ratio/low_min": 0.0025144438550341874, "clip_ratio/region_mean": 0.004249755438650027, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 785.5, "completions/mean_terminated_length": 785.5, "completions/min_length": 673.0, "completions/min_terminated_length": 673.0, "entropy": 0.0861493507400155, "epoch": 9.73469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.017870675772428513, "learning_rate": 2.8571428571428575e-07, "loss": -0.0241, "num_tokens": 23307540.0, "reward": 0.6356894969940186, "reward_std": 0.14946207404136658, "rewards/reward_commands_completeness/mean": 0.11625000089406967, "rewards/reward_commands_completeness/std": 0.05572252720594406, "rewards/reward_commands_correctness/mean": 0.07000000029802322, "rewards/reward_commands_correctness/std": 0.03119829297065735, "rewards/reward_diversity/mean": 0.14506444334983826, "rewards/reward_diversity/std": 0.06226065009832382, "rewards/reward_format/mean": 0.08124999701976776, "rewards/reward_format/std": 0.040311288088560104, "rewards/reward_problem_validity/mean": 0.07500000298023224, "rewards/reward_problem_validity/std": 0.03224903345108032, "rewards/reward_solution_effectiveness/mean": 0.14812499284744263, "rewards/reward_solution_effectiveness/std": 0.07704706490039825, "sampling/importance_sampling_ratio/max": 0.5055487751960754, "sampling/importance_sampling_ratio/mean": 0.06644268333911896, "sampling/importance_sampling_ratio/min": 7.486569524040032e-12, "sampling/sampling_logp_difference/max": 20.561725616455078, "sampling/sampling_logp_difference/mean": 0.028475502505898476, "step": 477, "step_time": 126.91524473018944 }, { "clip_ratio/high_max": 0.0012385192312649451, "clip_ratio/high_mean": 0.0012385192312649451, "clip_ratio/low_mean": 0.0005244319690973498, "clip_ratio/low_min": 0.0005244319690973498, "clip_ratio/region_mean": 0.001762951200362295, "completions/clipped_ratio": 0.125, "completions/max_length": 4096.0, "completions/max_terminated_length": 929.0, "completions/mean_length": 1157.1875, "completions/mean_terminated_length": 737.357177734375, "completions/min_length": 680.0, "completions/min_terminated_length": 680.0, "entropy": 0.08340965351089835, "epoch": 9.755102040816327, "frac_reward_zero_std": 0.25, "grad_norm": 0.017918527126312256, "learning_rate": 2.653061224489796e-07, "loss": 0.0222, "num_tokens": 23347803.0, "reward": 0.5309265851974487, "reward_std": 0.06578756868839264, "rewards/reward_commands_completeness/mean": 0.09624999761581421, "rewards/reward_commands_completeness/std": 0.05572253093123436, "rewards/reward_commands_correctness/mean": 0.05937500298023224, "rewards/reward_commands_correctness/std": 0.03530227020382881, "rewards/reward_diversity/mean": 0.12342655658721924, "rewards/reward_diversity/std": 0.06600797176361084, "rewards/reward_format/mean": 0.07500000298023224, "rewards/reward_format/std": 0.04472136124968529, "rewards/reward_problem_validity/mean": 0.06625000387430191, "rewards/reward_problem_validity/std": 0.033837851136922836, "rewards/reward_solution_effectiveness/mean": 0.1106249988079071, "rewards/reward_solution_effectiveness/std": 0.07075485587120056, "sampling/importance_sampling_ratio/max": 0.8292324542999268, "sampling/importance_sampling_ratio/mean": 0.07483875751495361, "sampling/importance_sampling_ratio/min": 3.835005554719828e-05, "sampling/sampling_logp_difference/max": 5.631042957305908, "sampling/sampling_logp_difference/mean": 0.016583213582634926, "step": 478, "step_time": 258.28691386803985 }, { "clip_ratio/high_max": 0.002665452309884131, "clip_ratio/high_mean": 0.002665452309884131, "clip_ratio/low_mean": 0.0007797659927746281, "clip_ratio/low_min": 0.0007797659927746281, "clip_ratio/region_mean": 0.003445218288106844, "completions/clipped_ratio": 0.0, "completions/max_length": 1627.0, "completions/max_terminated_length": 1627.0, "completions/mean_length": 850.625, "completions/mean_terminated_length": 850.625, "completions/min_length": 696.0, "completions/min_terminated_length": 696.0, "entropy": 0.0964798154309392, "epoch": 9.775510204081632, "frac_reward_zero_std": 0.0, "grad_norm": 0.03791247680783272, "learning_rate": 2.4489795918367347e-07, "loss": -0.0788, "num_tokens": 23393613.0, "reward": 0.716275691986084, "reward_std": 0.095810666680336, "rewards/reward_commands_completeness/mean": 0.13249999284744263, "rewards/reward_commands_completeness/std": 0.032557643949985504, "rewards/reward_commands_correctness/mean": 0.06812499463558197, "rewards/reward_commands_correctness/std": 0.019050372764468193, "rewards/reward_diversity/mean": 0.17565065622329712, "rewards/reward_diversity/std": 0.01937970332801342, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.1537500023841858, "rewards/reward_solution_effectiveness/std": 0.05783597752451897, "sampling/importance_sampling_ratio/max": 0.7749959230422974, "sampling/importance_sampling_ratio/mean": 0.16464394330978394, "sampling/importance_sampling_ratio/min": 4.0930233808467165e-05, "sampling/sampling_logp_difference/max": 3.012068271636963, "sampling/sampling_logp_difference/mean": 0.02151394635438919, "step": 479, "step_time": 182.5316512081772 }, { "clip_ratio/high_max": 0.0019441611038928386, "clip_ratio/high_mean": 0.0019441611038928386, "clip_ratio/low_mean": 0.0017533998397993855, "clip_ratio/low_min": 0.0017533998397993855, "clip_ratio/region_mean": 0.003697560998261906, "completions/clipped_ratio": 0.0, "completions/max_length": 1479.0, "completions/max_terminated_length": 1479.0, "completions/mean_length": 799.5625, "completions/mean_terminated_length": 799.5625, "completions/min_length": 682.0, "completions/min_terminated_length": 682.0, "entropy": 0.10006214212626219, "epoch": 9.795918367346939, "frac_reward_zero_std": 0.0, "grad_norm": 0.012610548175871372, "learning_rate": 2.2448979591836735e-07, "loss": 0.0023, "num_tokens": 23438758.0, "reward": 0.6609722375869751, "reward_std": 0.1555517613887787, "rewards/reward_commands_completeness/mean": 0.11874999850988388, "rewards/reward_commands_completeness/std": 0.046457868069410324, "rewards/reward_commands_correctness/mean": 0.06562499701976776, "rewards/reward_commands_correctness/std": 0.023935679346323013, "rewards/reward_diversity/mean": 0.1634722203016281, "rewards/reward_diversity/std": 0.05812980979681015, "rewards/reward_format/mean": 0.08750000596046448, "rewards/reward_format/std": 0.03415650501847267, "rewards/reward_problem_validity/mean": 0.07750000059604645, "rewards/reward_problem_validity/std": 0.02670830301940441, "rewards/reward_solution_effectiveness/mean": 0.14812499284744263, "rewards/reward_solution_effectiveness/std": 0.07138802856206894, "sampling/importance_sampling_ratio/max": 0.19247384369373322, "sampling/importance_sampling_ratio/mean": 0.044617898762226105, "sampling/importance_sampling_ratio/min": 1.628179916224326e-06, "sampling/sampling_logp_difference/max": 2.6043267250061035, "sampling/sampling_logp_difference/mean": 0.023181773722171783, "step": 480, "step_time": 169.17439588159323 }, { "clip_ratio/high_max": 0.0025559133136994205, "clip_ratio/high_mean": 0.0025559133136994205, "clip_ratio/low_mean": 0.0017628037385293283, "clip_ratio/low_min": 0.0017628037385293283, "clip_ratio/region_mean": 0.004318717044952791, "completions/clipped_ratio": 0.0, "completions/max_length": 882.0, "completions/max_terminated_length": 882.0, "completions/mean_length": 743.1875, "completions/mean_terminated_length": 743.1875, "completions/min_length": 643.0, "completions/min_terminated_length": 643.0, "entropy": 0.1028298307210207, "epoch": 9.816326530612244, "frac_reward_zero_std": 0.0, "grad_norm": 0.12937860190868378, "learning_rate": 2.0408163265306121e-07, "loss": -0.0331, "num_tokens": 23473301.0, "reward": 0.7267456650733948, "reward_std": 0.09717081487178802, "rewards/reward_commands_completeness/mean": 0.13374999165534973, "rewards/reward_commands_completeness/std": 0.037036024034023285, "rewards/reward_commands_correctness/mean": 0.07874999940395355, "rewards/reward_commands_correctness/std": 0.015438050031661987, "rewards/reward_diversity/mean": 0.16487067937850952, "rewards/reward_diversity/std": 0.018493937328457832, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08625000715255737, "rewards/reward_problem_validity/std": 0.005000002682209015, "rewards/reward_solution_effectiveness/mean": 0.16312500834465027, "rewards/reward_solution_effectiveness/std": 0.07345917075872421, "sampling/importance_sampling_ratio/max": 2.585711717605591, "sampling/importance_sampling_ratio/mean": 0.3045397400856018, "sampling/importance_sampling_ratio/min": 1.16240489660413e-05, "sampling/sampling_logp_difference/max": 4.927235126495361, "sampling/sampling_logp_difference/mean": 0.027647364884614944, "step": 481, "step_time": 117.66430759988725 }, { "clip_ratio/high_max": 0.0020607272599590942, "clip_ratio/high_mean": 0.0020607272599590942, "clip_ratio/low_mean": 0.0020420874207047746, "clip_ratio/low_min": 0.0020420874207047746, "clip_ratio/region_mean": 0.004102814680663869, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 809.0, "completions/mean_length": 938.75, "completions/mean_terminated_length": 728.2667236328125, "completions/min_length": 640.0, "completions/min_terminated_length": 640.0, "entropy": 0.0970634138211608, "epoch": 9.83673469387755, "frac_reward_zero_std": 0.0, "grad_norm": 0.025454623624682426, "learning_rate": 1.8367346938775513e-07, "loss": -0.0404, "num_tokens": 23512793.0, "reward": 0.7135600447654724, "reward_std": 0.07385624945163727, "rewards/reward_commands_completeness/mean": 0.13499999046325684, "rewards/reward_commands_completeness/std": 0.03306559473276138, "rewards/reward_commands_correctness/mean": 0.06624999642372131, "rewards/reward_commands_correctness/std": 0.01668332703411579, "rewards/reward_diversity/mean": 0.17168505489826202, "rewards/reward_diversity/std": 0.015365218743681908, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687500655651093, "rewards/reward_problem_validity/std": 0.0047871386632323265, "rewards/reward_solution_effectiveness/mean": 0.1537500023841858, "rewards/reward_solution_effectiveness/std": 0.06086871027946472, "sampling/importance_sampling_ratio/max": 2.435816764831543, "sampling/importance_sampling_ratio/mean": 0.1834351122379303, "sampling/importance_sampling_ratio/min": 6.837940600235015e-05, "sampling/sampling_logp_difference/max": 6.028493404388428, "sampling/sampling_logp_difference/mean": 0.01971079781651497, "step": 482, "step_time": 266.35535011067986 }, { "clip_ratio/high_max": 0.0014224131737137213, "clip_ratio/high_mean": 0.0014224131737137213, "clip_ratio/low_mean": 0.0010730043504736386, "clip_ratio/low_min": 0.0010730043504736386, "clip_ratio/region_mean": 0.00249541752418736, "completions/clipped_ratio": 0.0, "completions/max_length": 897.0, "completions/max_terminated_length": 897.0, "completions/mean_length": 771.0625, "completions/mean_terminated_length": 771.0625, "completions/min_length": 692.0, "completions/min_terminated_length": 692.0, "entropy": 0.10364964418113232, "epoch": 9.857142857142858, "frac_reward_zero_std": 0.0, "grad_norm": 0.1502760648727417, "learning_rate": 1.6326530612244901e-07, "loss": -0.2611, "num_tokens": 23557954.0, "reward": 0.7653461694717407, "reward_std": 0.06381857395172119, "rewards/reward_commands_completeness/mean": 0.14124999940395355, "rewards/reward_commands_completeness/std": 0.021252451464533806, "rewards/reward_commands_correctness/mean": 0.07562500238418579, "rewards/reward_commands_correctness/std": 0.010307765565812588, "rewards/reward_diversity/mean": 0.1822212040424347, "rewards/reward_diversity/std": 0.012874318286776543, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.17812499403953552, "rewards/reward_solution_effectiveness/std": 0.043084222823381424, "sampling/importance_sampling_ratio/max": 2.578310012817383, "sampling/importance_sampling_ratio/mean": 0.3716241419315338, "sampling/importance_sampling_ratio/min": 3.6038509416275177e-12, "sampling/sampling_logp_difference/max": 17.74881935119629, "sampling/sampling_logp_difference/mean": 0.026441853493452072, "step": 483, "step_time": 136.3787358198315 }, { "clip_ratio/high_max": 0.002102848855429329, "clip_ratio/high_mean": 0.002102848855429329, "clip_ratio/low_mean": 0.001384673138090875, "clip_ratio/low_min": 0.001384673138090875, "clip_ratio/region_mean": 0.003487521957140416, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 998.0625, "completions/mean_terminated_length": 791.5333862304688, "completions/min_length": 700.0, "completions/min_terminated_length": 700.0, "entropy": 0.08230381552129984, "epoch": 9.877551020408163, "frac_reward_zero_std": 0.0, "grad_norm": 0.03813162073493004, "learning_rate": 1.4285714285714287e-07, "loss": 0.0259, "num_tokens": 23600607.0, "reward": 0.7565563917160034, "reward_std": 0.07840286195278168, "rewards/reward_commands_completeness/mean": 0.14249999821186066, "rewards/reward_commands_completeness/std": 0.027202943339943886, "rewards/reward_commands_correctness/mean": 0.06812499463558197, "rewards/reward_commands_correctness/std": 0.012763881124556065, "rewards/reward_diversity/mean": 0.1865563839673996, "rewards/reward_diversity/std": 0.009759287349879742, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08687499910593033, "rewards/reward_problem_validity/std": 0.004787138197571039, "rewards/reward_solution_effectiveness/mean": 0.17249999940395355, "rewards/reward_solution_effectiveness/std": 0.047116875648498535, "sampling/importance_sampling_ratio/max": 0.9380287528038025, "sampling/importance_sampling_ratio/mean": 0.15144039690494537, "sampling/importance_sampling_ratio/min": 6.83857942931354e-05, "sampling/sampling_logp_difference/max": 2.890354633331299, "sampling/sampling_logp_difference/mean": 0.017932778224349022, "step": 484, "step_time": 277.9883100669831 }, { "clip_ratio/high_max": 0.0013816032733302563, "clip_ratio/high_mean": 0.0013816032733302563, "clip_ratio/low_mean": 0.001796039672626648, "clip_ratio/low_min": 0.001796039672626648, "clip_ratio/region_mean": 0.0031776429095771164, "completions/clipped_ratio": 0.0, "completions/max_length": 934.0, "completions/max_terminated_length": 934.0, "completions/mean_length": 718.6875, "completions/mean_terminated_length": 718.6875, "completions/min_length": 617.0, "completions/min_terminated_length": 617.0, "entropy": 0.10578685440123081, "epoch": 9.89795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.03204866871237755, "learning_rate": 1.2244897959183673e-07, "loss": 0.0398, "num_tokens": 23637766.0, "reward": 0.7611560821533203, "reward_std": 0.10070258378982544, "rewards/reward_commands_completeness/mean": 0.14374999701976776, "rewards/reward_commands_completeness/std": 0.03283798694610596, "rewards/reward_commands_correctness/mean": 0.07375000417232513, "rewards/reward_commands_correctness/std": 0.020615529268980026, "rewards/reward_diversity/mean": 0.1780310571193695, "rewards/reward_diversity/std": 0.023276517167687416, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08750000596046448, "rewards/reward_problem_validity/std": 0.004472138360142708, "rewards/reward_solution_effectiveness/mean": 0.17812499403953552, "rewards/reward_solution_effectiveness/std": 0.07467429339885712, "sampling/importance_sampling_ratio/max": 0.9812117218971252, "sampling/importance_sampling_ratio/mean": 0.12689073383808136, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6450493335723877, "sampling/sampling_logp_difference/mean": 0.02474232204258442, "step": 485, "step_time": 123.473304733634 }, { "clip_ratio/high_max": 0.0017112443165387958, "clip_ratio/high_mean": 0.0017112443165387958, "clip_ratio/low_mean": 0.0004978098513674922, "clip_ratio/low_min": 0.0004978098513674922, "clip_ratio/region_mean": 0.0022090541606303304, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 980.0, "completions/mean_length": 1000.8125, "completions/mean_terminated_length": 794.4667358398438, "completions/min_length": 609.0, "completions/min_terminated_length": 609.0, "entropy": 0.10256121447309852, "epoch": 9.918367346938776, "frac_reward_zero_std": 0.0, "grad_norm": 0.002672966569662094, "learning_rate": 1.0204081632653061e-07, "loss": -0.0066, "num_tokens": 23673927.0, "reward": 0.7699188590049744, "reward_std": 0.08035193383693695, "rewards/reward_commands_completeness/mean": 0.14500001072883606, "rewards/reward_commands_completeness/std": 0.028751812875270844, "rewards/reward_commands_correctness/mean": 0.08312499523162842, "rewards/reward_commands_correctness/std": 0.013022419065237045, "rewards/reward_diversity/mean": 0.16991882026195526, "rewards/reward_diversity/std": 0.02670321613550186, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.08812500536441803, "rewards/reward_problem_validity/std": 0.0040311310440301895, "rewards/reward_solution_effectiveness/mean": 0.1837500035762787, "rewards/reward_solution_effectiveness/std": 0.046314142644405365, "sampling/importance_sampling_ratio/max": 0.1255238652229309, "sampling/importance_sampling_ratio/mean": 0.02424893155694008, "sampling/importance_sampling_ratio/min": 3.445782638117452e-17, "sampling/sampling_logp_difference/max": 17.298507690429688, "sampling/sampling_logp_difference/mean": 0.023518340662121773, "step": 486, "step_time": 268.50853003002703 }, { "clip_ratio/high_max": 0.0009859471683739685, "clip_ratio/high_mean": 0.0009859471683739685, "clip_ratio/low_mean": 0.0019021261687157676, "clip_ratio/low_min": 0.0019021261687157676, "clip_ratio/region_mean": 0.0028880733443656936, "completions/clipped_ratio": 0.0625, "completions/max_length": 4096.0, "completions/max_terminated_length": 1922.0, "completions/mean_length": 1014.0, "completions/mean_terminated_length": 808.5333862304688, "completions/min_length": 658.0, "completions/min_terminated_length": 658.0, "entropy": 0.09266865300014615, "epoch": 9.938775510204081, "frac_reward_zero_std": 0.0, "grad_norm": 0.028141682967543602, "learning_rate": 8.163265306122451e-08, "loss": -0.0145, "num_tokens": 23723859.0, "reward": 0.4927777647972107, "reward_std": 0.22136835753917694, "rewards/reward_commands_completeness/mean": 0.08624999970197678, "rewards/reward_commands_completeness/std": 0.05200961232185364, "rewards/reward_commands_correctness/mean": 0.05000000074505806, "rewards/reward_commands_correctness/std": 0.03245509788393974, "rewards/reward_diversity/mean": 0.12777778506278992, "rewards/reward_diversity/std": 0.07637465000152588, "rewards/reward_format/mean": 0.06875000149011612, "rewards/reward_format/std": 0.04787135869264603, "rewards/reward_problem_validity/mean": 0.0625, "rewards/reward_problem_validity/std": 0.03678768128156662, "rewards/reward_solution_effectiveness/mean": 0.09749999642372131, "rewards/reward_solution_effectiveness/std": 0.06244998052716255, "sampling/importance_sampling_ratio/max": 0.7075430750846863, "sampling/importance_sampling_ratio/mean": 0.18501931428909302, "sampling/importance_sampling_ratio/min": 0.0011871057795360684, "sampling/sampling_logp_difference/max": 4.291441917419434, "sampling/sampling_logp_difference/mean": 0.016399646177887917, "step": 487, "step_time": 272.85713142715394 }, { "clip_ratio/high_max": 0.0025837369030341506, "clip_ratio/high_mean": 0.0025837369030341506, "clip_ratio/low_mean": 0.001399307067913469, "clip_ratio/low_min": 0.001399307067913469, "clip_ratio/region_mean": 0.003983043920015916, "completions/clipped_ratio": 0.0, "completions/max_length": 3251.0, "completions/max_terminated_length": 3251.0, "completions/mean_length": 948.0625, "completions/mean_terminated_length": 948.0625, "completions/min_length": 649.0, "completions/min_terminated_length": 649.0, "entropy": 0.10400942759588361, "epoch": 9.959183673469388, "frac_reward_zero_std": 0.0, "grad_norm": 0.011259865947067738, "learning_rate": 6.122448979591837e-08, "loss": -0.0007, "num_tokens": 23762024.0, "reward": 0.6762585639953613, "reward_std": 0.14253777265548706, "rewards/reward_commands_completeness/mean": 0.1237500011920929, "rewards/reward_commands_completeness/std": 0.03879433125257492, "rewards/reward_commands_correctness/mean": 0.06937500834465027, "rewards/reward_commands_correctness/std": 0.025157837197184563, "rewards/reward_diversity/mean": 0.1637585461139679, "rewards/reward_diversity/std": 0.04682067409157753, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08124999701976776, "rewards/reward_problem_validity/std": 0.01962141878902912, "rewards/reward_solution_effectiveness/mean": 0.1443749964237213, "rewards/reward_solution_effectiveness/std": 0.06217917427420616, "sampling/importance_sampling_ratio/max": 0.2594126760959625, "sampling/importance_sampling_ratio/mean": 0.048950277268886566, "sampling/importance_sampling_ratio/min": 2.44625097955975e-10, "sampling/sampling_logp_difference/max": 18.241382598876953, "sampling/sampling_logp_difference/mean": 0.024318166077136993, "step": 488, "step_time": 228.1518321763724 }, { "clip_ratio/high_max": 0.0019394392147660255, "clip_ratio/high_mean": 0.0019394392147660255, "clip_ratio/low_mean": 0.000977380339463707, "clip_ratio/low_min": 0.000977380339463707, "clip_ratio/region_mean": 0.0029168195324018598, "completions/clipped_ratio": 0.0, "completions/max_length": 958.0, "completions/max_terminated_length": 958.0, "completions/mean_length": 735.9375, "completions/mean_terminated_length": 735.9375, "completions/min_length": 602.0, "completions/min_terminated_length": 602.0, "entropy": 0.09472176805138588, "epoch": 9.979591836734693, "frac_reward_zero_std": 0.0, "grad_norm": 0.10561031103134155, "learning_rate": 4.0816326530612253e-08, "loss": 0.1156, "num_tokens": 23795235.0, "reward": 0.7793229222297668, "reward_std": 0.14679445326328278, "rewards/reward_commands_completeness/mean": 0.1537500023841858, "rewards/reward_commands_completeness/std": 0.04112987965345383, "rewards/reward_commands_correctness/mean": 0.0793749988079071, "rewards/reward_commands_correctness/std": 0.024891432374715805, "rewards/reward_diversity/mean": 0.16432291269302368, "rewards/reward_diversity/std": 0.043255507946014404, "rewards/reward_format/mean": 0.09375, "rewards/reward_format/std": 0.02500000037252903, "rewards/reward_problem_validity/mean": 0.08375000208616257, "rewards/reward_problem_validity/std": 0.019958291202783585, "rewards/reward_solution_effectiveness/mean": 0.2043749988079071, "rewards/reward_solution_effectiveness/std": 0.06860212236642838, "sampling/importance_sampling_ratio/max": 2.4395124912261963, "sampling/importance_sampling_ratio/mean": 0.43380388617515564, "sampling/importance_sampling_ratio/min": 2.0543051505228505e-05, "sampling/sampling_logp_difference/max": 4.45493221282959, "sampling/sampling_logp_difference/mean": 0.0246244128793478, "step": 489, "step_time": 108.00551187060773 }, { "clip_ratio/high_max": 0.0015274708566721529, "clip_ratio/high_mean": 0.0015274708566721529, "clip_ratio/low_mean": 0.001133353274781257, "clip_ratio/low_min": 0.001133353274781257, "clip_ratio/region_mean": 0.002660824146005325, "completions/clipped_ratio": 0.0, "completions/max_length": 981.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 773.75, "completions/mean_terminated_length": 773.75, "completions/min_length": 666.0, "completions/min_terminated_length": 666.0, "entropy": 0.08828313369303942, "epoch": 10.0, "frac_reward_zero_std": 0.0, "grad_norm": 0.04290207102894783, "learning_rate": 2.0408163265306127e-08, "loss": 0.0413, "num_tokens": 23842291.0, "reward": 0.8273300528526306, "reward_std": 0.06581485271453857, "rewards/reward_commands_completeness/mean": 0.16249999403953552, "rewards/reward_commands_completeness/std": 0.027202943339943886, "rewards/reward_commands_correctness/mean": 0.08562500029802322, "rewards/reward_commands_correctness/std": 0.011528950184583664, "rewards/reward_diversity/mean": 0.16045504808425903, "rewards/reward_diversity/std": 0.02738291211426258, "rewards/reward_format/mean": 0.10000000149011612, "rewards/reward_format/std": 0.0, "rewards/reward_problem_validity/mean": 0.09000000357627869, "rewards/reward_problem_validity/std": 0.0, "rewards/reward_solution_effectiveness/mean": 0.22875002026557922, "rewards/reward_solution_effectiveness/std": 0.056789085268974304, "sampling/importance_sampling_ratio/max": 0.8224916458129883, "sampling/importance_sampling_ratio/mean": 0.19343778491020203, "sampling/importance_sampling_ratio/min": 0.00011223353067180142, "sampling/sampling_logp_difference/max": 4.670352458953857, "sampling/sampling_logp_difference/mean": 0.021431904286146164, "step": 490, "step_time": 149.47226720117033 } ], "logging_steps": 1, "max_steps": 490, "num_input_tokens_seen": 23842291, "num_train_epochs": 10, "save_steps": 10, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }