End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.632707830547588e-06,
-    "train_runtime": 249.6853,
+    "train_loss": 5.740017877542414e-06,
+    "train_runtime": 244.1952,
     "train_samples": 160,
-    "train_samples_per_second": 1.282,
-    "train_steps_per_second": 0.08
+    "train_samples_per_second": 1.31,
+    "train_steps_per_second": 0.082
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63be79087db633953e9920f97f970b806482f2c3b73192058c415010e183d215
+oid sha256:7d4abc4be2746d60e457d4d55a7d42234d18c2c466f8bc4fea5399f339f26446
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.632707830547588e-06,
-    "train_runtime": 249.6853,
+    "train_loss": 5.740017877542414e-06,
+    "train_runtime": 244.1952,
     "train_samples": 160,
-    "train_samples_per_second": 1.282,
-    "train_steps_per_second": 0.08
+    "train_samples_per_second": 1.31,
+    "train_steps_per_second": 0.082
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 423.375,
+      "completion_length": 273.6875,
       "epoch": 0.1,
-      "grad_norm": 4.712019920349121,
+      "grad_norm": 5.613799095153809,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
-      "loss": -0.0,
-      "reward": 1.2723155594430864,
-      "reward_std": 1.2591764808166772,
-      "rewards/concensus_correctness_reward_func": 0.17881250008940697,
-      "rewards/consensus_reward_func": 0.125,
+      "loss": 0.0,
+      "reward": 1.2698225686326623,
+      "reward_std": 0.4694367880001664,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5106592940865085,
+      "rewards/question_recreation_reward_func": 0.4705725513631478,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2703437595628202,
+      "rewards/xmlcount_reward_func": 0.6117499978281558,
       "step": 2
     },
     {
-      "completion_length": 340.90625,
+      "completion_length": 395.5,
       "epoch": 0.2,
-      "grad_norm": 5.716149806976318,
-      "kl": 0.0012523982550192159,
+      "grad_norm": 5.6916728019714355,
+      "kl": 0.001478342423979484,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 1.9341710209846497,
-      "reward_std": 0.8787517766613746,
-      "rewards/concensus_correctness_reward_func": 0.12725000083446503,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 2.052010379731655,
+      "reward_std": 1.2824568781652488,
+      "rewards/concensus_correctness_reward_func": 0.27250000461935997,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5809522373601794,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.646854140330106,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7884687539190054,
+      "rewards/xmlcount_reward_func": 0.6951562445610762,
       "step": 4
     },
     {
-      "completion_length": 452.4375,
+      "completion_length": 354.4375,
       "epoch": 0.3,
-      "grad_norm": 4.414357662200928,
-      "kl": 0.0034979822776222136,
+      "grad_norm": 4.427953243255615,
+      "kl": 0.002940741087513743,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 1.9987164307385683,
-      "reward_std": 1.4319329850841314,
-      "rewards/concensus_correctness_reward_func": 0.6856250017881393,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.88736292719841,
+      "reward_std": 1.0095689775771461,
+      "rewards/concensus_correctness_reward_func": 0.23024999350309372,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.481903936015442,
+      "rewards/question_recreation_reward_func": 0.5639879088848829,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.581187498755753,
+      "rewards/xmlcount_reward_func": 0.718125008046627,
       "step": 6
     },
     {
-      "completion_length": 300.0,
+      "completion_length": 339.5625,
       "epoch": 0.4,
-      "grad_norm": 4.599290370941162,
-      "kl": 0.0027592763981374446,
+      "grad_norm": 4.619532108306885,
+      "kl": 0.0038217708715819754,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 1.96402383223176,
-      "reward_std": 1.2454729605233297,
-      "rewards/concensus_correctness_reward_func": 0.3191250003874302,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.5609666481614113,
+      "reward_std": 0.7928959767086781,
+      "rewards/concensus_correctness_reward_func": 0.0963749997317791,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5841176057001576,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.44527911697514355,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.685781255364418,
+      "rewards/xmlcount_reward_func": 0.7068125046789646,
       "step": 8
     },
     {
-      "completion_length": 279.28125,
+      "completion_length": 255.0625,
       "epoch": 0.5,
-      "grad_norm": 5.2565388679504395,
-      "kl": 0.0031722134008305147,
+      "grad_norm": 5.806175231933594,
+      "kl": 0.008361328698811121,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 2.6547921895980835,
-      "reward_std": 1.8105847786646336,
-      "rewards/concensus_correctness_reward_func": 0.5421874988824129,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 2.214765891432762,
+      "reward_std": 1.0743446820415556,
+      "rewards/concensus_correctness_reward_func": 0.28068750351667404,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5164796938188374,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5725158285349607,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8461250029504299,
+      "rewards/xmlcount_reward_func": 0.9240625128149986,
       "step": 10
     },
     {
-      "completion_length": 445.21875,
+      "completion_length": 260.15625,
       "epoch": 0.6,
-      "grad_norm": 4.314850330352783,
-      "kl": 0.0035405759590503294,
+      "grad_norm": 5.134825229644775,
+      "kl": 0.0073645137454150245,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 1.4517545327544212,
-      "reward_std": 1.3524839248275384,
-      "rewards/concensus_correctness_reward_func": 0.15493750013411045,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 2.138347875326872,
+      "reward_std": 1.3769710529595613,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.510785786435008,
+      "rewards/question_recreation_reward_func": 0.5701290960423648,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4735312592238188,
+      "rewards/xmlcount_reward_func": 0.8807187471538782,
       "step": 12
     },
     {
-      "completion_length": 321.25,
+      "completion_length": 280.46875,
       "epoch": 0.7,
-      "grad_norm": 5.351323127746582,
-      "kl": 0.005599002040980849,
+      "grad_norm": 5.826389789581299,
+      "kl": 0.008991127862827852,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 2.4296363964676857,
-      "reward_std": 1.554591533727944,
-      "rewards/concensus_correctness_reward_func": 0.47974999994039536,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 3.125959627330303,
+      "reward_std": 2.497214440256357,
+      "rewards/concensus_correctness_reward_func": 1.3846875000745058,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5946988976793364,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5701471695210785,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7926875092089176,
+      "rewards/xmlcount_reward_func": 0.7961250096559525,
       "step": 14
     },
     {
-      "completion_length": 363.375,
+      "completion_length": 391.8125,
       "epoch": 0.8,
-      "grad_norm": 4.644340991973877,
-      "kl": 0.005632679956761422,
+      "grad_norm": 4.483850955963135,
+      "kl": 0.009203597175655887,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 2.5521909408271313,
-      "reward_std": 1.5568754894193262,
-      "rewards/concensus_correctness_reward_func": 0.346562496968545,
+      "reward": 2.4715022817254066,
+      "reward_std": 1.433350381674245,
+      "rewards/concensus_correctness_reward_func": 0.4466249980032444,
       "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.595378384925425,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5975959962233901,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7352500092238188,
+      "rewards/xmlcount_reward_func": 0.7397812604904175,
       "step": 16
     },
     {
-      "completion_length": 362.125,
+      "completion_length": 381.375,
       "epoch": 0.9,
-      "grad_norm": 4.9260430335998535,
-      "kl": 0.00453515207482269,
+      "grad_norm": 5.820269584655762,
+      "kl": 0.007357210670306813,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 2.1402398254722357,
-      "reward_std": 0.8473401258233935,
-      "rewards/concensus_correctness_reward_func": 0.2888124994933605,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.1668984480202198,
+      "reward_std": 1.2981863738968968,
+      "rewards/concensus_correctness_reward_func": 0.12124999985098839,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5473335776478052,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5702734417282045,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7415937688201666,
+      "rewards/xmlcount_reward_func": 0.2878750003874302,
       "step": 18
     },
     {
-      "completion_length": 294.65625,
+      "completion_length": 453.5625,
       "epoch": 1.0,
-      "grad_norm": 5.505785942077637,
-      "kl": 0.00645016050839331,
+      "grad_norm": 4.375577926635742,
+      "kl": 0.008185279730241746,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 2.8503429237753153,
-      "reward_std": 2.0548626030795276,
-      "rewards/concensus_correctness_reward_func": 0.8583124987781048,
+      "reward": 1.2048217430710793,
+      "reward_std": 1.6855253364774399,
+      "rewards/concensus_correctness_reward_func": 0.24656249955296516,
       "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6864366899244487,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.522352964617312,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6805937539320439,
+      "rewards/xmlcount_reward_func": -0.0015937648713588715,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.632707830547588e-06,
-      "train_runtime": 249.6853,
-      "train_samples_per_second": 1.282,
-      "train_steps_per_second": 0.08
+      "train_loss": 5.740017877542414e-06,
+      "train_runtime": 244.1952,
+      "train_samples_per_second": 1.31,
+      "train_steps_per_second": 0.082
     }
   ],
   "logging_steps": 2,