End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.740017877542414e-06,
-    "train_runtime": 244.1952,
-    "train_samples": 160,
-    "train_samples_per_second": 1.31,
-    "train_steps_per_second": 0.082
+    "train_loss": 3.6508403718471526e-05,
+    "train_runtime": 593.8392,
+    "train_samples": 340,
+    "train_samples_per_second": 0.539,
+    "train_steps_per_second": 0.034
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d4abc4be2746d60e457d4d55a7d42234d18c2c466f8bc4fea5399f339f26446
+oid sha256:b2c7564b8a4a30480aceacfc3b46f3165a0b1df07ed80d567414fdbb4a3c4e71
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.740017877542414e-06,
-    "train_runtime": 244.1952,
-    "train_samples": 160,
-    "train_samples_per_second": 1.31,
-    "train_steps_per_second": 0.082
+    "train_loss": 3.6508403718471526e-05,
+    "train_runtime": 593.8392,
+    "train_samples": 340,
+    "train_samples_per_second": 0.539,
+    "train_steps_per_second": 0.034
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 0.47058823529411764,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 273.6875,
-      "epoch": 0.1,
-      "grad_norm": 5.613799095153809,
+      "completion_length": 342.78125,
+      "epoch": 0.047058823529411764,
+      "grad_norm": 105.05091857910156,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 1.2698225686326623,
-      "reward_std": 0.4694367880001664,
+      "reward": 0.09535764821339399,
+      "reward_std": 0.49169618816813454,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.4705725513631478,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.15682640310842544,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6117499978281558,
+      "rewards/xmlcount_reward_func": -0.061468751868233085,
       "step": 2
     },
     {
-      "completion_length": 395.5,
-      "epoch": 0.2,
-      "grad_norm": 5.6916728019714355,
-      "kl": 0.001478342423979484,
+      "completion_length": 331.8125,
+      "epoch": 0.09411764705882353,
+      "grad_norm": 21.300691604614258,
+      "kl": 0.0020324239685578505,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 2.052010379731655,
-      "reward_std": 1.2824568781652488,
-      "rewards/concensus_correctness_reward_func": 0.27250000461935997,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.6017368285683915,
+      "reward_std": 0.48451118558296,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.646854140330106,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.26783057244028896,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6951562445610762,
+      "rewards/xmlcount_reward_func": 0.08628125162795186,
       "step": 4
     },
     {
-      "completion_length": 354.4375,
-      "epoch": 0.3,
-      "grad_norm": 4.427953243255615,
-      "kl": 0.002940741087513743,
+      "completion_length": 402.5625,
+      "epoch": 0.1411764705882353,
+      "grad_norm": 8.743898391723633,
+      "kl": 0.001346926739643095,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 1.88736292719841,
-      "reward_std": 1.0095689775771461,
-      "rewards/concensus_correctness_reward_func": 0.23024999350309372,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.33102574956137687,
+      "reward_std": 0.6741682634819881,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5639879088848829,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3189320039236918,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.718125008046627,
+      "rewards/xmlcount_reward_func": 0.01209374819882214,
       "step": 6
     },
     {
-      "completion_length": 339.5625,
-      "epoch": 0.4,
-      "grad_norm": 4.619532108306885,
-      "kl": 0.0038217708715819754,
+      "completion_length": 375.3125,
+      "epoch": 0.18823529411764706,
+      "grad_norm": 13.352766990661621,
+      "kl": 0.002045651930529857,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 1.5609666481614113,
-      "reward_std": 0.7928959767086781,
-      "rewards/concensus_correctness_reward_func": 0.0963749997317791,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.7190865158918314,
+      "reward_std": 0.6706056380644441,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.44527911697514355,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.2746177712106146,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7068125046789646,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.17884374968707561,
       "step": 8
     },
     {
-      "completion_length": 255.0625,
-      "epoch": 0.5,
-      "grad_norm": 5.806175231933594,
-      "kl": 0.008361328698811121,
+      "completion_length": 394.875,
+      "epoch": 0.23529411764705882,
+      "grad_norm": 8.22746467590332,
+      "kl": 0.0018384897648502374,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 2.214765891432762,
-      "reward_std": 1.0743446820415556,
-      "rewards/concensus_correctness_reward_func": 0.28068750351667404,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 0.414787842775695,
+      "reward_std": 0.3971917493036017,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5725158285349607,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.1551003473578021,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9240625128149986,
+      "rewards/xmlcount_reward_func": 0.07218750240281224,
       "step": 10
     },
     {
-      "completion_length": 260.15625,
-      "epoch": 0.6,
-      "grad_norm": 5.134825229644775,
-      "kl": 0.0073645137454150245,
+      "completion_length": 374.4375,
+      "epoch": 0.2823529411764706,
+      "grad_norm": 9.52830696105957,
+      "kl": 0.0018439871018927079,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 2.138347875326872,
-      "reward_std": 1.3769710529595613,
-      "rewards/concensus_correctness_reward_func": 0.625,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.47312507111928426,
+      "reward_std": 0.5195176128909225,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5701290960423648,
+      "rewards/question_recreation_reward_func": 0.19587506385869347,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8807187471538782,
+      "rewards/xmlcount_reward_func": 0.08975000167265534,
       "step": 12
     },
     {
-      "completion_length": 280.46875,
-      "epoch": 0.7,
-      "grad_norm": 5.826389789581299,
-      "kl": 0.008991127862827852,
+      "completion_length": 233.0,
+      "epoch": 0.32941176470588235,
+      "grad_norm": 59.9680290222168,
+      "kl": 0.03755158399872016,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 3.125959627330303,
-      "reward_std": 2.497214440256357,
-      "rewards/concensus_correctness_reward_func": 1.3846875000745058,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.3624184874643106,
+      "reward_std": 0.607328199519543,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5701471695210785,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.17566850411822088,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7961250096559525,
+      "rewards/xmlcount_reward_func": -0.06325000803917646,
       "step": 14
     },
     {
-      "completion_length": 391.8125,
-      "epoch": 0.8,
-      "grad_norm": 4.483850955963135,
-      "kl": 0.009203597175655887,
+      "completion_length": 338.28125,
+      "epoch": 0.3764705882352941,
+      "grad_norm": 15.39432144165039,
+      "kl": 0.005397441258537583,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 2.4715022817254066,
-      "reward_std": 1.433350381674245,
-      "rewards/concensus_correctness_reward_func": 0.4466249980032444,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 0.656184867140837,
+      "reward_std": 0.5379592185490765,
+      "rewards/concensus_correctness_reward_func": 0.024562500417232513,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5975959962233901,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.22346611449029297,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7397812604904175,
+      "rewards/xmlcount_reward_func": 0.15815624897368252,
       "step": 16
     },
     {
-      "completion_length": 381.375,
-      "epoch": 0.9,
-      "grad_norm": 5.820269584655762,
-      "kl": 0.007357210670306813,
+      "completion_length": 384.9375,
+      "epoch": 0.4235294117647059,
+      "grad_norm": 64.03816986083984,
+      "kl": 0.030782449430262204,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 1.1668984480202198,
-      "reward_std": 1.2981863738968968,
-      "rewards/concensus_correctness_reward_func": 0.12124999985098839,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.0119954891270027,
+      "reward_std": 0.8672709579986986,
+      "rewards/concensus_correctness_reward_func": 0.23118750005960464,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5702734417282045,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.24224546540062875,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2878750003874302,
+      "rewards/xmlcount_reward_func": 0.22606249852105975,
       "step": 18
     },
     {
-      "completion_length": 453.5625,
-      "epoch": 1.0,
-      "grad_norm": 4.375577926635742,
-      "kl": 0.008185279730241746,
+      "completion_length": 388.21875,
+      "epoch": 0.47058823529411764,
+      "grad_norm": 172.61647033691406,
+      "kl": 0.2819034291896969,
       "learning_rate": 0.0,
-      "loss": 0.0,
-      "reward": 1.2048217430710793,
-      "reward_std": 1.6855253364774399,
-      "rewards/concensus_correctness_reward_func": 0.24656249955296516,
-      "rewards/consensus_reward_func": 0.375,
+      "loss": 0.0003,
+      "reward": 0.27144061494618654,
+      "reward_std": 0.2920520424377173,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.522352964617312,
+      "rewards/question_recreation_reward_func": 0.15850311041867826,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.0015937648713588715,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.03481250023469329,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 0.47058823529411764,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 5.740017877542414e-06,
-      "train_runtime": 244.1952,
-      "train_samples_per_second": 1.31,
-      "train_steps_per_second": 0.082
+      "train_loss": 3.6508403718471526e-05,
+      "train_runtime": 593.8392,
+      "train_samples_per_second": 0.539,
+      "train_steps_per_second": 0.034
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc9374eb6f20be21c4b4b970f30fc27750ebbd47d3e3fae8476e65272df79fa0
+oid sha256:39ee23cf846813ec861713d4ed959f3e69f501717102a8a41870e79416d5b06a
 size 6008