End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.5534832436969735e-05,
-    "train_runtime": 364.9312,
+    "train_loss": 4.7611911895728555e-06,
+    "train_runtime": 1288.5548,
     "train_samples": 160,
-    "train_samples_per_second": 0.877,
-    "train_steps_per_second": 0.055
+    "train_samples_per_second": 0.248,
+    "train_steps_per_second": 0.016
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:703db15e2be829f2b89f8d0b0c4ca73ee340bf9fbf0a6af4827f3620f7e32f08
+oid sha256:ebb1b68f087bfc0372c9195f43e27c6719372cbf069849129e597881d0ae3ce4
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.5534832436969735e-05,
-    "train_runtime": 364.9312,
+    "train_loss": 4.7611911895728555e-06,
+    "train_runtime": 1288.5548,
     "train_samples": 160,
-    "train_samples_per_second": 0.877,
-    "train_steps_per_second": 0.055
+    "train_samples_per_second": 0.248,
+    "train_steps_per_second": 0.016
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 119.40625,
+      "completion_length": 277.25,
       "epoch": 0.1,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "grad_norm": 10.197137832641602,
+      "kl": -6.773255289971303e-10,
       "learning_rate": 4.965903258506806e-07,
-      "loss": 0.0,
-      "reward": 7.420198768377304,
-      "reward_std": 7.232399366330355e-05,
-      "rewards/concensus_correctness_reward_func": 2.2952499985694885,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": -0.0,
+      "reward": 0.06830399160389788,
+      "reward_std": 0.03100924338650657,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.9999488517642021,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.06439774183672853,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.5,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.00390625,
       "step": 2
     },
     {
-      "completion_length": 118.53125,
+      "completion_length": 308.875,
       "epoch": 0.2,
-      "grad_norm": 0.00973969791084528,
-      "kl": 0.011504065854751389,
+      "grad_norm": 9.83273696899414,
+      "kl": 0.0006185987587059572,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 6.565430022776127,
-      "reward_std": 0.03401839733123779,
-      "rewards/concensus_correctness_reward_func": 1.9239999875426292,
-      "rewards/consensus_reward_func": 1.875,
+      "reward": 0.16232500248588622,
+      "reward_std": 0.1747917605098337,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9856800884008408,
+      "rewards/question_recreation_reward_func": 0.06829375738743693,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.46875,
-      "rewards/xmlcount_reward_func": 1.1869999999180436,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.030968749662861228,
       "step": 4
     },
     {
-      "completion_length": 111.03125,
+      "completion_length": 238.1875,
       "epoch": 0.3,
-      "grad_norm": 0.004112224094569683,
-      "kl": 0.014096516541030724,
+      "grad_norm": 6.625319957733154,
+      "kl": 0.002304944671323028,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 6.748906537890434,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 2.045499972999096,
-      "rewards/consensus_reward_func": 2.0,
+      "reward": 0.17766900575952604,
+      "reward_std": 0.15545232097792905,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9377815315383486,
+      "rewards/question_recreation_reward_func": 0.07545025536091998,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.46875,
-      "rewards/xmlcount_reward_func": 1.171875,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.022781249252147973,
       "step": 6
     },
     {
-      "completion_length": 104.96875,
+      "completion_length": 234.375,
       "epoch": 0.4,
-      "grad_norm": 0.004763758275657892,
-      "kl": 0.013947516650659963,
+      "grad_norm": 37.45527648925781,
+      "kl": 0.011934110838410561,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 6.827690407633781,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 2.1699999943375587,
-      "rewards/consensus_reward_func": 2.0,
+      "reward": 0.07559840637259185,
+      "reward_std": 0.03572613595315488,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8764404423709493,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.07559840637259185,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.09375,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.0,
       "step": 8
     },
     {
-      "completion_length": 115.78125,
+      "completion_length": 128.6875,
       "epoch": 0.5,
-      "grad_norm": 0.0074347881600260735,
-      "kl": 0.02546092786360532,
+      "grad_norm": 7.554455757141113,
+      "kl": 0.0033416283295082394,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 7.4185000360012054,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 2.2934999838471413,
-      "rewards/consensus_reward_func": 2.0,
+      "reward": 0.1688448223285377,
+      "reward_std": 0.11179644273215672,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 1.0,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.11162607255391777,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.5,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.005281249526888132,
       "step": 10
     },
     {
-      "completion_length": 112.65625,
+      "completion_length": 170.0,
       "epoch": 0.6,
-      "grad_norm": 2.8312203884124756,
-      "kl": 0.02281197946285829,
+      "grad_norm": 10.444393157958984,
+      "kl": 0.004899180180473195,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 6.453548967838287,
-      "reward_std": 0.06348194181919098,
-      "rewards/concensus_correctness_reward_func": 1.9199999794363976,
-      "rewards/consensus_reward_func": 2.0,
+      "reward": 0.16101158899255097,
+      "reward_std": 0.11239912232849747,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9280802386929281,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.09179283422417939,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.15234375,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.006718750344589353,
       "step": 12
     },
     {
-      "completion_length": 109.03125,
+      "completion_length": 194.65625,
       "epoch": 0.7,
-      "grad_norm": 13.77484130859375,
-      "kl": 0.025941459636669606,
+      "grad_norm": 7.662856101989746,
+      "kl": 0.005340235337826016,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 6.516690209507942,
-      "reward_std": 0.6806851476430893,
-      "rewards/concensus_correctness_reward_func": 1.9266250059008598,
-      "rewards/consensus_reward_func": 1.8125,
+      "reward": 0.07461975858313963,
+      "reward_std": 0.022769286635593744,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9338152839336544,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.07461975858313963,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.15625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.0,
       "step": 14
     },
     {
-      "completion_length": 96.0625,
+      "completion_length": 154.65625,
       "epoch": 0.8,
-      "grad_norm": 2.659135580062866,
-      "kl": 0.019892732321750373,
+      "grad_norm": 20.92414665222168,
+      "kl": 0.005203944640015834,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 6.349273964762688,
-      "reward_std": 0.055694911075988784,
-      "rewards/concensus_correctness_reward_func": 1.9817499965429306,
-      "rewards/consensus_reward_func": 2.0,
+      "reward": 0.2567264976678416,
+      "reward_std": 0.21568255053716712,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8753678416251205,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.1382577515905723,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.0859062522649765,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.006531250197440386,
       "step": 16
     },
     {
-      "completion_length": 103.28125,
+      "completion_length": 207.8125,
       "epoch": 0.9,
-      "grad_norm": 0.011885394342243671,
-      "kl": 0.023440142394974828,
+      "grad_norm": 9.581371307373047,
+      "kl": 0.0037136046616979,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 6.983782038092613,
-      "reward_std": 0.022097086533904076,
-      "rewards/concensus_correctness_reward_func": 2.1709999963641167,
-      "rewards/consensus_reward_func": 2.0,
+      "reward": 0.11271083337487653,
+      "reward_std": 0.045449698125594296,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9377821670495905,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.11271083337487653,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.171875,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.0,
       "step": 18
     },
     {
-      "completion_length": 102.78125,
+      "completion_length": 141.46875,
       "epoch": 1.0,
-      "grad_norm": 0.005043835379183292,
-      "kl": 0.018489947833586484,
+      "grad_norm": 16.01797103881836,
+      "kl": 0.010725417636422208,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 6.994738191366196,
-      "reward_std": 0.11422449508245336,
-      "rewards/concensus_correctness_reward_func": 2.233749996870756,
-      "rewards/consensus_reward_func": 1.9375,
+      "reward": 0.1010746251558885,
+      "reward_std": 0.02244083739060443,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.9016132599354023,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.09173087438102812,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.109375,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.009343750338302925,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.5534832436969735e-05,
-      "train_runtime": 364.9312,
-      "train_samples_per_second": 0.877,
-      "train_steps_per_second": 0.055
+      "train_loss": 4.7611911895728555e-06,
+      "train_runtime": 1288.5548,
+      "train_samples_per_second": 0.248,
+      "train_steps_per_second": 0.016
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c0c2951d250f3e4bf073daf943746586b431737de433430548740dcf8bc1294
+oid sha256:b5dc00c9f68771408ac6551bf785c04391a281c02dd4e7662c09b83242adb765
 size 6008