End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0010977147303135788,
-    "train_runtime": 1029.9631,
-    "train_samples": 41,
-    "train_samples_per_second": 0.078,
-    "train_steps_per_second": 0.019
+    "train_loss": 0.0,
+    "train_runtime": 1185.7286,
+    "train_samples": 4,
+    "train_samples_per_second": 0.067,
+    "train_steps_per_second": 0.017
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce139ecd3281a7380df2c3936f21b3420de47386afe270772bb502c4acd4e901
+oid sha256:2697fc174e28904067fcab74d824745b5ab1e8819842bc47e81595810ea4dab8
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0010977147303135788,
-    "train_runtime": 1029.9631,
-    "train_samples": 41,
-    "train_samples_per_second": 0.078,
-    "train_steps_per_second": 0.019
+    "train_loss": 0.0,
+    "train_runtime": 1185.7286,
+    "train_samples": 4,
+    "train_samples_per_second": 0.067,
+    "train_steps_per_second": 0.017
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.975609756097561,
+  "epoch": 10.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -16,31 +16,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 11.0,
-      "completions/max_terminated_length": 11.0,
-      "completions/mean_length": 7.0,
-      "completions/mean_terminated_length": 7.0,
+      "completions/max_length": 4.0,
+      "completions/max_terminated_length": 4.0,
+      "completions/mean_length": 4.0,
+      "completions/mean_terminated_length": 4.0,
       "completions/min_length": 4.0,
       "completions/min_terminated_length": 4.0,
-      "epoch": 0.0975609756097561,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 71.97881317138672,
+      "epoch": 1.0,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0,
       "kl": 0.0,
       "learning_rate": 1e-06,
-      "loss": -0.0071,
-      "num_tokens": 2104.0,
-      "reward": 1.305239737033844,
-      "reward_std": 0.10550488950684667,
-      "rewards/concensus_correctness_reward_func/mean": 0.7842499911785126,
-      "rewards/concensus_correctness_reward_func/std": 0.9670694470405579,
-      "rewards/consensus_reward_func/mean": 0.5,
-      "rewards/consensus_reward_func/std": 0.5773502588272095,
+      "loss": 0.0,
+      "num_tokens": 2080.0,
+      "reward": 2.215920627117157,
+      "reward_std": 0.0,
+      "rewards/concensus_correctness_reward_func/mean": 2.1505000591278076,
+      "rewards/concensus_correctness_reward_func/std": 0.9416583180427551,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.02098984457552433,
-      "rewards/question_recreation_reward_func/std": 0.014506918843835592,
+      "rewards/question_recreation_reward_func/mean": 0.06542056053876877,
+      "rewards/question_recreation_reward_func/std": 0.0,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -62,25 +62,25 @@
       "completions/mean_terminated_length": 4.0,
       "completions/min_length": 4.0,
       "completions/min_terminated_length": 4.0,
-      "epoch": 0.1951219512195122,
+      "epoch": 2.0,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.450817065546289e-06,
-      "kl": 0.00017640739679336548,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 9.729086208503173e-07,
       "loss": 0.0,
-      "num_tokens": 4184.0,
-      "reward": 2.4889785051345825,
+      "num_tokens": 4160.0,
+      "reward": 0.6780470022931695,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 1.940999984741211,
-      "rewards/concensus_correctness_reward_func/std": 2.2412737607955933,
-      "rewards/consensus_reward_func/mean": 0.5,
-      "rewards/consensus_reward_func/std": 0.5773502588272095,
+      "rewards/concensus_correctness_reward_func/mean": 0.6675000190734863,
+      "rewards/concensus_correctness_reward_func/std": 0.7707626819610596,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.047978651942685246,
-      "rewards/question_recreation_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.010547009063884616,
+      "rewards/question_recreation_reward_func/std": 0.002309401286765933,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -102,25 +102,25 @@
       "completions/mean_terminated_length": 4.0,
       "completions/min_length": 4.0,
       "completions/min_terminated_length": 4.0,
-      "epoch": 0.2926829268292683,
+      "epoch": 3.0,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00392628088593483,
-      "kl": 0.00011070072650909424,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 8.945702546981968e-07,
       "loss": 0.0,
-      "num_tokens": 6264.0,
-      "reward": 2.96940016746521,
+      "num_tokens": 6240.0,
+      "reward": 2.3659205436706543,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 1.4599999785423279,
-      "rewards/concensus_correctness_reward_func/std": 1.1085125207901,
-      "rewards/consensus_reward_func/mean": 1.5,
-      "rewards/consensus_reward_func/std": 0.5773502588272095,
+      "rewards/concensus_correctness_reward_func/mean": 2.2985000610351562,
+      "rewards/concensus_correctness_reward_func/std": 0.7707626819610596,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.009400193579494953,
-      "rewards/question_recreation_reward_func/std": 0.0027492870576679707,
+      "rewards/question_recreation_reward_func/mean": 0.06742056063376367,
+      "rewards/question_recreation_reward_func/std": 0.002309401286765933,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -136,31 +136,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 5.0,
-      "completions/max_terminated_length": 5.0,
-      "completions/mean_length": 4.25,
-      "completions/mean_terminated_length": 4.25,
+      "completions/max_length": 4.0,
+      "completions/max_terminated_length": 4.0,
+      "completions/mean_length": 4.0,
+      "completions/mean_terminated_length": 4.0,
       "completions/min_length": 4.0,
       "completions/min_terminated_length": 4.0,
-      "epoch": 0.3902439024390244,
-      "frac_reward_zero_std": 0.75,
-      "grad_norm": 1.0253148730043904e-06,
-      "kl": 0.28631603345274925,
+      "epoch": 4.0,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 7.734740790612136e-07,
-      "loss": 0.018,
-      "num_tokens": 8346.0,
-      "reward": 2.461086630821228,
-      "reward_std": 0.6597800850868225,
-      "rewards/concensus_correctness_reward_func/mean": 1.448500007390976,
-      "rewards/concensus_correctness_reward_func/std": 2.0686065554618835,
-      "rewards/consensus_reward_func/mean": 1.0,
-      "rewards/consensus_reward_func/std": 1.154700517654419,
+      "loss": 0.0,
+      "num_tokens": 8320.0,
+      "reward": 3.709144711494446,
+      "reward_std": 0.0,
+      "rewards/concensus_correctness_reward_func/mean": 3.5920000076293945,
+      "rewards/concensus_correctness_reward_func/std": 0.0011546856258064508,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.012586628086864948,
-      "rewards/question_recreation_reward_func/std": 0.004967818967998028,
+      "rewards/question_recreation_reward_func/mean": 0.11714470013976097,
+      "rewards/question_recreation_reward_func/std": 0.0,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -182,25 +182,25 @@
       "completions/mean_terminated_length": 4.0,
       "completions/min_length": 4.0,
       "completions/min_terminated_length": 4.0,
-      "epoch": 0.4878048780487805,
+      "epoch": 5.0,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.06691131740808487,
-      "kl": 0.06441881880164146,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 6.227427435703995e-07,
-      "loss": 0.0001,
-      "num_tokens": 10426.0,
-      "reward": 2.5516220331192017,
+      "loss": 0.0,
+      "num_tokens": 10400.0,
+      "reward": 0.6780470022931695,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 1.0370000302791595,
-      "rewards/concensus_correctness_reward_func/std": 1.1974244713783264,
-      "rewards/consensus_reward_func/mean": 1.5,
-      "rewards/consensus_reward_func/std": 0.5773502588272095,
+      "rewards/concensus_correctness_reward_func/mean": 0.6675000190734863,
+      "rewards/concensus_correctness_reward_func/std": 0.7707626819610596,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.014621995389461517,
-      "rewards/question_recreation_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.010547009063884616,
+      "rewards/question_recreation_reward_func/std": 0.002309401286765933,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -222,24 +222,24 @@
       "completions/mean_terminated_length": 4.0,
       "completions/min_length": 4.0,
       "completions/min_terminated_length": 4.0,
-      "epoch": 0.5853658536585366,
+      "epoch": 6.0,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 1.1406209523556754e-06,
-      "kl": 7.622689008712769e-05,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 4.5871032726383385e-07,
       "loss": 0.0,
-      "num_tokens": 12506.0,
-      "reward": 1.0048940181732178,
+      "num_tokens": 12480.0,
+      "reward": 3.709144711494446,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 0.5,
-      "rewards/concensus_correctness_reward_func/std": 0.5773502588272095,
-      "rewards/consensus_reward_func/mean": 0.5,
-      "rewards/consensus_reward_func/std": 0.5773502588272095,
+      "rewards/concensus_correctness_reward_func/mean": 3.5920000076293945,
+      "rewards/concensus_correctness_reward_func/std": 0.0011546856258064508,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.004893964156508446,
+      "rewards/question_recreation_reward_func/mean": 0.11714470013976097,
       "rewards/question_recreation_reward_func/std": 0.0,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
@@ -262,24 +262,24 @@
       "completions/mean_terminated_length": 4.0,
       "completions/min_length": 4.0,
       "completions/min_terminated_length": 4.0,
-      "epoch": 0.6829268292682927,
+      "epoch": 7.0,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 5.935615990892984e-05,
-      "kl": 0.00036910921335220337,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 2.9915228767351535e-07,
       "loss": 0.0,
-      "num_tokens": 14586.0,
-      "reward": 0.9908410720527172,
+      "num_tokens": 14560.0,
+      "reward": 0.889467597939074,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 0.9620000123977661,
-      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/concensus_correctness_reward_func/mean": 0.815500020980835,
+      "rewards/concensus_correctness_reward_func/std": 0.9416583180427551,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.02884104219265282,
+      "rewards/question_recreation_reward_func/mean": 0.07396756950765848,
       "rewards/question_recreation_reward_func/std": 0.0,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
@@ -302,25 +302,25 @@
       "completions/mean_terminated_length": 4.0,
       "completions/min_length": 4.0,
       "completions/min_terminated_length": 4.0,
-      "epoch": 0.7804878048780488,
+      "epoch": 8.0,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 4.315011778999178e-07,
-      "kl": 1.2665987014770508e-07,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 1.6135921418712955e-07,
       "loss": 0.0,
-      "num_tokens": 16666.0,
-      "reward": 3.3268758058547974,
+      "num_tokens": 16640.0,
+      "reward": 1.7049675593152642,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 2.2880000174045563,
-      "rewards/concensus_correctness_reward_func/std": 0.7355442643165588,
-      "rewards/consensus_reward_func/mean": 1.0,
-      "rewards/consensus_reward_func/std": 1.154700517654419,
+      "rewards/concensus_correctness_reward_func/mean": 1.63100004196167,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.03887587878853083,
-      "rewards/question_recreation_reward_func/std": 0.0031549197155982256,
+      "rewards/question_recreation_reward_func/mean": 0.07396756950765848,
+      "rewards/question_recreation_reward_func/std": 0.0,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -342,25 +342,25 @@
       "completions/mean_terminated_length": 4.0,
       "completions/min_length": 4.0,
       "completions/min_terminated_length": 4.0,
-      "epoch": 0.8780487804878049,
+      "epoch": 9.0,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 5.20552760008286e-07,
-      "kl": 0.001730598509311676,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 6.026312439675551e-08,
       "loss": 0.0,
-      "num_tokens": 18746.0,
-      "reward": 3.8770835399627686,
+      "num_tokens": 18720.0,
+      "reward": 0.889467597939074,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 2.825500011444092,
-      "rewards/concensus_correctness_reward_func/std": 1.0502002239227295,
-      "rewards/consensus_reward_func/mean": 1.0,
-      "rewards/consensus_reward_func/std": 1.154700517654419,
+      "rewards/concensus_correctness_reward_func/mean": 0.815500020980835,
+      "rewards/concensus_correctness_reward_func/std": 0.9416583180427551,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.05158371292054653,
-      "rewards/question_recreation_reward_func/std": 0.003396179061383009,
+      "rewards/question_recreation_reward_func/mean": 0.07396756950765848,
+      "rewards/question_recreation_reward_func/std": 0.0,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -382,24 +382,24 @@
       "completions/mean_terminated_length": 4.0,
       "completions/min_length": 4.0,
       "completions/min_terminated_length": 4.0,
-      "epoch": 0.975609756097561,
+      "epoch": 10.0,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 2.5999270292231813e-06,
-      "kl": 0.047765836119651794,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 6.819348298638839e-09,
       "loss": 0.0,
-      "num_tokens": 20826.0,
-      "reward": 1.658750057220459,
+      "num_tokens": 20800.0,
+      "reward": 2.021271170116961,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 1.0805000066757202,
-      "rewards/concensus_correctness_reward_func/std": 1.2476539611816406,
-      "rewards/consensus_reward_func/mean": 0.5,
-      "rewards/consensus_reward_func/std": 0.5773502588272095,
+      "rewards/concensus_correctness_reward_func/mean": 1.9609999656677246,
+      "rewards/concensus_correctness_reward_func/std": 0.0011546856258064508,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.0782500971108675,
+      "rewards/question_recreation_reward_func/mean": 0.060271148569881916,
       "rewards/question_recreation_reward_func/std": 0.0,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
@@ -410,19 +410,19 @@
       "step": 20
     },
     {
-      "epoch": 0.975609756097561,
+      "epoch": 10.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.0010977147303135788,
-      "train_runtime": 1029.9631,
-      "train_samples_per_second": 0.078,
-      "train_steps_per_second": 0.019
+      "train_loss": 0.0,
+      "train_runtime": 1185.7286,
+      "train_samples_per_second": 0.067,
+      "train_steps_per_second": 0.017
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
-  "num_input_tokens_seen": 20826,
-  "num_train_epochs": 1,
+  "num_input_tokens_seen": 20800,
+  "num_train_epochs": 10,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {