End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 4.618498860509135e-06,
-    "train_runtime": 5730.7706,
+    "train_loss": 2.7680349262482195e-06,
+    "train_runtime": 10731.1849,
     "train_samples": 17,
-    "train_samples_per_second": 0.028,
-    "train_steps_per_second": 0.002
+    "train_samples_per_second": 0.015,
+    "train_steps_per_second": 0.001
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60da45bd453fb0812a5514acd122207919386f16c17274cf9684c111f9ff929b
+oid sha256:aaa97d6bafa9c8206fe3c8ca8ac2f56892c467b11549c18885409cedd514c0ec
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 4.618498860509135e-06,
-    "train_runtime": 5730.7706,
+    "train_loss": 2.7680349262482195e-06,
+    "train_runtime": 10731.1849,
     "train_samples": 17,
-    "train_samples_per_second": 0.028,
-    "train_steps_per_second": 0.002
+    "train_samples_per_second": 0.015,
+    "train_steps_per_second": 0.001
 }

trainer_state.json

@@ -10,108 +10,108 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 159.3125,
+      "completion_length": 144.1875,
       "epoch": 0.9411764705882353,
-      "grad_norm": 3.968330144882202,
+      "grad_norm": 6.789586067199707,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 5.479410901665688,
-      "reward_std": 0.7337492129590828,
-      "rewards/concensus_correctness_reward_func": 1.743625022470951,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": -0.0,
+      "reward": 5.241227939724922,
+      "reward_std": 0.9851414754521102,
+      "rewards/concensus_correctness_reward_func": 1.6835000216960907,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9558484070003033,
+      "rewards/question_recreation_reward_func": 0.8487593084573746,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9674374908208847,
+      "rewards/xmlcount_reward_func": 0.9589687455445528,
       "step": 2
     },
     {
-      "completion_length": 148.11111111111111,
+      "completion_length": 161.27777777777777,
       "epoch": 1.4705882352941178,
-      "grad_norm": 3.661262273788452,
-      "kl": 0.0017332102547192739,
+      "grad_norm": 3.3434903621673584,
+      "kl": 0.001027053564838651,
       "learning_rate": 4.415111107797445e-07,
       "loss": 0.0,
-      "reward": 5.273355695936415,
-      "reward_std": 0.7181890833760716,
-      "rewards/concensus_correctness_reward_func": 1.7102222442626953,
-      "rewards/consensus_reward_func": 1.7777777777777777,
+      "reward": 5.637641244464451,
+      "reward_std": 0.28302601642078823,
+      "rewards/concensus_correctness_reward_func": 1.8171111345291138,
+      "rewards/consensus_reward_func": 1.8888888888888888,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8104112578762902,
+      "rewards/question_recreation_reward_func": 0.9391969243685404,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9749444391992357,
+      "rewards/xmlcount_reward_func": 0.9924444357554117,
       "step": 4
     },
     {
-      "completion_length": 160.16666666666666,
+      "completion_length": 153.05555555555554,
       "epoch": 2.0,
-      "grad_norm": 1.6980197429656982,
-      "kl": 0.004528235217245917,
+      "grad_norm": 0.0001829353568609804,
+      "kl": 0.003993511400444226,
       "learning_rate": 2.934120444167326e-07,
       "loss": 0.0,
-      "reward": 5.121453099780613,
-      "reward_std": 1.137750490879019,
-      "rewards/concensus_correctness_reward_func": 1.6033333539962769,
-      "rewards/consensus_reward_func": 1.6666666666666667,
+      "reward": 5.428056240081787,
+      "reward_std": 0.8201641109254625,
+      "rewards/concensus_correctness_reward_func": 1.7102222442626953,
+      "rewards/consensus_reward_func": 1.7777777777777777,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8694530559910668,
+      "rewards/question_recreation_reward_func": 0.8786119355095757,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9819999999470181,
+      "rewards/xmlcount_reward_func": 1.0614444414774578,
       "step": 6
     },
     {
-      "completion_length": 147.46875,
+      "completion_length": 149.0625,
       "epoch": 2.9411764705882355,
-      "grad_norm": 11.758064270019531,
-      "kl": 0.00643002160359174,
+      "grad_norm": 2.8968610763549805,
+      "kl": 0.004341503452451434,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": -0.0,
-      "reward": 5.572311516152695,
-      "reward_std": 0.08049981834483333,
-      "rewards/concensus_correctness_reward_func": 1.8037500232458115,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0,
+      "reward": 5.723421677947044,
+      "reward_std": 0.27014742519531865,
+      "rewards/concensus_correctness_reward_func": 1.8638750240206718,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9087179258931428,
+      "rewards/question_recreation_reward_func": 0.8951405128464103,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.984843734651804,
+      "rewards/xmlcount_reward_func": 1.0269062500447035,
       "step": 8
     },
     {
-      "completion_length": 152.72222222222223,
+      "completion_length": 167.44444444444446,
       "epoch": 3.4705882352941178,
-      "grad_norm": 4.525729656219482,
-      "kl": 0.005293702939525247,
+      "grad_norm": 2.8172271251678467,
+      "kl": 0.0033239201422677273,
       "learning_rate": 1.507684480352292e-08,
       "loss": 0.0,
-      "reward": 5.598444435331556,
-      "reward_std": 0.38582915231947684,
-      "rewards/concensus_correctness_reward_func": 1.8171111345291138,
-      "rewards/consensus_reward_func": 1.8888888888888888,
+      "reward": 5.154782348208958,
+      "reward_std": 0.39541297778487206,
+      "rewards/concensus_correctness_reward_func": 1.6033333539962769,
+      "rewards/consensus_reward_func": 1.6666666666666667,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.883000040685551,
+      "rewards/question_recreation_reward_func": 0.9082267946667142,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0094444420602586,
+      "rewards/xmlcount_reward_func": 0.9765555461247762,
       "step": 10
     },
     {
       "epoch": 3.4705882352941178,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 4.618498860509135e-06,
-      "train_runtime": 5730.7706,
-      "train_samples_per_second": 0.028,
-      "train_steps_per_second": 0.002
+      "train_loss": 2.7680349262482195e-06,
+      "train_runtime": 10731.1849,
+      "train_samples_per_second": 0.015,
+      "train_steps_per_second": 0.001
     }
   ],
   "logging_steps": 2,