End of training

all_results.json

@@ -1,7 +1,7 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.124013000568084e-06,
-    "train_runtime": 8806.9064,
+    "train_loss": 1.2116685013552341e-06,
+    "train_runtime": 8717.96,
     "train_samples": 17,
     "train_samples_per_second": 0.018,
     "train_steps_per_second": 0.001

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd2d9fd1d5d8fce1a1ba71b3804bfb6c0658e4235bff3d3f69b9430c56f4dd67
+oid sha256:2fafb79166df88241f64171d02eea5850229c38f0f755b4b40d7168cca5ff84f
 size 1976163472

train_results.json

@@ -1,7 +1,7 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.124013000568084e-06,
-    "train_runtime": 8806.9064,
+    "train_loss": 1.2116685013552341e-06,
+    "train_runtime": 8717.96,
     "train_samples": 17,
     "train_samples_per_second": 0.018,
     "train_steps_per_second": 0.001

trainer_state.json

@@ -10,106 +10,106 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 144.46875,
+      "completion_length": 172.875,
       "epoch": 0.9411764705882353,
-      "grad_norm": 0.9362114667892456,
+      "grad_norm": 3.068678617477417,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 5.535559568554163,
-      "reward_std": 0.23417601641267538,
-      "rewards/concensus_correctness_reward_func": 1.743625022470951,
-      "rewards/consensus_reward_func": 1.8125,
+      "reward": 5.741971403360367,
+      "reward_std": 0.38871440006187186,
+      "rewards/concensus_correctness_reward_func": 1.8638750240206718,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.929309664235916,
+      "rewards/question_recreation_reward_func": 0.9205340519547462,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.050124991685152,
+      "rewards/xmlcount_reward_func": 1.0200624950230122,
       "step": 2
     },
     {
-      "completion_length": 164.27777777777777,
+      "completion_length": 165.33333333333334,
       "epoch": 1.4705882352941178,
-      "grad_norm": 3.365419387817383,
-      "kl": 0.003109261640929617,
+      "grad_norm": 2.847506284713745,
+      "kl": 0.0008035562301908309,
       "learning_rate": 4.415111107797445e-07,
       "loss": 0.0,
-      "reward": 5.484833982255724,
-      "reward_std": 0.7263522694508234,
-      "rewards/concensus_correctness_reward_func": 1.7102222442626953,
-      "rewards/consensus_reward_func": 1.7777777777777777,
+      "reward": 5.709931479560004,
+      "reward_std": 0.4099043897715294,
+      "rewards/concensus_correctness_reward_func": 1.8171111345291138,
+      "rewards/consensus_reward_func": 1.8888888888888888,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9279452264308929,
+      "rewards/question_recreation_reward_func": 0.9730427000257704,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.027777777777777776,
-      "rewards/xmlcount_reward_func": 1.0411111116409302,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 1.0308888753255208,
       "step": 4
     },
     {
-      "completion_length": 160.0,
+      "completion_length": 156.94444444444446,
       "epoch": 2.0,
-      "grad_norm": 1.2870244979858398,
-      "kl": 0.005767702642414305,
+      "grad_norm": 0.0002546820032875985,
+      "kl": 0.0018556694008616938,
       "learning_rate": 2.934120444167326e-07,
       "loss": 0.0,
-      "reward": 5.786555502149794,
-      "reward_std": 0.31316973646688795,
+      "reward": 5.694922553168403,
+      "reward_std": 0.4427582581475791,
       "rewards/concensus_correctness_reward_func": 1.8171111345291138,
       "rewards/consensus_reward_func": 1.8888888888888888,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.9273114999135336,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0805555449591742,
+      "rewards/xmlcount_reward_func": 1.0616111026869879,
       "step": 6
     },
     {
-      "completion_length": 152.375,
+      "completion_length": 152.09375,
       "epoch": 2.9411764705882355,
-      "grad_norm": 1.4321972131729126,
-      "kl": 0.008710413501830772,
+      "grad_norm": 1.5246340036392212,
+      "kl": 0.004192383479676209,
       "learning_rate": 1.2500000000000005e-07,
       "loss": 0.0,
-      "reward": 5.777823477983475,
-      "reward_std": 0.2957839351875009,
-      "rewards/concensus_correctness_reward_func": 1.8638750240206718,
-      "rewards/consensus_reward_func": 1.9375,
+      "reward": 5.592076659202576,
+      "reward_std": 0.5399532936280593,
+      "rewards/concensus_correctness_reward_func": 1.743625022470951,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9471985623240471,
+      "rewards/question_recreation_reward_func": 0.9527017837390304,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0292499922215939,
+      "rewards/xmlcount_reward_func": 1.0832500010728836,
       "step": 8
     },
     {
-      "completion_length": 167.22222222222223,
+      "completion_length": 165.77777777777777,
       "epoch": 3.4705882352941178,
-      "grad_norm": 2.3002288341522217,
-      "kl": 0.006044962702112066,
+      "grad_norm": 1.8249889612197876,
+      "kl": 0.004116886476468708,
       "learning_rate": 1.507684480352292e-08,
       "loss": 0.0,
-      "reward": 5.7980002297295465,
-      "reward_std": 0.1421050899144676,
-      "rewards/concensus_correctness_reward_func": 1.9240000247955322,
-      "rewards/consensus_reward_func": 2.0,
+      "reward": 5.675918738047282,
+      "reward_std": 0.34448517662369543,
+      "rewards/concensus_correctness_reward_func": 1.8171111345291138,
+      "rewards/consensus_reward_func": 1.8888888888888888,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8351114358132085,
+      "rewards/question_recreation_reward_func": 0.8998077147536807,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.038888884915246,
+      "rewards/xmlcount_reward_func": 1.0701111025280423,
       "step": 10
     },
     {
       "epoch": 3.4705882352941178,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 5.124013000568084e-06,
-      "train_runtime": 8806.9064,
+      "train_loss": 1.2116685013552341e-06,
+      "train_runtime": 8717.96,
       "train_samples_per_second": 0.018,
       "train_steps_per_second": 0.001
     }