End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab385243c34cb3b32b5b0a5932fb372a0684f901728153c2471a5d42e40836a9
+oid sha256:720815e59d7ba1418e94fae4ff14c22382a99028d20bebb3942f4b3277342de6
 size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.5607625815997987e-05,
-    "train_runtime": 1068.9447,
+    "train_loss": 6.524305323788818e-06,
+    "train_runtime": 1123.4392,
     "train_samples": 24,
-    "train_samples_per_second": 0.449,
-    "train_steps_per_second": 0.028
+    "train_samples_per_second": 0.427,
+    "train_steps_per_second": 0.027
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.5607625815997987e-05,
-    "train_runtime": 1068.9447,
+    "train_loss": 6.524305323788818e-06,
+    "train_runtime": 1123.4392,
     "train_samples": 24,
-    "train_samples_per_second": 0.449,
-    "train_steps_per_second": 0.028
+    "train_samples_per_second": 0.427,
+    "train_steps_per_second": 0.027
 }

trainer_state.json

@@ -9,298 +9,298 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 504.65625,
+      "completion_length": 361.625,
       "epoch": 0.3333333333333333,
-      "grad_norm": 1.4403740167617798,
-      "kl": 0.0050917769003717694,
+      "grad_norm": 8.351390838623047,
+      "kl": 0.006714582836139016,
       "learning_rate": 4.985344892885899e-07,
       "loss": 0.0,
-      "reward": 1.4946137741208076,
-      "reward_std": 2.029614135622978,
-      "rewards/concensus_correctness_reward_func": 0.752750039100647,
+      "reward": 0.37743089627474546,
+      "reward_std": 0.8290271172299981,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.40905123949050903,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.29689964605495334,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.08281250391155481,
+      "rewards/xmlcount_reward_func": 0.010218747425824404,
       "step": 2
     },
     {
-      "completion_length": 408.6875,
+      "completion_length": 446.53125,
       "epoch": 0.6666666666666666,
-      "grad_norm": 2.176173448562622,
-      "kl": 0.006444787635700777,
+      "grad_norm": 2.6079986095428467,
+      "kl": 0.003864512938889675,
       "learning_rate": 4.869132927957006e-07,
       "loss": 0.0,
-      "reward": 0.19803515821695328,
-      "reward_std": 0.9823504760861397,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.42519605346024036,
+      "reward_std": 0.5993584310635924,
+      "rewards/concensus_correctness_reward_func": 0.03125,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.24097266141325235,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2344147926196456,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.10543750016950071,
+      "rewards/xmlcount_reward_func": 0.1595312487334013,
       "step": 4
     },
     {
-      "completion_length": 328.28125,
+      "completion_length": 498.875,
       "epoch": 1.0,
-      "grad_norm": 27.15877914428711,
-      "kl": 0.06361144967377186,
+      "grad_norm": 2.200446367263794,
+      "kl": 0.006163155565445777,
       "learning_rate": 4.642142940418973e-07,
-      "loss": 0.0001,
-      "reward": 0.8097957521677017,
-      "reward_std": 0.9731377065181732,
-      "rewards/concensus_correctness_reward_func": 0.05925000086426735,
-      "rewards/consensus_reward_func": 0.0625,
+      "loss": 0.0,
+      "reward": 0.4711222779005766,
+      "reward_std": 0.8597253574989736,
+      "rewards/concensus_correctness_reward_func": 0.013249999843537807,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4571082442998886,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.21531250048428774,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.24005977483466268,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.014687508693896234,
       "step": 6
     },
     {
-      "completion_length": 424.21875,
+      "completion_length": 380.09375,
       "epoch": 1.3333333333333333,
-      "grad_norm": 1.9393937587738037,
-      "kl": 0.005781132036645431,
+      "grad_norm": 4.7573957443237305,
+      "kl": 0.007023236306849867,
       "learning_rate": 4.314988729807827e-07,
       "loss": 0.0,
-      "reward": 0.49797048792243004,
-      "reward_std": 0.7629107250832021,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.5695265596732497,
+      "reward_std": 0.752269237767905,
+      "rewards/concensus_correctness_reward_func": 0.0054374998435378075,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.2030642405152321,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.33280782122164965,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10740624624304473,
+      "rewards/xmlcount_reward_func": 0.1687812516465783,
       "step": 8
     },
     {
-      "completion_length": 440.71875,
+      "completion_length": 417.34375,
       "epoch": 1.6666666666666665,
-      "grad_norm": 2.385334014892578,
-      "kl": 0.0059448932006489486,
+      "grad_norm": 2.3667070865631104,
+      "kl": 0.006902311972226016,
       "learning_rate": 3.902967663405956e-07,
       "loss": 0.0,
-      "reward": 0.7348218522965908,
-      "reward_std": 0.988620568998158,
-      "rewards/concensus_correctness_reward_func": 0.08099999651312828,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.7031246684491634,
+      "reward_std": 0.8944519199430943,
+      "rewards/concensus_correctness_reward_func": 0.04337500035762787,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.38029062608256936,
+      "rewards/question_recreation_reward_func": 0.40178092010319233,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.07040623761713505,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.13296874426305294,
       "step": 10
     },
     {
-      "completion_length": 415.84375,
+      "completion_length": 353.3125,
       "epoch": 2.0,
-      "grad_norm": 5.587430000305176,
-      "kl": 0.008983468462247401,
+      "grad_norm": 4.754609107971191,
+      "kl": 0.006206598438438959,
       "learning_rate": 3.4253453883497864e-07,
       "loss": 0.0,
-      "reward": 1.5599827533587813,
-      "reward_std": 2.116942808032036,
-      "rewards/concensus_correctness_reward_func": 0.6625000014901161,
+      "reward": 0.5522835701704025,
+      "reward_std": 0.8312764503061771,
+      "rewards/concensus_correctness_reward_func": 0.01693749986588955,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.40367025416344404,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.24353354331105947,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.24381249770522118,
+      "rewards/xmlcount_reward_func": 0.16681250650435686,
       "step": 12
     },
     {
-      "completion_length": 476.96875,
+      "completion_length": 363.90625,
       "epoch": 2.3333333333333335,
-      "grad_norm": 61.25277328491211,
-      "kl": 0.007211562857264653,
+      "grad_norm": 2.000690460205078,
+      "kl": 0.006510635066661052,
       "learning_rate": 2.9044549913819124e-07,
       "loss": 0.0,
-      "reward": 0.16547351004555821,
-      "reward_std": 0.9365631502587348,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.5496439021080732,
+      "reward_std": 0.47898848820477724,
+      "rewards/concensus_correctness_reward_func": 0.019062500447034836,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3197860182262957,
+      "rewards/question_recreation_reward_func": 0.30370639509055763,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.1543124858289957,
+      "rewards/xmlcount_reward_func": 0.22687499690800905,
       "step": 14
     },
     {
-      "completion_length": 459.90625,
+      "completion_length": 360.5,
       "epoch": 2.6666666666666665,
-      "grad_norm": 3.288461446762085,
-      "kl": 0.003235837444663048,
+      "grad_norm": 24.43448829650879,
+      "kl": 0.0047017102770041674,
       "learning_rate": 2.3646527285364563e-07,
       "loss": 0.0,
-      "reward": 0.6074534058570862,
-      "reward_std": 1.0632685758173466,
-      "rewards/concensus_correctness_reward_func": 0.05968749802559614,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.3255016878247261,
+      "reward_std": 0.758862366899848,
+      "rewards/concensus_correctness_reward_func": 0.021687500178813934,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.34573464188724756,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.25115791661664844,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.04796875827014446,
+      "rewards/xmlcount_reward_func": -0.009843747597187757,
       "step": 16
     },
     {
-      "completion_length": 486.15625,
+      "completion_length": 423.0,
       "epoch": 3.0,
-      "grad_norm": 9.365442276000977,
-      "kl": 0.007338304363656789,
+      "grad_norm": 1.6484004259109497,
+      "kl": 0.008576760621508583,
       "learning_rate": 1.8311791536769483e-07,
       "loss": 0.0,
-      "reward": 0.6128235310316086,
-      "reward_std": 0.7765713827684522,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 1.3712667990475893,
+      "reward_std": 1.8687430834397674,
+      "rewards/concensus_correctness_reward_func": 0.625,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.2919485382735729,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.4197980733588338,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.19587500393390656,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": -0.00165625661611557,
       "step": 18
     },
     {
-      "completion_length": 439.78125,
+      "completion_length": 297.34375,
       "epoch": 3.3333333333333335,
-      "grad_norm": 2.5282180309295654,
-      "kl": 0.00842256513715256,
+      "grad_norm": 2.9706649780273438,
+      "kl": 0.004152192239416763,
       "learning_rate": 1.328978898250525e-07,
       "loss": 0.0,
-      "reward": 0.4443195927888155,
-      "reward_std": 0.6411480270326138,
+      "reward": 0.29801630694419146,
+      "reward_std": 0.4195643085986376,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2506633587181568,
+      "rewards/question_recreation_reward_func": 0.2764850608073175,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.19365624990314245,
+      "rewards/xmlcount_reward_func": 0.02153125242330134,
       "step": 20
     },
     {
-      "completion_length": 509.0,
+      "completion_length": 408.25,
       "epoch": 3.6666666666666665,
-      "grad_norm": 4.149728298187256,
-      "kl": 0.004497723566601053,
+      "grad_norm": 2.1173388957977295,
+      "kl": 0.007231552786834072,
       "learning_rate": 8.81534288045431e-08,
       "loss": 0.0,
-      "reward": 0.5238850563764572,
-      "reward_std": 0.6654719486832619,
+      "reward": 0.5215826937928796,
+      "reward_std": 0.579944210126996,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.30360380560159683,
+      "rewards/question_recreation_reward_func": 0.31623893650248647,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.15778124891221523,
+      "rewards/xmlcount_reward_func": 0.08034375123679638,
       "step": 22
     },
     {
-      "completion_length": 416.4375,
+      "completion_length": 384.625,
       "epoch": 4.0,
-      "grad_norm": 2.2486772537231445,
-      "kl": 0.00601585180265829,
+      "grad_norm": 2.203369617462158,
+      "kl": 0.009177996675134636,
       "learning_rate": 5.097673357358906e-08,
       "loss": 0.0,
-      "reward": 1.212707320228219,
-      "reward_std": 2.214023731648922,
-      "rewards/concensus_correctness_reward_func": 0.6842500008642673,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.7585753174498677,
+      "reward_std": 0.9127043741755188,
+      "rewards/concensus_correctness_reward_func": 0.12924999929964542,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3643635520711541,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.30348155926913023,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10159375052899122,
+      "rewards/xmlcount_reward_func": 0.13834375515580177,
       "step": 24
     },
     {
-      "completion_length": 450.96875,
+      "completion_length": 403.09375,
       "epoch": 4.333333333333333,
-      "grad_norm": 2.4227540493011475,
-      "kl": 0.006006023977533914,
+      "grad_norm": 2.9469876289367676,
+      "kl": 0.008016096995561384,
       "learning_rate": 2.3106145082260774e-08,
       "loss": 0.0,
-      "reward": 0.44052465446293354,
-      "reward_std": 0.9205197803676128,
-      "rewards/concensus_correctness_reward_func": 0.06637499714270234,
+      "reward": 0.4874296057969332,
+      "reward_std": 0.8158589489758015,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.2899934018496424,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.41186710400506854,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.10334375500679016,
+      "rewards/xmlcount_reward_func": 0.013062505051493645,
       "step": 26
     },
     {
-      "completion_length": 399.3125,
+      "completion_length": 388.625,
       "epoch": 4.666666666666667,
-      "grad_norm": 1.1664119958877563,
-      "kl": 0.0036647580709541216,
+      "grad_norm": 3.6871232986450195,
+      "kl": 0.005408280456322245,
       "learning_rate": 5.844861072478335e-09,
       "loss": 0.0,
-      "reward": 0.24491316895000637,
-      "reward_std": 0.33329145796597004,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.688326888717711,
+      "reward_std": 0.8707571625709534,
+      "rewards/concensus_correctness_reward_func": 0.019500000402331352,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.18856942281126976,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.33163937414065003,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.05634374776855111,
+      "rewards/xmlcount_reward_func": 0.2121874988079071,
       "step": 28
     },
     {
-      "completion_length": 318.15625,
+      "completion_length": 457.8125,
       "epoch": 5.0,
-      "grad_norm": 28.999483108520508,
-      "kl": 0.09184785535035189,
+      "grad_norm": 3.5897397994995117,
+      "kl": 0.0072338285244768485,
       "learning_rate": 0.0,
-      "loss": 0.0001,
-      "reward": 1.9683829322457314,
-      "reward_std": 2.9943294394761324,
-      "rewards/concensus_correctness_reward_func": 1.25,
-      "rewards/consensus_reward_func": 0.0,
+      "loss": 0.0,
+      "reward": 1.5410773381590843,
+      "reward_std": 2.2354293540120125,
+      "rewards/concensus_correctness_reward_func": 0.6743749994784594,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.418382840231061,
+      "rewards/question_recreation_reward_func": 0.4282960817217827,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17499999841675162,
+      "rewards/xmlcount_reward_func": 0.250906256493181,
       "step": 30
     },
     {
       "epoch": 5.0,
       "step": 30,
       "total_flos": 0.0,
-      "train_loss": 1.5607625815997987e-05,
-      "train_runtime": 1068.9447,
-      "train_samples_per_second": 0.449,
-      "train_steps_per_second": 0.028
+      "train_loss": 6.524305323788818e-06,
+      "train_runtime": 1123.4392,
+      "train_samples_per_second": 0.427,
+      "train_steps_per_second": 0.027
     }
   ],
   "logging_steps": 2,