End of training

adapter_config.json

@@ -24,13 +24,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "o_proj",
     "q_proj",
-    "gate_proj",
+    "v_proj",
+    "down_proj",
     "up_proj",
     "k_proj",
-    "down_proj"
+    "gate_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d23003d951dfe0fe85203d809ee9331314bd8897ab3b9ff0206192b2b1c0a1f8
+oid sha256:08f4abc4995d4c51526a79742269052d307599496ee1a835ba3942536d71beb4
 size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.5961448712526666e-05,
-    "train_runtime": 902.7895,
+    "train_loss": 2.2229773397460423e-06,
+    "train_runtime": 1482.5235,
     "train_samples": 24,
-    "train_samples_per_second": 0.532,
-    "train_steps_per_second": 0.033
+    "train_samples_per_second": 0.324,
+    "train_steps_per_second": 0.02
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.5961448712526666e-05,
-    "train_runtime": 902.7895,
+    "train_loss": 2.2229773397460423e-06,
+    "train_runtime": 1482.5235,
     "train_samples": 24,
-    "train_samples_per_second": 0.532,
-    "train_steps_per_second": 0.033
+    "train_samples_per_second": 0.324,
+    "train_steps_per_second": 0.02
 }

trainer_state.json

@@ -9,298 +9,298 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 466.21875,
+      "completion_length": 316.15625,
       "epoch": 0.3333333333333333,
-      "grad_norm": 2.818002700805664,
-      "kl": 0.024589125911006704,
+      "grad_norm": 21.52219009399414,
+      "kl": 0.0030751565063837916,
       "learning_rate": 4.985344892885899e-07,
       "loss": 0.0,
-      "reward": 1.0728617757558823,
-      "reward_std": 1.0374318063259125,
-      "rewards/concensus_correctness_reward_func": 0.05012499913573265,
+      "reward": 0.7553889602422714,
+      "reward_std": 0.8830663803964853,
+      "rewards/concensus_correctness_reward_func": 0.12793750315904617,
       "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.495455514639616,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3625452145934105,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.339781254529953,
+      "rewards/xmlcount_reward_func": 0.13990625739097595,
       "step": 2
     },
     {
-      "completion_length": 364.875,
+      "completion_length": 300.5625,
       "epoch": 0.6666666666666666,
-      "grad_norm": 1.6754084825515747,
-      "kl": 0.02502421976532787,
+      "grad_norm": 1.9447593688964844,
+      "kl": 0.0016620483074802905,
       "learning_rate": 4.869132927957006e-07,
       "loss": 0.0,
-      "reward": 1.3312416300177574,
-      "reward_std": 2.140560135245323,
-      "rewards/concensus_correctness_reward_func": 0.625,
+      "reward": 0.46813742350786924,
+      "reward_std": 0.5111670058686286,
+      "rewards/concensus_correctness_reward_func": 0.0390625,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.375085380859673,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.25704367691650987,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.01865624636411667,
+      "rewards/xmlcount_reward_func": 0.1720312472898513,
       "step": 4
     },
     {
-      "completion_length": 428.9375,
+      "completion_length": 356.59375,
       "epoch": 1.0,
-      "grad_norm": 2.6106526851654053,
-      "kl": 0.019527853466570377,
+      "grad_norm": 6.489116668701172,
+      "kl": 0.002859230269677937,
       "learning_rate": 4.642142940418973e-07,
       "loss": 0.0,
-      "reward": 0.8669472727924585,
-      "reward_std": 1.043245192617178,
-      "rewards/concensus_correctness_reward_func": 0.023874999955296516,
+      "reward": 0.12191218649968505,
+      "reward_std": 0.35503741540014744,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5792285241186619,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.24194344179704785,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": -0.0017812494188547134,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.1825312376022339,
       "step": 6
     },
     {
-      "completion_length": 382.65625,
+      "completion_length": 345.09375,
       "epoch": 1.3333333333333333,
-      "grad_norm": 3.19954776763916,
-      "kl": 0.02953332115430385,
+      "grad_norm": 2.4612724781036377,
+      "kl": 0.0013941506476840004,
       "learning_rate": 4.314988729807827e-07,
       "loss": 0.0,
-      "reward": 0.7912007421255112,
-      "reward_std": 1.1522618383169174,
-      "rewards/concensus_correctness_reward_func": 0.02250000089406967,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.33836980583146214,
+      "reward_std": 0.2510320176370442,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.44829449243843555,
+      "rewards/question_recreation_reward_func": 0.1744322977028787,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1954062469303608,
+      "rewards/xmlcount_reward_func": 0.10143750347197056,
       "step": 8
     },
     {
-      "completion_length": 454.5,
+      "completion_length": 339.03125,
       "epoch": 1.6666666666666665,
-      "grad_norm": 1.2625623941421509,
-      "kl": 0.02400887871044688,
+      "grad_norm": 1.9401979446411133,
+      "kl": 0.0020870345033472404,
       "learning_rate": 3.902967663405956e-07,
       "loss": 0.0,
-      "reward": 1.6547736041247845,
-      "reward_std": 2.4716445207595825,
-      "rewards/concensus_correctness_reward_func": 0.6788749992847443,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.46584965474903584,
+      "reward_std": 0.5341893364675343,
+      "rewards/concensus_correctness_reward_func": 0.049937501549720764,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.47183603793382645,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.33859966415911913,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1915624924004078,
+      "rewards/xmlcount_reward_func": 0.07731248997151852,
       "step": 10
     },
     {
-      "completion_length": 374.78125,
+      "completion_length": 375.65625,
       "epoch": 2.0,
-      "grad_norm": 2.689986228942871,
-      "kl": 0.028676623362116516,
+      "grad_norm": 6.633189678192139,
+      "kl": 0.0027604274946497753,
       "learning_rate": 3.4253453883497864e-07,
       "loss": 0.0,
-      "reward": 2.5205069594085217,
-      "reward_std": 3.358973853290081,
-      "rewards/concensus_correctness_reward_func": 1.3355000000447035,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.49713892582803965,
+      "reward_std": 0.6947258803993464,
+      "rewards/concensus_correctness_reward_func": 0.07800000160932541,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.49960072338581085,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.300607668235898,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.24790625181049109,
+      "rewards/xmlcount_reward_func": 0.05603125458583236,
       "step": 12
     },
     {
-      "completion_length": 351.0625,
+      "completion_length": 317.78125,
       "epoch": 2.3333333333333335,
-      "grad_norm": 2.329031229019165,
-      "kl": 0.022683687566313893,
+      "grad_norm": 22.7032413482666,
+      "kl": 0.004153315079747699,
       "learning_rate": 2.9044549913819124e-07,
       "loss": 0.0,
-      "reward": 0.8053823411464691,
-      "reward_std": 0.9377600699663162,
-      "rewards/concensus_correctness_reward_func": 0.0494375005364418,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.25570555380545557,
+      "reward_std": 0.4283748477464542,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3073198366910219,
+      "rewards/question_recreation_reward_func": 0.18479929422028363,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3236249890178442,
+      "rewards/xmlcount_reward_func": 0.07090624794363976,
       "step": 14
     },
     {
-      "completion_length": 402.5625,
+      "completion_length": 462.53125,
       "epoch": 2.6666666666666665,
-      "grad_norm": 2.642098903656006,
-      "kl": 0.03899826877750456,
+      "grad_norm": 1.7028603553771973,
+      "kl": 0.0016381583554903045,
       "learning_rate": 2.3646527285364563e-07,
       "loss": 0.0,
-      "reward": 1.048937451094389,
-      "reward_std": 1.4023225009441376,
-      "rewards/concensus_correctness_reward_func": 0.1349375005811453,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.3854512879624963,
+      "reward_std": 0.6286731185391545,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.3677499322220683,
+      "rewards/question_recreation_reward_func": 0.2168888095766306,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17124998942017555,
+      "rewards/xmlcount_reward_func": -0.0267499890178442,
       "step": 16
     },
     {
-      "completion_length": 454.9375,
+      "completion_length": 428.8125,
       "epoch": 3.0,
-      "grad_norm": 1.4734266996383667,
-      "kl": 0.015171069288044237,
+      "grad_norm": 7.420510292053223,
+      "kl": 0.0014154443051666021,
       "learning_rate": 1.8311791536769483e-07,
       "loss": 0.0,
-      "reward": 1.820919245481491,
-      "reward_std": 1.901282200589776,
-      "rewards/concensus_correctness_reward_func": 1.2738749999552965,
+      "reward": 1.9474243009462953,
+      "reward_std": 2.0522735831327736,
+      "rewards/concensus_correctness_reward_func": 1.25,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.40041930228471756,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.3706743409857154,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13099999353289604,
+      "rewards/xmlcount_reward_func": 0.2017499953508377,
       "step": 18
     },
     {
-      "completion_length": 447.5625,
+      "completion_length": 335.375,
       "epoch": 3.3333333333333335,
-      "grad_norm": 1.9561309814453125,
-      "kl": 0.027638212544843554,
+      "grad_norm": 1.8302611112594604,
+      "kl": 0.0017498906381661072,
       "learning_rate": 1.328978898250525e-07,
       "loss": 0.0,
-      "reward": 1.6138426065444946,
-      "reward_std": 2.08666705340147,
-      "rewards/concensus_correctness_reward_func": 0.625,
+      "reward": 0.29314853716641665,
+      "reward_std": 0.4034676030278206,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.3899675915017724,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.22377353068441153,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.41137499921023846,
+      "rewards/xmlcount_reward_func": 0.0693750036880374,
       "step": 20
     },
     {
-      "completion_length": 313.5,
+      "completion_length": 375.75,
       "epoch": 3.6666666666666665,
-      "grad_norm": 2.4282801151275635,
-      "kl": 0.02601044374750927,
+      "grad_norm": 2.6359448432922363,
+      "kl": 0.001499448495451361,
       "learning_rate": 8.81534288045431e-08,
       "loss": 0.0,
-      "reward": 0.7636516839265823,
-      "reward_std": 1.200943410396576,
-      "rewards/concensus_correctness_reward_func": 0.05081249959766865,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.40015758108347654,
+      "reward_std": 0.5831130938604474,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3807766861282289,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.2579075903631747,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2070624940097332,
+      "rewards/xmlcount_reward_func": 0.0797500004991889,
       "step": 22
     },
     {
-      "completion_length": 348.625,
+      "completion_length": 384.8125,
       "epoch": 4.0,
-      "grad_norm": 3.557086706161499,
-      "kl": 0.025902074296027422,
+      "grad_norm": 1.4408454895019531,
+      "kl": 0.002384145453106612,
       "learning_rate": 5.097673357358906e-08,
       "loss": 0.0,
-      "reward": 0.7689573541283607,
-      "reward_std": 1.0340721681714058,
-      "rewards/concensus_correctness_reward_func": 0.024937499314546585,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.5307273641228676,
+      "reward_std": 0.6833547167479992,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.43936359975486994,
+      "rewards/question_recreation_reward_func": 0.37400862388312817,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1171562522649765,
+      "rewards/xmlcount_reward_func": 0.031718747690320015,
       "step": 24
     },
     {
-      "completion_length": 443.53125,
+      "completion_length": 345.78125,
       "epoch": 4.333333333333333,
-      "grad_norm": 1.6169641017913818,
-      "kl": 0.01715872809290886,
+      "grad_norm": 2.0862185955047607,
+      "kl": 0.002453506240271963,
       "learning_rate": 2.3106145082260774e-08,
       "loss": 0.0,
-      "reward": 0.7657004399225116,
-      "reward_std": 0.7439820282161236,
-      "rewards/concensus_correctness_reward_func": 0.0078125,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.30404016049578786,
+      "reward_std": 0.45155784906819463,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5021067075431347,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.21141517208889127,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06828123982995749,
+      "rewards/xmlcount_reward_func": 0.09262499958276749,
       "step": 26
     },
     {
-      "completion_length": 301.0,
+      "completion_length": 450.09375,
       "epoch": 4.666666666666667,
-      "grad_norm": 2.6919360160827637,
-      "kl": 0.022670619655400515,
+      "grad_norm": 1.684699535369873,
+      "kl": 0.0021325907873688266,
       "learning_rate": 5.844861072478335e-09,
       "loss": 0.0,
-      "reward": 0.8254948575049639,
-      "reward_std": 0.9493589596822858,
-      "rewards/concensus_correctness_reward_func": 0.02250000089406967,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.05113972723484039,
+      "reward_std": 1.3334482870995998,
+      "rewards/concensus_correctness_reward_func": 0.016124999150633812,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.43468233942985535,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4341397238895297,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.18081250321120024,
+      "rewards/xmlcount_reward_func": -0.39912501722574234,
       "step": 28
     },
     {
-      "completion_length": 339.40625,
+      "completion_length": 348.8125,
       "epoch": 5.0,
-      "grad_norm": 3.3629519939422607,
-      "kl": 0.04183358867885545,
+      "grad_norm": 20.362266540527344,
+      "kl": 0.001994746329728514,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 1.6037534587085247,
-      "reward_std": 1.9255643365904689,
-      "rewards/concensus_correctness_reward_func": 0.6506249997764826,
+      "reward": 1.1518285004422069,
+      "reward_std": 2.1777954126009718,
+      "rewards/concensus_correctness_reward_func": 0.6850625015795231,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.432628508657217,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.20567227923311293,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3173750042915344,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.13609375664964318,
       "step": 30
     },
     {
       "epoch": 5.0,
       "step": 30,
       "total_flos": 0.0,
-      "train_loss": 2.5961448712526666e-05,
-      "train_runtime": 902.7895,
-      "train_samples_per_second": 0.532,
-      "train_steps_per_second": 0.033
+      "train_loss": 2.2229773397460423e-06,
+      "train_runtime": 1482.5235,
+      "train_samples_per_second": 0.324,
+      "train_steps_per_second": 0.02
     }
   ],
   "logging_steps": 2,