End of training

README.md

@@ -37,9 +37,9 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 
 ### Framework versions
 
-- TRL: 0.18.1
+- TRL: 0.18.2
 - Transformers: 4.52.4
-- Pytorch: 2.7.0
+- Pytorch: 2.7.1
 - Datasets: 3.6.0
 - Tokenizers: 0.21.1
 

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.13003649711608886,
-    "train_runtime": 4843.1597,
-    "train_samples": 33,
-    "train_samples_per_second": 0.004,
-    "train_steps_per_second": 0.001
+    "train_loss": 8.335709571838379e-06,
+    "train_runtime": 2249.7087,
+    "train_samples": 2,
+    "train_samples_per_second": 0.009,
+    "train_steps_per_second": 0.002
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29162337a7219006047e083f7c12d6a9388c43c8dd4348cab92422efb60f915b
+oid sha256:dd2f90d31f60841357e40a7d80a8590c8439aab55aea403847ffe115bba02aa8
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.13003649711608886,
-    "train_runtime": 4843.1597,
-    "train_samples": 33,
-    "train_samples_per_second": 0.004,
-    "train_steps_per_second": 0.001
+    "train_loss": 8.335709571838379e-06,
+    "train_runtime": 2249.7087,
+    "train_samples": 2,
+    "train_samples_per_second": 0.009,
+    "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.30303030303030304,
+  "epoch": 5.0,
   "eval_steps": 500,
   "global_step": 5,
   "is_hyper_param_search": false,
@@ -16,31 +16,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 573.5,
-      "completions/max_terminated_length": 573.5,
-      "completions/mean_length": 274.25,
-      "completions/mean_terminated_length": 274.25,
-      "completions/min_length": 71.0,
-      "completions/min_terminated_length": 71.0,
-      "epoch": 0.12121212121212122,
+      "completions/max_length": 214.5,
+      "completions/max_terminated_length": 214.5,
+      "completions/mean_length": 100.0,
+      "completions/mean_terminated_length": 100.0,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "epoch": 2.0,
       "frac_reward_zero_std": 0.25,
-      "grad_norm": 7.713528156280518,
+      "grad_norm": 62.43251037597656,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.131,
-      "num_tokens": 4242.0,
-      "reward": 0.08580746594816446,
-      "reward_std": 0.0030113481334410608,
-      "rewards/concensus_correctness_reward_func/mean": 0.0,
-      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "loss": -0.0,
+      "num_tokens": 2848.0,
+      "reward": 0.39153189957141876,
+      "reward_std": 0.17964571295306087,
+      "rewards/concensus_correctness_reward_func/mean": 0.375,
+      "rewards/concensus_correctness_reward_func/std": 0.5386751294136047,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.08580746594816446,
-      "rewards/question_recreation_reward_func/std": 0.01734682370442897,
+      "rewards/question_recreation_reward_func/mean": 0.016531903762370348,
+      "rewards/question_recreation_reward_func/std": 0.00948757166042924,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -55,32 +55,32 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 795.5,
-      "completions/max_terminated_length": 684.5,
-      "completions/mean_length": 331.875,
-      "completions/mean_terminated_length": 241.0,
-      "completions/min_length": 22.5,
-      "completions/min_terminated_length": 22.5,
-      "epoch": 0.24242424242424243,
-      "frac_reward_zero_std": 0.25,
-      "grad_norm": 6.215135097503662,
-      "kl": 0.0003396936699573416,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 188.0,
+      "completions/max_terminated_length": 188.0,
+      "completions/mean_length": 78.375,
+      "completions/mean_terminated_length": 78.375,
+      "completions/min_length": 10.5,
+      "completions/min_terminated_length": 10.5,
+      "epoch": 4.0,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 25.1249942779541,
+      "kl": 0.006960342208913062,
       "learning_rate": 2.5e-07,
-      "loss": 0.0075,
-      "num_tokens": 8945.0,
-      "reward": 0.016037299297749996,
-      "reward_std": 0.007742004003375769,
-      "rewards/concensus_correctness_reward_func/mean": 0.0,
-      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "loss": 0.0,
+      "num_tokens": 5523.0,
+      "reward": 0.14147359877824783,
+      "reward_std": 0.18529291450977325,
+      "rewards/concensus_correctness_reward_func/mean": 0.125,
+      "rewards/concensus_correctness_reward_func/std": 0.25,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.016037299297749996,
-      "rewards/question_recreation_reward_func/std": 0.016464148182421923,
+      "rewards/question_recreation_reward_func/mean": 0.016473600640892982,
+      "rewards/question_recreation_reward_func/std": 0.012013186700642109,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -95,29 +95,29 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 348.0,
-      "completions/max_terminated_length": 348.0,
-      "completions/mean_length": 220.0,
-      "completions/mean_terminated_length": 220.0,
-      "completions/min_length": 89.0,
-      "completions/min_terminated_length": 89.0,
-      "epoch": 0.30303030303030304,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 8.0,
+      "completions/mean_length": 260.0,
+      "completions/mean_terminated_length": 5.333333492279053,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "epoch": 5.0,
       "frac_reward_zero_std": 0.0,
-      "kl": 0.0014414309989660978,
-      "num_tokens": 10849.0,
-      "reward": 0.009446130134165287,
-      "reward_std": 0.005829450208693743,
-      "rewards/concensus_correctness_reward_func/mean": 0.0,
-      "rewards/concensus_correctness_reward_func/std": 0.0,
-      "rewards/consensus_reward_func/mean": 0.0,
-      "rewards/consensus_reward_func/std": 0.0,
+      "kl": 0.027669312112266198,
+      "num_tokens": 7587.0,
+      "reward": 0.7625923156738281,
+      "reward_std": 1.0524399280548096,
+      "rewards/concensus_correctness_reward_func/mean": 0.25,
+      "rewards/concensus_correctness_reward_func/std": 0.5,
+      "rewards/consensus_reward_func/mean": 0.5,
+      "rewards/consensus_reward_func/std": 1.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.009446130134165287,
-      "rewards/question_recreation_reward_func/std": 0.00640238169580698,
+      "rewards/question_recreation_reward_func/mean": 0.012592295184731483,
+      "rewards/question_recreation_reward_func/std": 0.009031646884977818,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -126,16 +126,16 @@
       "rewards/xmlcount_reward_func/std": 0.0,
       "step": 5,
       "total_flos": 0.0,
-      "train_loss": 0.13003649711608886,
-      "train_runtime": 4843.1597,
-      "train_samples_per_second": 0.004,
-      "train_steps_per_second": 0.001
+      "train_loss": 8.335709571838379e-06,
+      "train_runtime": 2249.7087,
+      "train_samples_per_second": 0.009,
+      "train_steps_per_second": 0.002
     }
   ],
   "logging_steps": 2,
   "max_steps": 5,
-  "num_input_tokens_seen": 10849,
-  "num_train_epochs": 1,
+  "num_input_tokens_seen": 7587,
+  "num_train_epochs": 5,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -150,7 +150,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 2,
+  "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37d7dd9760e2f0b9fc0ecb1bb7fe855a2ba96420c98f7b3fdad2eb40e8a0b8be
+oid sha256:6696c1da0590b9246244a1e4ddc0545787b5190ca665df5ee07fe4357d032fec
 size 6865