End of training

README.md

@@ -37,10 +37,10 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 
 ### Framework versions
 
-- TRL: 0.15.2
-- Transformers: 4.51.2
-- Pytorch: 2.6.0
-- Datasets: 3.5.0
+- TRL: 0.18.2
+- Transformers: 4.52.4
+- Pytorch: 2.7.1
+- Datasets: 3.6.0
 - Tokenizers: 0.21.1
 
 ## Citations
@@ -62,7 +62,7 @@ Cite TRL as:
 ```bibtex
 @misc{vonwerra2022trl,
 	title        = {{TRL: Transformer Reinforcement Learning}},
-	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
 	year         = 2020,
 	journal      = {GitHub repository},
 	publisher    = {GitHub},

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 560.398,
-    "train_samples": 5,
-    "train_samples_per_second": 0.071,
-    "train_steps_per_second": 0.018
+    "train_loss": 0.0012827153317630292,
+    "train_runtime": 1313.7208,
+    "train_samples": 2,
+    "train_samples_per_second": 0.03,
+    "train_steps_per_second": 0.008
 }

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

config.json

@@ -21,7 +21,7 @@
   "sliding_window": null,
   "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.51.2",
+  "transformers_version": "4.52.4",
   "unsloth_fixed": true,
   "use_cache": true,
   "use_sliding_window": false,

generation_config.json

@@ -11,5 +11,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.51.2"
+  "transformers_version": "4.52.4"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:97246949e1438194f4cd2ae49f5cf58dc124e51f4b54b3dfb3b8960c26dcbb1c
+oid sha256:6347e2dfdbe7717ba9e9c6aadd257224c8d6cf8d9c313596d00d8ffa25d00dd6
 size 1976163472

tokenizer_config.json

@@ -195,7 +195,6 @@
     "<|video_pad|>"
   ],
   "bos_token": null,
-  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 560.398,
-    "train_samples": 5,
-    "train_samples_per_second": 0.071,
-    "train_steps_per_second": 0.018
+    "train_loss": 0.0012827153317630292,
+    "train_runtime": 1313.7208,
+    "train_samples": 2,
+    "train_samples_per_second": 0.03,
+    "train_steps_per_second": 0.008
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.4,
+  "epoch": 10.0,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -10,114 +10,219 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 4.0,
-      "epoch": 0.8,
-      "grad_norm": 0.0,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 199.0,
+      "completions/mean_length": 194.375,
+      "completions/mean_terminated_length": 172.5,
+      "completions/min_length": 151.5,
+      "completions/min_terminated_length": 151.5,
+      "epoch": 2.0,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 8.42098331451416,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 2.9176148772239685,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 2.886499971151352,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.031114899553358555,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "loss": 0.0055,
+      "num_tokens": 2579.0,
+      "reward": 0.014096824452280998,
+      "reward_std": 0.00433365476783365,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.014096824452280998,
+      "rewards/question_recreation_reward_func/std": 0.007435560924932361,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 2
     },
     {
-      "completion_length": 4.0,
-      "epoch": 1.4,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.375,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 167.5,
+      "completions/mean_length": 182.375,
+      "completions/mean_terminated_length": 134.91666412353516,
+      "completions/min_length": 102.5,
+      "completions/min_terminated_length": 102.5,
+      "epoch": 4.0,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 9.635068893432617,
+      "kl": 0.0003269764692959143,
       "learning_rate": 4.415111107797445e-07,
-      "loss": 0.0,
-      "reward": 2.01388156414032,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 1.9226666688919067,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.09121479839086533,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "loss": 0.0833,
+      "num_tokens": 5062.0,
+      "reward": 0.012714702636003494,
+      "reward_std": 0.004921070067211986,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.012714702636003494,
+      "rewards/question_recreation_reward_func/std": 0.011344576952978969,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 4
     },
     {
-      "completion_length": 4.0,
-      "epoch": 2.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 237.5,
+      "completions/max_terminated_length": 190.5,
+      "completions/mean_length": 136.75,
+      "completions/mean_terminated_length": 116.41666793823242,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "epoch": 6.0,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 14.077054977416992,
+      "kl": 0.0014197196724126115,
       "learning_rate": 2.934120444167326e-07,
-      "loss": 0.0,
-      "reward": 3.2245121399561563,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 3.2086666425069175,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.015845506141583126,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "loss": 0.0104,
+      "num_tokens": 7180.0,
+      "reward": 0.019166121259331703,
+      "reward_std": 0.013739365618675947,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.019166121259331703,
+      "rewards/question_recreation_reward_func/std": 0.018057925160974264,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 6
     },
     {
-      "completion_length": 4.0,
-      "epoch": 2.8,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 239.5,
+      "completions/max_terminated_length": 151.0,
+      "completions/mean_length": 134.25,
+      "completions/mean_terminated_length": 108.08333396911621,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "epoch": 8.0,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 10.22560977935791,
+      "kl": 0.004310431773774326,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": 0.0,
-      "reward": 2.9239743053913116,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 2.887499988079071,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.036474294029176235,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "loss": -0.054,
+      "num_tokens": 9278.0,
+      "reward": 0.013700914569199085,
+      "reward_std": 0.00266791571630165,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.013700914569199085,
+      "rewards/question_recreation_reward_func/std": 0.006989771965891123,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 8
     },
     {
-      "completion_length": 4.0,
-      "epoch": 3.4,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 240.5,
+      "completions/max_terminated_length": 211.5,
+      "completions/mean_length": 145.5,
+      "completions/mean_terminated_length": 133.5,
+      "completions/min_length": 72.5,
+      "completions/min_terminated_length": 72.5,
+      "epoch": 10.0,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 10.423093795776367,
+      "kl": 0.0022090721467975527,
       "learning_rate": 1.507684480352292e-08,
-      "loss": 0.0,
-      "reward": 2.5760087966918945,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 2.540666619936625,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.035342139502366386,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "loss": -0.0387,
+      "num_tokens": 11466.0,
+      "reward": 0.02149433922022581,
+      "reward_std": 0.007891800487414002,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.021494340151548386,
+      "rewards/question_recreation_reward_func/std": 0.010832870844751596,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 10
     },
     {
-      "epoch": 3.4,
+      "epoch": 10.0,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 0.0,
-      "train_runtime": 560.398,
-      "train_samples_per_second": 0.071,
-      "train_steps_per_second": 0.018
+      "train_loss": 0.0012827153317630292,
+      "train_runtime": 1313.7208,
+      "train_samples_per_second": 0.03,
+      "train_steps_per_second": 0.008
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
-  "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_input_tokens_seen": 11466,
+  "num_train_epochs": 10,
   "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b20224c3db5138686a08b11bf17aaf1e466a77dbcdc3a3ad0f3e200ce206d334
-size 5944
+oid sha256:adf026899a97fda9014186ee4051de9b477d823c1d6bf872414ea85471beed35
+size 6865