Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,87 @@
+{%- if messages[0]["role"] == "system" %}
+    {%- set system_message = messages[0]["content"] %}
+    {%- set loop_messages = messages[1:] %}
+{%- else %}
+    {%- set loop_messages = messages %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+{%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %}
+
+{#- This block checks for alternating user/assistant messages, skipping tool calling messages #}
+{%- set ns = namespace() %}
+{%- set ns.index = 0 %}
+{%- for message in loop_messages %}
+    {%- if not (message.role == "tool" or message.role == "tool_results" or (message.tool_calls is defined and message.tool_calls is not none)) %}
+        {%- if (message["role"] == "user") != (ns.index % 2 == 0) %}
+            {{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }}
+        {%- endif %}
+        {%- set ns.index = ns.index + 1 %}
+    {%- endif %}
+{%- endfor %}
+
+{{- bos_token }}
+{%- for message in loop_messages %}
+    {%- if message["role"] == "user" %}
+        {%- if tools is not none and (message == user_messages[-1]) %}
+            {{- "[AVAILABLE_TOOLS][" }}
+            {%- for tool in tools %}
+                {%- set tool = tool.function %}
+                {{- '{"type": "function", "function": {' }}
+                {%- for key, val in tool.items() if key != "return" %}
+                    {%- if val is string %}
+                        {{- '"' + key + '": "' + val + '"' }}
+                    {%- else %}
+                        {{- '"' + key + '": ' + val|tojson }}
+                    {%- endif %}
+                    {%- if not loop.last %}
+                        {{- ", " }}
+                    {%- endif %}
+                {%- endfor %}
+                {{- "}}" }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- else %}
+                    {{- "]" }}
+                {%- endif %}
+            {%- endfor %}
+            {{- "[/AVAILABLE_TOOLS]" }}
+            {%- endif %}
+        {%- if loop.last and system_message is defined %}
+            {{- "[INST]" + system_message + "\n\n" + message["content"] + "[/INST]" }}
+        {%- else %}
+            {{- "[INST]" + message["content"] + "[/INST]" }}
+        {%- endif %}
+    {%- elif (message.tool_calls is defined and message.tool_calls is not none) %}
+        {{- "[TOOL_CALLS][" }}
+        {%- for tool_call in message.tool_calls %}
+            {%- set out = tool_call.function|tojson %}
+            {{- out[:-1] }}
+            {%- if not tool_call.id is defined or tool_call.id|length != 9 %}
+                {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }}
+            {%- endif %}
+            {{- ', "id": "' + tool_call.id + '"}' }}
+            {%- if not loop.last %}
+                {{- ", " }}
+            {%- else %}
+                {{- "]" + eos_token }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif message["role"] == "assistant" %}
+        {{- message["content"] + eos_token}}
+    {%- elif message["role"] == "tool_results" or message["role"] == "tool" %}
+        {%- if message.content is defined and message.content.content is defined %}
+            {%- set content = message.content.content %}
+        {%- else %}
+            {%- set content = message.content %}
+        {%- endif %}
+        {{- '[TOOL_RESULTS]{"content": ' + content|string + ", " }}
+        {%- if not message.tool_call_id is defined or message.tool_call_id|length != 9 %}
+            {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }}
+        {%- endif %}
+        {{- '"call_id": "' + message.tool_call_id + '"}[/TOOL_RESULTS]' }}
+    {%- else %}
+        {{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}
+    {%- endif %}
+{%- endfor %}

config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 131072,
+  "eos_token_id": 131073,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.4",
+  "use_cache": false,
+  "vocab_size": 131074
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "transformers_version": "4.55.4"
+}

model-00001-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:341d9f2f7f6345dff49cb5e2980c3f06a67ea1fa0166b79933387d4d0e7ffbf1
+size 4865542976

model-00002-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecd7ead7d90f84080487b7323d5c31c26ce62db0b2b2336c6e1a2053a394f714
+size 4907529424

model-00003-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:330b1f8840f22f55a24203ec7f9b8f388ec87d4c9a29db75c8a184130eccff56
+size 4907529456

model-00004-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb65a2c1ed9cd02ebe4686852743e0677f7327d6073be0866b703251dfffb1ff
+size 4907529456

model-00005-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50fd83fcd1a0279b935a27d7f092c30b0ef01339733903aadaf48358f0e27f52
+size 4907516752

model.safetensors.index.json

@@ -0,0 +1,371 @@
+{
+  "metadata": {
+    "total_parameters": 12247802880,
+    "total_size": 24495605760
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00005-of-00005.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.norm.weight": "model-00005-of-00005.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48130f8c042761b84abbfbf10ad07efa7c26108a14e7a2a0402daa06e447a47a
+size 17078668

trainer_state.json

@@ -0,0 +1,2034 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.13614703880190607,
+  "eval_steps": 500,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0005104645227156713,
+      "grad_norm": 24.05441665649414,
+      "learning_rate": 0.0,
+      "loss": 2.0061,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 139.53,
+      "memory/max_allocated (GiB)": 139.53,
+      "step": 1
+    },
+    {
+      "epoch": 0.0010209290454313426,
+      "grad_norm": 7.6078009605407715,
+      "learning_rate": 1.0309278350515465e-07,
+      "loss": 1.8967,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 2
+    },
+    {
+      "epoch": 0.0015313935681470138,
+      "grad_norm": 8.540678977966309,
+      "learning_rate": 2.061855670103093e-07,
+      "loss": 1.7939,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 3
+    },
+    {
+      "epoch": 0.002041858090862685,
+      "grad_norm": 16.590530395507812,
+      "learning_rate": 3.0927835051546394e-07,
+      "loss": 1.9599,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 4
+    },
+    {
+      "epoch": 0.002552322613578356,
+      "grad_norm": 7.192079544067383,
+      "learning_rate": 4.123711340206186e-07,
+      "loss": 1.8068,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 5
+    },
+    {
+      "epoch": 0.0030627871362940277,
+      "grad_norm": 8.165936470031738,
+      "learning_rate": 5.154639175257732e-07,
+      "loss": 1.8497,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 6
+    },
+    {
+      "epoch": 0.0035732516590096988,
+      "grad_norm": 8.299514770507812,
+      "learning_rate": 6.185567010309279e-07,
+      "loss": 1.7926,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 7
+    },
+    {
+      "epoch": 0.00408371618172537,
+      "grad_norm": 6.091303825378418,
+      "learning_rate": 7.216494845360824e-07,
+      "loss": 1.8695,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 8
+    },
+    {
+      "epoch": 0.004594180704441042,
+      "grad_norm": 16.246517181396484,
+      "learning_rate": 8.247422680412372e-07,
+      "loss": 1.9153,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 9
+    },
+    {
+      "epoch": 0.005104645227156712,
+      "grad_norm": 7.1781325340271,
+      "learning_rate": 9.278350515463919e-07,
+      "loss": 1.7798,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 10
+    },
+    {
+      "epoch": 0.005615109749872384,
+      "grad_norm": 2.5561985969543457,
+      "learning_rate": 1.0309278350515464e-06,
+      "loss": 1.8484,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 11
+    },
+    {
+      "epoch": 0.006125574272588055,
+      "grad_norm": 2.5366039276123047,
+      "learning_rate": 1.134020618556701e-06,
+      "loss": 1.8163,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 12
+    },
+    {
+      "epoch": 0.006636038795303726,
+      "grad_norm": 2.0043585300445557,
+      "learning_rate": 1.2371134020618557e-06,
+      "loss": 1.8295,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 13
+    },
+    {
+      "epoch": 0.0071465033180193975,
+      "grad_norm": 1.26850426197052,
+      "learning_rate": 1.3402061855670104e-06,
+      "loss": 1.8045,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 14
+    },
+    {
+      "epoch": 0.007656967840735069,
+      "grad_norm": 1.0957386493682861,
+      "learning_rate": 1.4432989690721649e-06,
+      "loss": 1.7863,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 15
+    },
+    {
+      "epoch": 0.00816743236345074,
+      "grad_norm": 1.302420735359192,
+      "learning_rate": 1.5463917525773197e-06,
+      "loss": 1.7876,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 16
+    },
+    {
+      "epoch": 0.008677896886166412,
+      "grad_norm": 0.9507644176483154,
+      "learning_rate": 1.6494845360824744e-06,
+      "loss": 1.7966,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 17
+    },
+    {
+      "epoch": 0.009188361408882083,
+      "grad_norm": 1.1378878355026245,
+      "learning_rate": 1.7525773195876288e-06,
+      "loss": 1.8161,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 18
+    },
+    {
+      "epoch": 0.009698825931597753,
+      "grad_norm": 1.7316354513168335,
+      "learning_rate": 1.8556701030927837e-06,
+      "loss": 1.7439,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 19
+    },
+    {
+      "epoch": 0.010209290454313425,
+      "grad_norm": 1.0558087825775146,
+      "learning_rate": 1.9587628865979384e-06,
+      "loss": 1.7623,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 20
+    },
+    {
+      "epoch": 0.010719754977029096,
+      "grad_norm": 0.9555009603500366,
+      "learning_rate": 2.061855670103093e-06,
+      "loss": 1.6405,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 21
+    },
+    {
+      "epoch": 0.011230219499744768,
+      "grad_norm": 1.0347907543182373,
+      "learning_rate": 2.1649484536082477e-06,
+      "loss": 1.7348,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 22
+    },
+    {
+      "epoch": 0.01174068402246044,
+      "grad_norm": 1.1328850984573364,
+      "learning_rate": 2.268041237113402e-06,
+      "loss": 1.847,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 23
+    },
+    {
+      "epoch": 0.01225114854517611,
+      "grad_norm": 1.0314826965332031,
+      "learning_rate": 2.3711340206185566e-06,
+      "loss": 1.7493,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 24
+    },
+    {
+      "epoch": 0.012761613067891782,
+      "grad_norm": 1.158893346786499,
+      "learning_rate": 2.4742268041237115e-06,
+      "loss": 1.7491,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 25
+    },
+    {
+      "epoch": 0.013272077590607452,
+      "grad_norm": 1.144657015800476,
+      "learning_rate": 2.577319587628866e-06,
+      "loss": 1.8281,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 26
+    },
+    {
+      "epoch": 0.013782542113323124,
+      "grad_norm": 0.9864128828048706,
+      "learning_rate": 2.680412371134021e-06,
+      "loss": 1.7979,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 27
+    },
+    {
+      "epoch": 0.014293006636038795,
+      "grad_norm": 0.9023644328117371,
+      "learning_rate": 2.7835051546391757e-06,
+      "loss": 1.7567,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 28
+    },
+    {
+      "epoch": 0.014803471158754467,
+      "grad_norm": 1.0202115774154663,
+      "learning_rate": 2.8865979381443297e-06,
+      "loss": 1.7217,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 29
+    },
+    {
+      "epoch": 0.015313935681470138,
+      "grad_norm": 1.4286682605743408,
+      "learning_rate": 2.9896907216494846e-06,
+      "loss": 1.7248,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 30
+    },
+    {
+      "epoch": 0.01582440020418581,
+      "grad_norm": 0.8136070966720581,
+      "learning_rate": 3.0927835051546395e-06,
+      "loss": 1.6771,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 31
+    },
+    {
+      "epoch": 0.01633486472690148,
+      "grad_norm": 0.9729646444320679,
+      "learning_rate": 3.195876288659794e-06,
+      "loss": 1.7832,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 32
+    },
+    {
+      "epoch": 0.016845329249617153,
+      "grad_norm": 1.1746621131896973,
+      "learning_rate": 3.298969072164949e-06,
+      "loss": 1.7123,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 33
+    },
+    {
+      "epoch": 0.017355793772332824,
+      "grad_norm": 0.813364565372467,
+      "learning_rate": 3.4020618556701037e-06,
+      "loss": 1.7838,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 34
+    },
+    {
+      "epoch": 0.017866258295048495,
+      "grad_norm": 0.9410524368286133,
+      "learning_rate": 3.5051546391752577e-06,
+      "loss": 1.7666,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 35
+    },
+    {
+      "epoch": 0.018376722817764167,
+      "grad_norm": 0.7961885929107666,
+      "learning_rate": 3.6082474226804126e-06,
+      "loss": 1.7823,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 36
+    },
+    {
+      "epoch": 0.018887187340479835,
+      "grad_norm": 0.8009322881698608,
+      "learning_rate": 3.7113402061855674e-06,
+      "loss": 1.7066,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 37
+    },
+    {
+      "epoch": 0.019397651863195507,
+      "grad_norm": 0.9726182222366333,
+      "learning_rate": 3.814432989690722e-06,
+      "loss": 1.7448,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 38
+    },
+    {
+      "epoch": 0.019908116385911178,
+      "grad_norm": 0.9403437972068787,
+      "learning_rate": 3.917525773195877e-06,
+      "loss": 1.8017,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 39
+    },
+    {
+      "epoch": 0.02041858090862685,
+      "grad_norm": 0.9949467182159424,
+      "learning_rate": 4.020618556701032e-06,
+      "loss": 1.6682,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 40
+    },
+    {
+      "epoch": 0.02092904543134252,
+      "grad_norm": 0.8237268328666687,
+      "learning_rate": 4.123711340206186e-06,
+      "loss": 1.7786,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 41
+    },
+    {
+      "epoch": 0.021439509954058193,
+      "grad_norm": 0.774246871471405,
+      "learning_rate": 4.2268041237113405e-06,
+      "loss": 1.7255,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 42
+    },
+    {
+      "epoch": 0.021949974476773864,
+      "grad_norm": 1.031959891319275,
+      "learning_rate": 4.329896907216495e-06,
+      "loss": 1.7181,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 43
+    },
+    {
+      "epoch": 0.022460438999489536,
+      "grad_norm": 1.1743037700653076,
+      "learning_rate": 4.4329896907216494e-06,
+      "loss": 1.7398,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 44
+    },
+    {
+      "epoch": 0.022970903522205207,
+      "grad_norm": 0.8437848091125488,
+      "learning_rate": 4.536082474226804e-06,
+      "loss": 1.8505,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 45
+    },
+    {
+      "epoch": 0.02348136804492088,
+      "grad_norm": 0.971718430519104,
+      "learning_rate": 4.639175257731959e-06,
+      "loss": 1.7237,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 46
+    },
+    {
+      "epoch": 0.02399183256763655,
+      "grad_norm": 0.9551330208778381,
+      "learning_rate": 4.742268041237113e-06,
+      "loss": 1.6944,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 47
+    },
+    {
+      "epoch": 0.02450229709035222,
+      "grad_norm": 1.007398009300232,
+      "learning_rate": 4.845360824742268e-06,
+      "loss": 1.744,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 48
+    },
+    {
+      "epoch": 0.025012761613067893,
+      "grad_norm": 0.7662305235862732,
+      "learning_rate": 4.948453608247423e-06,
+      "loss": 1.684,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 49
+    },
+    {
+      "epoch": 0.025523226135783564,
+      "grad_norm": 0.8253554105758667,
+      "learning_rate": 5.051546391752578e-06,
+      "loss": 1.7711,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 50
+    },
+    {
+      "epoch": 0.026033690658499236,
+      "grad_norm": 0.8845403790473938,
+      "learning_rate": 5.154639175257732e-06,
+      "loss": 1.8203,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 51
+    },
+    {
+      "epoch": 0.026544155181214904,
+      "grad_norm": 0.8738449215888977,
+      "learning_rate": 5.257731958762888e-06,
+      "loss": 1.6307,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 52
+    },
+    {
+      "epoch": 0.027054619703930576,
+      "grad_norm": 0.711958646774292,
+      "learning_rate": 5.360824742268042e-06,
+      "loss": 1.7172,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 53
+    },
+    {
+      "epoch": 0.027565084226646247,
+      "grad_norm": 0.7549559473991394,
+      "learning_rate": 5.463917525773196e-06,
+      "loss": 1.755,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 54
+    },
+    {
+      "epoch": 0.02807554874936192,
+      "grad_norm": 0.7762870192527771,
+      "learning_rate": 5.567010309278351e-06,
+      "loss": 1.6802,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 55
+    },
+    {
+      "epoch": 0.02858601327207759,
+      "grad_norm": 0.8023381233215332,
+      "learning_rate": 5.670103092783505e-06,
+      "loss": 1.698,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 56
+    },
+    {
+      "epoch": 0.02909647779479326,
+      "grad_norm": 0.865761399269104,
+      "learning_rate": 5.7731958762886594e-06,
+      "loss": 1.6885,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 57
+    },
+    {
+      "epoch": 0.029606942317508933,
+      "grad_norm": 1.0811809301376343,
+      "learning_rate": 5.876288659793815e-06,
+      "loss": 1.6592,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 58
+    },
+    {
+      "epoch": 0.030117406840224605,
+      "grad_norm": 0.9070002436637878,
+      "learning_rate": 5.979381443298969e-06,
+      "loss": 1.7564,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 59
+    },
+    {
+      "epoch": 0.030627871362940276,
+      "grad_norm": 1.04336416721344,
+      "learning_rate": 6.082474226804124e-06,
+      "loss": 1.7126,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 60
+    },
+    {
+      "epoch": 0.031138335885655948,
+      "grad_norm": 1.3839117288589478,
+      "learning_rate": 6.185567010309279e-06,
+      "loss": 1.6491,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 61
+    },
+    {
+      "epoch": 0.03164880040837162,
+      "grad_norm": 0.7790964841842651,
+      "learning_rate": 6.288659793814433e-06,
+      "loss": 1.7819,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 62
+    },
+    {
+      "epoch": 0.03215926493108729,
+      "grad_norm": 1.3247586488723755,
+      "learning_rate": 6.391752577319588e-06,
+      "loss": 1.7544,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 63
+    },
+    {
+      "epoch": 0.03266972945380296,
+      "grad_norm": 1.272275686264038,
+      "learning_rate": 6.494845360824743e-06,
+      "loss": 1.7166,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 64
+    },
+    {
+      "epoch": 0.033180193976518634,
+      "grad_norm": 1.0625935792922974,
+      "learning_rate": 6.597938144329898e-06,
+      "loss": 1.8027,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 65
+    },
+    {
+      "epoch": 0.033690658499234305,
+      "grad_norm": 1.3712449073791504,
+      "learning_rate": 6.701030927835052e-06,
+      "loss": 1.6856,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 66
+    },
+    {
+      "epoch": 0.034201123021949976,
+      "grad_norm": 13.220693588256836,
+      "learning_rate": 6.804123711340207e-06,
+      "loss": 1.8061,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 67
+    },
+    {
+      "epoch": 0.03471158754466565,
+      "grad_norm": 1.3873469829559326,
+      "learning_rate": 6.907216494845361e-06,
+      "loss": 1.6946,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 68
+    },
+    {
+      "epoch": 0.03522205206738132,
+      "grad_norm": 0.8954477310180664,
+      "learning_rate": 7.010309278350515e-06,
+      "loss": 1.5911,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 69
+    },
+    {
+      "epoch": 0.03573251659009699,
+      "grad_norm": 1.6247202157974243,
+      "learning_rate": 7.113402061855671e-06,
+      "loss": 1.6588,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 70
+    },
+    {
+      "epoch": 0.03624298111281266,
+      "grad_norm": 1.2285557985305786,
+      "learning_rate": 7.216494845360825e-06,
+      "loss": 1.7729,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 71
+    },
+    {
+      "epoch": 0.036753445635528334,
+      "grad_norm": 1.0797786712646484,
+      "learning_rate": 7.319587628865979e-06,
+      "loss": 1.6978,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 72
+    },
+    {
+      "epoch": 0.037263910158244005,
+      "grad_norm": 1.8204232454299927,
+      "learning_rate": 7.422680412371135e-06,
+      "loss": 1.6912,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 73
+    },
+    {
+      "epoch": 0.03777437468095967,
+      "grad_norm": 3.366705894470215,
+      "learning_rate": 7.525773195876289e-06,
+      "loss": 1.7523,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 74
+    },
+    {
+      "epoch": 0.03828483920367534,
+      "grad_norm": 1.5358184576034546,
+      "learning_rate": 7.628865979381444e-06,
+      "loss": 1.7172,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 75
+    },
+    {
+      "epoch": 0.03879530372639101,
+      "grad_norm": 0.8976320028305054,
+      "learning_rate": 7.731958762886599e-06,
+      "loss": 1.7073,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 76
+    },
+    {
+      "epoch": 0.039305768249106685,
+      "grad_norm": 0.8933356404304504,
+      "learning_rate": 7.835051546391754e-06,
+      "loss": 1.6702,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 77
+    },
+    {
+      "epoch": 0.039816232771822356,
+      "grad_norm": 1.2677210569381714,
+      "learning_rate": 7.938144329896907e-06,
+      "loss": 1.7786,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 78
+    },
+    {
+      "epoch": 0.04032669729453803,
+      "grad_norm": 0.8787310719490051,
+      "learning_rate": 8.041237113402063e-06,
+      "loss": 1.6648,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 79
+    },
+    {
+      "epoch": 0.0408371618172537,
+      "grad_norm": 1.032422661781311,
+      "learning_rate": 8.144329896907216e-06,
+      "loss": 1.7612,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 80
+    },
+    {
+      "epoch": 0.04134762633996937,
+      "grad_norm": 0.8892786502838135,
+      "learning_rate": 8.247422680412371e-06,
+      "loss": 1.7551,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 81
+    },
+    {
+      "epoch": 0.04185809086268504,
+      "grad_norm": 1.0008693933486938,
+      "learning_rate": 8.350515463917526e-06,
+      "loss": 1.7573,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 82
+    },
+    {
+      "epoch": 0.042368555385400714,
+      "grad_norm": 0.9513587951660156,
+      "learning_rate": 8.453608247422681e-06,
+      "loss": 1.7052,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 83
+    },
+    {
+      "epoch": 0.042879019908116385,
+      "grad_norm": 0.8307821154594421,
+      "learning_rate": 8.556701030927836e-06,
+      "loss": 1.7634,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 84
+    },
+    {
+      "epoch": 0.043389484430832057,
+      "grad_norm": 0.8338541388511658,
+      "learning_rate": 8.65979381443299e-06,
+      "loss": 1.699,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 85
+    },
+    {
+      "epoch": 0.04389994895354773,
+      "grad_norm": 0.9660134315490723,
+      "learning_rate": 8.762886597938146e-06,
+      "loss": 1.6446,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 86
+    },
+    {
+      "epoch": 0.0444104134762634,
+      "grad_norm": 1.0152493715286255,
+      "learning_rate": 8.865979381443299e-06,
+      "loss": 1.7269,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 87
+    },
+    {
+      "epoch": 0.04492087799897907,
+      "grad_norm": 1.0614452362060547,
+      "learning_rate": 8.969072164948455e-06,
+      "loss": 1.6373,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 88
+    },
+    {
+      "epoch": 0.04543134252169474,
+      "grad_norm": 4.689784526824951,
+      "learning_rate": 9.072164948453609e-06,
+      "loss": 1.8024,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 89
+    },
+    {
+      "epoch": 0.045941807044410414,
+      "grad_norm": 2.7629218101501465,
+      "learning_rate": 9.175257731958764e-06,
+      "loss": 1.7774,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 90
+    },
+    {
+      "epoch": 0.046452271567126086,
+      "grad_norm": 1.2467944622039795,
+      "learning_rate": 9.278350515463918e-06,
+      "loss": 1.7005,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 91
+    },
+    {
+      "epoch": 0.04696273608984176,
+      "grad_norm": 1.175451636314392,
+      "learning_rate": 9.381443298969073e-06,
+      "loss": 1.708,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 92
+    },
+    {
+      "epoch": 0.04747320061255743,
+      "grad_norm": 36.13218307495117,
+      "learning_rate": 9.484536082474226e-06,
+      "loss": 1.9211,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 93
+    },
+    {
+      "epoch": 0.0479836651352731,
+      "grad_norm": 1.6346375942230225,
+      "learning_rate": 9.587628865979383e-06,
+      "loss": 1.6819,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 94
+    },
+    {
+      "epoch": 0.04849412965798877,
+      "grad_norm": 1.2270629405975342,
+      "learning_rate": 9.690721649484536e-06,
+      "loss": 1.5809,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 95
+    },
+    {
+      "epoch": 0.04900459418070444,
+      "grad_norm": 1.4751657247543335,
+      "learning_rate": 9.793814432989691e-06,
+      "loss": 1.7517,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 96
+    },
+    {
+      "epoch": 0.049515058703420115,
+      "grad_norm": 1.174103856086731,
+      "learning_rate": 9.896907216494846e-06,
+      "loss": 1.6645,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 97
+    },
+    {
+      "epoch": 0.050025523226135786,
+      "grad_norm": 1.0809050798416138,
+      "learning_rate": 1e-05,
+      "loss": 1.7526,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 98
+    },
+    {
+      "epoch": 0.05053598774885146,
+      "grad_norm": 0.9332408308982849,
+      "learning_rate": 9.999992883273144e-06,
+      "loss": 1.717,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 99
+    },
+    {
+      "epoch": 0.05104645227156713,
+      "grad_norm": 0.9629523754119873,
+      "learning_rate": 9.999971533112832e-06,
+      "loss": 1.6674,
+      "memory/device_reserved (GiB)": 192.97,
+      "memory/max_active (GiB)": 151.03,
+      "memory/max_allocated (GiB)": 151.03,
+      "step": 100
+    },
+    {
+      "epoch": 0.06875425459496257,
+      "grad_norm": 0.9861069321632385,
+      "learning_rate": 9.999935949579843e-06,
+      "loss": 1.7583,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 101
+    },
+    {
+      "epoch": 0.06943498978897208,
+      "grad_norm": 1.425486445426941,
+      "learning_rate": 9.898346201737316e-06,
+      "loss": 1.7852,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 102
+    },
+    {
+      "epoch": 0.07011572498298162,
+      "grad_norm": 1.648892879486084,
+      "learning_rate": 9.896179406315343e-06,
+      "loss": 1.6976,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 103
+    },
+    {
+      "epoch": 0.07079646017699115,
+      "grad_norm": 1.883970022201538,
+      "learning_rate": 9.893990002892777e-06,
+      "loss": 1.7738,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 104
+    },
+    {
+      "epoch": 0.07147719537100068,
+      "grad_norm": 1.1324390172958374,
+      "learning_rate": 9.891778001579136e-06,
+      "loss": 1.7525,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 105
+    },
+    {
+      "epoch": 0.07215793056501021,
+      "grad_norm": 1.5833842754364014,
+      "learning_rate": 9.88954341258829e-06,
+      "loss": 1.7709,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 106
+    },
+    {
+      "epoch": 0.07283866575901975,
+      "grad_norm": 1.1700878143310547,
+      "learning_rate": 9.887286246238406e-06,
+      "loss": 1.7962,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 107
+    },
+    {
+      "epoch": 0.07351940095302927,
+      "grad_norm": 1.573387861251831,
+      "learning_rate": 9.885006512951898e-06,
+      "loss": 1.7308,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 108
+    },
+    {
+      "epoch": 0.0742001361470388,
+      "grad_norm": 2.2107796669006348,
+      "learning_rate": 9.882704223255383e-06,
+      "loss": 1.7296,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 109
+    },
+    {
+      "epoch": 0.07488087134104833,
+      "grad_norm": 0.7726507186889648,
+      "learning_rate": 9.880379387779637e-06,
+      "loss": 1.7369,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 110
+    },
+    {
+      "epoch": 0.07556160653505786,
+      "grad_norm": 1.3989717960357666,
+      "learning_rate": 9.878032017259533e-06,
+      "loss": 1.7628,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 111
+    },
+    {
+      "epoch": 0.0762423417290674,
+      "grad_norm": 0.7562165260314941,
+      "learning_rate": 9.875662122534004e-06,
+      "loss": 1.6657,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 112
+    },
+    {
+      "epoch": 0.07692307692307693,
+      "grad_norm": 1.1455652713775635,
+      "learning_rate": 9.873269714545986e-06,
+      "loss": 1.7242,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 113
+    },
+    {
+      "epoch": 0.07760381211708646,
+      "grad_norm": 0.7659608721733093,
+      "learning_rate": 9.87085480434237e-06,
+      "loss": 1.6782,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 114
+    },
+    {
+      "epoch": 0.07828454731109598,
+      "grad_norm": 1.2639816999435425,
+      "learning_rate": 9.868417403073954e-06,
+      "loss": 1.7197,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 115
+    },
+    {
+      "epoch": 0.07896528250510551,
+      "grad_norm": 0.8840866088867188,
+      "learning_rate": 9.86595752199538e-06,
+      "loss": 1.7588,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 116
+    },
+    {
+      "epoch": 0.07964601769911504,
+      "grad_norm": 0.9605395793914795,
+      "learning_rate": 9.863475172465096e-06,
+      "loss": 1.746,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 117
+    },
+    {
+      "epoch": 0.08032675289312458,
+      "grad_norm": 1.208536148071289,
+      "learning_rate": 9.8609703659453e-06,
+      "loss": 1.8172,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 118
+    },
+    {
+      "epoch": 0.08100748808713411,
+      "grad_norm": 0.9458699226379395,
+      "learning_rate": 9.858443114001876e-06,
+      "loss": 1.6307,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 119
+    },
+    {
+      "epoch": 0.08168822328114364,
+      "grad_norm": 6.640661239624023,
+      "learning_rate": 9.855893428304357e-06,
+      "loss": 1.7085,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 120
+    },
+    {
+      "epoch": 0.08236895847515316,
+      "grad_norm": 4.13405704498291,
+      "learning_rate": 9.853321320625859e-06,
+      "loss": 1.7143,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 121
+    },
+    {
+      "epoch": 0.08304969366916269,
+      "grad_norm": 1.9490628242492676,
+      "learning_rate": 9.850726802843035e-06,
+      "loss": 1.7832,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 122
+    },
+    {
+      "epoch": 0.08373042886317222,
+      "grad_norm": 0.9990502595901489,
+      "learning_rate": 9.848109886936011e-06,
+      "loss": 1.6934,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 123
+    },
+    {
+      "epoch": 0.08441116405718176,
+      "grad_norm": 1.1732097864151,
+      "learning_rate": 9.84547058498834e-06,
+      "loss": 1.7131,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 124
+    },
+    {
+      "epoch": 0.08509189925119129,
+      "grad_norm": 0.9427511692047119,
+      "learning_rate": 9.842808909186941e-06,
+      "loss": 1.73,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 125
+    },
+    {
+      "epoch": 0.08577263444520082,
+      "grad_norm": 1.0290261507034302,
+      "learning_rate": 9.84012487182204e-06,
+      "loss": 1.7099,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 126
+    },
+    {
+      "epoch": 0.08645336963921035,
+      "grad_norm": 0.9603204727172852,
+      "learning_rate": 9.837418485287126e-06,
+      "loss": 1.7028,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 127
+    },
+    {
+      "epoch": 0.08713410483321987,
+      "grad_norm": 1.0807061195373535,
+      "learning_rate": 9.834689762078877e-06,
+      "loss": 1.6857,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 128
+    },
+    {
+      "epoch": 0.0878148400272294,
+      "grad_norm": 1.2012745141983032,
+      "learning_rate": 9.83193871479711e-06,
+      "loss": 1.7612,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 129
+    },
+    {
+      "epoch": 0.08849557522123894,
+      "grad_norm": 1.2601819038391113,
+      "learning_rate": 9.829165356144728e-06,
+      "loss": 1.7485,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 130
+    },
+    {
+      "epoch": 0.08917631041524847,
+      "grad_norm": 0.7284013628959656,
+      "learning_rate": 9.82636969892765e-06,
+      "loss": 1.7126,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 131
+    },
+    {
+      "epoch": 0.089857045609258,
+      "grad_norm": 1.1513925790786743,
+      "learning_rate": 9.823551756054768e-06,
+      "loss": 1.7005,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 132
+    },
+    {
+      "epoch": 0.09053778080326753,
+      "grad_norm": 0.660559356212616,
+      "learning_rate": 9.820711540537866e-06,
+      "loss": 1.6811,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 133
+    },
+    {
+      "epoch": 0.09121851599727705,
+      "grad_norm": 1.0490988492965698,
+      "learning_rate": 9.817849065491576e-06,
+      "loss": 1.7433,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 134
+    },
+    {
+      "epoch": 0.09189925119128659,
+      "grad_norm": 0.8521629571914673,
+      "learning_rate": 9.814964344133318e-06,
+      "loss": 1.7245,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 135
+    },
+    {
+      "epoch": 0.09257998638529612,
+      "grad_norm": 0.8048623204231262,
+      "learning_rate": 9.812057389783225e-06,
+      "loss": 1.7915,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 136
+    },
+    {
+      "epoch": 0.09326072157930565,
+      "grad_norm": 0.8406012654304504,
+      "learning_rate": 9.809128215864096e-06,
+      "loss": 1.7127,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 137
+    },
+    {
+      "epoch": 0.09394145677331518,
+      "grad_norm": 1.0638079643249512,
+      "learning_rate": 9.806176835901329e-06,
+      "loss": 1.6889,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 138
+    },
+    {
+      "epoch": 0.09462219196732471,
+      "grad_norm": 0.8079673647880554,
+      "learning_rate": 9.803203263522854e-06,
+      "loss": 1.7144,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 139
+    },
+    {
+      "epoch": 0.09530292716133425,
+      "grad_norm": 0.6343026161193848,
+      "learning_rate": 9.800207512459076e-06,
+      "loss": 1.6674,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 140
+    },
+    {
+      "epoch": 0.09598366235534377,
+      "grad_norm": 0.8282296657562256,
+      "learning_rate": 9.797189596542809e-06,
+      "loss": 1.6704,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 141
+    },
+    {
+      "epoch": 0.0966643975493533,
+      "grad_norm": 0.7079451084136963,
+      "learning_rate": 9.794149529709217e-06,
+      "loss": 1.7305,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 142
+    },
+    {
+      "epoch": 0.09734513274336283,
+      "grad_norm": 0.6755410432815552,
+      "learning_rate": 9.791087325995737e-06,
+      "loss": 1.6668,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 143
+    },
+    {
+      "epoch": 0.09802586793737236,
+      "grad_norm": 1.338319182395935,
+      "learning_rate": 9.78800299954203e-06,
+      "loss": 1.674,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 144
+    },
+    {
+      "epoch": 0.0987066031313819,
+      "grad_norm": 0.8062757253646851,
+      "learning_rate": 9.784896564589905e-06,
+      "loss": 1.6971,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 145
+    },
+    {
+      "epoch": 0.09938733832539143,
+      "grad_norm": 0.7266926169395447,
+      "learning_rate": 9.781768035483256e-06,
+      "loss": 1.7531,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 146
+    },
+    {
+      "epoch": 0.10006807351940095,
+      "grad_norm": 0.946113109588623,
+      "learning_rate": 9.778617426667998e-06,
+      "loss": 1.7512,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 147
+    },
+    {
+      "epoch": 0.10074880871341048,
+      "grad_norm": 0.7514466643333435,
+      "learning_rate": 9.775444752691998e-06,
+      "loss": 1.6771,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 148
+    },
+    {
+      "epoch": 0.10142954390742001,
+      "grad_norm": 1.0235679149627686,
+      "learning_rate": 9.772250028205009e-06,
+      "loss": 1.6477,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 149
+    },
+    {
+      "epoch": 0.10211027910142954,
+      "grad_norm": 0.7400968074798584,
+      "learning_rate": 9.769033267958598e-06,
+      "loss": 1.7386,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 150
+    },
+    {
+      "epoch": 0.10279101429543908,
+      "grad_norm": 0.8439730405807495,
+      "learning_rate": 9.765794486806089e-06,
+      "loss": 1.6702,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 151
+    },
+    {
+      "epoch": 0.10347174948944861,
+      "grad_norm": 0.9278808236122131,
+      "learning_rate": 9.76253369970248e-06,
+      "loss": 1.756,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 152
+    },
+    {
+      "epoch": 0.10415248468345814,
+      "grad_norm": 0.7247333526611328,
+      "learning_rate": 9.759250921704382e-06,
+      "loss": 1.7133,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 153
+    },
+    {
+      "epoch": 0.10483321987746766,
+      "grad_norm": 0.7844570279121399,
+      "learning_rate": 9.755946167969952e-06,
+      "loss": 1.6426,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 154
+    },
+    {
+      "epoch": 0.10551395507147719,
+      "grad_norm": 0.878671407699585,
+      "learning_rate": 9.752619453758818e-06,
+      "loss": 1.7451,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 155
+    },
+    {
+      "epoch": 0.10619469026548672,
+      "grad_norm": 0.8259819149971008,
+      "learning_rate": 9.749270794432011e-06,
+      "loss": 1.7338,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 156
+    },
+    {
+      "epoch": 0.10687542545949626,
+      "grad_norm": 1.142391324043274,
+      "learning_rate": 9.745900205451888e-06,
+      "loss": 1.6637,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 157
+    },
+    {
+      "epoch": 0.10755616065350579,
+      "grad_norm": 0.8899725079536438,
+      "learning_rate": 9.74250770238207e-06,
+      "loss": 1.6494,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 158
+    },
+    {
+      "epoch": 0.10823689584751532,
+      "grad_norm": 0.7415008544921875,
+      "learning_rate": 9.73909330088737e-06,
+      "loss": 1.6499,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 159
+    },
+    {
+      "epoch": 0.10891763104152484,
+      "grad_norm": 0.7182707786560059,
+      "learning_rate": 9.735657016733706e-06,
+      "loss": 1.7394,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 160
+    },
+    {
+      "epoch": 0.10959836623553437,
+      "grad_norm": 0.9890943169593811,
+      "learning_rate": 9.732198865788047e-06,
+      "loss": 1.7266,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 161
+    },
+    {
+      "epoch": 0.1102791014295439,
+      "grad_norm": 0.9935751557350159,
+      "learning_rate": 9.72871886401833e-06,
+      "loss": 1.6527,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 162
+    },
+    {
+      "epoch": 0.11095983662355344,
+      "grad_norm": 0.7096329927444458,
+      "learning_rate": 9.725217027493383e-06,
+      "loss": 1.667,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 163
+    },
+    {
+      "epoch": 0.11164057181756297,
+      "grad_norm": 0.8332772850990295,
+      "learning_rate": 9.721693372382863e-06,
+      "loss": 1.7337,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 164
+    },
+    {
+      "epoch": 0.1123213070115725,
+      "grad_norm": 0.7919803857803345,
+      "learning_rate": 9.718147914957166e-06,
+      "loss": 1.576,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 165
+    },
+    {
+      "epoch": 0.11300204220558203,
+      "grad_norm": 0.8618379235267639,
+      "learning_rate": 9.714580671587366e-06,
+      "loss": 1.6249,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 166
+    },
+    {
+      "epoch": 0.11368277739959155,
+      "grad_norm": 0.7667165994644165,
+      "learning_rate": 9.71099165874513e-06,
+      "loss": 1.6761,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 167
+    },
+    {
+      "epoch": 0.11436351259360109,
+      "grad_norm": 0.6841418147087097,
+      "learning_rate": 9.707380893002647e-06,
+      "loss": 1.6868,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 168
+    },
+    {
+      "epoch": 0.11504424778761062,
+      "grad_norm": 0.6731845140457153,
+      "learning_rate": 9.703748391032548e-06,
+      "loss": 1.7487,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 169
+    },
+    {
+      "epoch": 0.11572498298162015,
+      "grad_norm": 0.7406771779060364,
+      "learning_rate": 9.700094169607828e-06,
+      "loss": 1.7071,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 170
+    },
+    {
+      "epoch": 0.11640571817562968,
+      "grad_norm": 0.7292412519454956,
+      "learning_rate": 9.696418245601779e-06,
+      "loss": 1.706,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 171
+    },
+    {
+      "epoch": 0.11708645336963922,
+      "grad_norm": 0.8011760711669922,
+      "learning_rate": 9.692720635987893e-06,
+      "loss": 1.7142,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 172
+    },
+    {
+      "epoch": 0.11776718856364875,
+      "grad_norm": 0.9184378385543823,
+      "learning_rate": 9.689001357839807e-06,
+      "loss": 1.7051,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 173
+    },
+    {
+      "epoch": 0.11844792375765827,
+      "grad_norm": 0.6435485482215881,
+      "learning_rate": 9.685260428331203e-06,
+      "loss": 1.736,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 174
+    },
+    {
+      "epoch": 0.1191286589516678,
+      "grad_norm": 0.8926677107810974,
+      "learning_rate": 9.68149786473574e-06,
+      "loss": 1.6661,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 175
+    },
+    {
+      "epoch": 0.11980939414567733,
+      "grad_norm": 0.6705238819122314,
+      "learning_rate": 9.677713684426973e-06,
+      "loss": 1.6917,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 176
+    },
+    {
+      "epoch": 0.12049012933968686,
+      "grad_norm": 0.6798558831214905,
+      "learning_rate": 9.673907904878272e-06,
+      "loss": 1.6402,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 177
+    },
+    {
+      "epoch": 0.1211708645336964,
+      "grad_norm": 0.6277421712875366,
+      "learning_rate": 9.670080543662742e-06,
+      "loss": 1.6738,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 178
+    },
+    {
+      "epoch": 0.12185159972770593,
+      "grad_norm": 0.6391667723655701,
+      "learning_rate": 9.666231618453135e-06,
+      "loss": 1.6823,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 179
+    },
+    {
+      "epoch": 0.12253233492171545,
+      "grad_norm": 0.6664396524429321,
+      "learning_rate": 9.66236114702178e-06,
+      "loss": 1.7058,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 180
+    },
+    {
+      "epoch": 0.12321307011572498,
+      "grad_norm": 1.5816713571548462,
+      "learning_rate": 9.658469147240494e-06,
+      "loss": 1.7435,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 181
+    },
+    {
+      "epoch": 0.12389380530973451,
+      "grad_norm": 8.72884750366211,
+      "learning_rate": 9.654555637080503e-06,
+      "loss": 1.6991,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 182
+    },
+    {
+      "epoch": 0.12457454050374404,
+      "grad_norm": 1.1152446269989014,
+      "learning_rate": 9.65062063461235e-06,
+      "loss": 1.7147,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 183
+    },
+    {
+      "epoch": 0.12525527569775358,
+      "grad_norm": 0.953542172908783,
+      "learning_rate": 9.646664158005823e-06,
+      "loss": 1.7064,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 184
+    },
+    {
+      "epoch": 0.1259360108917631,
+      "grad_norm": 0.8553478717803955,
+      "learning_rate": 9.642686225529864e-06,
+      "loss": 1.683,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 185
+    },
+    {
+      "epoch": 0.12661674608577264,
+      "grad_norm": 1.008264422416687,
+      "learning_rate": 9.638686855552494e-06,
+      "loss": 1.6093,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 186
+    },
+    {
+      "epoch": 0.12729748127978216,
+      "grad_norm": 0.8719388246536255,
+      "learning_rate": 9.634666066540713e-06,
+      "loss": 1.7384,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 187
+    },
+    {
+      "epoch": 0.1279782164737917,
+      "grad_norm": 1.1259270906448364,
+      "learning_rate": 9.630623877060423e-06,
+      "loss": 1.7397,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 188
+    },
+    {
+      "epoch": 0.12865895166780122,
+      "grad_norm": 1.1336485147476196,
+      "learning_rate": 9.626560305776349e-06,
+      "loss": 1.6912,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 189
+    },
+    {
+      "epoch": 0.12933968686181074,
+      "grad_norm": 1.1066447496414185,
+      "learning_rate": 9.62247537145194e-06,
+      "loss": 1.667,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 190
+    },
+    {
+      "epoch": 0.1300204220558203,
+      "grad_norm": 0.8986105918884277,
+      "learning_rate": 9.618369092949289e-06,
+      "loss": 1.7657,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 191
+    },
+    {
+      "epoch": 0.1307011572498298,
+      "grad_norm": 0.7317138314247131,
+      "learning_rate": 9.61424148922905e-06,
+      "loss": 1.6715,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 192
+    },
+    {
+      "epoch": 0.13138189244383935,
+      "grad_norm": 0.6420246362686157,
+      "learning_rate": 9.610092579350339e-06,
+      "loss": 1.6346,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 193
+    },
+    {
+      "epoch": 0.13206262763784887,
+      "grad_norm": 0.7444735765457153,
+      "learning_rate": 9.605922382470659e-06,
+      "loss": 1.6846,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 194
+    },
+    {
+      "epoch": 0.13274336283185842,
+      "grad_norm": 0.9247352480888367,
+      "learning_rate": 9.601730917845798e-06,
+      "loss": 1.6513,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 195
+    },
+    {
+      "epoch": 0.13342409802586794,
+      "grad_norm": 0.7954132556915283,
+      "learning_rate": 9.597518204829755e-06,
+      "loss": 1.7392,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 196
+    },
+    {
+      "epoch": 0.13410483321987746,
+      "grad_norm": 0.7020078301429749,
+      "learning_rate": 9.59328426287464e-06,
+      "loss": 1.6942,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 197
+    },
+    {
+      "epoch": 0.134785568413887,
+      "grad_norm": 0.7420036792755127,
+      "learning_rate": 9.589029111530585e-06,
+      "loss": 1.7682,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 198
+    },
+    {
+      "epoch": 0.13546630360789652,
+      "grad_norm": 0.7332531213760376,
+      "learning_rate": 9.584752770445658e-06,
+      "loss": 1.7433,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 199
+    },
+    {
+      "epoch": 0.13614703880190607,
+      "grad_norm": 0.6352435946464539,
+      "learning_rate": 9.58045525936577e-06,
+      "loss": 1.706,
+      "memory/device_reserved (GiB)": 234.63,
+      "memory/max_active (GiB)": 186.82,
+      "memory/max_allocated (GiB)": 186.82,
+      "step": 200
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 1469,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.5488598918372524e+19,
+  "train_batch_size": 12,
+  "trial_name": null,
+  "trial_params": null
+}