Model save

README.md

@@ -0,0 +1,74 @@
+---
+base_model: dmis-lab/selfbiorag_7b
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: selfbiorag-7b-dpo-full-wo-kqa_silver_wogold-ep3
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# selfbiorag-7b-dpo-full-wo-kqa_silver_wogold-ep3
+
+This model is a fine-tuned version of [dmis-lab/selfbiorag_7b](https://huggingface.co/dmis-lab/selfbiorag_7b) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6403
+- Rewards/chosen: 0.1251
+- Rewards/rejected: 0.0020
+- Rewards/accuracies: 0.7554
+- Rewards/margins: 0.1232
+- Logps/rejected: -171.1355
+- Logps/chosen: -150.7145
+- Logits/rejected: -1.5597
+- Logits/chosen: -1.7911
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6608        | 0.25  | 100  | 0.6631          | 0.1074         | 0.0395           | 0.7107             | 0.0680          | -167.3843      | -152.4830    | -1.5362         | -1.7612       |
+| 0.6271        | 0.51  | 200  | 0.6474          | 0.1331         | 0.0272           | 0.7455             | 0.1060          | -168.6118      | -149.9109    | -1.5243         | -1.7495       |
+| 0.61          | 0.76  | 300  | 0.6403          | 0.1251         | 0.0020           | 0.7554             | 0.1232          | -171.1355      | -150.7145    | -1.5597         | -1.7911       |
+
+
+### Framework versions
+
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.2
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6365382266896111,
+    "train_runtime": 6736.0228,
+    "train_samples": 25108,
+    "train_samples_per_second": 3.727,
+    "train_steps_per_second": 0.058
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "max_length": 4096,
+  "pad_token_id": 0,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.39.0.dev0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8617038beb841fd579481c4717c83f5e6fbfdc1cbd3c9649eec52b0dec40470
+size 4939116424

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc289d59aa25a76d3f2a73f45d0c20ed7355f97bfd95b2baed921bfe9ed784e3
+size 4947390880

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f19ed34b77b232cbd2102bc668843b39e8a5eaf9abc5e36ce001ecbf5cdddfa6
+size 3590619888

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 13477093376
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6365382266896111,
+    "train_runtime": 6736.0228,
+    "train_samples": 25108,
+    "train_samples_per_second": 3.727,
+    "train_steps_per_second": 0.058
+}

trainer_state.json

@@ -0,0 +1,678 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9987261146496815,
+  "eval_steps": 100,
+  "global_step": 392,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 6.623563152590816,
+      "learning_rate": 1.25e-08,
+      "logits/chosen": -1.7762709856033325,
+      "logits/rejected": -1.7055308818817139,
+      "logps/chosen": -158.75875854492188,
+      "logps/rejected": -156.2900848388672,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 6.268846680880051,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": -1.6677038669586182,
+      "logits/rejected": -1.8711202144622803,
+      "logps/chosen": -168.4319305419922,
+      "logps/rejected": -168.21697998046875,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": 0.00022323739540297538,
+      "rewards/margins": -0.00025546393590047956,
+      "rewards/rejected": 0.0004787013167515397,
+      "step": 10
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 6.916196723084192,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -1.7691844701766968,
+      "logits/rejected": -1.8732593059539795,
+      "logps/chosen": -171.44699096679688,
+      "logps/rejected": -199.4316864013672,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -7.604262646054849e-05,
+      "rewards/margins": -0.0008163767051883042,
+      "rewards/rejected": 0.0007403340423479676,
+      "step": 20
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 6.050258223468227,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": -1.725830078125,
+      "logits/rejected": -1.911584496498108,
+      "logps/chosen": -142.91587829589844,
+      "logps/rejected": -139.6880340576172,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.004078424070030451,
+      "rewards/margins": 0.0037065320648252964,
+      "rewards/rejected": 0.00037189191789366305,
+      "step": 30
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 6.53529321403207,
+      "learning_rate": 5e-07,
+      "logits/chosen": -1.6931381225585938,
+      "logits/rejected": -1.8671138286590576,
+      "logps/chosen": -165.03321838378906,
+      "logps/rejected": -160.69021606445312,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.012477820739150047,
+      "rewards/margins": 0.005718561355024576,
+      "rewards/rejected": 0.006759260781109333,
+      "step": 40
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 7.752844837075934,
+      "learning_rate": 4.990049701020115e-07,
+      "logits/chosen": -1.7152957916259766,
+      "logits/rejected": -1.7908411026000977,
+      "logps/chosen": -164.02880859375,
+      "logps/rejected": -157.40158081054688,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.03030446171760559,
+      "rewards/margins": 0.01812821254134178,
+      "rewards/rejected": 0.012176251038908958,
+      "step": 50
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 5.916611846644029,
+      "learning_rate": 4.96027801084029e-07,
+      "logits/chosen": -1.5954145193099976,
+      "logits/rejected": -1.8138660192489624,
+      "logps/chosen": -167.0249481201172,
+      "logps/rejected": -183.04986572265625,
+      "loss": 0.6808,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.04540279507637024,
+      "rewards/margins": 0.027045052498579025,
+      "rewards/rejected": 0.018357738852500916,
+      "step": 60
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 5.675116917052438,
+      "learning_rate": 4.910921919235267e-07,
+      "logits/chosen": -1.6030914783477783,
+      "logits/rejected": -1.6948573589324951,
+      "logps/chosen": -148.21978759765625,
+      "logps/rejected": -147.9906005859375,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.06823595613241196,
+      "rewards/margins": 0.04401998221874237,
+      "rewards/rejected": 0.024215972051024437,
+      "step": 70
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 6.566490638157099,
+      "learning_rate": 4.842374312499405e-07,
+      "logits/chosen": -1.560091257095337,
+      "logits/rejected": -1.6131219863891602,
+      "logps/chosen": -160.62417602539062,
+      "logps/rejected": -157.3994903564453,
+      "loss": 0.6682,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.0820702612400055,
+      "rewards/margins": 0.045181743800640106,
+      "rewards/rejected": 0.03688852861523628,
+      "step": 80
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 5.772231174341092,
+      "learning_rate": 4.755180845977803e-07,
+      "logits/chosen": -1.6046760082244873,
+      "logits/rejected": -1.7161102294921875,
+      "logps/chosen": -171.00735473632812,
+      "logps/rejected": -150.57485961914062,
+      "loss": 0.662,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.104692243039608,
+      "rewards/margins": 0.07025764882564545,
+      "rewards/rejected": 0.03443457931280136,
+      "step": 90
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 5.7911295820151185,
+      "learning_rate": 4.6500356005192507e-07,
+      "logits/chosen": -1.5439958572387695,
+      "logits/rejected": -1.6122440099716187,
+      "logps/chosen": -150.26260375976562,
+      "logps/rejected": -159.27491760253906,
+      "loss": 0.6608,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.12373924255371094,
+      "rewards/margins": 0.08463168889284134,
+      "rewards/rejected": 0.03910756856203079,
+      "step": 100
+    },
+    {
+      "epoch": 0.25,
+      "eval_logits/chosen": -1.761186122894287,
+      "eval_logits/rejected": -1.5362099409103394,
+      "eval_logps/chosen": -152.4829864501953,
+      "eval_logps/rejected": -167.38433837890625,
+      "eval_loss": 0.6631277203559875,
+      "eval_rewards/accuracies": 0.7107142806053162,
+      "eval_rewards/chosen": 0.10742609202861786,
+      "eval_rewards/margins": 0.0679633766412735,
+      "eval_rewards/rejected": 0.039462704211473465,
+      "eval_runtime": 429.7832,
+      "eval_samples_per_second": 10.405,
+      "eval_steps_per_second": 0.326,
+      "step": 100
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 5.829928418813387,
+      "learning_rate": 4.5277755574266476e-07,
+      "logits/chosen": -1.563971757888794,
+      "logits/rejected": -1.5954703092575073,
+      "logps/chosen": -140.49835205078125,
+      "logps/rejected": -151.38461303710938,
+      "loss": 0.6565,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.12113968282938004,
+      "rewards/margins": 0.07626114040613174,
+      "rewards/rejected": 0.04487854987382889,
+      "step": 110
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 5.592803590186937,
+      "learning_rate": 4.3893739358856455e-07,
+      "logits/chosen": -1.5266677141189575,
+      "logits/rejected": -1.6740039587020874,
+      "logps/chosen": -142.7982177734375,
+      "logps/rejected": -159.28396606445312,
+      "loss": 0.6551,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.10721515119075775,
+      "rewards/margins": 0.061263542622327805,
+      "rewards/rejected": 0.045951612293720245,
+      "step": 120
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 5.735849419245511,
+      "learning_rate": 4.235932445907152e-07,
+      "logits/chosen": -1.4909614324569702,
+      "logits/rejected": -1.639513611793518,
+      "logps/chosen": -138.89341735839844,
+      "logps/rejected": -163.13583374023438,
+      "loss": 0.6501,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.14001671969890594,
+      "rewards/margins": 0.09246806800365448,
+      "rewards/rejected": 0.04754864051938057,
+      "step": 130
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 6.001903159162729,
+      "learning_rate": 4.0686725184521296e-07,
+      "logits/chosen": -1.470578670501709,
+      "logits/rejected": -1.6052910089492798,
+      "logps/chosen": -153.43539428710938,
+      "logps/rejected": -159.36370849609375,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.1407071053981781,
+      "rewards/margins": 0.10678094625473022,
+      "rewards/rejected": 0.033926136791706085,
+      "step": 140
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 5.784632122034035,
+      "learning_rate": 3.8889255825490053e-07,
+      "logits/chosen": -1.4748151302337646,
+      "logits/rejected": -1.6701900959014893,
+      "logps/chosen": -161.1116943359375,
+      "logps/rejected": -157.26760864257812,
+      "loss": 0.6425,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.12853461503982544,
+      "rewards/margins": 0.10510917007923126,
+      "rewards/rejected": 0.023425456136465073,
+      "step": 150
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 5.43601562892169,
+      "learning_rate": 3.698122466800142e-07,
+      "logits/chosen": -1.4563039541244507,
+      "logits/rejected": -1.5854458808898926,
+      "logps/chosen": -153.89834594726562,
+      "logps/rejected": -158.865234375,
+      "loss": 0.6412,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.14952915906906128,
+      "rewards/margins": 0.1120678037405014,
+      "rewards/rejected": 0.03746134787797928,
+      "step": 160
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 5.971873303211828,
+      "learning_rate": 3.4977820096439347e-07,
+      "logits/chosen": -1.575829029083252,
+      "logits/rejected": -1.693624496459961,
+      "logps/chosen": -150.4434814453125,
+      "logps/rejected": -162.12527465820312,
+      "loss": 0.6335,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.15023693442344666,
+      "rewards/margins": 0.13257721066474915,
+      "rewards/rejected": 0.017659714445471764,
+      "step": 170
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 6.184425133655195,
+      "learning_rate": 3.2894989690375624e-07,
+      "logits/chosen": -1.5169557332992554,
+      "logits/rejected": -1.6756654977798462,
+      "logps/chosen": -145.499755859375,
+      "logps/rejected": -182.90296936035156,
+      "loss": 0.6308,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.17053017020225525,
+      "rewards/margins": 0.13632676005363464,
+      "rewards/rejected": 0.03420340269804001,
+      "step": 180
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 5.8236364772206235,
+      "learning_rate": 3.0749313278022014e-07,
+      "logits/chosen": -1.5066370964050293,
+      "logits/rejected": -1.5730130672454834,
+      "logps/chosen": -142.18922424316406,
+      "logps/rejected": -174.89212036132812,
+      "loss": 0.6321,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 0.16863951086997986,
+      "rewards/margins": 0.1373470276594162,
+      "rewards/rejected": 0.03129247575998306,
+      "step": 190
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 5.574873360285366,
+      "learning_rate": 2.8557870956832133e-07,
+      "logits/chosen": -1.512538194656372,
+      "logits/rejected": -1.660465955734253,
+      "logps/chosen": -153.04806518554688,
+      "logps/rejected": -159.74850463867188,
+      "loss": 0.6271,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.14079943299293518,
+      "rewards/margins": 0.13594433665275574,
+      "rewards/rejected": 0.004855105187743902,
+      "step": 200
+    },
+    {
+      "epoch": 0.51,
+      "eval_logits/chosen": -1.7494703531265259,
+      "eval_logits/rejected": -1.5242888927459717,
+      "eval_logps/chosen": -149.910888671875,
+      "eval_logps/rejected": -168.61181640625,
+      "eval_loss": 0.647361159324646,
+      "eval_rewards/accuracies": 0.7455357313156128,
+      "eval_rewards/chosen": 0.1331472396850586,
+      "eval_rewards/margins": 0.10595925897359848,
+      "eval_rewards/rejected": 0.02718796581029892,
+      "eval_runtime": 429.8203,
+      "eval_samples_per_second": 10.404,
+      "eval_steps_per_second": 0.326,
+      "step": 200
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 5.16275032219013,
+      "learning_rate": 2.6338107131840375e-07,
+      "logits/chosen": -1.5131943225860596,
+      "logits/rejected": -1.5957306623458862,
+      "logps/chosen": -138.80398559570312,
+      "logps/rejected": -163.5387420654297,
+      "loss": 0.6294,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.14896559715270996,
+      "rewards/margins": 0.13558414578437805,
+      "rewards/rejected": 0.013381436467170715,
+      "step": 210
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 5.612241311663525,
+      "learning_rate": 2.410769165402549e-07,
+      "logits/chosen": -1.572727918624878,
+      "logits/rejected": -1.746077537536621,
+      "logps/chosen": -157.94985961914062,
+      "logps/rejected": -178.78587341308594,
+      "loss": 0.6189,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 0.1848190724849701,
+      "rewards/margins": 0.1625775396823883,
+      "rewards/rejected": 0.0222414992749691,
+      "step": 220
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 5.752003030987183,
+      "learning_rate": 2.188437916407035e-07,
+      "logits/chosen": -1.4695866107940674,
+      "logits/rejected": -1.631110429763794,
+      "logps/chosen": -133.35543823242188,
+      "logps/rejected": -154.0459747314453,
+      "loss": 0.6209,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 0.16957566142082214,
+      "rewards/margins": 0.13986818492412567,
+      "rewards/rejected": 0.029707470908761024,
+      "step": 230
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 6.200324302208738,
+      "learning_rate": 1.968586776117558e-07,
+      "logits/chosen": -1.565099835395813,
+      "logits/rejected": -1.6545028686523438,
+      "logps/chosen": -144.25076293945312,
+      "logps/rejected": -193.65194702148438,
+      "loss": 0.6206,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1510196179151535,
+      "rewards/margins": 0.1540193259716034,
+      "rewards/rejected": -0.0029997043311595917,
+      "step": 240
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 5.743380374146985,
+      "learning_rate": 1.7529658121956775e-07,
+      "logits/chosen": -1.512157678604126,
+      "logits/rejected": -1.6696420907974243,
+      "logps/chosen": -140.75936889648438,
+      "logps/rejected": -168.1226043701172,
+      "loss": 0.611,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 0.178760826587677,
+      "rewards/margins": 0.17773476243019104,
+      "rewards/rejected": 0.0010260592680424452,
+      "step": 250
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 5.561168918717955,
+      "learning_rate": 1.5432914190872756e-07,
+      "logits/chosen": -1.496914029121399,
+      "logits/rejected": -1.6344115734100342,
+      "logps/chosen": -142.13116455078125,
+      "logps/rejected": -150.83810424804688,
+      "loss": 0.6132,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 0.18571367859840393,
+      "rewards/margins": 0.18682031333446503,
+      "rewards/rejected": -0.001106642885133624,
+      "step": 260
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 5.259324607320041,
+      "learning_rate": 1.3412326551122364e-07,
+      "logits/chosen": -1.548492670059204,
+      "logits/rejected": -1.7842861413955688,
+      "logps/chosen": -168.30926513671875,
+      "logps/rejected": -187.5395050048828,
+      "loss": 0.6118,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.17960253357887268,
+      "rewards/margins": 0.18946436047554016,
+      "rewards/rejected": -0.009861818514764309,
+      "step": 270
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 6.754783024987692,
+      "learning_rate": 1.1483979563610069e-07,
+      "logits/chosen": -1.4931293725967407,
+      "logits/rejected": -1.6094744205474854,
+      "logps/chosen": -142.0960235595703,
+      "logps/rejected": -157.36795043945312,
+      "loss": 0.6125,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1815401315689087,
+      "rewards/margins": 0.18247996270656586,
+      "rewards/rejected": -0.000939844932872802,
+      "step": 280
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 5.454358787814826,
+      "learning_rate": 9.663223331586017e-08,
+      "logits/chosen": -1.5075619220733643,
+      "logits/rejected": -1.6393429040908813,
+      "logps/chosen": -145.65135192871094,
+      "logps/rejected": -148.68853759765625,
+      "loss": 0.6099,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.16045069694519043,
+      "rewards/margins": 0.16450481116771698,
+      "rewards/rejected": -0.004054122604429722,
+      "step": 290
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 5.831755464788406,
+      "learning_rate": 7.96455151015272e-08,
+      "logits/chosen": -1.529247522354126,
+      "logits/rejected": -1.6624796390533447,
+      "logps/chosen": -141.4434356689453,
+      "logps/rejected": -175.48023986816406,
+      "loss": 0.61,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.18716564774513245,
+      "rewards/margins": 0.17878592014312744,
+      "rewards/rejected": 0.00837971456348896,
+      "step": 300
+    },
+    {
+      "epoch": 0.76,
+      "eval_logits/chosen": -1.7910784482955933,
+      "eval_logits/rejected": -1.5597336292266846,
+      "eval_logps/chosen": -150.71453857421875,
+      "eval_logps/rejected": -171.13546752929688,
+      "eval_loss": 0.6403496265411377,
+      "eval_rewards/accuracies": 0.7553571462631226,
+      "eval_rewards/chosen": 0.12511074542999268,
+      "eval_rewards/margins": 0.12315954267978668,
+      "eval_rewards/rejected": 0.0019512074068188667,
+      "eval_runtime": 430.712,
+      "eval_samples_per_second": 10.383,
+      "eval_steps_per_second": 0.325,
+      "step": 300
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 5.994460021417267,
+      "learning_rate": 6.40148593330376e-08,
+      "logits/chosen": -1.5582424402236938,
+      "logits/rejected": -1.6306709051132202,
+      "logps/chosen": -155.81874084472656,
+      "logps/rejected": -172.6529541015625,
+      "loss": 0.6124,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 0.18409350514411926,
+      "rewards/margins": 0.20004510879516602,
+      "rewards/rejected": -0.015951596200466156,
+      "step": 310
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 6.01317607272459,
+      "learning_rate": 4.986468976890992e-08,
+      "logits/chosen": -1.5403350591659546,
+      "logits/rejected": -1.607718825340271,
+      "logps/chosen": -153.20468139648438,
+      "logps/rejected": -156.13182067871094,
+      "loss": 0.612,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 0.19601142406463623,
+      "rewards/margins": 0.19289278984069824,
+      "rewards/rejected": 0.0031186500564217567,
+      "step": 320
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 6.168162138638437,
+      "learning_rate": 3.7307645143367316e-08,
+      "logits/chosen": -1.5653051137924194,
+      "logits/rejected": -1.6613670587539673,
+      "logps/chosen": -140.8334503173828,
+      "logps/rejected": -182.35919189453125,
+      "loss": 0.6095,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.16427811980247498,
+      "rewards/margins": 0.176461860537529,
+      "rewards/rejected": -0.01218376588076353,
+      "step": 330
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 6.281797381891087,
+      "learning_rate": 2.6443682535072175e-08,
+      "logits/chosen": -1.508976936340332,
+      "logits/rejected": -1.6643750667572021,
+      "logps/chosen": -153.21690368652344,
+      "logps/rejected": -161.5585479736328,
+      "loss": 0.6016,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 0.18692052364349365,
+      "rewards/margins": 0.21301814913749695,
+      "rewards/rejected": -0.026097631081938744,
+      "step": 340
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 5.653935577395501,
+      "learning_rate": 1.7359281684871607e-08,
+      "logits/chosen": -1.503671646118164,
+      "logits/rejected": -1.6055927276611328,
+      "logps/chosen": -139.28070068359375,
+      "logps/rejected": -191.95657348632812,
+      "loss": 0.6051,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.1942298710346222,
+      "rewards/margins": 0.22690966725349426,
+      "rewards/rejected": -0.03267978876829147,
+      "step": 350
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 6.459157803791203,
+      "learning_rate": 1.0126756596375685e-08,
+      "logits/chosen": -1.5701175928115845,
+      "logits/rejected": -1.7649949789047241,
+      "logps/chosen": -151.25030517578125,
+      "logps/rejected": -167.69241333007812,
+      "loss": 0.6017,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.1774672269821167,
+      "rewards/margins": 0.21254825592041016,
+      "rewards/rejected": -0.03508101403713226,
+      "step": 360
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 6.83494669909883,
+      "learning_rate": 4.803679899192392e-09,
+      "logits/chosen": -1.651423692703247,
+      "logits/rejected": -1.6732279062271118,
+      "logps/chosen": -154.01516723632812,
+      "logps/rejected": -187.14566040039062,
+      "loss": 0.6014,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.1780395209789276,
+      "rewards/margins": 0.19665344059467316,
+      "rewards/rejected": -0.01861393079161644,
+      "step": 370
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 5.6174610094071245,
+      "learning_rate": 1.4324245570256632e-09,
+      "logits/chosen": -1.5403234958648682,
+      "logits/rejected": -1.6710277795791626,
+      "logps/chosen": -132.97093200683594,
+      "logps/rejected": -171.85385131835938,
+      "loss": 0.6114,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 0.16198143362998962,
+      "rewards/margins": 0.17665724456310272,
+      "rewards/rejected": -0.014675813727080822,
+      "step": 380
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 5.967863071568072,
+      "learning_rate": 3.982656874917945e-11,
+      "logits/chosen": -1.5084283351898193,
+      "logits/rejected": -1.657230019569397,
+      "logps/chosen": -153.21353149414062,
+      "logps/rejected": -168.41329956054688,
+      "loss": 0.6037,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.18791179358959198,
+      "rewards/margins": 0.20547135174274445,
+      "rewards/rejected": -0.017559560015797615,
+      "step": 390
+    },
+    {
+      "epoch": 1.0,
+      "step": 392,
+      "total_flos": 0.0,
+      "train_loss": 0.6365382266896111,
+      "train_runtime": 6736.0228,
+      "train_samples_per_second": 3.727,
+      "train_steps_per_second": 0.058
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 392,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}