Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+lm_eval.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -1,20 +1,61 @@
 {
-  "architectures": [
-    "SkipMiddleModel"
-  ],
-  "dim": 768,
-  "ffn_dim_multiplier": 4,
-  "initializer_range": 0.02,
-  "max_seq_len": 1024,
-  "multiple_of": 256,
-  "n_heads": 12,
-  "n_kv_heads": 12,
-  "n_layers": 2,
-  "norm_eps": 1e-05,
-  "rope_theta": 10000,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.51.1",
-  "use_scaled_rope": false,
-  "vocab_size": 50257,
-  "zero_init_masks": false
-}
+  "data": {
+    "train_files": "data/fineweb_10B_gpt2/fineweb_train_*.bin",
+    "val_files": "data/fineweb_10B_gpt2/fineweb_val_*.bin",
+    "batch_size": 512,
+    "device_batch_size": 32
+  },
+  "model": {
+    "dim": 768,
+    "n_layers": 2,
+    "n_heads": 12,
+    "n_kv_heads": 12,
+    "vocab_size": 50257,
+    "multiple_of": 256,
+    "ffn_dim_multiplier": 4,
+    "norm_eps": 1e-05,
+    "rope_theta": 10000,
+    "use_scaled_rope": false,
+    "max_seq_len": 1024,
+    "initializer_range": 0.02,
+    "zero_init_masks": false
+  },
+  "optimizer": {
+    "default": {
+      "lr": 0.001,
+      "beta1": 0.8,
+      "beta2": 0.95,
+      "eps": 1e-10,
+      "weight_decay": 0
+    },
+    "masks": {
+      "lr": 0.001,
+      "beta1": 0.8,
+      "beta2": 0.95,
+      "eps": 1e-10,
+      "weight_decay": 0
+    },
+    "norms": {
+      "lr": 0.001,
+      "beta1": 0.8,
+      "beta2": 0.95,
+      "eps": 1e-10,
+      "weight_decay": 0
+    }
+  },
+  "scheduler": {
+    "warmup_steps": 0.1,
+    "start_factor": 0.1
+  },
+  "gates": {},
+  "gates_zero_eps": 1e-08,
+  "seed": 0,
+  "project": "fineweb-nocontrol",
+  "run_id": null,
+  "logdir": "logs/fineweb-nocontrol",
+  "log_gradients": false,
+  "log_params": false,
+  "log_every_steps": 1,
+  "val_every_steps": 100,
+  "save_every_steps": -1
+}

lm_eval.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00c63c800503bb61e688c51aa577983dc98f5e29348edf4528216fa91b278d81
+size 447535065

lm_eval_results.json

@@ -0,0 +1,381 @@
+{
+  "arc_easy": {
+    "alias": "arc_easy",
+    "acc,none": 0.3484848484848485,
+    "acc_stderr,none": 0.009777377947106498,
+    "acc_norm,none": 0.3312289562289562,
+    "acc_norm_stderr,none": 0.009657641311350737
+  },
+  "blimp": {
+    "acc,none": 0.7066119402985074,
+    "acc_stderr,none": 0.0016056416481442229,
+    "alias": "blimp"
+  },
+  "blimp_adjunct_island": {
+    "alias": " - blimp_adjunct_island",
+    "acc,none": 0.768,
+    "acc_stderr,none": 0.013354937452281623
+  },
+  "blimp_anaphor_gender_agreement": {
+    "alias": " - blimp_anaphor_gender_agreement",
+    "acc,none": 0.889,
+    "acc_stderr,none": 0.009938701010583716
+  },
+  "blimp_anaphor_number_agreement": {
+    "alias": " - blimp_anaphor_number_agreement",
+    "acc,none": 0.969,
+    "acc_stderr,none": 0.005483527064679181
+  },
+  "blimp_animate_subject_passive": {
+    "alias": " - blimp_animate_subject_passive",
+    "acc,none": 0.682,
+    "acc_stderr,none": 0.014734079309311903
+  },
+  "blimp_animate_subject_trans": {
+    "alias": " - blimp_animate_subject_trans",
+    "acc,none": 0.814,
+    "acc_stderr,none": 0.012310790208412926
+  },
+  "blimp_causative": {
+    "alias": " - blimp_causative",
+    "acc,none": 0.785,
+    "acc_stderr,none": 0.01299784381903183
+  },
+  "blimp_complex_NP_island": {
+    "alias": " - blimp_complex_NP_island",
+    "acc,none": 0.641,
+    "acc_stderr,none": 0.01517726422479853
+  },
+  "blimp_coordinate_structure_constraint_complex_left_branch": {
+    "alias": " - blimp_coordinate_structure_constraint_complex_left_branch",
+    "acc,none": 0.658,
+    "acc_stderr,none": 0.015008706182121804
+  },
+  "blimp_coordinate_structure_constraint_object_extraction": {
+    "alias": " - blimp_coordinate_structure_constraint_object_extraction",
+    "acc,none": 0.755,
+    "acc_stderr,none": 0.01360735683959821
+  },
+  "blimp_determiner_noun_agreement_1": {
+    "alias": " - blimp_determiner_noun_agreement_1",
+    "acc,none": 0.953,
+    "acc_stderr,none": 0.006695956678163013
+  },
+  "blimp_determiner_noun_agreement_2": {
+    "alias": " - blimp_determiner_noun_agreement_2",
+    "acc,none": 0.955,
+    "acc_stderr,none": 0.006558812241406063
+  },
+  "blimp_determiner_noun_agreement_irregular_1": {
+    "alias": " - blimp_determiner_noun_agreement_irregular_1",
+    "acc,none": 0.845,
+    "acc_stderr,none": 0.011450157470799522
+  },
+  "blimp_determiner_noun_agreement_irregular_2": {
+    "alias": " - blimp_determiner_noun_agreement_irregular_2",
+    "acc,none": 0.923,
+    "acc_stderr,none": 0.00843458014024066
+  },
+  "blimp_determiner_noun_agreement_with_adj_2": {
+    "alias": " - blimp_determiner_noun_agreement_with_adj_2",
+    "acc,none": 0.873,
+    "acc_stderr,none": 0.010534798620855644
+  },
+  "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+    "alias": " - blimp_determiner_noun_agreement_with_adj_irregular_1",
+    "acc,none": 0.793,
+    "acc_stderr,none": 0.012818553557844009
+  },
+  "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+    "alias": " - blimp_determiner_noun_agreement_with_adj_irregular_2",
+    "acc,none": 0.819,
+    "acc_stderr,none": 0.012181436179178027
+  },
+  "blimp_determiner_noun_agreement_with_adjective_1": {
+    "alias": " - blimp_determiner_noun_agreement_with_adjective_1",
+    "acc,none": 0.875,
+    "acc_stderr,none": 0.010463483381956722
+  },
+  "blimp_distractor_agreement_relational_noun": {
+    "alias": " - blimp_distractor_agreement_relational_noun",
+    "acc,none": 0.723,
+    "acc_stderr,none": 0.014158794845306273
+  },
+  "blimp_distractor_agreement_relative_clause": {
+    "alias": " - blimp_distractor_agreement_relative_clause",
+    "acc,none": 0.579,
+    "acc_stderr,none": 0.015620595475301287
+  },
+  "blimp_drop_argument": {
+    "alias": " - blimp_drop_argument",
+    "acc,none": 0.638,
+    "acc_stderr,none": 0.015204840912919383
+  },
+  "blimp_ellipsis_n_bar_1": {
+    "alias": " - blimp_ellipsis_n_bar_1",
+    "acc,none": 0.677,
+    "acc_stderr,none": 0.01479492784334854
+  },
+  "blimp_ellipsis_n_bar_2": {
+    "alias": " - blimp_ellipsis_n_bar_2",
+    "acc,none": 0.834,
+    "acc_stderr,none": 0.011772110370812133
+  },
+  "blimp_existential_there_object_raising": {
+    "alias": " - blimp_existential_there_object_raising",
+    "acc,none": 0.773,
+    "acc_stderr,none": 0.013253174964763977
+  },
+  "blimp_existential_there_quantifiers_1": {
+    "alias": " - blimp_existential_there_quantifiers_1",
+    "acc,none": 0.849,
+    "acc_stderr,none": 0.011328165223341657
+  },
+  "blimp_existential_there_quantifiers_2": {
+    "alias": " - blimp_existential_there_quantifiers_2",
+    "acc,none": 0.189,
+    "acc_stderr,none": 0.01238678458811779
+  },
+  "blimp_existential_there_subject_raising": {
+    "alias": " - blimp_existential_there_subject_raising",
+    "acc,none": 0.722,
+    "acc_stderr,none": 0.014174516461485272
+  },
+  "blimp_expletive_it_object_raising": {
+    "alias": " - blimp_expletive_it_object_raising",
+    "acc,none": 0.677,
+    "acc_stderr,none": 0.01479492784334854
+  },
+  "blimp_inchoative": {
+    "alias": " - blimp_inchoative",
+    "acc,none": 0.783,
+    "acc_stderr,none": 0.013041513757270706
+  },
+  "blimp_intransitive": {
+    "alias": " - blimp_intransitive",
+    "acc,none": 0.823,
+    "acc_stderr,none": 0.012075463420375087
+  },
+  "blimp_irregular_past_participle_adjectives": {
+    "alias": " - blimp_irregular_past_participle_adjectives",
+    "acc,none": 0.962,
+    "acc_stderr,none": 0.0060491811505849775
+  },
+  "blimp_irregular_past_participle_verbs": {
+    "alias": " - blimp_irregular_past_participle_verbs",
+    "acc,none": 0.794,
+    "acc_stderr,none": 0.012795613612786522
+  },
+  "blimp_irregular_plural_subject_verb_agreement_1": {
+    "alias": " - blimp_irregular_plural_subject_verb_agreement_1",
+    "acc,none": 0.796,
+    "acc_stderr,none": 0.012749374359024311
+  },
+  "blimp_irregular_plural_subject_verb_agreement_2": {
+    "alias": " - blimp_irregular_plural_subject_verb_agreement_2",
+    "acc,none": 0.814,
+    "acc_stderr,none": 0.012310790208412926
+  },
+  "blimp_left_branch_island_echo_question": {
+    "alias": " - blimp_left_branch_island_echo_question",
+    "acc,none": 0.3,
+    "acc_stderr,none": 0.014498627873361335
+  },
+  "blimp_left_branch_island_simple_question": {
+    "alias": " - blimp_left_branch_island_simple_question",
+    "acc,none": 0.708,
+    "acc_stderr,none": 0.014385511563477433
+  },
+  "blimp_matrix_question_npi_licensor_present": {
+    "alias": " - blimp_matrix_question_npi_licensor_present",
+    "acc,none": 0.629,
+    "acc_stderr,none": 0.015283736211823096
+  },
+  "blimp_npi_present_1": {
+    "alias": " - blimp_npi_present_1",
+    "acc,none": 0.281,
+    "acc_stderr,none": 0.01422115470843494
+  },
+  "blimp_npi_present_2": {
+    "alias": " - blimp_npi_present_2",
+    "acc,none": 0.286,
+    "acc_stderr,none": 0.014297146862518002
+  },
+  "blimp_only_npi_licensor_present": {
+    "alias": " - blimp_only_npi_licensor_present",
+    "acc,none": 0.724,
+    "acc_stderr,none": 0.01414298497574059
+  },
+  "blimp_only_npi_scope": {
+    "alias": " - blimp_only_npi_scope",
+    "acc,none": 0.48,
+    "acc_stderr,none": 0.015806639423035177
+  },
+  "blimp_passive_1": {
+    "alias": " - blimp_passive_1",
+    "acc,none": 0.764,
+    "acc_stderr,none": 0.013434451402438602
+  },
+  "blimp_passive_2": {
+    "alias": " - blimp_passive_2",
+    "acc,none": 0.809,
+    "acc_stderr,none": 0.012436787112179482
+  },
+  "blimp_principle_A_c_command": {
+    "alias": " - blimp_principle_A_c_command",
+    "acc,none": 0.514,
+    "acc_stderr,none": 0.01581309754773093
+  },
+  "blimp_principle_A_case_1": {
+    "alias": " - blimp_principle_A_case_1",
+    "acc,none": 1.0,
+    "acc_stderr,none": 0.0
+  },
+  "blimp_principle_A_case_2": {
+    "alias": " - blimp_principle_A_case_2",
+    "acc,none": 0.876,
+    "acc_stderr,none": 0.010427498872343873
+  },
+  "blimp_principle_A_domain_1": {
+    "alias": " - blimp_principle_A_domain_1",
+    "acc,none": 0.889,
+    "acc_stderr,none": 0.009938701010583716
+  },
+  "blimp_principle_A_domain_2": {
+    "alias": " - blimp_principle_A_domain_2",
+    "acc,none": 0.68,
+    "acc_stderr,none": 0.014758652303574938
+  },
+  "blimp_principle_A_domain_3": {
+    "alias": " - blimp_principle_A_domain_3",
+    "acc,none": 0.545,
+    "acc_stderr,none": 0.015755101498347232
+  },
+  "blimp_principle_A_reconstruction": {
+    "alias": " - blimp_principle_A_reconstruction",
+    "acc,none": 0.69,
+    "acc_stderr,none": 0.014632638658632766
+  },
+  "blimp_regular_plural_subject_verb_agreement_1": {
+    "alias": " - blimp_regular_plural_subject_verb_agreement_1",
+    "acc,none": 0.83,
+    "acc_stderr,none": 0.011884495834541793
+  },
+  "blimp_regular_plural_subject_verb_agreement_2": {
+    "alias": " - blimp_regular_plural_subject_verb_agreement_2",
+    "acc,none": 0.847,
+    "acc_stderr,none": 0.011389500459665504
+  },
+  "blimp_sentential_negation_npi_licensor_present": {
+    "alias": " - blimp_sentential_negation_npi_licensor_present",
+    "acc,none": 0.941,
+    "acc_stderr,none": 0.007454835650406693
+  },
+  "blimp_sentential_negation_npi_scope": {
+    "alias": " - blimp_sentential_negation_npi_scope",
+    "acc,none": 0.22,
+    "acc_stderr,none": 0.013106173040661863
+  },
+  "blimp_sentential_subject_island": {
+    "alias": " - blimp_sentential_subject_island",
+    "acc,none": 0.411,
+    "acc_stderr,none": 0.01556667341859915
+  },
+  "blimp_superlative_quantifiers_1": {
+    "alias": " - blimp_superlative_quantifiers_1",
+    "acc,none": 0.477,
+    "acc_stderr,none": 0.015802554246726087
+  },
+  "blimp_superlative_quantifiers_2": {
+    "alias": " - blimp_superlative_quantifiers_2",
+    "acc,none": 0.48,
+    "acc_stderr,none": 0.015806639423035177
+  },
+  "blimp_tough_vs_raising_1": {
+    "alias": " - blimp_tough_vs_raising_1",
+    "acc,none": 0.733,
+    "acc_stderr,none": 0.013996674851796299
+  },
+  "blimp_tough_vs_raising_2": {
+    "alias": " - blimp_tough_vs_raising_2",
+    "acc,none": 0.647,
+    "acc_stderr,none": 0.015120172605483696
+  },
+  "blimp_transitive": {
+    "alias": " - blimp_transitive",
+    "acc,none": 0.744,
+    "acc_stderr,none": 0.013807775152234145
+  },
+  "blimp_wh_island": {
+    "alias": " - blimp_wh_island",
+    "acc,none": 0.879,
+    "acc_stderr,none": 0.010318210380946179
+  },
+  "blimp_wh_questions_object_gap": {
+    "alias": " - blimp_wh_questions_object_gap",
+    "acc,none": 0.42,
+    "acc_stderr,none": 0.015615500115072968
+  },
+  "blimp_wh_questions_subject_gap": {
+    "alias": " - blimp_wh_questions_subject_gap",
+    "acc,none": 0.713,
+    "acc_stderr,none": 0.014312087053810001
+  },
+  "blimp_wh_questions_subject_gap_long_distance": {
+    "alias": " - blimp_wh_questions_subject_gap_long_distance",
+    "acc,none": 0.634,
+    "acc_stderr,none": 0.0152406127264056
+  },
+  "blimp_wh_vs_that_no_gap": {
+    "alias": " - blimp_wh_vs_that_no_gap",
+    "acc,none": 0.661,
+    "acc_stderr,none": 0.014976758771620224
+  },
+  "blimp_wh_vs_that_no_gap_long_distance": {
+    "alias": " - blimp_wh_vs_that_no_gap_long_distance",
+    "acc,none": 0.688,
+    "acc_stderr,none": 0.014658474370509057
+  },
+  "blimp_wh_vs_that_with_gap": {
+    "alias": " - blimp_wh_vs_that_with_gap",
+    "acc,none": 0.697,
+    "acc_stderr,none": 0.014539683710535193
+  },
+  "blimp_wh_vs_that_with_gap_long_distance": {
+    "alias": " - blimp_wh_vs_that_with_gap_long_distance",
+    "acc,none": 0.516,
+    "acc_stderr,none": 0.015811198373114985
+  },
+  "c4": {
+    "alias": "c4",
+    "word_perplexity,none": 1325.5495280973184,
+    "word_perplexity_stderr,none": "N/A",
+    "byte_perplexity,none": 3.324064373292853,
+    "byte_perplexity_stderr,none": "N/A",
+    "bits_per_byte,none": 1.7329483213775403,
+    "bits_per_byte_stderr,none": "N/A"
+  },
+  "hellaswag": {
+    "alias": "hellaswag",
+    "acc,none": 0.27375024895439154,
+    "acc_stderr,none": 0.004449710700861623,
+    "acc_norm,none": 0.29247161919936265,
+    "acc_norm_stderr,none": 0.00453968076414204
+  },
+  "piqa": {
+    "alias": "piqa",
+    "acc,none": 0.6104461371055495,
+    "acc_stderr,none": 0.011377657455130346,
+    "acc_norm,none": 0.5968443960826986,
+    "acc_norm_stderr,none": 0.011444908701768758
+  },
+  "wikitext": {
+    "alias": "wikitext",
+    "word_perplexity,none": 842.7463841759351,
+    "word_perplexity_stderr,none": "N/A",
+    "byte_perplexity,none": 3.5246820326371133,
+    "byte_perplexity_stderr,none": "N/A",
+    "bits_per_byte,none": 1.8174931154891987,
+    "bits_per_byte_stderr,none": "N/A"
+  }
+}

train_config.json

@@ -1 +1,61 @@
-{"data": {"train_files": "data/fineweb_10B_gpt2/fineweb_train_*.bin", "val_files": "data/fineweb_10B_gpt2/fineweb_val_*.bin", "batch_size": 512, "device_batch_size": 32}, "model": {"dim": 768, "n_layers": 2, "n_heads": 12, "n_kv_heads": 12, "vocab_size": 50257, "multiple_of": 256, "ffn_dim_multiplier": 4, "norm_eps": 1e-05, "rope_theta": 10000, "use_scaled_rope": false, "max_seq_len": 1024, "initializer_range": 0.02, "zero_init_masks": false}, "optimizer": {"default": {"lr": 0.001, "beta1": 0.8, "beta2": 0.95, "eps": 1e-10, "weight_decay": 0}, "masks": {"lr": 0.001, "beta1": 0.8, "beta2": 0.95, "eps": 1e-10, "weight_decay": 0}, "norms": {"lr": 0.001, "beta1": 0.8, "beta2": 0.95, "eps": 1e-10, "weight_decay": 0}}, "scheduler": {"warmup_steps": 0.1, "start_factor": 0.1}, "gates": {}, "gates_zero_eps": 1e-08, "seed": 0, "project": "fineweb-nocontrol", "run_id": null, "logdir": "logs/fineweb-nocontrol", "log_gradients": false, "log_params": false, "log_every_steps": 1, "val_every_steps": 100, "save_every_steps": -1}
+{
+  "data": {
+    "train_files": "data/fineweb_10B_gpt2/fineweb_train_*.bin",
+    "val_files": "data/fineweb_10B_gpt2/fineweb_val_*.bin",
+    "batch_size": 512,
+    "device_batch_size": 32
+  },
+  "model": {
+    "dim": 768,
+    "n_layers": 2,
+    "n_heads": 12,
+    "n_kv_heads": 12,
+    "vocab_size": 50257,
+    "multiple_of": 256,
+    "ffn_dim_multiplier": 4,
+    "norm_eps": 1e-05,
+    "rope_theta": 10000,
+    "use_scaled_rope": false,
+    "max_seq_len": 1024,
+    "initializer_range": 0.02,
+    "zero_init_masks": false
+  },
+  "optimizer": {
+    "default": {
+      "lr": 0.001,
+      "beta1": 0.8,
+      "beta2": 0.95,
+      "eps": 1e-10,
+      "weight_decay": 0
+    },
+    "masks": {
+      "lr": 0.001,
+      "beta1": 0.8,
+      "beta2": 0.95,
+      "eps": 1e-10,
+      "weight_decay": 0
+    },
+    "norms": {
+      "lr": 0.001,
+      "beta1": 0.8,
+      "beta2": 0.95,
+      "eps": 1e-10,
+      "weight_decay": 0
+    }
+  },
+  "scheduler": {
+    "warmup_steps": 0.1,
+    "start_factor": 0.1
+  },
+  "gates": {},
+  "gates_zero_eps": 1e-08,
+  "seed": 0,
+  "project": "fineweb-nocontrol",
+  "run_id": null,
+  "logdir": "logs/fineweb-nocontrol",
+  "log_gradients": false,
+  "log_params": false,
+  "log_every_steps": 1,
+  "val_every_steps": 100,
+  "save_every_steps": -1
+}