| { |
| "model": "d0rj/t5-moe-55M-base", |
| "step": 15000, |
| "source_tokens": 3932160000, |
| "core_protocol": "UL2 S-mode: encoder S+context+sentinel+EOS; decoder BOS+sentinel+answer; score answer text only, full vocabulary", |
| "core_device": "RTX 5070 Ti", |
| "core_dtype": "bfloat16", |
| "harness": "lm-eval 0.4.12", |
| "date": "2026-09-20", |
| "tasks": { |
| "hellaswag": { |
| "samples": 10042, |
| "shots": 0, |
| "dataset": "Rowan/hellaswag", |
| "config": "default", |
| "split": "validation", |
| "metrics": { |
| "acc": { |
| "value": 0.27106154152559253, |
| "stderr": 0.004435993492583661, |
| "ci95": [ |
| 0.26245635297740455, |
| 0.2798418189728654 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| }, |
| "acc_norm": { |
| "value": 0.2779326827325234, |
| "stderr": 0.004470644845243263, |
| "ci95": [ |
| 0.2692569969516344, |
| 0.28677820246076313 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| } |
| } |
| }, |
| "arc_easy": { |
| "samples": 2376, |
| "shots": 0, |
| "dataset": "allenai/ai2_arc", |
| "config": "ARC-Easy", |
| "split": "test", |
| "metrics": { |
| "acc": { |
| "value": 0.4158249158249158, |
| "stderr": 0.01011334824464792, |
| "ci95": [ |
| 0.39615869651760094, |
| 0.4357628802254919 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| }, |
| "acc_norm": { |
| "value": 0.38930976430976433, |
| "stderr": 0.010005212782878208, |
| "ci95": [ |
| 0.3698977221380954, |
| 0.40907915127907174 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| } |
| } |
| }, |
| "arc_challenge": { |
| "samples": 1172, |
| "shots": 0, |
| "dataset": "allenai/ai2_arc", |
| "config": "ARC-Challenge", |
| "split": "test", |
| "metrics": { |
| "acc": { |
| "value": 0.18515358361774745, |
| "stderr": 0.011350774438389583, |
| "ci95": [ |
| 0.16395710645113426, |
| 0.20840725911548352 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| }, |
| "acc_norm": { |
| "value": 0.2380546075085324, |
| "stderr": 0.012445770028026118, |
| "ci95": [ |
| 0.21455235182352161, |
| 0.26326840760453163 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| } |
| } |
| }, |
| "piqa": { |
| "samples": 1838, |
| "shots": 0, |
| "dataset": "baber/piqa", |
| "config": "default", |
| "split": "validation", |
| "metrics": { |
| "acc": { |
| "value": 0.5799782372143635, |
| "stderr": 0.011515615810587429, |
| "ci95": [ |
| 0.5572703036721319, |
| 0.6023525556092767 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| }, |
| "acc_norm": { |
| "value": 0.5560391730141458, |
| "stderr": 0.011592322161277677, |
| "ci95": [ |
| 0.5332313404997734, |
| 0.5786132479763156 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| } |
| } |
| }, |
| "winogrande": { |
| "samples": 1267, |
| "shots": 0, |
| "dataset": "allenai/winogrande", |
| "config": "winogrande_xl", |
| "split": "validation", |
| "metrics": { |
| "acc": { |
| "value": 0.4846093133385951, |
| "stderr": 0.014045826789783585, |
| "ci95": [ |
| 0.4571789703267686, |
| 0.512132701298903 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| } |
| } |
| }, |
| "openbookqa": { |
| "samples": 500, |
| "shots": 0, |
| "dataset": "allenai/openbookqa", |
| "config": "main", |
| "split": "test", |
| "metrics": { |
| "acc": { |
| "value": 0.152, |
| "stderr": 0.016071982367911835, |
| "ci95": [ |
| 0.12319239267380164, |
| 0.18611414828743875 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| }, |
| "acc_norm": { |
| "value": 0.262, |
| "stderr": 0.019684688820194637, |
| "ci95": [ |
| 0.22537626941723765, |
| 0.30225291664246134 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| } |
| } |
| }, |
| "boolq": { |
| "samples": 3270, |
| "shots": 0, |
| "dataset": "aps/super_glue", |
| "config": "boolq", |
| "split": "validation", |
| "metrics": { |
| "acc": { |
| "value": 0.499388379204893, |
| "stderr": 0.00874504843369544, |
| "ci95": [ |
| 0.48226179513749795, |
| 0.5165163985990305 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| } |
| } |
| }, |
| "lambada_openai": { |
| "samples": 5153, |
| "shots": 0, |
| "dataset": "EleutherAI/lambada_openai", |
| "config": "default", |
| "split": "test", |
| "metrics": { |
| "acc": { |
| "value": 0.14826314768096255, |
| "stderr": 0.004950868601262541, |
| "ci95": [ |
| 0.13882265856941103, |
| 0.1582276717641896 |
| ], |
| "ci_method": "Wilson score, z=1.959963984540054" |
| }, |
| "perplexity": { |
| "value": 586.4175184468266, |
| "stderr": 30.041658031622973 |
| } |
| } |
| }, |
| "arithmark3": { |
| "label": "ArithMark-3", |
| "dataset": "AxiomicLabs/Arithmark-3.0", |
| "config": "default", |
| "split": "train", |
| "samples": 1000, |
| "shots": 0, |
| "primary": "acc_norm", |
| "metrics": { |
| "acc": { |
| "value": 0.346, |
| "stderr": 0.015050266127564339, |
| "ci95": [ |
| 0.3171566601020642, |
| 0.3760219815114867 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| }, |
| "acc_norm": { |
| "value": 0.342, |
| "stderr": 0.015008706182121804, |
| "ci95": [ |
| 0.3132528988050859, |
| 0.37195635687634954 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/arithmark3.json", |
| "result_sha256": "0d0d9bc0918204a22db4c2e9acbafcddd494e433c46ae82d0dedfde50aea3eef", |
| "categories": { |
| "elementary_school_math_continuation::addition::grades_1_2::easy": { |
| "count": 128, |
| "acc_norm,none": 0.25, |
| "acc_norm_stderr,none": 0.038423663968391995 |
| }, |
| "elementary_school_math_continuation::comparison::grades_2_3::medium": { |
| "count": 44, |
| "acc_norm,none": 0.2727272727272727, |
| "acc_norm_stderr,none": 0.06791703342160257 |
| }, |
| "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": { |
| "count": 48, |
| "acc_norm,none": 0.2916666666666667, |
| "acc_norm_stderr,none": 0.06629996666059657 |
| }, |
| "elementary_school_math_continuation::data::grades_2_3::easy": { |
| "count": 43, |
| "acc_norm,none": 0.2558139534883721, |
| "acc_norm_stderr,none": 0.06732528971681358 |
| }, |
| "elementary_school_math_continuation::division::grades_3_4::medium": { |
| "count": 54, |
| "acc_norm,none": 0.42592592592592593, |
| "acc_norm_stderr,none": 0.06792240738867397 |
| }, |
| "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": { |
| "count": 50, |
| "acc_norm,none": 0.22, |
| "acc_norm_stderr,none": 0.05917804336345137 |
| }, |
| "elementary_school_math_continuation::geometry_area::grades_4_5::medium": { |
| "count": 52, |
| "acc_norm,none": 0.5384615384615384, |
| "acc_norm_stderr,none": 0.06980655484407924 |
| }, |
| "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": { |
| "count": 45, |
| "acc_norm,none": 0.5555555555555556, |
| "acc_norm_stderr,none": 0.07491109582924912 |
| }, |
| "elementary_school_math_continuation::measurement::grades_2_3::easy": { |
| "count": 76, |
| "acc_norm,none": 0.35526315789473684, |
| "acc_norm_stderr,none": 0.05526315789473684 |
| }, |
| "elementary_school_math_continuation::money::grades_3_4::medium": { |
| "count": 64, |
| "acc_norm,none": 0.296875, |
| "acc_norm_stderr,none": 0.05756159356351619 |
| }, |
| "elementary_school_math_continuation::multiplication::grades_3_4::medium": { |
| "count": 74, |
| "acc_norm,none": 0.5, |
| "acc_norm_stderr,none": 0.05852057359806528 |
| }, |
| "elementary_school_math_continuation::patterns::grades_3_4::medium": { |
| "count": 53, |
| "acc_norm,none": 0.20754716981132076, |
| "acc_norm_stderr,none": 0.05623975840347623 |
| }, |
| "elementary_school_math_continuation::subtraction::grades_1_2::easy": { |
| "count": 117, |
| "acc_norm,none": 0.1794871794871795, |
| "acc_norm_stderr,none": 0.035631196604084675 |
| }, |
| "elementary_school_math_continuation::time::grades_2_3::easy": { |
| "count": 55, |
| "acc_norm,none": 0.9090909090909091, |
| "acc_norm_stderr,none": 0.03912104390108503 |
| }, |
| "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": { |
| "count": 46, |
| "acc_norm,none": 0.1956521739130435, |
| "acc_norm_stderr,none": 0.05913682829884975 |
| }, |
| "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": { |
| "count": 19, |
| "acc_norm,none": 0.10526315789473684, |
| "acc_norm_stderr,none": 0.07233518641434492 |
| }, |
| "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": { |
| "count": 32, |
| "acc_norm,none": 0.3125, |
| "acc_norm_stderr,none": 0.08324928557283298 |
| } |
| } |
| }, |
| "balanced_copa": { |
| "label": "Balanced COPA", |
| "dataset": "pkavumba/balanced-copa", |
| "config": "default", |
| "split": "train", |
| "samples": 1000, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.538, |
| "stderr": 0.015773547629015002, |
| "ci95": [ |
| 0.5070132970793998, |
| 0.5686958692756982 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/balanced_copa.json", |
| "result_sha256": "62c4a30d3f5847a68094e91c764dc79ba3e9a2e2a9c67236af6d5f5266018f78", |
| "categories": {} |
| }, |
| "commonsense_qa": { |
| "label": "CommonsenseQA", |
| "dataset": "tau/commonsense_qa", |
| "config": "default", |
| "split": "validation", |
| "samples": 1221, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.20065520065520065, |
| "stderr": 0.011466011466011467, |
| "ci95": [ |
| 0.17914588148536167, |
| 0.2240421844184298 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/commonsense_qa.json", |
| "result_sha256": "8c3ba38b4757c3678e717a81217c7c304c74309f0cec30ca34f28222347d0ec4", |
| "categories": {} |
| }, |
| "sciq": { |
| "label": "SciQ (with support)", |
| "dataset": "allenai/sciq", |
| "config": "default", |
| "split": "test", |
| "samples": 1000, |
| "shots": 0, |
| "primary": "acc_norm", |
| "metrics": { |
| "acc": { |
| "value": 0.703, |
| "stderr": 0.01445683229480096, |
| "ci95": [ |
| 0.6739460359438715, |
| 0.730500300110993 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| }, |
| "acc_norm": { |
| "value": 0.654, |
| "stderr": 0.01505026612756434, |
| "ci95": [ |
| 0.6239780184885133, |
| 0.6828433398979359 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/sciq.json", |
| "result_sha256": "d2aa221220038ac172272d4f598ade4df4841839b2de315e4438bb8f3da0c031", |
| "categories": {} |
| }, |
| "truthfulqa_mc2": { |
| "label": "TruthfulQA MC2", |
| "dataset": "truthfulqa/truthful_qa", |
| "config": "multiple_choice", |
| "split": "validation", |
| "samples": 817, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.4656644987453161, |
| "stderr": 0.016000577772697092 |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/truthfulqa_mc2.json", |
| "result_sha256": "951d4dd4cfeba277592bdc041d88516a36c8f3ee1c21ab1bb20668bbc7b6d329", |
| "categories": {} |
| }, |
| "bananamind_base": { |
| "label": "BananaMind Base 1.1", |
| "dataset": "BananaMind/BananaMind-Base-Bench-1.1", |
| "config": "default", |
| "split": "test", |
| "samples": 350, |
| "shots": 0, |
| "primary": "raw_accuracy", |
| "metrics": { |
| "raw_accuracy": { |
| "value": 0.36, |
| "stderr": 0.025693810923465083, |
| "ci95": [ |
| 0.3114835744745811, |
| 0.41155622892601307 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/bananamind_base.json", |
| "result_sha256": "473694fd9a25bf2721607c4ef28aa17f19f6850af3760f1c90a42c1dfb59a071", |
| "categories": { |
| "code_completion": { |
| "count": 50, |
| "raw_accuracy,none": 0.24, |
| "raw_accuracy_stderr,none": 0.06101187572589321 |
| }, |
| "commonsense": { |
| "count": 50, |
| "raw_accuracy,none": 0.36, |
| "raw_accuracy_stderr,none": 0.06857142857142857 |
| }, |
| "context_tracking": { |
| "count": 50, |
| "raw_accuracy,none": 0.36, |
| "raw_accuracy_stderr,none": 0.06857142857142857 |
| }, |
| "language_completion": { |
| "count": 50, |
| "raw_accuracy,none": 0.42, |
| "raw_accuracy_stderr,none": 0.07050835816716038 |
| }, |
| "logical_reasoning": { |
| "count": 50, |
| "raw_accuracy,none": 0.34, |
| "raw_accuracy_stderr,none": 0.0676726816132972 |
| }, |
| "quantitative": { |
| "count": 50, |
| "raw_accuracy,none": 0.18, |
| "raw_accuracy_stderr,none": 0.054883922035138706 |
| }, |
| "world_knowledge": { |
| "count": 50, |
| "raw_accuracy,none": 0.62, |
| "raw_accuracy_stderr,none": 0.06934092056863769 |
| } |
| } |
| }, |
| "mmlu_continuation": { |
| "label": "MMLU continuation", |
| "dataset": "cais/mmlu", |
| "config": "57 subjects", |
| "split": "test", |
| "samples": 14042, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.24932345819683804, |
| "stderr": 0.003640204268569072 |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/mmlu_continuation.json", |
| "result_sha256": "8e939afec45700c2d5f70dafa5df87b52841ace350b74969bbfcacc9fc1e54f8", |
| "categories": {} |
| }, |
| "blimp": { |
| "label": "BLiMP", |
| "dataset": "nyu-mll/blimp", |
| "config": "67 minimal-pair subsets", |
| "split": "train", |
| "samples": 67000, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.6988955223880597, |
| "stderr": 0.0015386258071003309 |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-002/blimp.json", |
| "result_sha256": "58646e23f3b9a6fbeca3b50b503fe9893fe83c34306a830f520761e225bb4c2f", |
| "categories": {} |
| } |
| }, |
| "continuation_cpu_fp32": { |
| "nll": 3.9203571982400116, |
| "nll_ci95": [ |
| 3.8792989141547807, |
| 3.960741957038948 |
| ], |
| "ppl": 50.41845094392994, |
| "ppl_ci95": [ |
| 48.39027743758022, |
| 52.496261473788515 |
| ], |
| "blocks": 291, |
| "scored_tokens": 148992, |
| "block_nll": [ |
| 3.0761337280273438, |
| 4.079192042350769, |
| 3.792433500289917, |
| 3.8495718240737915, |
| 4.15901792049408, |
| 4.1957807540893555, |
| 4.858497619628906, |
| 4.304347634315491, |
| 3.9739487171173096, |
| 4.084920883178711, |
| 3.858888030052185, |
| 3.356896996498108, |
| 3.2847559452056885, |
| 3.7786415815353394, |
| 4.163232326507568, |
| 3.811104655265808, |
| 4.206327676773071, |
| 4.170447587966919, |
| 3.968077778816223, |
| 4.145242929458618, |
| 4.085895895957947, |
| 3.6578187942504883, |
| 4.47190523147583, |
| 3.5634177923202515, |
| 3.5364233255386353, |
| 3.7972182035446167, |
| 3.872505307197571, |
| 3.7728580236434937, |
| 3.7172998189926147, |
| 3.6024192571640015, |
| 4.207360506057739, |
| 4.442894697189331, |
| 3.5926103591918945, |
| 3.5468727350234985, |
| 3.7207106351852417, |
| 4.447509288787842, |
| 3.945070743560791, |
| 3.9589524269104004, |
| 3.5569419860839844, |
| 4.00419807434082, |
| 3.7097017765045166, |
| 4.105579733848572, |
| 3.6655616760253906, |
| 3.983635187149048, |
| 3.7981103658676147, |
| 4.071430444717407, |
| 4.23374605178833, |
| 3.9315249919891357, |
| 4.139315128326416, |
| 3.9565144777297974, |
| 3.826423764228821, |
| 3.715441346168518, |
| 4.19908881187439, |
| 4.244288682937622, |
| 3.9537326097488403, |
| 4.429767608642578, |
| 3.784157872200012, |
| 3.5398250818252563, |
| 3.979478597640991, |
| 3.5311062335968018, |
| 3.677019715309143, |
| 3.4096540212631226, |
| 3.8282769918441772, |
| 2.798559308052063, |
| 3.6455711126327515, |
| 3.704820990562439, |
| 3.529540181159973, |
| 3.3359416723251343, |
| 3.9118762016296387, |
| 3.68520987033844, |
| 3.822246789932251, |
| 3.5766007900238037, |
| 3.986479640007019, |
| 3.719839096069336, |
| 4.152796506881714, |
| 3.8391287326812744, |
| 3.9557178020477295, |
| 3.9554009437561035, |
| 3.8190184831619263, |
| 4.023033976554871, |
| 3.796515941619873, |
| 3.8765863180160522, |
| 3.74442195892334, |
| 4.405590295791626, |
| 4.193913459777832, |
| 4.437158107757568, |
| 4.129748582839966, |
| 4.139063119888306, |
| 4.294118762016296, |
| 4.159630656242371, |
| 3.840534210205078, |
| 3.985872268676758, |
| 4.093213796615601, |
| 3.9668267965316772, |
| 3.9052979946136475, |
| 4.020379185676575, |
| 3.784497857093811, |
| 3.7786608934402466, |
| 3.653836727142334, |
| 4.237546920776367, |
| 3.967842698097229, |
| 3.8961071968078613, |
| 3.021655559539795, |
| 4.051250219345093, |
| 3.7720279693603516, |
| 3.488795757293701, |
| 3.8025426864624023, |
| 3.3990745544433594, |
| 3.6348395347595215, |
| 3.699379801750183, |
| 3.5647464990615845, |
| 3.5748815536499023, |
| 3.038840889930725, |
| 3.4079902172088623, |
| 3.8965002298355103, |
| 3.7620593309402466, |
| 3.5435174703598022, |
| 4.014705657958984, |
| 3.691099524497986, |
| 3.8808759450912476, |
| 4.210118293762207, |
| 4.09901762008667, |
| 4.0313684940338135, |
| 3.9684605598449707, |
| 4.259034156799316, |
| 4.205959320068359, |
| 3.8939430713653564, |
| 4.025556564331055, |
| 4.235677003860474, |
| 4.029173374176025, |
| 4.173335790634155, |
| 4.074177026748657, |
| 4.02467155456543, |
| 3.647403359413147, |
| 4.111065983772278, |
| 4.035062909126282, |
| 3.959086298942566, |
| 4.163747787475586, |
| 3.796730875968933, |
| 3.989443302154541, |
| 3.65218448638916, |
| 3.4024927616119385, |
| 4.547068119049072, |
| 4.1708152294158936, |
| 4.254725694656372, |
| 4.341301441192627, |
| 4.585696458816528, |
| 4.2345826625823975, |
| 3.738357663154602, |
| 3.9488762617111206, |
| 4.143638610839844, |
| 4.082504153251648, |
| 3.439720392227173, |
| 3.882248044013977, |
| 3.354142904281616, |
| 3.808571934700012, |
| 4.034612536430359, |
| 4.118360161781311, |
| 4.4715986251831055, |
| 3.6331266164779663, |
| 3.961649179458618, |
| 4.019700050354004, |
| 3.1727192401885986, |
| 3.1752305030822754, |
| 2.9871737957000732, |
| 3.4513983726501465, |
| 3.255954146385193, |
| 3.7564663887023926, |
| 3.329494833946228, |
| 4.367681503295898, |
| 4.036498308181763, |
| 4.403446197509766, |
| 4.157774090766907, |
| 4.7906129360198975, |
| 4.744508266448975, |
| 4.22991418838501, |
| 4.3303468227386475, |
| 3.9489530324935913, |
| 3.6769163608551025, |
| 5.098318815231323, |
| 4.181112766265869, |
| 4.56702995300293, |
| 4.325042486190796, |
| 4.491414546966553, |
| 4.616647005081177, |
| 4.4533233642578125, |
| 4.20113468170166, |
| 4.4946160316467285, |
| 4.579037427902222, |
| 4.378486156463623, |
| 4.784059047698975, |
| 4.476404428482056, |
| 3.531564235687256, |
| 4.690818786621094, |
| 4.406581401824951, |
| 3.944377064704895, |
| 4.071741223335266, |
| 3.789208173751831, |
| 4.132367134094238, |
| 4.115847587585449, |
| 4.100175142288208, |
| 3.4047350883483887, |
| 3.820371150970459, |
| 3.6308525800704956, |
| 3.9466482400894165, |
| 4.010139107704163, |
| 3.771423578262329, |
| 4.069459676742554, |
| 3.654417634010315, |
| 3.859255313873291, |
| 3.913407325744629, |
| 3.664337635040283, |
| 3.8283824920654297, |
| 3.8213530778884888, |
| 3.708220601081848, |
| 4.080739378929138, |
| 4.100520133972168, |
| 3.6943432092666626, |
| 3.4650919437408447, |
| 3.4190703630447388, |
| 3.6568729877471924, |
| 3.8066530227661133, |
| 3.6321038007736206, |
| 3.596403121948242, |
| 3.9770424365997314, |
| 3.6597766876220703, |
| 3.3995832204818726, |
| 4.028826355934143, |
| 4.148176074028015, |
| 3.718005895614624, |
| 4.120266675949097, |
| 3.9619749784469604, |
| 3.852488875389099, |
| 4.052497625350952, |
| 4.091804027557373, |
| 4.02939510345459, |
| 4.256818771362305, |
| 3.8955929279327393, |
| 3.5858232975006104, |
| 3.992673635482788, |
| 3.9264363050460815, |
| 3.8334593772888184, |
| 3.611130475997925, |
| 3.2467005252838135, |
| 3.6779762506484985, |
| 3.5896599292755127, |
| 3.6830077171325684, |
| 3.872781276702881, |
| 4.432836532592773, |
| 3.8607051372528076, |
| 4.347602844238281, |
| 4.423125982284546, |
| 3.9818156957626343, |
| 3.795296311378479, |
| 4.2359819412231445, |
| 4.199247121810913, |
| 3.8171900510787964, |
| 4.0548319816589355, |
| 4.342285871505737, |
| 3.9213531017303467, |
| 3.7376575469970703, |
| 3.651806950569153, |
| 3.3956466913223267, |
| 3.7015174627304077, |
| 3.8937771320343018, |
| 4.029564738273621, |
| 3.5710394382476807, |
| 4.138579845428467, |
| 3.918642520904541, |
| 3.4925676584243774, |
| 4.080316543579102, |
| 4.114610314369202, |
| 3.332297444343567, |
| 3.4136807918548584, |
| 3.2582297325134277, |
| 2.9236204624176025, |
| 3.892964005470276, |
| 3.701326012611389, |
| 4.241790771484375, |
| 4.124042749404907, |
| 4.0206745862960815, |
| 4.105286717414856, |
| 3.647748827934265, |
| 3.912590742111206, |
| 4.15474271774292, |
| 3.690990924835205, |
| 3.896915555000305, |
| 4.0754547119140625, |
| 4.880402088165283, |
| 4.429782867431641, |
| 3.8676462173461914 |
| ] |
| }, |
| "continuation_ci_method": "percentile bootstrap of nonoverlapping 1024-token blocks; conditional suffix NLL; exponentiate interval endpoints for PPL", |
| "continuation_bootstrap_samples": 10000, |
| "continuation_bootstrap_seed": 2026, |
| "corpus_sha256": "aca2f46735043bcfd0a44eca981d04627b9cdf74c4c9a04bf0856d04066f58fc", |
| "weights_sha256": "1efcd3aeed90f8b87ff06495f29f2cffa36435db1885d34a5e237450d2404b9f", |
| "source_sha256": { |
| "src/tiny_llm/__init__.py": "8c7f075d51006a53fe1d645ef5a84d65db58ff4251e7544074fe29ee7549c14c", |
| "src/tiny_llm/benchmarks.py": "5d9f461366389bd1f926c6a6468192f3e2313c87caf5b1de27f407d563a858a4", |
| "src/tiny_llm/checkpoint.py": "ac9df5f8c981a47eb1f1e59d0687ff41a1f6c38ca74ced50f71ac446a58d6ef5", |
| "src/tiny_llm/continuation.py": "548b7e76401cd545af3e7ff1aedf041fc96a330c8c9cc80266e0e20830f0e95d", |
| "src/tiny_llm/data.py": "973e4ff3e7b6e80c74bd99173f89cb8ce94c6ef3a21ff54acf1a42eab9654d89", |
| "src/tiny_llm/evaluation/__init__.py": "e49e80f7be3ab6e200f0db14b028e465de9caa8f6f705430eabce0c41b8322ba", |
| "src/tiny_llm/evaluation/adapters.py": "5a881b82bf6938ca953be751de021889f2c51a559df88f537da1e6ce4dfc6e15", |
| "src/tiny_llm/evaluation/catalog.py": "5681c9433c67890644488f7f73280e18662f28a5de8c18374443bd7c84ba4be9", |
| "src/tiny_llm/evaluation/cli.py": "b4875211ed8390c72ee96e2cb5f0abc02eb9ea7c5e4828b7c3eea7d6ad682f11", |
| "src/tiny_llm/losses.py": "592a05287369f6e5077b3faca404a9392de4772cf06389fb8aa7e36924e74f86", |
| "src/tiny_llm/models/__init__.py": "55f88bbd8cf8bfd70e5e17c1d0050299412528385c3714bb712623f40b0a681d", |
| "src/tiny_llm/models/diffusion/__init__.py": "c92eaf5986d604d8aaf1894a862a217cefa32ab25359175b88110e45a71c1b25", |
| "src/tiny_llm/models/diffusion/configuration_diffusion_lm.py": "8f796fb3bb105dfedc5d14d939ffc1c0cb3a119904fe90ca0feb76ffee9fc9a2", |
| "src/tiny_llm/models/diffusion/modeling_diffusion_lm.py": "71162bf7fbe92cbf08cf3bf64f59fc11098c5c302787fc1c7fe9fcb14c9479e5", |
| "src/tiny_llm/models/looped/__init__.py": "4b46b426021c59ed313cb2f49afc1926ad2793d771ef90117e3fef0462957cff", |
| "src/tiny_llm/models/looped/configuration_looped_lm.py": "b8d99c52b98e0a97a4cae76ca4c20affd8f4d019ec38e0a9beefd5fad864dc25", |
| "src/tiny_llm/models/looped/modeling_looped_lm.py": "9bdb3c5e3aaae663d09e98055f95df1adbc42e0a4bf5dffb7f6c39242af41b9e", |
| "src/tiny_llm/models/moe_t5/__init__.py": "18e33fe38b2049521f573512ac8796afa209e1b25c2052784e1c927a7160d66c", |
| "src/tiny_llm/models/moe_t5/configuration_alicet5_moe.py": "4647b1e9ab9ce07887bffc1abc3beb3e691962a380b4cc237aae376490bb56b7", |
| "src/tiny_llm/models/moe_t5/modeling_alicet5_moe.py": "c7bbba2f01dc889cc604c7bc161354397239b28503d05f7e00665658be84751d", |
| "src/tiny_llm/models/prefixlm/__init__.py": "c94c3c861acba8d2147a3a6c46bccfc71ac8adc8d70757bede009598029d31ea", |
| "src/tiny_llm/models/prefixlm/configuration_prefix_lm.py": "20b3a62ae3628b5a64566ef19d2947c90f10c6159daad00dd13d18fc5910c42c", |
| "src/tiny_llm/models/prefixlm/modeling_prefix_lm.py": "f62a38211df175696f64945173d1e41d9f7fcf6152a7648020da8c91788e98fd", |
| "src/tiny_llm/models/q50m/__init__.py": "9d0dfec6fa0cfa6967e2f17a73d8fbeca69ba93b4a93651eda4d95626d5f5eb5", |
| "src/tiny_llm/models/q50m/configuration_q50m.py": "796806b2d5d982b633796bc7de53c3073b1b6e52d259f44ef798b63f55eaea92", |
| "src/tiny_llm/models/q50m/modeling_q50m.py": "b7a712050167fad24471193b0ca7baf4ca7e5ebdf8036c91b506d3fff5b49c8b", |
| "src/tiny_llm/telemetry.py": "f5a35de68da1a2304159fd3613fa3773bc368982ed18712791c847285b895c09", |
| "src/tiny_llm/ul2.py": "f4d948d89d5e106965612971dd473a54b57719f811761f18457c6221d763b876" |
| }, |
| "packages": { |
| "lm_eval": "0.4.12", |
| "torch": "2.11.0+cu128", |
| "transformers": "5.17.0", |
| "datasets": "5.0.1", |
| "accelerate": "1.15.0" |
| }, |
| "updated": "2026-10-01", |
| "additional_evaluation": { |
| "path": "evaluation/comparison-20261001/results.json", |
| "weight_sha256": "1efcd3aeed90f8b87ff06495f29f2cffa36435db1885d34a5e237450d2404b9f", |
| "date": "2026-10-01", |
| "dtype": "bfloat16", |
| "tasks": [ |
| "arithmark3", |
| "balanced_copa", |
| "commonsense_qa", |
| "sciq", |
| "truthfulqa_mc2", |
| "bananamind_base", |
| "mmlu_continuation", |
| "blimp" |
| ] |
| } |
| } |
|
|
