| { |
| "date": "2026-10-01", |
| "model": "d0rj/t5-moe-55M-base", |
| "protocol": "Full selected splits; lm-eval 0.4.12; seed 1234; BF16 on RTX 5070 Ti;\ncontext cap 2048 (ArithMark 1024), TF32 disabled, no chat template and no added\nfew-shot examples. TruthfulQA retains the harness's fixed six-QA preamble.\nArithMark and BananaMind normalize by continuation token count; ordinary harness\nacc_norm uses its own length normalization. BananaMind is raw accuracy, not Elo.\nSciQ includes the support passage. Balanced COPA uses the mirrored 1000-item\ntrain-named evaluation split; cRia's split was inferred, not confirmed.\nMMLU scores full answer continuations across 57 subjects, weighted by item count;\nBLiMP averages 67 equal-sized minimal-pair subsets. Standard errors are retained\nfrom each evaluator. Wilson intervals are reported only where the runner logged\nbinary item accuracy; MC2 is probability mass, not binary accuracy. These\nintervals do not model dependence between paired/templated examples or training\nseed variation. UL2, PrefixLM and experimental diffusion PLL use their documented\nconditional scoring protocols; PLL exposes the other answer tokens and is not\nautoregressive likelihood. cRia's published scores used a different precision\nand benchmark-adapted checkpoint; this completes our comparison coverage, not\nan independent reproduction of cRia or an official leaderboard submission.", |
| "results": { |
| "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/cria-backfill-20261001/checkpoints/t5", |
| "checkpoint_sha256": { |
| "config.json": "8031de3caf99e9d424970b69701b1077d9dff5151bafd4ce214936a50fbd3bd4", |
| "generation_config.json": "8f1ce3b416c7f594951681c73e2b3c7933baac0a09a4f8d649aad27af21941f6", |
| "model.safetensors": "1efcd3aeed90f8b87ff06495f29f2cffa36435db1885d34a5e237450d2404b9f", |
| "tokenizer.json": "1d1f7409de3e53ae51d7085b3009aa5cd491dc08fbe114794c8d314f3fba2af4", |
| "tokenizer_config.json": "879515efa0fe331af8eeb0ede00fdd13c0bd3c827bff8bb6f4b7f959c02159ec" |
| }, |
| "core": "/mnt/d/Projects/tiny_llm/runs/eval/t5-core-full-20260920", |
| "training_run": "moe_t5-gpu-v1", |
| "training_step": 15000, |
| "hf": "d0rj/t5-moe-55M-base", |
| "tokenizer": "runs/moe_t5-gpu-v1/checkpoint-last", |
| "tb": "runs/moe_t5-gpu-v1/tb", |
| "protocol": "UL2 S-mode: encoder S+context+sentinel+EOS; decoder BOS+sentinel+answer; score answer text only, full vocabulary", |
| "kind": "seq2seq", |
| "tasks": { |
| "arithmark3": { |
| "label": "ArithMark-3", |
| "dataset": "AxiomicLabs/Arithmark-3.0", |
| "config": "default", |
| "split": "train", |
| "samples": 1000, |
| "shots": 0, |
| "primary": "acc_norm", |
| "metrics": { |
| "acc": { |
| "value": 0.346, |
| "stderr": 0.015050266127564339, |
| "ci95": [ |
| 0.3171566601020642, |
| 0.3760219815114867 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| }, |
| "acc_norm": { |
| "value": 0.342, |
| "stderr": 0.015008706182121804, |
| "ci95": [ |
| 0.3132528988050859, |
| 0.37195635687634954 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/arithmark3.json", |
| "result_sha256": "0d0d9bc0918204a22db4c2e9acbafcddd494e433c46ae82d0dedfde50aea3eef", |
| "categories": { |
| "elementary_school_math_continuation::addition::grades_1_2::easy": { |
| "count": 128, |
| "acc_norm,none": 0.25, |
| "acc_norm_stderr,none": 0.038423663968391995 |
| }, |
| "elementary_school_math_continuation::comparison::grades_2_3::medium": { |
| "count": 44, |
| "acc_norm,none": 0.2727272727272727, |
| "acc_norm_stderr,none": 0.06791703342160257 |
| }, |
| "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": { |
| "count": 48, |
| "acc_norm,none": 0.2916666666666667, |
| "acc_norm_stderr,none": 0.06629996666059657 |
| }, |
| "elementary_school_math_continuation::data::grades_2_3::easy": { |
| "count": 43, |
| "acc_norm,none": 0.2558139534883721, |
| "acc_norm_stderr,none": 0.06732528971681358 |
| }, |
| "elementary_school_math_continuation::division::grades_3_4::medium": { |
| "count": 54, |
| "acc_norm,none": 0.42592592592592593, |
| "acc_norm_stderr,none": 0.06792240738867397 |
| }, |
| "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": { |
| "count": 50, |
| "acc_norm,none": 0.22, |
| "acc_norm_stderr,none": 0.05917804336345137 |
| }, |
| "elementary_school_math_continuation::geometry_area::grades_4_5::medium": { |
| "count": 52, |
| "acc_norm,none": 0.5384615384615384, |
| "acc_norm_stderr,none": 0.06980655484407924 |
| }, |
| "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": { |
| "count": 45, |
| "acc_norm,none": 0.5555555555555556, |
| "acc_norm_stderr,none": 0.07491109582924912 |
| }, |
| "elementary_school_math_continuation::measurement::grades_2_3::easy": { |
| "count": 76, |
| "acc_norm,none": 0.35526315789473684, |
| "acc_norm_stderr,none": 0.05526315789473684 |
| }, |
| "elementary_school_math_continuation::money::grades_3_4::medium": { |
| "count": 64, |
| "acc_norm,none": 0.296875, |
| "acc_norm_stderr,none": 0.05756159356351619 |
| }, |
| "elementary_school_math_continuation::multiplication::grades_3_4::medium": { |
| "count": 74, |
| "acc_norm,none": 0.5, |
| "acc_norm_stderr,none": 0.05852057359806528 |
| }, |
| "elementary_school_math_continuation::patterns::grades_3_4::medium": { |
| "count": 53, |
| "acc_norm,none": 0.20754716981132076, |
| "acc_norm_stderr,none": 0.05623975840347623 |
| }, |
| "elementary_school_math_continuation::subtraction::grades_1_2::easy": { |
| "count": 117, |
| "acc_norm,none": 0.1794871794871795, |
| "acc_norm_stderr,none": 0.035631196604084675 |
| }, |
| "elementary_school_math_continuation::time::grades_2_3::easy": { |
| "count": 55, |
| "acc_norm,none": 0.9090909090909091, |
| "acc_norm_stderr,none": 0.03912104390108503 |
| }, |
| "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": { |
| "count": 46, |
| "acc_norm,none": 0.1956521739130435, |
| "acc_norm_stderr,none": 0.05913682829884975 |
| }, |
| "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": { |
| "count": 19, |
| "acc_norm,none": 0.10526315789473684, |
| "acc_norm_stderr,none": 0.07233518641434492 |
| }, |
| "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": { |
| "count": 32, |
| "acc_norm,none": 0.3125, |
| "acc_norm_stderr,none": 0.08324928557283298 |
| } |
| } |
| }, |
| "balanced_copa": { |
| "label": "Balanced COPA", |
| "dataset": "pkavumba/balanced-copa", |
| "config": "default", |
| "split": "train", |
| "samples": 1000, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.538, |
| "stderr": 0.015773547629015002, |
| "ci95": [ |
| 0.5070132970793998, |
| 0.5686958692756982 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/balanced_copa.json", |
| "result_sha256": "62c4a30d3f5847a68094e91c764dc79ba3e9a2e2a9c67236af6d5f5266018f78", |
| "categories": {} |
| }, |
| "commonsense_qa": { |
| "label": "CommonsenseQA", |
| "dataset": "tau/commonsense_qa", |
| "config": "default", |
| "split": "validation", |
| "samples": 1221, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.20065520065520065, |
| "stderr": 0.011466011466011467, |
| "ci95": [ |
| 0.17914588148536167, |
| 0.2240421844184298 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/commonsense_qa.json", |
| "result_sha256": "8c3ba38b4757c3678e717a81217c7c304c74309f0cec30ca34f28222347d0ec4", |
| "categories": {} |
| }, |
| "sciq": { |
| "label": "SciQ (with support)", |
| "dataset": "allenai/sciq", |
| "config": "default", |
| "split": "test", |
| "samples": 1000, |
| "shots": 0, |
| "primary": "acc_norm", |
| "metrics": { |
| "acc": { |
| "value": 0.703, |
| "stderr": 0.01445683229480096, |
| "ci95": [ |
| 0.6739460359438715, |
| 0.730500300110993 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| }, |
| "acc_norm": { |
| "value": 0.654, |
| "stderr": 0.01505026612756434, |
| "ci95": [ |
| 0.6239780184885133, |
| 0.6828433398979359 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/sciq.json", |
| "result_sha256": "d2aa221220038ac172272d4f598ade4df4841839b2de315e4438bb8f3da0c031", |
| "categories": {} |
| }, |
| "truthfulqa_mc2": { |
| "label": "TruthfulQA MC2", |
| "dataset": "truthfulqa/truthful_qa", |
| "config": "multiple_choice", |
| "split": "validation", |
| "samples": 817, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.4656644987453161, |
| "stderr": 0.016000577772697092 |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/truthfulqa_mc2.json", |
| "result_sha256": "951d4dd4cfeba277592bdc041d88516a36c8f3ee1c21ab1bb20668bbc7b6d329", |
| "categories": {} |
| }, |
| "bananamind_base": { |
| "label": "BananaMind Base 1.1", |
| "dataset": "BananaMind/BananaMind-Base-Bench-1.1", |
| "config": "default", |
| "split": "test", |
| "samples": 350, |
| "shots": 0, |
| "primary": "raw_accuracy", |
| "metrics": { |
| "raw_accuracy": { |
| "value": 0.36, |
| "stderr": 0.025693810923465083, |
| "ci95": [ |
| 0.3114835744745811, |
| 0.41155622892601307 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/bananamind_base.json", |
| "result_sha256": "473694fd9a25bf2721607c4ef28aa17f19f6850af3760f1c90a42c1dfb59a071", |
| "categories": { |
| "code_completion": { |
| "count": 50, |
| "raw_accuracy,none": 0.24, |
| "raw_accuracy_stderr,none": 0.06101187572589321 |
| }, |
| "commonsense": { |
| "count": 50, |
| "raw_accuracy,none": 0.36, |
| "raw_accuracy_stderr,none": 0.06857142857142857 |
| }, |
| "context_tracking": { |
| "count": 50, |
| "raw_accuracy,none": 0.36, |
| "raw_accuracy_stderr,none": 0.06857142857142857 |
| }, |
| "language_completion": { |
| "count": 50, |
| "raw_accuracy,none": 0.42, |
| "raw_accuracy_stderr,none": 0.07050835816716038 |
| }, |
| "logical_reasoning": { |
| "count": 50, |
| "raw_accuracy,none": 0.34, |
| "raw_accuracy_stderr,none": 0.0676726816132972 |
| }, |
| "quantitative": { |
| "count": 50, |
| "raw_accuracy,none": 0.18, |
| "raw_accuracy_stderr,none": 0.054883922035138706 |
| }, |
| "world_knowledge": { |
| "count": 50, |
| "raw_accuracy,none": 0.62, |
| "raw_accuracy_stderr,none": 0.06934092056863769 |
| } |
| } |
| }, |
| "mmlu_continuation": { |
| "label": "MMLU continuation", |
| "dataset": "cais/mmlu", |
| "config": "57 subjects", |
| "split": "test", |
| "samples": 14042, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.24932345819683804, |
| "stderr": 0.003640204268569072 |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-001/mmlu_continuation.json", |
| "result_sha256": "8e939afec45700c2d5f70dafa5df87b52841ace350b74969bbfcacc9fc1e54f8", |
| "categories": {} |
| }, |
| "blimp": { |
| "label": "BLiMP", |
| "dataset": "nyu-mll/blimp", |
| "config": "67 minimal-pair subsets", |
| "split": "train", |
| "samples": 67000, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.6988955223880597, |
| "stderr": 0.0015386258071003309 |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/t5/attempt-002/blimp.json", |
| "result_sha256": "58646e23f3b9a6fbeca3b50b503fe9893fe83c34306a830f520761e225bb4c2f", |
| "categories": {} |
| } |
| }, |
| "batch_size": 8, |
| "label": "T5 MoE UL2", |
| "manifests": { |
| "runs/eval/cria-backfill-20261001/t5/attempt-001": { |
| "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/cria-backfill-20261001/checkpoints/t5", |
| "local_model": "moe_t5", |
| "kind": "seq2seq", |
| "tokenizer": "runs/moe_t5-gpu-v1/checkpoint-last", |
| "revision": "main", |
| "tokenizer_revision": null, |
| "protocol": "UL2 S-mode: encoder S+context+sentinel+EOS; decoder BOS+sentinel+answer; score answer text only, full vocabulary", |
| "harness_version": "0.4.12", |
| "smoke_only": false, |
| "tasks": [ |
| { |
| "task": "arithmark3", |
| "shots": 0, |
| "description": "ArithMark-3.0 train-named evaluation split; token-normalized acc_norm", |
| "unsafe_code": false, |
| "rolling": false, |
| "engine": "token_continuation" |
| }, |
| { |
| "task": "balanced_copa", |
| "shots": 0, |
| "description": "Balanced COPA mirrored 1000-item train split; acc; cRia split inferred", |
| "unsafe_code": false, |
| "rolling": false, |
| "engine": "lm_eval" |
| }, |
| { |
| "task": "commonsense_qa", |
| "shots": 0, |
| "description": "CommonsenseQA; acc", |
| "unsafe_code": false, |
| "rolling": false, |
| "engine": "lm_eval" |
| }, |
| { |
| "task": "sciq", |
| "shots": 0, |
| "description": "SciQ test with support passage; acc and acc_norm", |
| "unsafe_code": false, |
| "rolling": false, |
| "engine": "lm_eval" |
| }, |
| { |
| "task": "truthfulqa_mc2", |
| "shots": 0, |
| "description": "TruthfulQA multiple choice; mc2", |
| "unsafe_code": false, |
| "rolling": false, |
| "engine": "lm_eval" |
| }, |
| { |
| "task": "bananamind_base", |
| "shots": 0, |
| "description": "BananaMind Base Bench 1.1 test; mean-token raw accuracy; gated", |
| "unsafe_code": false, |
| "rolling": false, |
| "engine": "token_continuation" |
| }, |
| { |
| "task": "mmlu_continuation", |
| "shots": 0, |
| "description": "MMLU cloze: full answer continuations; acc", |
| "unsafe_code": false, |
| "rolling": false, |
| "engine": "lm_eval" |
| }, |
| { |
| "task": "blimp", |
| "shots": 0, |
| "description": "Full BLiMP group; grammatical minimal pairs", |
| "unsafe_code": false, |
| "rolling": false, |
| "engine": "lm_eval" |
| } |
| ], |
| "options": { |
| "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/cria-backfill-20261001/checkpoints/t5", |
| "model": null, |
| "tokenizer": "runs/moe_t5-gpu-v1/checkpoint-last", |
| "revision": "main", |
| "tokenizer_revision": null, |
| "suite": "core", |
| "tasks": "arithmark3,balanced_copa,commonsense_qa,sciq,truthfulqa_mc2,bananamind_base,mmlu_continuation,blimp", |
| "num_fewshot": null, |
| "device": "cuda:0", |
| "dtype": "bfloat16", |
| "allow_tf32": false, |
| "batch_size": 8, |
| "max_length": 2048, |
| "max_gen_toks": null, |
| "limit": null, |
| "seed": 1234, |
| "threads": 4, |
| "bootstrap_iters": 1000, |
| "trust_remote_code": false, |
| "allow_code_execution": false, |
| "apply_chat_template": false, |
| "log_samples": false, |
| "output": "/mnt/d/Projects/tiny_llm/runs/eval/cria-backfill-20261001/t5/attempt-001", |
| "tb_dir": "runs/moe_t5-gpu-v1/tb", |
| "training_step": 15000, |
| "list": false, |
| "dry_run": false |
| }, |
| "checkpoint_stamp": { |
| "model.safetensors": [ |
| 219483760, |
| 1789897063000000000 |
| ], |
| "config.json": [ |
| 7837, |
| 1789897062000000000 |
| ], |
| "generation_config.json": [ |
| 226, |
| 1789897062000000000 |
| ], |
| "tokenizer.json": [ |
| 2420528, |
| 1789897063000000000 |
| ], |
| "tokenizer_config.json": [ |
| 9425, |
| 1789897063000000000 |
| ] |
| }, |
| "checkpoint_sha256": { |
| "config.json": "8031de3caf99e9d424970b69701b1077d9dff5151bafd4ce214936a50fbd3bd4", |
| "generation_config.json": "8f1ce3b416c7f594951681c73e2b3c7933baac0a09a4f8d649aad27af21941f6", |
| "model.safetensors": "1efcd3aeed90f8b87ff06495f29f2cffa36435db1885d34a5e237450d2404b9f", |
| "tokenizer.json": "1d1f7409de3e53ae51d7085b3009aa5cd491dc08fbe114794c8d314f3fba2af4", |
| "tokenizer_config.json": "879515efa0fe331af8eeb0ede00fdd13c0bd3c827bff8bb6f4b7f959c02159ec" |
| }, |
| "source_sha256": { |
| "src/tiny_llm/__init__.py": "8c7f075d51006a53fe1d645ef5a84d65db58ff4251e7544074fe29ee7549c14c", |
| "src/tiny_llm/benchmarks.py": "5d9f461366389bd1f926c6a6468192f3e2313c87caf5b1de27f407d563a858a4", |
| "src/tiny_llm/checkpoint.py": "ac9df5f8c981a47eb1f1e59d0687ff41a1f6c38ca74ced50f71ac446a58d6ef5", |
| "src/tiny_llm/comet_sync.py": "ea802bfb58dacad66f83b9e5ac5ccb330c190204a81c90c9e14aafe44c893c37", |
| "src/tiny_llm/comet_views.py": "3e03f4ae69f8edc2eaf5f946f5cc90f373ca1b9cc91f9dd2e9adae6d6a672689", |
| "src/tiny_llm/continuation.py": "4f4143c828d4f94a851bcbf8a466d3e1b3e9eaec1027970a426208cc0ff4f4db", |
| "src/tiny_llm/data.py": "1a52e38afef9af32463779cc2f471d9b7174a7c4c73753e74f485d526d3c6733", |
| "src/tiny_llm/diffusion_diagnostics.py": "2ae667a076b1d3f057a29d20f00559068eb73a073b3f99030e1d5b4c2f478fa5", |
| "src/tiny_llm/evaluation/__init__.py": "e49e80f7be3ab6e200f0db14b028e465de9caa8f6f705430eabce0c41b8322ba", |
| "src/tiny_llm/evaluation/adapters.py": "8e64e5542bbcb0f9ad448b320e712916a9559823987e298b22321a8d8e2f727f", |
| "src/tiny_llm/evaluation/catalog.py": "2fd29084ec780539023a38bcbfd7302931fa6be48624e227875f13518a1b986f", |
| "src/tiny_llm/evaluation/cli.py": "a322e39bf58ba52132249dae524e9772c3a8e1a5ba388f5c0267f8dc1ea18882", |
| "src/tiny_llm/evaluation/tasks/balanced_copa.yaml": "f47baa6a713e15f7a5600ed257f316f8c54252bf4977bbc716a6da7aa580f05d", |
| "src/tiny_llm/evaluation/tasks/balanced_copa_test.yaml": "dce21ad70b395d8d576c584be1b87ebae846e360e59878030645215a387b28e6", |
| "src/tiny_llm/evaluation/telemetry.py": "f4dad30574753739ea10ba71ef9a7ed9d9118a89a8202dee224961be27f13a48", |
| "src/tiny_llm/evaluation/token_continuation.py": "e8d88cd4aca51ca2fda07f2b799a54612d0e2964a98e05824a135bea1a533550", |
| "src/tiny_llm/losses.py": "592a05287369f6e5077b3faca404a9392de4772cf06389fb8aa7e36924e74f86", |
| "src/tiny_llm/models/__init__.py": "b94ab167b6eb9f9b455823eea382e67b5ac2cc7f9ef5f6de183a09b9650b9718", |
| "src/tiny_llm/models/diffusion/__init__.py": "c92eaf5986d604d8aaf1894a862a217cefa32ab25359175b88110e45a71c1b25", |
| "src/tiny_llm/models/diffusion/configuration_diffusion_lm.py": "3b303aba60f0a831c966f5cfa7f5441daa2584814e335646195bd6ceecfa6d0c", |
| "src/tiny_llm/models/diffusion/modeling_diffusion_lm.py": "4fc21c2aed1d5a231afe6f7634d9d8c3682b4e2abfc8fecef27b52136de77b2f", |
| "src/tiny_llm/models/looped/__init__.py": "4b46b426021c59ed313cb2f49afc1926ad2793d771ef90117e3fef0462957cff", |
| "src/tiny_llm/models/looped/configuration_looped_lm.py": "b8d99c52b98e0a97a4cae76ca4c20affd8f4d019ec38e0a9beefd5fad864dc25", |
| "src/tiny_llm/models/looped/modeling_looped_lm.py": "9bdb3c5e3aaae663d09e98055f95df1adbc42e0a4bf5dffb7f6c39242af41b9e", |
| "src/tiny_llm/models/moe_t5/__init__.py": "18e33fe38b2049521f573512ac8796afa209e1b25c2052784e1c927a7160d66c", |
| "src/tiny_llm/models/moe_t5/configuration_alicet5_moe.py": "4647b1e9ab9ce07887bffc1abc3beb3e691962a380b4cc237aae376490bb56b7", |
| "src/tiny_llm/models/moe_t5/modeling_alicet5_moe.py": "5f899c4726b120eed6c9bf6258ad319bb1ccea734645f27e7e6ccbc5057222ff", |
| "src/tiny_llm/models/prefixlm/__init__.py": "c94c3c861acba8d2147a3a6c46bccfc71ac8adc8d70757bede009598029d31ea", |
| "src/tiny_llm/models/prefixlm/configuration_prefix_lm.py": "20b3a62ae3628b5a64566ef19d2947c90f10c6159daad00dd13d18fc5910c42c", |
| "src/tiny_llm/models/prefixlm/modeling_prefix_lm.py": "f62a38211df175696f64945173d1e41d9f7fcf6152a7648020da8c91788e98fd", |
| "src/tiny_llm/models/q50m/__init__.py": "9d0dfec6fa0cfa6967e2f17a73d8fbeca69ba93b4a93651eda4d95626d5f5eb5", |
| "src/tiny_llm/models/q50m/configuration_q50m.py": "796806b2d5d982b633796bc7de53c3073b1b6e52d259f44ef798b63f55eaea92", |
| "src/tiny_llm/models/q50m/modeling_q50m.py": "b68abce1f19a36deb39c4ac4e520ffaab0c879d8a7cc2abef35f014bacc011f2", |
| "src/tiny_llm/models/qmod/__init__.py": "e8c3e10710ac1cceac72ea329db480e680e8bbaae2a92ec26c36c92a778552ad", |
| "src/tiny_llm/models/qmod/configuration_qmod.py": "0dfc9a4540697a9b15e423b9592df9fc435dcfc437ddad20dc6b6377ebebea40", |
| "src/tiny_llm/models/qmod/modeling_qmod.py": "3831e34a86309b91a7a0787b9471eaceb91aa4c267f8030eb2e6722a0fcab6f8", |
| "src/tiny_llm/models/qmoe/__init__.py": "cfb2bb24bc2f3d5b72b3890a393b0386cb25f127e375f5454255478a36dec32f", |
| "src/tiny_llm/models/qmoe/configuration_qmoe.py": "bba7c9ac16f444146a78c99e7a26b4a11864b215d2b673bf658d9930f2f4de2f", |
| "src/tiny_llm/models/qmoe/modeling_qmoe.py": "1d713e5e287f46dbf13b7cf9c562b4539efbf6231f7ceaa261f8cea1acd2768c", |
| "src/tiny_llm/models/qt5/__init__.py": "df943b42a9f339e006687bcd30a666838c1dec15c79751b944e5f7a2bc69186c", |
| "src/tiny_llm/models/qt5/configuration_qt5.py": "59709fe2c9fe08ccecea6633b29b3b47170fd30abcbee37952507d676446cd44", |
| "src/tiny_llm/models/qt5/modeling_qt5.py": "6269829cca6631824622df034ff7b544dbbe47790556c60b26e356b5279cceff", |
| "src/tiny_llm/models/t5/__init__.py": "e4d8838253800edbcac1f9a9b6dc875bcb3ffa2962a0c9044d066448ea10488e", |
| "src/tiny_llm/models/t5/configuration_alicet5.py": "81b1f594139f201af9afb25ba58df862432c15dedf57f6e340ee2af80ed0d44f", |
| "src/tiny_llm/models/t5/modeling_alicet5.py": "bd52ce0022668f2defb57770df229f148e070ea73becca9a2e681325114bfc20", |
| "src/tiny_llm/models/zarya/__init__.py": "651b19cdba422680ff16bf2b2f63be0c3e9045c44c410fb17dbf51df2a23bd55", |
| "src/tiny_llm/models/zarya/configuration_zarya_lm.py": "42a3e7a1b11b687bac34e00f9cd3cb59d1cd573eec7492e1c55834662d37ed3f", |
| "src/tiny_llm/models/zarya/modeling_zarya_lm.py": "73ab033860288f7367d5b912c79de908fd81a4e88f369a658b96dd2bcfcd7bfe", |
| "src/tiny_llm/telemetry.py": "7ee8f14f4545bfa0e5ac632440d49c7f86fd6812a16429f862ebe3cfe09e3545", |
| "src/tiny_llm/ul2.py": "f4d948d89d5e106965612971dd473a54b57719f811761f18457c6221d763b876", |
| "src/tiny_llm/zarya.py": "8b5084ab5086c75745115d3a9ba4f2fa9cfda8ad713f65d3f8a22e77db8c0941" |
| }, |
| "packages": { |
| "lm_eval": "0.4.12", |
| "torch": "2.11.0+cu128", |
| "transformers": "5.17.0", |
| "datasets": "5.0.1", |
| "accelerate": "1.15.0" |
| }, |
| "git_commit": "fb65467388c90a7d94d50fbcb711abfd8a568102", |
| "git_dirty": true |
| }, |
| "runs/eval/cria-backfill-20261001/t5/attempt-002": { |
| "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/cria-backfill-20261001/checkpoints/t5", |
| "local_model": "moe_t5", |
| "kind": "seq2seq", |
| "tokenizer": "runs/moe_t5-gpu-v1/checkpoint-last", |
| "revision": "main", |
| "tokenizer_revision": null, |
| "protocol": "UL2 S-mode: encoder S+context+sentinel+EOS; decoder BOS+sentinel+answer; score answer text only, full vocabulary", |
| "harness_version": "0.4.12", |
| "smoke_only": false, |
| "tasks": [ |
| { |
| "task": "blimp", |
| "shots": 0, |
| "description": "Full BLiMP group; grammatical minimal pairs", |
| "unsafe_code": false, |
| "rolling": false, |
| "engine": "lm_eval" |
| } |
| ], |
| "options": { |
| "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/cria-backfill-20261001/checkpoints/t5", |
| "model": null, |
| "tokenizer": "runs/moe_t5-gpu-v1/checkpoint-last", |
| "revision": "main", |
| "tokenizer_revision": null, |
| "suite": "core", |
| "tasks": "blimp", |
| "num_fewshot": null, |
| "device": "cuda:0", |
| "dtype": "bfloat16", |
| "allow_tf32": false, |
| "batch_size": 8, |
| "max_length": 2048, |
| "max_gen_toks": null, |
| "limit": null, |
| "seed": 1234, |
| "threads": 4, |
| "bootstrap_iters": 1000, |
| "trust_remote_code": false, |
| "allow_code_execution": false, |
| "apply_chat_template": false, |
| "log_samples": false, |
| "output": "/mnt/d/Projects/tiny_llm/runs/eval/cria-backfill-20261001/t5/attempt-002", |
| "tb_dir": "runs/moe_t5-gpu-v1/tb", |
| "training_step": 15000, |
| "list": false, |
| "dry_run": false |
| }, |
| "checkpoint_stamp": { |
| "model.safetensors": [ |
| 219483760, |
| 1789897063000000000 |
| ], |
| "config.json": [ |
| 7837, |
| 1789897062000000000 |
| ], |
| "generation_config.json": [ |
| 226, |
| 1789897062000000000 |
| ], |
| "tokenizer.json": [ |
| 2420528, |
| 1789897063000000000 |
| ], |
| "tokenizer_config.json": [ |
| 9425, |
| 1789897063000000000 |
| ] |
| }, |
| "checkpoint_sha256": { |
| "config.json": "8031de3caf99e9d424970b69701b1077d9dff5151bafd4ce214936a50fbd3bd4", |
| "generation_config.json": "8f1ce3b416c7f594951681c73e2b3c7933baac0a09a4f8d649aad27af21941f6", |
| "model.safetensors": "1efcd3aeed90f8b87ff06495f29f2cffa36435db1885d34a5e237450d2404b9f", |
| "tokenizer.json": "1d1f7409de3e53ae51d7085b3009aa5cd491dc08fbe114794c8d314f3fba2af4", |
| "tokenizer_config.json": "879515efa0fe331af8eeb0ede00fdd13c0bd3c827bff8bb6f4b7f959c02159ec" |
| }, |
| "source_sha256": { |
| "src/tiny_llm/__init__.py": "8c7f075d51006a53fe1d645ef5a84d65db58ff4251e7544074fe29ee7549c14c", |
| "src/tiny_llm/benchmarks.py": "5d9f461366389bd1f926c6a6468192f3e2313c87caf5b1de27f407d563a858a4", |
| "src/tiny_llm/checkpoint.py": "ac9df5f8c981a47eb1f1e59d0687ff41a1f6c38ca74ced50f71ac446a58d6ef5", |
| "src/tiny_llm/comet_sync.py": "ea802bfb58dacad66f83b9e5ac5ccb330c190204a81c90c9e14aafe44c893c37", |
| "src/tiny_llm/comet_views.py": "3e03f4ae69f8edc2eaf5f946f5cc90f373ca1b9cc91f9dd2e9adae6d6a672689", |
| "src/tiny_llm/continuation.py": "4f4143c828d4f94a851bcbf8a466d3e1b3e9eaec1027970a426208cc0ff4f4db", |
| "src/tiny_llm/data.py": "1a52e38afef9af32463779cc2f471d9b7174a7c4c73753e74f485d526d3c6733", |
| "src/tiny_llm/diffusion_diagnostics.py": "2ae667a076b1d3f057a29d20f00559068eb73a073b3f99030e1d5b4c2f478fa5", |
| "src/tiny_llm/evaluation/__init__.py": "e49e80f7be3ab6e200f0db14b028e465de9caa8f6f705430eabce0c41b8322ba", |
| "src/tiny_llm/evaluation/adapters.py": "8e64e5542bbcb0f9ad448b320e712916a9559823987e298b22321a8d8e2f727f", |
| "src/tiny_llm/evaluation/catalog.py": "2fd29084ec780539023a38bcbfd7302931fa6be48624e227875f13518a1b986f", |
| "src/tiny_llm/evaluation/cli.py": "a322e39bf58ba52132249dae524e9772c3a8e1a5ba388f5c0267f8dc1ea18882", |
| "src/tiny_llm/evaluation/tasks/balanced_copa.yaml": "f47baa6a713e15f7a5600ed257f316f8c54252bf4977bbc716a6da7aa580f05d", |
| "src/tiny_llm/evaluation/tasks/balanced_copa_test.yaml": "dce21ad70b395d8d576c584be1b87ebae846e360e59878030645215a387b28e6", |
| "src/tiny_llm/evaluation/telemetry.py": "f4dad30574753739ea10ba71ef9a7ed9d9118a89a8202dee224961be27f13a48", |
| "src/tiny_llm/evaluation/token_continuation.py": "e8d88cd4aca51ca2fda07f2b799a54612d0e2964a98e05824a135bea1a533550", |
| "src/tiny_llm/losses.py": "592a05287369f6e5077b3faca404a9392de4772cf06389fb8aa7e36924e74f86", |
| "src/tiny_llm/models/__init__.py": "b94ab167b6eb9f9b455823eea382e67b5ac2cc7f9ef5f6de183a09b9650b9718", |
| "src/tiny_llm/models/diffusion/__init__.py": "c92eaf5986d604d8aaf1894a862a217cefa32ab25359175b88110e45a71c1b25", |
| "src/tiny_llm/models/diffusion/configuration_diffusion_lm.py": "3b303aba60f0a831c966f5cfa7f5441daa2584814e335646195bd6ceecfa6d0c", |
| "src/tiny_llm/models/diffusion/modeling_diffusion_lm.py": "4fc21c2aed1d5a231afe6f7634d9d8c3682b4e2abfc8fecef27b52136de77b2f", |
| "src/tiny_llm/models/looped/__init__.py": "4b46b426021c59ed313cb2f49afc1926ad2793d771ef90117e3fef0462957cff", |
| "src/tiny_llm/models/looped/configuration_looped_lm.py": "b8d99c52b98e0a97a4cae76ca4c20affd8f4d019ec38e0a9beefd5fad864dc25", |
| "src/tiny_llm/models/looped/modeling_looped_lm.py": "9bdb3c5e3aaae663d09e98055f95df1adbc42e0a4bf5dffb7f6c39242af41b9e", |
| "src/tiny_llm/models/moe_t5/__init__.py": "18e33fe38b2049521f573512ac8796afa209e1b25c2052784e1c927a7160d66c", |
| "src/tiny_llm/models/moe_t5/configuration_alicet5_moe.py": "4647b1e9ab9ce07887bffc1abc3beb3e691962a380b4cc237aae376490bb56b7", |
| "src/tiny_llm/models/moe_t5/modeling_alicet5_moe.py": "5f899c4726b120eed6c9bf6258ad319bb1ccea734645f27e7e6ccbc5057222ff", |
| "src/tiny_llm/models/prefixlm/__init__.py": "c94c3c861acba8d2147a3a6c46bccfc71ac8adc8d70757bede009598029d31ea", |
| "src/tiny_llm/models/prefixlm/configuration_prefix_lm.py": "20b3a62ae3628b5a64566ef19d2947c90f10c6159daad00dd13d18fc5910c42c", |
| "src/tiny_llm/models/prefixlm/modeling_prefix_lm.py": "f62a38211df175696f64945173d1e41d9f7fcf6152a7648020da8c91788e98fd", |
| "src/tiny_llm/models/q50m/__init__.py": "9d0dfec6fa0cfa6967e2f17a73d8fbeca69ba93b4a93651eda4d95626d5f5eb5", |
| "src/tiny_llm/models/q50m/configuration_q50m.py": "796806b2d5d982b633796bc7de53c3073b1b6e52d259f44ef798b63f55eaea92", |
| "src/tiny_llm/models/q50m/modeling_q50m.py": "b68abce1f19a36deb39c4ac4e520ffaab0c879d8a7cc2abef35f014bacc011f2", |
| "src/tiny_llm/models/qmod/__init__.py": "e8c3e10710ac1cceac72ea329db480e680e8bbaae2a92ec26c36c92a778552ad", |
| "src/tiny_llm/models/qmod/configuration_qmod.py": "0dfc9a4540697a9b15e423b9592df9fc435dcfc437ddad20dc6b6377ebebea40", |
| "src/tiny_llm/models/qmod/modeling_qmod.py": "3831e34a86309b91a7a0787b9471eaceb91aa4c267f8030eb2e6722a0fcab6f8", |
| "src/tiny_llm/models/qmoe/__init__.py": "cfb2bb24bc2f3d5b72b3890a393b0386cb25f127e375f5454255478a36dec32f", |
| "src/tiny_llm/models/qmoe/configuration_qmoe.py": "bba7c9ac16f444146a78c99e7a26b4a11864b215d2b673bf658d9930f2f4de2f", |
| "src/tiny_llm/models/qmoe/modeling_qmoe.py": "1d713e5e287f46dbf13b7cf9c562b4539efbf6231f7ceaa261f8cea1acd2768c", |
| "src/tiny_llm/models/qt5/__init__.py": "df943b42a9f339e006687bcd30a666838c1dec15c79751b944e5f7a2bc69186c", |
| "src/tiny_llm/models/qt5/configuration_qt5.py": "59709fe2c9fe08ccecea6633b29b3b47170fd30abcbee37952507d676446cd44", |
| "src/tiny_llm/models/qt5/modeling_qt5.py": "6269829cca6631824622df034ff7b544dbbe47790556c60b26e356b5279cceff", |
| "src/tiny_llm/models/t5/__init__.py": "e4d8838253800edbcac1f9a9b6dc875bcb3ffa2962a0c9044d066448ea10488e", |
| "src/tiny_llm/models/t5/configuration_alicet5.py": "81b1f594139f201af9afb25ba58df862432c15dedf57f6e340ee2af80ed0d44f", |
| "src/tiny_llm/models/t5/modeling_alicet5.py": "bd52ce0022668f2defb57770df229f148e070ea73becca9a2e681325114bfc20", |
| "src/tiny_llm/models/zarya/__init__.py": "651b19cdba422680ff16bf2b2f63be0c3e9045c44c410fb17dbf51df2a23bd55", |
| "src/tiny_llm/models/zarya/configuration_zarya_lm.py": "42a3e7a1b11b687bac34e00f9cd3cb59d1cd573eec7492e1c55834662d37ed3f", |
| "src/tiny_llm/models/zarya/modeling_zarya_lm.py": "73ab033860288f7367d5b912c79de908fd81a4e88f369a658b96dd2bcfcd7bfe", |
| "src/tiny_llm/telemetry.py": "7ee8f14f4545bfa0e5ac632440d49c7f86fd6812a16429f862ebe3cfe09e3545", |
| "src/tiny_llm/ul2.py": "f4d948d89d5e106965612971dd473a54b57719f811761f18457c6221d763b876", |
| "src/tiny_llm/zarya.py": "8b5084ab5086c75745115d3a9ba4f2fa9cfda8ad713f65d3f8a22e77db8c0941" |
| }, |
| "packages": { |
| "lm_eval": "0.4.12", |
| "torch": "2.11.0+cu128", |
| "transformers": "5.17.0", |
| "datasets": "5.0.1", |
| "accelerate": "1.15.0" |
| }, |
| "git_commit": "fb65467388c90a7d94d50fbcb711abfd8a568102", |
| "git_dirty": true |
| } |
| }, |
| "tokenizer_sha256": { |
| "tokenizer.json": "1d1f7409de3e53ae51d7085b3009aa5cd491dc08fbe114794c8d314f3fba2af4", |
| "tokenizer_config.json": "879515efa0fe331af8eeb0ede00fdd13c0bd3c827bff8bb6f4b7f959c02159ec" |
| }, |
| "tensorboard_event_files": { |
| "runs/moe_t5-gpu-v1/tb/events.out.tfevents.1790866065.DESKTOP-OU3A33R.18959.0.eval": "7eea85c0d3f0054571717d24651da3cff3342a3028b07c4bfb24ab4923bacff8", |
| "runs/moe_t5-gpu-v1/tb/events.out.tfevents.1790866080.DESKTOP-OU3A33R.18959.1.eval": "d13ad2fcca6784100fca5121ac1a8252e7b096b15022fd1cc9d163e645f93491", |
| "runs/moe_t5-gpu-v1/tb/events.out.tfevents.1790866112.DESKTOP-OU3A33R.18959.2.eval": "49704476a69178e92c19346ef026115f71eeeda9a685c670a58efd75941ef15d", |
| "runs/moe_t5-gpu-v1/tb/events.out.tfevents.1790866137.DESKTOP-OU3A33R.18959.3.eval": "9d2032c4df62dbd5b81ba7e6561ea413322ff5bbc9d56d03d990f0b30e768bce", |
| "runs/moe_t5-gpu-v1/tb/events.out.tfevents.1790866174.DESKTOP-OU3A33R.18959.4.eval": "209958c83b2c6b092ce334f461c0f7ee00b8ff7266fbca818cccbaa147ca5090", |
| "runs/moe_t5-gpu-v1/tb/events.out.tfevents.1790866180.DESKTOP-OU3A33R.18959.5.eval": "4e77ff5d52036d37678147b767e53bf51e8fde20583934d13b28d4f2cbd8d9b3", |
| "runs/moe_t5-gpu-v1/tb/events.out.tfevents.1790866494.DESKTOP-OU3A33R.18959.6.eval": "5cbfd8342dca93812e864b3c1789c0afebcb1f6e56338ae68a003394f9d946bd", |
| "runs/moe_t5-gpu-v1/tb/events.out.tfevents.1790868998.DESKTOP-OU3A33R.20360.0.eval": "48e3f3505d57a16c34141e7758bd6d09979ba71dde024b1b34cefa63c3dbafaa" |
| }, |
| "tensorboard_verified_points": 1166 |
| } |
| } |
|
|
