Publish final trained checkpoint, sixteen benchmarks and training logs

README.md

@@ -0,0 +1,445 @@
+---
+language:
+- en
+library_name: transformers
+pipeline_tag: text-generation
+tags:
+- pretrained
+- from-scratch
+- tiny-llm-ablation
+- custom_code
+- tensorboard
+- causal-lm
+datasets:
+- HuggingFaceFW/fineweb-edu
+model-index:
+- name: q-mod-51M-base
+  results:
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: Rowan/hellaswag
+      name: HellaSwag
+      config: default
+      split: validation
+    metrics:
+    - type: acc_norm
+      name: acc_norm (fraction; full comparison protocol)
+      value: 0.2874925313682533
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.004516681953878919
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.27872301915216174
+        ci95_high: 0.29642456629450026
+        ci_method: Wilson 95%; item independence approximation
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: allenai/ai2_arc
+      name: ARC-Easy
+      config: ARC-Easy
+      split: test
+    metrics:
+    - type: acc_norm
+      name: acc_norm (fraction; full comparison protocol)
+      value: 0.41919191919191917
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.010124905282491126
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.3994976736933545
+        ci95_high: 0.43914703998001464
+        ci_method: Wilson 95%; item independence approximation
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: allenai/ai2_arc
+      name: ARC-Challenge
+      config: ARC-Challenge
+      split: test
+    metrics:
+    - type: acc_norm
+      name: acc_norm (fraction; full comparison protocol)
+      value: 0.23122866894197952
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.01232085883477234
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.2079920763450836
+        ci95_high: 0.2562214064556877
+        ci_method: Wilson 95%; item independence approximation
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: baber/piqa
+      name: PIQA
+      config: default
+      split: validation
+    metrics:
+    - type: acc_norm
+      name: acc_norm (fraction; full comparison protocol)
+      value: 0.5875952121871599
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.011485407152743092
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.5649303971157125
+        ci95_high: 0.6098946392401159
+        ci_method: Wilson 95%; item independence approximation
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: allenai/winogrande
+      name: WinoGrande
+      config: winogrande_xl
+      split: validation
+    metrics:
+    - type: acc
+      name: acc (fraction; full comparison protocol)
+      value: 0.5138121546961326
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.014047122916440271
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.4862910104211901
+        ci95_high: 0.5412497970942327
+        ci_method: Wilson 95%; item independence approximation
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: allenai/openbookqa
+      name: OpenBookQA
+      config: main
+      split: test
+    metrics:
+    - type: acc_norm
+      name: acc_norm (fraction; full comparison protocol)
+      value: 0.306
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.020629569998345414
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.26721346373113636
+        ci95_high: 0.3477447803679459
+        ci_method: Wilson 95%; item independence approximation
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: aps/super_glue
+      name: BoolQ
+      config: boolq
+      split: validation
+    metrics:
+    - type: acc
+      name: acc (fraction; full comparison protocol)
+      value: 0.5577981651376147
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.008686430526114761
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.5407177130610681
+        ci95_high: 0.5747429788369947
+        ci_method: Wilson 95%; item independence approximation
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: EleutherAI/lambada_openai
+      name: LAMBADA OpenAI
+      config: default
+      split: test
+    metrics:
+    - type: acc
+      name: acc (fraction; full comparison protocol)
+      value: 0.16689307199689501
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.005194952730633079
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.1569610474392711
+        ci95_high: 0.17732137574286708
+        ci_method: Wilson 95%; item independence approximation
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: AxiomicLabs/Arithmark-3.0
+      name: ArithMark-3
+      config: default
+      split: train
+    metrics:
+    - type: acc_norm
+      name: acc_norm (fraction; full comparison protocol)
+      value: 0.341
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 1024
+        standard_error: 0.01499813134840284
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.31227731829126676
+        ci95_high: 0.37093959090713985
+        ci_method: Wilson 95%; item independence approximation
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: pkavumba/balanced-copa
+      name: Balanced COPA
+      config: default
+      split: train
+    metrics:
+    - type: acc
+      name: acc (fraction; full comparison protocol)
+      value: 0.533
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.015784807891138876
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.5020104989134991
+        ci95_high: 0.5637369350264544
+        ci_method: Wilson 95%; item independence approximation
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: nyu-mll/blimp
+      name: BLiMP
+      config: 67 minimal-pair subsets
+      split: train
+    metrics:
+    - type: acc
+      name: acc (fraction; full comparison protocol)
+      value: 0.7317910447761194
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.001517376416839114
+        evaluation_date: '2026-10-01'
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: tau/commonsense_qa
+      name: CommonsenseQA
+      config: default
+      split: validation
+    metrics:
+    - type: acc
+      name: acc (fraction; full comparison protocol)
+      value: 0.19492219492219492
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.011341478090883516
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.17367347122724505
+        ci95_high: 0.21808454395514162
+        ci_method: Wilson 95%; item independence approximation
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: cais/mmlu
+      name: MMLU continuation
+      config: 57 subjects
+      split: test
+    metrics:
+    - type: acc
+      name: acc (fraction; full comparison protocol)
+      value: 0.2521720552627831
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.0036528205915989775
+        evaluation_date: '2026-10-01'
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: allenai/sciq
+      name: SciQ (with support)
+      config: default
+      split: test
+    metrics:
+    - type: acc_norm
+      name: acc_norm (fraction; full comparison protocol)
+      value: 0.612
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.015417317979911216
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.5814239245764211
+        ci95_high: 0.6417188815228144
+        ci_method: Wilson 95%; item independence approximation
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: truthfulqa/truthful_qa
+      name: TruthfulQA MC2
+      config: multiple_choice
+      split: validation
+    metrics:
+    - type: acc
+      name: acc (fraction; full comparison protocol)
+      value: 0.4417644031264386
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.01519901667714673
+        evaluation_date: '2026-10-01'
+  - task:
+      type: text-generation
+      name: Continuation likelihood
+    dataset:
+      type: BananaMind/BananaMind-Base-Bench-1.1
+      name: BananaMind Base 1.1
+      config: default
+      split: test
+    metrics:
+    - type: raw_accuracy
+      name: raw_accuracy (fraction; full comparison protocol)
+      value: 0.45714285714285713
+      args:
+        dtype: bfloat16
+        num_few_shot: 0
+        max_length: 2048
+        standard_error: 0.02666588697376804
+        evaluation_date: '2026-10-01'
+        ci95_low: 0.40570065529637145
+        ci95_high: 0.5095156110507492
+        ci_method: Wilson 95%; item independence approximation
+---
+
+# Q-MoD
+
+A **50,919,168-parameter** English base model from the Tiny llm ablation experiment. Trained from random initialization for **15,000 optimizer steps**, processing **3,932,160,000 source tokens**. These are processed input tokens, not unique text or supervised target-token counts.
+
+## Architecture
+
+10 decoder layers, width 512, 8 query / 2 KV heads, head dimension 64, RoPE, QK normalization, gated residuals and tied 32,768-token embeddings. Five alternating whole blocks (zero-based layers 1, 3, 5, 7, 9) use Mixture-of-Depths token routing. Training selects the top 12.5% of valid tokens per sequence; evaluation selects each token causally with router logit > 0, without future-dependent top-k. Routed FFN width 1797 (ordinary blocks 1792) makes the total exactly 50,919,168 parameters. This routes tokens through blocks, not experts. Inference compute depends on selected tokens.
+
+Tokenizer foundation: [q-project/Q-50M-Base](https://huggingface.co/q-project/Q-50M-Base). This model is trained from scratch, not fine-tuned from the reference weights. [Exact trained configuration](config.json) and custom model code are included.
+
+## Training
+
+- Data: [FineWeb-Edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu), `sample-10BT`, local Parquet shards; shuffle buffer 100,000.
+- Objective: Causal next-token cross-entropy plus router binary classification auxiliary loss, coefficient 0.01.
+- Final recorded batch: 4 sequences × 32 gradient accumulation × 2048 tokens = 262,144 source tokens per update.
+- Fused AdamW, peak LR 0.001, betas (0.9, 0.95), weight decay 0.1 excluding bias/norm/1D parameters, gradient clipping 1.0. Warmup 150 steps, cosine decay to 10% of peak LR.
+- BF16 compute, one RTX 5070 Ti, seed 2026; released checkpoint weights remain FP32. [Training configuration](training_config.json).
+
+## Evaluation
+
+Full selected splits, lm-eval 0.4.12, no added few-shot examples or chat template, BF16 on RTX 5070 Ti, context cap 2048 (ArithMark 1024). New evaluations explicitly disable TF32; historical core manifests predate the explicit TF32 flag. Scores below are percentages; **± is one standard error**, and the separate interval column is 95% CI where available.
+
+| Dataset | Split | Examples | Metric | Score ± SE (%) | 95% CI (%) |
+|---|---|---:|---|---:|---:|
+| [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag) | validation | 10,042 | `acc_norm` | 28.75 ± 0.45 | [27.87, 29.64] |
+| [ARC-Easy](https://huggingface.co/datasets/allenai/ai2_arc) | test | 2,376 | `acc_norm` | 41.92 ± 1.01 | [39.95, 43.91] |
+| [ARC-Challenge](https://huggingface.co/datasets/allenai/ai2_arc) | test | 1,172 | `acc_norm` | 23.12 ± 1.23 | [20.80, 25.62] |
+| [PIQA](https://huggingface.co/datasets/baber/piqa) | validation | 1,838 | `acc_norm` | 58.76 ± 1.15 | [56.49, 60.99] |
+| [WinoGrande](https://huggingface.co/datasets/allenai/winogrande) | validation | 1,267 | `acc` | 51.38 ± 1.40 | [48.63, 54.12] |
+| [OpenBookQA](https://huggingface.co/datasets/allenai/openbookqa) | test | 500 | `acc_norm` | 30.60 ± 2.06 | [26.72, 34.77] |
+| [BoolQ](https://huggingface.co/datasets/aps/super_glue) | validation | 3,270 | `acc` | 55.78 ± 0.87 | [54.07, 57.47] |
+| [LAMBADA OpenAI](https://huggingface.co/datasets/EleutherAI/lambada_openai) | test | 5,153 | `acc` | 16.69 ± 0.52 | [15.70, 17.73] |
+| [ArithMark-3](https://huggingface.co/datasets/AxiomicLabs/Arithmark-3.0) | train | 1,000 | `acc_norm` | 34.10 ± 1.50 | [31.23, 37.09] |
+| [Balanced COPA](https://huggingface.co/datasets/pkavumba/balanced-copa) | train | 1,000 | `acc` | 53.30 ± 1.58 | [50.20, 56.37] |
+| [BLiMP](https://huggingface.co/datasets/nyu-mll/blimp) | train | 67,000 | `acc` | 73.18 ± 0.15 | — |
+| [CommonsenseQA](https://huggingface.co/datasets/tau/commonsense_qa) | validation | 1,221 | `acc` | 19.49 ± 1.13 | [17.37, 21.81] |
+| [MMLU continuation](https://huggingface.co/datasets/cais/mmlu) | test | 14,042 | `acc` | 25.22 ± 0.37 | — |
+| [SciQ (with support)](https://huggingface.co/datasets/allenai/sciq) | test | 1,000 | `acc_norm` | 61.20 ± 1.54 | [58.14, 64.17] |
+| [TruthfulQA MC2](https://huggingface.co/datasets/truthfulqa/truthful_qa) | validation | 817 | `acc` | 44.18 ± 1.52 | — |
+| [BananaMind Base 1.1](https://huggingface.co/datasets/BananaMind/BananaMind-Base-Bench-1.1) | test | 350 | `raw_accuracy` | 45.71 ± 2.67 | [40.57, 50.95] |
+
+LAMBADA requires all final-word tokens to match. The descriptive macro-average is **41.4613%**: one primary metric per each of the 16 tasks, with equal task weight. PPL, standard errors, CI bounds and MMLU/BLiMP leaf scores are excluded. It is not an official leaderboard score or a statistical ranking test.
+
+Full selected splits; lm-eval 0.4.12; seed 1234; BF16 on RTX 5070 Ti;
+context cap 2048 (ArithMark 1024), TF32 disabled on new runs (historical core flag unrecorded), no chat template and no added
+few-shot examples. TruthfulQA retains the harness's fixed six-QA preamble.
+ArithMark and BananaMind normalize by continuation token count; ordinary harness
+acc_norm uses its own length normalization. BananaMind is raw accuracy, not Elo.
+SciQ includes the support passage. Balanced COPA uses the mirrored 1000-item
+train-named evaluation split; cRia's split was inferred, not confirmed.
+MMLU scores full answer continuations across 57 subjects, weighted by item count;
+BLiMP averages 67 equal-sized minimal-pair subsets. Standard errors are retained
+from each evaluator. Wilson intervals are reported only where the runner logged
+binary item accuracy; MC2 is probability mass, not binary accuracy. These
+intervals do not model dependence between paired/templated examples or training
+seed variation. UL2, PrefixLM and experimental diffusion PLL use their documented
+conditional scoring protocols; PLL exposes the other answer tokens and is not
+autoregressive likelihood. cRia's published scores used a different precision
+and benchmark-adapted checkpoint; this completes our comparison coverage, not
+an independent reproduction of cRia or an official leaderboard submission.
+
+**WikiText-2 raw test continuation**, reported separately: 291 nonoverlapping blocks of 1024 tokens, 512-token prefix plus 512-token scored suffix; 148,992 scored tokens, 335 tail tokens excluded. GPU evaluation; percentile block bootstrap with 10,000 resamples and seed 2026. PPL intervals exponentiate NLL endpoints. This is not rolling or word perplexity.
+
+- `torch.bfloat16`: NLL **3.689238**, 95% CI [3.649657, 3.728491]; PPL **40.014**, 95% CI [38.461, 41.616].
+- `torch.float32`: NLL **3.688851**, 95% CI [3.649183, 3.728063]; PPL **39.999**, 95% CI [38.443, 41.598].
+
+[Structured results and provenance](evaluation/results.json), [raw task outputs](evaluation/raw/) and [per-block continuation scores](evaluation/continuation.json). HF metadata contains author-reported measurements; no verified benchmark badge is claimed. Historical and new measurements refer to identical weight hashes; individual run manifests retain their original dates and source hashes.
+
+## Usage
+
+Install `requirements.txt`; tested with PyTorch 2.11.0 and Transformers 5.17.0. Loading custom code requires `trust_remote_code=True`. Example on CPU:
+
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+repo = "d0rj/q-mod-51M-base"
+tokenizer = AutoTokenizer.from_pretrained(repo)
+model = AutoModelForCausalLM.from_pretrained(repo, trust_remote_code=True).eval()
+inputs = tokenizer("The capital of France is", return_tensors="pt")
+output = model.generate(**inputs, max_new_tokens=32, do_sample=False)
+print(tokenizer.decode(output[0], skip_special_tokens=True))
+```
+
+After downloading this repository, install the evaluation dependencies and authenticate for the gated BananaMind dataset after accepting its terms:
+
+```bash
+pip install -r evaluation/repro/requirements.txt
+hf auth login
+python evaluation/repro/run.py --device cuda:0 --dtype bfloat16 --batch-size 8 --output comparison-rerun
+python evaluation/repro/continuation.py --device cuda:0 --dtype bfloat16 --output continuation-bf16.json
+python evaluation/repro/continuation.py --device cuda:0 --dtype float32 --output continuation-fp32.json
+```
+
+`--limit` / `--limit-blocks` are smoke checks only. Dataset examples are not redistributed. The bundled runners use this published model and tokenizer.
+
+## Training logs and limitations
+
+[TensorBoard event files](tensorboard/) contain available training telemetry and the full evaluation at optimizer step 15,000. `train/loss` has 750 visible points spanning steps 20–15,000. Resume/purge records are retained. [Comet experiment](https://www.comet.com/d0rj/tiny-llm-ablation/1f1e28237a6241fbb5c369241d020481). Evaluation exports retain numeric metrics, uncertainty and task/subtask breakdowns.
+
+Equal source-token budgets do not imply equal target supervision, parameter count or FLOPs. Training dataset file revisions and overlap with benchmarks were not independently audited. Results come from one training seed; uncertainty intervals do not capture training-seed variability or dependence between templated examples. These are small base models, not instruction-tuned assistants.

config.json

@@ -0,0 +1,44 @@
+{
+  "architectures": [
+    "QMoDForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attention_scalar_gate": true,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "gate_multiplier": 2.0,
+  "head_dim": 64,
+  "hidden_size": 512,
+  "intermediate_size": 1792,
+  "max_position_embeddings": 2048,
+  "mlp_scalar_gate": true,
+  "mod_capacity": 0.125,
+  "mod_intermediate_size": 1797,
+  "mod_layers": [
+    1,
+    3,
+    5,
+    7,
+    9
+  ],
+  "model_type": "q_mod",
+  "nope_every_n": null,
+  "nope_layers": [],
+  "num_attention_heads": 8,
+  "num_hidden_layers": 10,
+  "num_key_value_heads": 2,
+  "pad_token_id": 0,
+  "qk_norm": true,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "router_aux_coef": 0.01,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.17.0",
+  "vocab_size": 32768,
+  "auto_map": {
+    "AutoConfig": "configuration_qmod.QMoDConfig",
+    "AutoModelForCausalLM": "modeling_qmod.QMoDForCausalLM"
+  },
+  "_name_or_path": "d0rj/q-mod-51M-base"
+}

configuration_q50m.py

@@ -0,0 +1,68 @@
+"""Configuration for the Q-50M architecture (replica of q-project/Q-50M-Base).
+
+512-wide, 10-layer decoder with GQA (8 q / 2 kv heads), per-head QK-Norm,
+content-dependent scalar residual gates on both branches, tied embeddings.
+"""
+
+from transformers import PretrainedConfig
+
+
+class Q50MConfig(PretrainedConfig):
+    model_type = "q50m"
+
+    def __init__(
+        self,
+        vocab_size: int = 32768,
+        hidden_size: int = 512,
+        intermediate_size: int = 1792,
+        num_hidden_layers: int = 10,
+        num_attention_heads: int = 8,
+        num_key_value_heads: int = 2,
+        head_dim: int = 64,
+        max_position_embeddings: int = 2048,
+        rope_theta: float = 10000.0,
+        rms_norm_eps: float = 1e-5,
+        qk_norm: bool = True,
+        attention_scalar_gate: bool = True,
+        mlp_scalar_gate: bool = True,
+        gate_multiplier: float = 2.0,
+        nope_every_n: int | None = None,
+        nope_layers: list[int] | None = None,
+        attention_dropout: float = 0.0,
+        tie_word_embeddings: bool = True,
+        bos_token_id: int = 1,
+        eos_token_id: int = 2,
+        pad_token_id: int = 0,
+        **kwargs,
+    ):
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            pad_token_id=pad_token_id,
+            **kwargs,
+        )
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.head_dim = head_dim
+        self.max_position_embeddings = max_position_embeddings
+        self.rope_theta = rope_theta
+        self.rms_norm_eps = rms_norm_eps
+        self.qk_norm = qk_norm
+        self.attention_scalar_gate = attention_scalar_gate
+        self.mlp_scalar_gate = mlp_scalar_gate
+        self.gate_multiplier = gate_multiplier
+        self.nope_every_n = nope_every_n
+        if nope_layers is not None:
+            self.nope_layers = list(nope_layers)
+        elif nope_every_n:
+            self.nope_layers = [
+                i for i in range(num_hidden_layers) if (i + 1) % nope_every_n == 0
+            ]
+        else:
+            self.nope_layers = []
+        self.attention_dropout = attention_dropout

configuration_qmod.py

@@ -0,0 +1,30 @@
+"""Q51M Mixture-of-Depths, exactly matched to Q-MoE's total budget."""
+import math
+
+from .configuration_q50m import Q50MConfig
+
+
+class QMoDConfig(Q50MConfig):
+    model_type = "q_mod"
+
+    def __init__(self, mod_capacity=0.125, mod_layers=None,
+                 mod_intermediate_size=None, router_aux_coef=0.01, **kwargs):
+        super().__init__(**kwargs)
+        self.mod_capacity = float(mod_capacity)
+        self.mod_layers = (list(range(1, self.num_hidden_layers, 2))
+                           if mod_layers is None else list(mod_layers))
+        # Five extra SwiGLU channels in each of five routed blocks, plus five
+        # scalar routers: 5 * (3 * 512 * 5 + 512) = 40,960 parameters.
+        self.mod_intermediate_size = (self.intermediate_size + 5
+                                      if mod_intermediate_size is None else mod_intermediate_size)
+        self.router_aux_coef = float(router_aux_coef)
+        if not 0 < self.mod_capacity <= 1:
+            raise ValueError("mod_capacity must be in (0, 1]")
+        if not math.isfinite(self.router_aux_coef) or self.router_aux_coef < 0:
+            raise ValueError("router_aux_coef must be finite and nonnegative")
+        if (len(set(self.mod_layers)) != len(self.mod_layers)
+                or any(not isinstance(i, int) or not 0 <= i < self.num_hidden_layers
+                       for i in self.mod_layers)):
+            raise ValueError("mod_layers must contain unique valid layer indices")
+        if not isinstance(self.mod_intermediate_size, int) or self.mod_intermediate_size < 1:
+            raise ValueError("mod_intermediate_size must be a positive integer")

evaluation/continuation.json

@@ -0,0 +1,624 @@
+{
+  "model": "q-mod",
+  "checkpoint_sha256": {
+    "model.safetensors": "e5be4b27775d846ab2e65293a83f647d89ba2e0a606c00645fa892b21f6d5cd3"
+  },
+  "corpus_sha256": "aca2f46735043bcfd0a44eca981d04627b9cdf74c4c9a04bf0856d04066f58fc",
+  "tokenizer_sha256": "fc71a154536f6c84a5bf729d06324032126603ef5183c0b4979407e4fad68a8d",
+  "blocks": 291,
+  "scored_tokens": 148992,
+  "dropped_tail_tokens": 335,
+  "protocol": "WikiText-2 raw test, newline-joined; tokenizer-specific nonoverlapping 1024-token blocks; 512 prefix + 512 scored suffix; full vocabulary. GPU BF16 and FP32 separately, TF32 off. Block bootstrap, 10000 resamples, seed 2026. PPL is not rolling or word PPL; different tokenizer results are not directly comparable.",
+  "measurements": {
+    "torch.bfloat16": {
+      "nll": 3.6892377994314502,
+      "nll_ci95": [
+        3.6496566288659666,
+        3.7284908163793307
+      ],
+      "ppl": 40.014336381234685,
+      "ppl_ci95": [
+        38.46145722721316,
+        41.616254177697236
+      ],
+      "block_nll": [
+        3.0334739685058594,
+        3.740220069885254,
+        3.628894805908203,
+        3.6683175563812256,
+        4.014724254608154,
+        3.891292095184326,
+        4.447037696838379,
+        4.0522780418396,
+        3.8349380493164062,
+        3.823822021484375,
+        3.633845329284668,
+        3.2401468753814697,
+        3.266814708709717,
+        3.5836172103881836,
+        3.8599278926849365,
+        3.666652202606201,
+        3.956968307495117,
+        3.8685550689697266,
+        3.7003097534179688,
+        3.9863815307617188,
+        3.7272982597351074,
+        3.4920151233673096,
+        4.134518623352051,
+        3.097790241241455,
+        3.377092123031616,
+        3.659186363220215,
+        3.691166877746582,
+        3.5041279792785645,
+        3.5620369911193848,
+        3.4219396114349365,
+        3.945758819580078,
+        4.074984073638916,
+        3.4408555030822754,
+        3.3952572345733643,
+        3.482008695602417,
+        4.128378868103027,
+        3.7565321922302246,
+        3.6840639114379883,
+        3.3337631225585938,
+        3.7233495712280273,
+        3.358358144760132,
+        3.8522372245788574,
+        3.4598348140716553,
+        3.858750343322754,
+        3.539834976196289,
+        3.7883243560791016,
+        3.972238302230835,
+        3.525768518447876,
+        3.943371295928955,
+        3.6065564155578613,
+        3.502178192138672,
+        3.6186389923095703,
+        3.925199270248413,
+        4.104352951049805,
+        3.556373357772827,
+        4.266332626342773,
+        3.2788915634155273,
+        3.275848150253296,
+        3.6967127323150635,
+        3.381783962249756,
+        3.3477587699890137,
+        3.2133116722106934,
+        3.3617265224456787,
+        2.6435787677764893,
+        3.3020172119140625,
+        3.4059677124023438,
+        3.1757893562316895,
+        3.1442549228668213,
+        3.4717979431152344,
+        3.4961135387420654,
+        3.559631824493408,
+        3.3489818572998047,
+        3.8820650577545166,
+        3.555074691772461,
+        3.8889122009277344,
+        3.5896503925323486,
+        3.769655227661133,
+        3.8484110832214355,
+        3.7232885360717773,
+        3.7955780029296875,
+        3.5623092651367188,
+        3.676947832107544,
+        3.684424877166748,
+        4.132134437561035,
+        4.085684776306152,
+        4.260656833648682,
+        3.877305507659912,
+        3.9520206451416016,
+        3.867370128631592,
+        4.114029884338379,
+        3.622154951095581,
+        3.7936620712280273,
+        3.7642085552215576,
+        3.8050858974456787,
+        3.628535270690918,
+        3.818166494369507,
+        3.595874309539795,
+        3.5458860397338867,
+        3.472426652908325,
+        4.054523468017578,
+        3.888446092605591,
+        3.6914849281311035,
+        3.0026016235351562,
+        3.6276140213012695,
+        3.5678868293762207,
+        3.2021985054016113,
+        3.4582393169403076,
+        3.1557910442352295,
+        3.4210400581359863,
+        3.4203357696533203,
+        3.337662696838379,
+        3.530256986618042,
+        2.94315767288208,
+        3.107532501220703,
+        3.6951563358306885,
+        3.2887697219848633,
+        3.3305134773254395,
+        3.8404202461242676,
+        3.448058605194092,
+        3.5578668117523193,
+        3.8797223567962646,
+        3.9302845001220703,
+        3.7769031524658203,
+        3.735970973968506,
+        4.003993511199951,
+        3.8857574462890625,
+        3.718710422515869,
+        3.696420669555664,
+        3.990670680999756,
+        3.836711883544922,
+        3.9563257694244385,
+        3.8588461875915527,
+        3.8682708740234375,
+        3.552096366882324,
+        4.007606506347656,
+        3.6597299575805664,
+        3.721909284591675,
+        3.916825532913208,
+        3.618872880935669,
+        4.049559593200684,
+        3.5828661918640137,
+        3.300783395767212,
+        4.254912376403809,
+        3.9667701721191406,
+        4.053187370300293,
+        4.266885757446289,
+        4.450619220733643,
+        4.25779914855957,
+        3.4460010528564453,
+        3.620326519012451,
+        3.8935182094573975,
+        3.7257468700408936,
+        3.1817684173583984,
+        3.5945611000061035,
+        3.14451265335083,
+        3.5998497009277344,
+        3.827526092529297,
+        3.8065614700317383,
+        4.318783760070801,
+        3.381446361541748,
+        3.701540470123291,
+        3.774550437927246,
+        3.0764577388763428,
+        3.0016181468963623,
+        2.8728439807891846,
+        3.309659242630005,
+        3.2308201789855957,
+        3.528296709060669,
+        3.1466598510742188,
+        4.0538530349731445,
+        3.837440252304077,
+        4.182062149047852,
+        4.07778263092041,
+        4.6475090980529785,
+        4.582226753234863,
+        3.9743824005126953,
+        4.288257598876953,
+        3.4931979179382324,
+        3.476074695587158,
+        4.683747291564941,
+        3.927859306335449,
+        4.299893379211426,
+        4.255132675170898,
+        4.288380146026611,
+        4.4493513107299805,
+        4.2332000732421875,
+        4.063198566436768,
+        4.357582092285156,
+        4.2598466873168945,
+        4.221273422241211,
+        4.55598258972168,
+        4.1570329666137695,
+        3.369076728820801,
+        4.477698802947998,
+        4.309594631195068,
+        3.729815721511841,
+        3.8138747215270996,
+        3.4916563034057617,
+        3.8918566703796387,
+        3.7358622550964355,
+        3.576416254043579,
+        3.2475414276123047,
+        3.634760856628418,
+        3.4506139755249023,
+        3.6861281394958496,
+        3.718350887298584,
+        3.434321880340576,
+        3.793008804321289,
+        3.359323501586914,
+        3.665397882461548,
+        3.6199021339416504,
+        3.3855361938476562,
+        3.4763875007629395,
+        3.5085606575012207,
+        3.3470630645751953,
+        3.6974496841430664,
+        3.754701614379883,
+        3.4645261764526367,
+        3.109065532684326,
+        3.254002094268799,
+        3.4032177925109863,
+        3.6319453716278076,
+        3.323887825012207,
+        3.2882790565490723,
+        3.724388837814331,
+        3.4665753841400146,
+        3.174009323120117,
+        3.791433095932007,
+        4.040876388549805,
+        3.4488320350646973,
+        3.8699331283569336,
+        3.6910126209259033,
+        3.5286316871643066,
+        3.798151731491089,
+        3.882476806640625,
+        3.844326972961426,
+        4.06152868270874,
+        3.8325114250183105,
+        3.337923526763916,
+        3.7876949310302734,
+        3.5918965339660645,
+        3.615245819091797,
+        3.2705817222595215,
+        3.1231560707092285,
+        3.385634422302246,
+        3.420309066772461,
+        3.39042329788208,
+        3.7529053688049316,
+        4.090065956115723,
+        3.57220196723938,
+        4.007654190063477,
+        4.212186813354492,
+        3.671332359313965,
+        3.585268974304199,
+        3.951240062713623,
+        3.822371006011963,
+        3.564332962036133,
+        3.556788921356201,
+        3.841912031173706,
+        3.7584228515625,
+        3.6320042610168457,
+        3.6407132148742676,
+        3.273137092590332,
+        3.4374094009399414,
+        3.547950506210327,
+        3.7734694480895996,
+        3.434028148651123,
+        3.8533287048339844,
+        3.884617567062378,
+        3.1127676963806152,
+        3.9401354789733887,
+        3.767160177230835,
+        3.10684871673584,
+        3.317154884338379,
+        3.160114288330078,
+        2.7552032470703125,
+        3.654399871826172,
+        3.568580389022827,
+        3.8528037071228027,
+        3.8489787578582764,
+        3.773097515106201,
+        3.8523926734924316,
+        3.409104108810425,
+        3.7001962661743164,
+        3.8040661811828613,
+        3.4476029872894287,
+        3.7821860313415527,
+        3.9857730865478516,
+        4.512575149536133,
+        4.251725673675537,
+        3.6333508491516113
+      ]
+    },
+    "torch.float32": {
+      "nll": 3.6888507895453277,
+      "nll_ci95": [
+        3.649182625079073,
+        3.728063138858559
+      ],
+      "ppl": 39.99885343368865,
+      "ppl_ci95": [
+        38.44323067090763,
+        41.598459646718396
+      ],
+      "block_nll": [
+        3.027667760848999,
+        3.738102912902832,
+        3.6281819343566895,
+        3.6695809364318848,
+        4.015194892883301,
+        3.894198417663574,
+        4.446779251098633,
+        4.050177574157715,
+        3.8330178260803223,
+        3.821993112564087,
+        3.6341664791107178,
+        3.2412233352661133,
+        3.2660343647003174,
+        3.5816807746887207,
+        3.8598198890686035,
+        3.66544771194458,
+        3.954576015472412,
+        3.8663859367370605,
+        3.700993299484253,
+        3.98309588432312,
+        3.726853370666504,
+        3.4925546646118164,
+        4.133160591125488,
+        3.097299098968506,
+        3.3766369819641113,
+        3.6585729122161865,
+        3.6912124156951904,
+        3.5034966468811035,
+        3.560659408569336,
+        3.421860456466675,
+        3.944514751434326,
+        4.076074123382568,
+        3.4416584968566895,
+        3.3954355716705322,
+        3.483755111694336,
+        4.1278228759765625,
+        3.755769968032837,
+        3.682060718536377,
+        3.33280086517334,
+        3.7252397537231445,
+        3.360107183456421,
+        3.8527755737304688,
+        3.460200786590576,
+        3.855809211730957,
+        3.5398995876312256,
+        3.7887706756591797,
+        3.971407890319824,
+        3.5239522457122803,
+        3.942335844039917,
+        3.6075119972229004,
+        3.4993295669555664,
+        3.6204614639282227,
+        3.9251532554626465,
+        4.106016159057617,
+        3.555708408355713,
+        4.265496730804443,
+        3.2797160148620605,
+        3.276913642883301,
+        3.6964378356933594,
+        3.384550094604492,
+        3.348212242126465,
+        3.213447332382202,
+        3.362593173980713,
+        2.643364429473877,
+        3.302009105682373,
+        3.4066994190216064,
+        3.1740009784698486,
+        3.146824836730957,
+        3.4738929271698,
+        3.4947752952575684,
+        3.5605530738830566,
+        3.346153497695923,
+        3.879917860031128,
+        3.5560824871063232,
+        3.8901753425598145,
+        3.5909934043884277,
+        3.768007516860962,
+        3.8438334465026855,
+        3.71993088722229,
+        3.7953481674194336,
+        3.5641112327575684,
+        3.6742324829101562,
+        3.6839911937713623,
+        4.1331987380981445,
+        4.086156845092773,
+        4.263089179992676,
+        3.8775670528411865,
+        3.950777530670166,
+        3.86856746673584,
+        4.112517833709717,
+        3.620612144470215,
+        3.7948694229125977,
+        3.7628049850463867,
+        3.8032991886138916,
+        3.6272482872009277,
+        3.817592144012451,
+        3.594144105911255,
+        3.547070264816284,
+        3.4730372428894043,
+        4.054729461669922,
+        3.8879971504211426,
+        3.6934356689453125,
+        3.000913381576538,
+        3.625133991241455,
+        3.568268060684204,
+        3.2034006118774414,
+        3.4547390937805176,
+        3.1572463512420654,
+        3.4198713302612305,
+        3.4206738471984863,
+        3.3378567695617676,
+        3.5324578285217285,
+        2.9415979385375977,
+        3.1075520515441895,
+        3.6945276260375977,
+        3.289034366607666,
+        3.333094596862793,
+        3.8393657207489014,
+        3.4496171474456787,
+        3.5584096908569336,
+        3.878268241882324,
+        3.929405689239502,
+        3.7761900424957275,
+        3.736541271209717,
+        3.9996914863586426,
+        3.8869969844818115,
+        3.717820167541504,
+        3.6949455738067627,
+        3.9903812408447266,
+        3.836249828338623,
+        3.959465980529785,
+        3.858759880065918,
+        3.868380308151245,
+        3.550398826599121,
+        4.00911808013916,
+        3.6598963737487793,
+        3.722276210784912,
+        3.916846990585327,
+        3.616698980331421,
+        4.050436019897461,
+        3.5778284072875977,
+        3.2959275245666504,
+        4.254424571990967,
+        3.9653892517089844,
+        4.0521240234375,
+        4.270989418029785,
+        4.4489850997924805,
+        4.254066467285156,
+        3.445030689239502,
+        3.61728572845459,
+        3.894674777984619,
+        3.7256832122802734,
+        3.181084632873535,
+        3.5941505432128906,
+        3.144176483154297,
+        3.5967330932617188,
+        3.829509735107422,
+        3.805874824523926,
+        4.318291664123535,
+        3.3804588317871094,
+        3.7016525268554688,
+        3.7745108604431152,
+        3.076606273651123,
+        2.9988205432891846,
+        2.872002601623535,
+        3.307823896408081,
+        3.2268385887145996,
+        3.525599956512451,
+        3.147146224975586,
+        4.053147315979004,
+        3.8385915756225586,
+        4.182605743408203,
+        4.073055267333984,
+        4.646452903747559,
+        4.581335544586182,
+        3.976539134979248,
+        4.287915229797363,
+        3.492595911026001,
+        3.4744696617126465,
+        4.682605743408203,
+        3.9282143115997314,
+        4.29866886138916,
+        4.253904342651367,
+        4.2884039878845215,
+        4.448787689208984,
+        4.234960556030273,
+        4.059430122375488,
+        4.357001781463623,
+        4.257026672363281,
+        4.221226692199707,
+        4.557419776916504,
+        4.155788898468018,
+        3.3654861450195312,
+        4.480966567993164,
+        4.309287071228027,
+        3.730501174926758,
+        3.8162360191345215,
+        3.4917783737182617,
+        3.889458656311035,
+        3.737387180328369,
+        3.574091911315918,
+        3.245443820953369,
+        3.635066032409668,
+        3.4508020877838135,
+        3.685305595397949,
+        3.7202224731445312,
+        3.436110019683838,
+        3.793421745300293,
+        3.3612964153289795,
+        3.664384603500366,
+        3.6176421642303467,
+        3.384169578552246,
+        3.4765501022338867,
+        3.50846004486084,
+        3.3470969200134277,
+        3.69547700881958,
+        3.752682685852051,
+        3.464268445968628,
+        3.1101222038269043,
+        3.250185489654541,
+        3.4053843021392822,
+        3.6328437328338623,
+        3.3243253231048584,
+        3.287609100341797,
+        3.7263553142547607,
+        3.465762138366699,
+        3.1718688011169434,
+        3.7949929237365723,
+        4.041664123535156,
+        3.4488463401794434,
+        3.8704917430877686,
+        3.690990924835205,
+        3.5286307334899902,
+        3.799253463745117,
+        3.8777198791503906,
+        3.8435721397399902,
+        4.059334754943848,
+        3.8340835571289062,
+        3.3388919830322266,
+        3.788696050643921,
+        3.5927886962890625,
+        3.6131863594055176,
+        3.2687082290649414,
+        3.120223045349121,
+        3.3863675594329834,
+        3.4221267700195312,
+        3.3879547119140625,
+        3.7518320083618164,
+        4.08969783782959,
+        3.574854850769043,
+        4.006902694702148,
+        4.210338592529297,
+        3.671088695526123,
+        3.586747407913208,
+        3.94930362701416,
+        3.821291923522949,
+        3.565288782119751,
+        3.5583267211914062,
+        3.840996742248535,
+        3.755756378173828,
+        3.632883071899414,
+        3.638172149658203,
+        3.272749185562134,
+        3.437190294265747,
+        3.547724723815918,
+        3.7728371620178223,
+        3.4354710578918457,
+        3.8511710166931152,
+        3.8856048583984375,
+        3.1140756607055664,
+        3.939563751220703,
+        3.765720844268799,
+        3.105600118637085,
+        3.3165178298950195,
+        3.1615777015686035,
+        2.7535500526428223,
+        3.6520907878875732,
+        3.569089889526367,
+        3.8519628047943115,
+        3.850273609161377,
+        3.775264263153076,
+        3.8534600734710693,
+        3.4068784713745117,
+        3.702049732208252,
+        3.802506923675537,
+        3.449129581451416,
+        3.781672954559326,
+        3.984006881713867,
+        4.511618137359619,
+        4.252767562866211,
+        3.6308560371398926
+      ]
+    }
+  }
+}

evaluation/raw/arc_challenge.json

@@ -0,0 +1,129 @@
+{
+  "results": {
+    "arc_challenge": {
+      "name": "arc_challenge",
+      "alias": "arc_challenge",
+      "sample_len": 1172,
+      "acc,none": 0.18344709897610922,
+      "acc_stderr,none": 0.011310170179554583,
+      "acc_norm,none": 0.23122866894197952,
+      "acc_norm_stderr,none": 0.01232085883477234
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "arc_challenge": {
+      "task": "arc_challenge",
+      "dataset_path": "allenai/ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "training_split": "train",
+      "validation_split": "validation",
+      "test_split": "test",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{choices.label.index(answerKey)}}",
+      "unsafe_code": false,
+      "doc_to_choice": "{{choices.text}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_choice": "{{choices.text}}",
+        "doc_to_target": "{{choices.label.index(answerKey)}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
+      "metadata": {
+        "version": 1.0,
+        "config_source": "/mnt/d/Projects/tiny_llm/.venv-eval/lib/python3.12/site-packages/lm_eval/tasks/arc/arc_challenge.yaml"
+      }
+    }
+  },
+  "versions": {
+    "arc_challenge": 1.0
+  },
+  "n-shot": {
+    "arc_challenge": 0
+  },
+  "higher_is_better": {
+    "arc_challenge": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "arc_challenge": {
+      "original": 1172,
+      "effective": 1172
+    }
+  },
+  "config": {
+    "model": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+    "model_args": null,
+    "model_num_parameters": 50919168,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": null,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 1000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 1234,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "d2b5d7b",
+  "date": 1790878760.1987338,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5070 Ti\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        48 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               8\nOn-line CPU(s) list:                  0-7\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 9800X3D 8-Core Processor\nCPU family:                           26\nModel:                                68\nThread(s) per core:                   2\nCore(s) per socket:                   4\nSocket(s):                            1\nStepping:                             0\nBogoMIPS:                             9399.75\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm avx512_vp2intersect\nVirtualization:                       AMD-V\nHypervisor vendor:                    Microsoft\nVirtualization type:                  full\nL1d cache:                            192 KiB (4 instances)\nL1i cache:                            128 KiB (4 instances)\nL2 cache:                             4 MiB (4 instances)\nL3 cache:                             96 MiB (1 instance)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-7\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Not affected\nVulnerability Spec rstack overflow:   Not affected\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.19.0.56\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.28.9\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.11.0+cu128\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.17.0",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/arc_easy.json

@@ -0,0 +1,129 @@
+{
+  "results": {
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.4701178451178451,
+      "acc_stderr,none": 0.010241444322886225,
+      "acc_norm,none": 0.41919191919191917,
+      "acc_norm_stderr,none": 0.010124905282491126
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "arc_easy": {
+      "task": "arc_easy",
+      "dataset_path": "allenai/ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "training_split": "train",
+      "validation_split": "validation",
+      "test_split": "test",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{choices.label.index(answerKey)}}",
+      "unsafe_code": false,
+      "doc_to_choice": "{{choices.text}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_choice": "{{choices.text}}",
+        "doc_to_target": "{{choices.label.index(answerKey)}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
+      "metadata": {
+        "version": 1.0,
+        "config_source": "/mnt/d/Projects/tiny_llm/.venv-eval/lib/python3.12/site-packages/lm_eval/tasks/arc/arc_easy.yaml"
+      }
+    }
+  },
+  "versions": {
+    "arc_easy": 1.0
+  },
+  "n-shot": {
+    "arc_easy": 0
+  },
+  "higher_is_better": {
+    "arc_easy": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "arc_easy": {
+      "original": 2376,
+      "effective": 2376
+    }
+  },
+  "config": {
+    "model": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+    "model_args": null,
+    "model_num_parameters": 50919168,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": null,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 1000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 1234,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "d2b5d7b",
+  "date": 1790878703.6323915,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5070 Ti\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        48 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               8\nOn-line CPU(s) list:                  0-7\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 9800X3D 8-Core Processor\nCPU family:                           26\nModel:                                68\nThread(s) per core:                   2\nCore(s) per socket:                   4\nSocket(s):                            1\nStepping:                             0\nBogoMIPS:                             9399.75\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm avx512_vp2intersect\nVirtualization:                       AMD-V\nHypervisor vendor:                    Microsoft\nVirtualization type:                  full\nL1d cache:                            192 KiB (4 instances)\nL1i cache:                            128 KiB (4 instances)\nL2 cache:                             4 MiB (4 instances)\nL3 cache:                             96 MiB (1 instance)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-7\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Not affected\nVulnerability Spec rstack overflow:   Not affected\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.19.0.56\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.28.9\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.11.0+cu128\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.17.0",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/arithmark3.json

@@ -0,0 +1,131 @@
+{
+  "results": {
+    "arithmark3": {
+      "acc_norm,none": 0.341,
+      "acc_norm_stderr,none": 0.01499813134840284,
+      "acc,none": 0.338,
+      "acc_stderr,none": 0.014965960710224385
+    }
+  },
+  "n-samples": {
+    "arithmark3": {
+      "original": 1000,
+      "effective": 1000
+    }
+  },
+  "configs": {
+    "arithmark3": {
+      "repo": "AxiomicLabs/Arithmark-3.0",
+      "revision": "6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746",
+      "file": "arithmark-3.jsonl",
+      "split": "train",
+      "count": 1000,
+      "context_field": "ctx",
+      "choices_field": "endings",
+      "category_field": "activity_label",
+      "context_cap": 1024,
+      "context_truncation": "per_choice",
+      "primary_metric": "acc_norm",
+      "dataset_sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435",
+      "num_fewshot": 0,
+      "effective_max_length": 1024,
+      "add_bos": false,
+      "tokenization": "context and continuation separately; add_special_tokens=False",
+      "normalization": "continuation token count; not character count",
+      "context_truncated_examples": 0,
+      "runner": "tiny_llm token_continuation v2; not an official leaderboard submission"
+    }
+  },
+  "categories": {
+    "elementary_school_math_continuation::addition::grades_1_2::easy": {
+      "count": 128,
+      "acc_norm,none": 0.234375,
+      "acc_norm_stderr,none": 0.037589092002846054
+    },
+    "elementary_school_math_continuation::comparison::grades_2_3::medium": {
+      "count": 44,
+      "acc_norm,none": 0.25,
+      "acc_norm_stderr,none": 0.06603381797442179
+    },
+    "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": {
+      "count": 48,
+      "acc_norm,none": 0.3125,
+      "acc_norm_stderr,none": 0.06761023307918171
+    },
+    "elementary_school_math_continuation::data::grades_2_3::easy": {
+      "count": 43,
+      "acc_norm,none": 0.13953488372093023,
+      "acc_norm_stderr,none": 0.05346670079545209
+    },
+    "elementary_school_math_continuation::division::grades_3_4::medium": {
+      "count": 54,
+      "acc_norm,none": 0.3888888888888889,
+      "acc_norm_stderr,none": 0.06696300316010247
+    },
+    "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": {
+      "count": 50,
+      "acc_norm,none": 0.22,
+      "acc_norm_stderr,none": 0.05917804336345137
+    },
+    "elementary_school_math_continuation::geometry_area::grades_4_5::medium": {
+      "count": 52,
+      "acc_norm,none": 0.5,
+      "acc_norm_stderr,none": 0.07001400420140048
+    },
+    "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": {
+      "count": 45,
+      "acc_norm,none": 0.4888888888888889,
+      "acc_norm_stderr,none": 0.0753592220347252
+    },
+    "elementary_school_math_continuation::measurement::grades_2_3::easy": {
+      "count": 76,
+      "acc_norm,none": 0.35526315789473684,
+      "acc_norm_stderr,none": 0.05526315789473684
+    },
+    "elementary_school_math_continuation::money::grades_3_4::medium": {
+      "count": 64,
+      "acc_norm,none": 0.3125,
+      "acc_norm_stderr,none": 0.058397074018894594
+    },
+    "elementary_school_math_continuation::multiplication::grades_3_4::medium": {
+      "count": 74,
+      "acc_norm,none": 0.5675675675675675,
+      "acc_norm_stderr,none": 0.057983774751431
+    },
+    "elementary_school_math_continuation::patterns::grades_3_4::medium": {
+      "count": 53,
+      "acc_norm,none": 0.2641509433962264,
+      "acc_norm_stderr,none": 0.06113906319252699
+    },
+    "elementary_school_math_continuation::subtraction::grades_1_2::easy": {
+      "count": 117,
+      "acc_norm,none": 0.21367521367521367,
+      "acc_norm_stderr,none": 0.038058271647936806
+    },
+    "elementary_school_math_continuation::time::grades_2_3::easy": {
+      "count": 55,
+      "acc_norm,none": 0.9090909090909091,
+      "acc_norm_stderr,none": 0.03912104390108503
+    },
+    "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": {
+      "count": 46,
+      "acc_norm,none": 0.21739130434782608,
+      "acc_norm_stderr,none": 0.06148754619013457
+    },
+    "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": {
+      "count": 19,
+      "acc_norm,none": 0.2631578947368421,
+      "acc_norm_stderr,none": 0.10379087338771256
+    },
+    "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": {
+      "count": 32,
+      "acc_norm,none": 0.1875,
+      "acc_norm_stderr,none": 0.07010217197868432
+    }
+  },
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/balanced_copa.json

@@ -0,0 +1,121 @@
+{
+  "results": {
+    "balanced_copa": {
+      "name": "balanced_copa",
+      "alias": "balanced_copa",
+      "sample_len": 1000,
+      "acc,none": 0.533,
+      "acc_stderr,none": 0.015784807891138876
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "balanced_copa": {
+      "task": "balanced_copa",
+      "dataset_path": "pkavumba/balanced-copa",
+      "dataset_kwargs": {
+        "revision": "813bd03cd6e07d9bd8d7333896ad5d40abb95ea9"
+      },
+      "test_split": "train",
+      "doc_to_text": "{{premise.strip()[:-1]}} {{'because' if question == 'cause' else 'therefore'}}",
+      "doc_to_target": "{{label}}",
+      "unsafe_code": false,
+      "doc_to_choice": "{{[choice1[0].lower() + choice1[1:], choice2[0].lower() + choice2[1:]]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{premise.strip()[:-1]}} {{'because' if question == 'cause' else 'therefore'}}",
+        "doc_to_choice": "{{[choice1[0].lower() + choice1[1:], choice2[0].lower() + choice2[1:]]}}",
+        "doc_to_target": "{{label}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1,
+        "reference_protocol": "cRia split and exact prompt unverified; COPA-style prompt",
+        "config_source": "/mnt/d/Projects/tiny_llm/src/tiny_llm/evaluation/tasks/balanced_copa.yaml"
+      }
+    }
+  },
+  "versions": {
+    "balanced_copa": 1
+  },
+  "n-shot": {
+    "balanced_copa": 0
+  },
+  "higher_is_better": {
+    "balanced_copa": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "balanced_copa": {
+      "original": 1000,
+      "effective": 1000
+    }
+  },
+  "config": {
+    "model": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+    "model_args": null,
+    "model_num_parameters": 50919168,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": null,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 1000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 1234,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "d2b5d7b",
+  "date": 1790878916.811857,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5070 Ti\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        48 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               8\nOn-line CPU(s) list:                  0-7\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 9800X3D 8-Core Processor\nCPU family:                           26\nModel:                                68\nThread(s) per core:                   2\nCore(s) per socket:                   4\nSocket(s):                            1\nStepping:                             0\nBogoMIPS:                             9399.75\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm avx512_vp2intersect\nVirtualization:                       AMD-V\nHypervisor vendor:                    Microsoft\nVirtualization type:                  full\nL1d cache:                            192 KiB (4 instances)\nL1i cache:                            128 KiB (4 instances)\nL2 cache:                             4 MiB (4 instances)\nL3 cache:                             96 MiB (1 instance)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-7\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Not affected\nVulnerability Spec rstack overflow:   Not affected\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.19.0.56\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.28.9\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.11.0+cu128\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.17.0",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/bananamind_base.json

@@ -0,0 +1,80 @@
+{
+  "results": {
+    "bananamind_base": {
+      "raw_accuracy,none": 0.45714285714285713,
+      "raw_accuracy_stderr,none": 0.02666588697376804
+    }
+  },
+  "n-samples": {
+    "bananamind_base": {
+      "original": 350,
+      "effective": 350
+    }
+  },
+  "configs": {
+    "bananamind_base": {
+      "repo": "BananaMind/BananaMind-Base-Bench-1.1",
+      "revision": "d4aade51312889e8580963e1ce960c6eaef1a450",
+      "file": "data/test.jsonl",
+      "split": "test",
+      "count": 350,
+      "sha256": "2f563bb46df778ca494fa20f994a8d3045d4c51fbbffeee433764e2813abea21",
+      "context_field": "context",
+      "choices_field": "continuations",
+      "category_field": "category",
+      "context_cap": null,
+      "context_truncation": "shared_longest_choice",
+      "primary_metric": "raw_accuracy",
+      "dataset_sha256": "2f563bb46df778ca494fa20f994a8d3045d4c51fbbffeee433764e2813abea21",
+      "num_fewshot": 0,
+      "effective_max_length": 2048,
+      "add_bos": false,
+      "tokenization": "context and continuation separately; add_special_tokens=False",
+      "normalization": "continuation token count; not character count",
+      "context_truncated_examples": 0,
+      "runner": "tiny_llm token_continuation v2; not an official leaderboard submission"
+    }
+  },
+  "categories": {
+    "code_completion": {
+      "count": 50,
+      "raw_accuracy,none": 0.28,
+      "raw_accuracy_stderr,none": 0.06414269805898186
+    },
+    "commonsense": {
+      "count": 50,
+      "raw_accuracy,none": 0.5,
+      "raw_accuracy_stderr,none": 0.07142857142857142
+    },
+    "context_tracking": {
+      "count": 50,
+      "raw_accuracy,none": 0.16,
+      "raw_accuracy_stderr,none": 0.05237229365663817
+    },
+    "language_completion": {
+      "count": 50,
+      "raw_accuracy,none": 0.9,
+      "raw_accuracy_stderr,none": 0.04285714285714285
+    },
+    "logical_reasoning": {
+      "count": 50,
+      "raw_accuracy,none": 0.44,
+      "raw_accuracy_stderr,none": 0.07091242083423346
+    },
+    "quantitative": {
+      "count": 50,
+      "raw_accuracy,none": 0.26,
+      "raw_accuracy_stderr,none": 0.06266203485560375
+    },
+    "world_knowledge": {
+      "count": 50,
+      "raw_accuracy,none": 0.66,
+      "raw_accuracy_stderr,none": 0.0676726816132972
+    }
+  },
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/boolq.json

@@ -0,0 +1,124 @@
+{
+  "results": {
+    "boolq": {
+      "name": "boolq",
+      "alias": "boolq",
+      "sample_len": 3270,
+      "acc,none": 0.5577981651376147,
+      "acc_stderr,none": 0.008686430526114761
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "boolq": {
+      "task": "boolq",
+      "dataset_path": "aps/super_glue",
+      "dataset_name": "boolq",
+      "training_split": "train",
+      "validation_split": "validation",
+      "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:",
+      "doc_to_target": "label",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "no",
+        "yes"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:",
+        "doc_to_choice": [
+          "no",
+          "yes"
+        ],
+        "doc_to_target": "label",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "passage",
+      "metadata": {
+        "version": 2.0,
+        "config_source": "/mnt/d/Projects/tiny_llm/.venv-eval/lib/python3.12/site-packages/lm_eval/tasks/super_glue/boolq/default.yaml"
+      }
+    }
+  },
+  "versions": {
+    "boolq": 2.0
+  },
+  "n-shot": {
+    "boolq": 0
+  },
+  "higher_is_better": {
+    "boolq": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "boolq": {
+      "original": 3270,
+      "effective": 3270
+    }
+  },
+  "config": {
+    "model": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+    "model_args": null,
+    "model_num_parameters": 50919168,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": null,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 1000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 1234,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "d2b5d7b",
+  "date": 1790878836.582125,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5070 Ti\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        48 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               8\nOn-line CPU(s) list:                  0-7\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 9800X3D 8-Core Processor\nCPU family:                           26\nModel:                                68\nThread(s) per core:                   2\nCore(s) per socket:                   4\nSocket(s):                            1\nStepping:                             0\nBogoMIPS:                             9399.75\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm avx512_vp2intersect\nVirtualization:                       AMD-V\nHypervisor vendor:                    Microsoft\nVirtualization type:                  full\nL1d cache:                            192 KiB (4 instances)\nL1i cache:                            128 KiB (4 instances)\nL2 cache:                             4 MiB (4 instances)\nL3 cache:                             96 MiB (1 instance)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-7\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Not affected\nVulnerability Spec rstack overflow:   Not affected\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.19.0.56\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.28.9\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.11.0+cu128\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.17.0",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/commonsense_qa.json

@@ -0,0 +1,129 @@
+{
+  "results": {
+    "commonsense_qa": {
+      "name": "commonsense_qa",
+      "alias": "commonsense_qa",
+      "sample_len": 1221,
+      "acc,none": 0.19492219492219492,
+      "acc_stderr,none": 0.011341478090883516
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "commonsense_qa": {
+      "task": "commonsense_qa",
+      "dataset_path": "tau/commonsense_qa",
+      "training_split": "train",
+      "validation_split": "validation",
+      "doc_to_text": "Question: {{ question.strip() }}\nA. {{choices['text'][0]}}\nB. {{choices['text'][1]}}\nC. {{choices['text'][2]}}\nD. {{choices['text'][3]}}\nE. {{choices['text'][4]}}\nAnswer:",
+      "doc_to_target": "answerKey",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D",
+        "E"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{ question.strip() }}\nA. {{choices['text'][0]}}\nB. {{choices['text'][1]}}\nC. {{choices['text'][2]}}\nD. {{choices['text'][3]}}\nE. {{choices['text'][4]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D",
+          "E"
+        ],
+        "doc_to_target": "answerKey",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "config_source": "/mnt/d/Projects/tiny_llm/.venv-eval/lib/python3.12/site-packages/lm_eval/tasks/commonsense_qa/default.yaml"
+      }
+    }
+  },
+  "versions": {
+    "commonsense_qa": "Yaml"
+  },
+  "n-shot": {
+    "commonsense_qa": 0
+  },
+  "higher_is_better": {
+    "commonsense_qa": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "commonsense_qa": {
+      "original": 1221,
+      "effective": 1221
+    }
+  },
+  "config": {
+    "model": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+    "model_args": null,
+    "model_num_parameters": 50919168,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": null,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 1000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 1234,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "d2b5d7b",
+  "date": 1790879411.5483394,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5070 Ti\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        48 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               8\nOn-line CPU(s) list:                  0-7\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 9800X3D 8-Core Processor\nCPU family:                           26\nModel:                                68\nThread(s) per core:                   2\nCore(s) per socket:                   4\nSocket(s):                            1\nStepping:                             0\nBogoMIPS:                             9399.75\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm avx512_vp2intersect\nVirtualization:                       AMD-V\nHypervisor vendor:                    Microsoft\nVirtualization type:                  full\nL1d cache:                            192 KiB (4 instances)\nL1i cache:                            128 KiB (4 instances)\nL2 cache:                             4 MiB (4 instances)\nL3 cache:                             96 MiB (1 instance)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-7\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Not affected\nVulnerability Spec rstack overflow:   Not affected\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.19.0.56\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.28.9\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.11.0+cu128\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.17.0",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/hellaswag.json

@@ -0,0 +1,127 @@
+{
+  "results": {
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.27633937462656843,
+      "acc_stderr,none": 0.004462727543055927,
+      "acc_norm,none": 0.2874925313682533,
+      "acc_norm_stderr,none": 0.004516681953878919
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "hellaswag": {
+      "task": "hellaswag",
+      "dataset_path": "Rowan/hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "unsafe_code": false,
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": "<function process_docs at 0x74bb6260d8a0>",
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{query}}",
+        "doc_to_choice": "choices",
+        "doc_to_target": "{{label}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "config_source": "/mnt/d/Projects/tiny_llm/.venv-eval/lib/python3.12/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml"
+      }
+    }
+  },
+  "versions": {
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    }
+  },
+  "config": {
+    "model": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+    "model_args": null,
+    "model_num_parameters": 50919168,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": null,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 1000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 1234,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "d2b5d7b",
+  "date": 1790878480.7127786,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5070 Ti\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        48 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               8\nOn-line CPU(s) list:                  0-7\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 9800X3D 8-Core Processor\nCPU family:                           26\nModel:                                68\nThread(s) per core:                   2\nCore(s) per socket:                   4\nSocket(s):                            1\nStepping:                             0\nBogoMIPS:                             9399.75\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm avx512_vp2intersect\nVirtualization:                       AMD-V\nHypervisor vendor:                    Microsoft\nVirtualization type:                  full\nL1d cache:                            192 KiB (4 instances)\nL1i cache:                            128 KiB (4 instances)\nL2 cache:                             4 MiB (4 instances)\nL3 cache:                             96 MiB (1 instance)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-7\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Not affected\nVulnerability Spec rstack overflow:   Not affected\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.19.0.56\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.28.9\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.11.0+cu128\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.17.0",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/lambada_openai.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "lambada_openai": {
+      "name": "lambada_openai",
+      "alias": "lambada_openai",
+      "sample_len": 5153,
+      "perplexity,none": 325.20143973341675,
+      "perplexity_stderr,none": 15.346256502549156,
+      "acc,none": 0.16689307199689501,
+      "acc_stderr,none": 0.005194952730633079
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "lambada_openai": {
+      "task": "lambada_openai",
+      "dataset_path": "EleutherAI/lambada_openai",
+      "dataset_name": "default",
+      "test_split": "test",
+      "doc_to_text": "{{text.split(' ')[:-1]|join(' ')}}",
+      "doc_to_target": "{{' '+text.split(' ')[-1]}}",
+      "unsafe_code": false,
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{text.split(' ')[:-1]|join(' ')}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{' '+text.split(' ')[-1]}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "perplexity",
+          "aggregation": "perplexity",
+          "higher_is_better": false
+        },
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "loglikelihood",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{text}}",
+      "metadata": {
+        "version": 1.0,
+        "config_source": "/mnt/d/Projects/tiny_llm/.venv-eval/lib/python3.12/site-packages/lm_eval/tasks/lambada/lambada_openai.yaml"
+      }
+    }
+  },
+  "versions": {
+    "lambada_openai": 1.0
+  },
+  "n-shot": {
+    "lambada_openai": 0
+  },
+  "higher_is_better": {
+    "lambada_openai": {
+      "perplexity": false,
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "lambada_openai": {
+      "original": 5153,
+      "effective": 5153
+    }
+  },
+  "config": {
+    "model": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+    "model_args": null,
+    "model_num_parameters": 50919168,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": null,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 1000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 1234,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "d2b5d7b",
+  "date": 1790878859.459506,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5070 Ti\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        48 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               8\nOn-line CPU(s) list:                  0-7\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 9800X3D 8-Core Processor\nCPU family:                           26\nModel:                                68\nThread(s) per core:                   2\nCore(s) per socket:                   4\nSocket(s):                            1\nStepping:                             0\nBogoMIPS:                             9399.75\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm avx512_vp2intersect\nVirtualization:                       AMD-V\nHypervisor vendor:                    Microsoft\nVirtualization type:                  full\nL1d cache:                            192 KiB (4 instances)\nL1i cache:                            128 KiB (4 instances)\nL2 cache:                             4 MiB (4 instances)\nL3 cache:                             96 MiB (1 instance)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-7\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Not affected\nVulnerability Spec rstack overflow:   Not affected\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.19.0.56\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.28.9\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.11.0+cu128\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.17.0",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/openbookqa.json

@@ -0,0 +1,129 @@
+{
+  "results": {
+    "openbookqa": {
+      "name": "openbookqa",
+      "alias": "openbookqa",
+      "sample_len": 500,
+      "acc,none": 0.168,
+      "acc_stderr,none": 0.01673655354154187,
+      "acc_norm,none": 0.306,
+      "acc_norm_stderr,none": 0.020629569998345414
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "openbookqa": {
+      "task": "openbookqa",
+      "dataset_path": "allenai/openbookqa",
+      "dataset_name": "main",
+      "training_split": "train",
+      "validation_split": "validation",
+      "test_split": "test",
+      "doc_to_text": "question_stem",
+      "doc_to_target": "{{choices.label.index(answerKey.lstrip())}}",
+      "unsafe_code": false,
+      "doc_to_choice": "{{choices.text}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "question_stem",
+        "doc_to_choice": "{{choices.text}}",
+        "doc_to_target": "{{choices.label.index(answerKey.lstrip())}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "question_stem",
+      "metadata": {
+        "version": 1.0,
+        "config_source": "/mnt/d/Projects/tiny_llm/.venv-eval/lib/python3.12/site-packages/lm_eval/tasks/openbookqa/openbookqa.yaml"
+      }
+    }
+  },
+  "versions": {
+    "openbookqa": 1.0
+  },
+  "n-shot": {
+    "openbookqa": 0
+  },
+  "higher_is_better": {
+    "openbookqa": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "openbookqa": {
+      "original": 500,
+      "effective": 500
+    }
+  },
+  "config": {
+    "model": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+    "model_args": null,
+    "model_num_parameters": 50919168,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": null,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 1000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 1234,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "d2b5d7b",
+  "date": 1790878826.5813043,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5070 Ti\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        48 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               8\nOn-line CPU(s) list:                  0-7\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 9800X3D 8-Core Processor\nCPU family:                           26\nModel:                                68\nThread(s) per core:                   2\nCore(s) per socket:                   4\nSocket(s):                            1\nStepping:                             0\nBogoMIPS:                             9399.75\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm avx512_vp2intersect\nVirtualization:                       AMD-V\nHypervisor vendor:                    Microsoft\nVirtualization type:                  full\nL1d cache:                            192 KiB (4 instances)\nL1i cache:                            128 KiB (4 instances)\nL2 cache:                             4 MiB (4 instances)\nL3 cache:                             96 MiB (1 instance)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-7\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Not affected\nVulnerability Spec rstack overflow:   Not affected\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.19.0.56\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.28.9\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.11.0+cu128\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.17.0",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/piqa.json

@@ -0,0 +1,127 @@
+{
+  "results": {
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.5924918389553863,
+      "acc_stderr,none": 0.01146449062565553,
+      "acc_norm,none": 0.5875952121871599,
+      "acc_norm_stderr,none": 0.011485407152743092
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "piqa": {
+      "task": "piqa",
+      "dataset_path": "baber/piqa",
+      "training_split": "train",
+      "validation_split": "validation",
+      "doc_to_text": "Question: {{goal}}\nAnswer:",
+      "doc_to_target": "label",
+      "unsafe_code": false,
+      "doc_to_choice": "{{[sol1, sol2]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{goal}}\nAnswer:",
+        "doc_to_choice": "{{[sol1, sol2]}}",
+        "doc_to_target": "label",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "goal",
+      "metadata": {
+        "version": 1.0,
+        "config_source": "/mnt/d/Projects/tiny_llm/.venv-eval/lib/python3.12/site-packages/lm_eval/tasks/piqa/piqa.yaml"
+      }
+    }
+  },
+  "versions": {
+    "piqa": 1.0
+  },
+  "n-shot": {
+    "piqa": 0
+  },
+  "higher_is_better": {
+    "piqa": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "piqa": {
+      "original": 1838,
+      "effective": 1838
+    }
+  },
+  "config": {
+    "model": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+    "model_args": null,
+    "model_num_parameters": 50919168,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": null,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 1000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 1234,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "d2b5d7b",
+  "date": 1790878791.5160453,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5070 Ti\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        48 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               8\nOn-line CPU(s) list:                  0-7\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 9800X3D 8-Core Processor\nCPU family:                           26\nModel:                                68\nThread(s) per core:                   2\nCore(s) per socket:                   4\nSocket(s):                            1\nStepping:                             0\nBogoMIPS:                             9399.75\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm avx512_vp2intersect\nVirtualization:                       AMD-V\nHypervisor vendor:                    Microsoft\nVirtualization type:                  full\nL1d cache:                            192 KiB (4 instances)\nL1i cache:                            128 KiB (4 instances)\nL2 cache:                             4 MiB (4 instances)\nL3 cache:                             96 MiB (1 instance)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-7\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Not affected\nVulnerability Spec rstack overflow:   Not affected\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.19.0.56\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.28.9\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.11.0+cu128\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.17.0",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/sciq.json

@@ -0,0 +1,128 @@
+{
+  "results": {
+    "sciq": {
+      "name": "sciq",
+      "alias": "sciq",
+      "sample_len": 1000,
+      "acc,none": 0.724,
+      "acc_stderr,none": 0.01414298497574059,
+      "acc_norm,none": 0.612,
+      "acc_norm_stderr,none": 0.015417317979911216
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "sciq": {
+      "task": "sciq",
+      "dataset_path": "allenai/sciq",
+      "training_split": "train",
+      "validation_split": "validation",
+      "test_split": "test",
+      "doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:",
+      "doc_to_target": 3,
+      "unsafe_code": false,
+      "doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:",
+        "doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}",
+        "doc_to_target": 3,
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{support}} {{question}}",
+      "metadata": {
+        "version": 1.0,
+        "config_source": "/mnt/d/Projects/tiny_llm/.venv-eval/lib/python3.12/site-packages/lm_eval/tasks/sciq/sciq.yaml"
+      }
+    }
+  },
+  "versions": {
+    "sciq": 1.0
+  },
+  "n-shot": {
+    "sciq": 0
+  },
+  "higher_is_better": {
+    "sciq": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "sciq": {
+      "original": 1000,
+      "effective": 1000
+    }
+  },
+  "config": {
+    "model": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+    "model_args": null,
+    "model_num_parameters": 50919168,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": null,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 1000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 1234,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "d2b5d7b",
+  "date": 1790879558.8372855,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5070 Ti\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        48 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               8\nOn-line CPU(s) list:                  0-7\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 9800X3D 8-Core Processor\nCPU family:                           26\nModel:                                68\nThread(s) per core:                   2\nCore(s) per socket:                   4\nSocket(s):                            1\nStepping:                             0\nBogoMIPS:                             9399.75\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm avx512_vp2intersect\nVirtualization:                       AMD-V\nHypervisor vendor:                    Microsoft\nVirtualization type:                  full\nL1d cache:                            192 KiB (4 instances)\nL1i cache:                            128 KiB (4 instances)\nL2 cache:                             4 MiB (4 instances)\nL3 cache:                             96 MiB (1 instance)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-7\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Not affected\nVulnerability Spec rstack overflow:   Not affected\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.19.0.56\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.28.9\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.11.0+cu128\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.17.0",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/truthfulqa_mc2.json

@@ -0,0 +1,120 @@
+{
+  "results": {
+    "truthfulqa_mc2": {
+      "name": "truthfulqa_mc2",
+      "alias": "truthfulqa_mc2",
+      "sample_len": 817,
+      "acc,none": 0.4417644031264386,
+      "acc_stderr,none": 0.01519901667714673
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "truthfulqa_mc2": {
+      "task": "truthfulqa_mc2",
+      "dataset_path": "truthfulqa/truthful_qa",
+      "dataset_name": "multiple_choice",
+      "validation_split": "validation",
+      "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}",
+      "doc_to_target": 0,
+      "unsafe_code": false,
+      "doc_to_choice": "{{mc2_targets.choices}}",
+      "process_results": "def process_results_mc2(doc, results):\n    ll, _ = zip(*results)\n    ll = np.array(ll)\n\n    # Convert log-likelihoods to probabilities.\n    probs = np.exp(ll)\n\n    # Normalize probabilities.\n    probs_norm = probs / np.sum(probs)\n\n    labels = np.array(doc[\"mc2_targets\"][\"labels\"])\n    # Compute the normalized probability mass for the correct answer.\n    pm_true = np.sum(probs_norm[labels == 1])\n\n    return {\"acc\": pm_true}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}",
+        "doc_to_choice": "{{mc2_targets.choices}}",
+        "doc_to_target": 0,
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "question",
+      "metadata": {
+        "version": 3.0,
+        "config_source": "/mnt/d/Projects/tiny_llm/.venv-eval/lib/python3.12/site-packages/lm_eval/tasks/truthfulqa/truthfulqa_mc2.yaml"
+      }
+    }
+  },
+  "versions": {
+    "truthfulqa_mc2": 3.0
+  },
+  "n-shot": {
+    "truthfulqa_mc2": 0
+  },
+  "higher_is_better": {
+    "truthfulqa_mc2": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "truthfulqa_mc2": {
+      "original": 817,
+      "effective": 817
+    }
+  },
+  "config": {
+    "model": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+    "model_args": null,
+    "model_num_parameters": 50919168,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": null,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 1000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 1234,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "d2b5d7b",
+  "date": 1790879569.56166,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5070 Ti\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        48 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               8\nOn-line CPU(s) list:                  0-7\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 9800X3D 8-Core Processor\nCPU family:                           26\nModel:                                68\nThread(s) per core:                   2\nCore(s) per socket:                   4\nSocket(s):                            1\nStepping:                             0\nBogoMIPS:                             9399.75\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm avx512_vp2intersect\nVirtualization:                       AMD-V\nHypervisor vendor:                    Microsoft\nVirtualization type:                  full\nL1d cache:                            192 KiB (4 instances)\nL1i cache:                            128 KiB (4 instances)\nL2 cache:                             4 MiB (4 instances)\nL3 cache:                             96 MiB (1 instance)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-7\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Not affected\nVulnerability Spec rstack overflow:   Not affected\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.19.0.56\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.28.9\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.11.0+cu128\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.17.0",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/raw/winogrande.json

@@ -0,0 +1,120 @@
+{
+  "results": {
+    "winogrande": {
+      "name": "winogrande",
+      "alias": "winogrande",
+      "sample_len": 1267,
+      "acc,none": 0.5138121546961326,
+      "acc_stderr,none": 0.014047122916440271
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "winogrande": {
+      "task": "winogrande",
+      "dataset_path": "allenai/winogrande",
+      "dataset_name": "winogrande_xl",
+      "training_split": "train",
+      "validation_split": "validation",
+      "doc_to_text": "def doc_to_text(doc):\n    answer_to_num = {\"1\": 0, \"2\": 1}\n    return answer_to_num[doc[\"answer\"]]\n",
+      "doc_to_target": "def doc_to_target(doc):\n    idx = doc[\"sentence\"].index(\"_\") + 1\n    return doc[\"sentence\"][idx:].strip()\n",
+      "unsafe_code": false,
+      "doc_to_choice": "def doc_to_choice(doc):\n    idx = doc[\"sentence\"].index(\"_\")\n    options = [doc[\"option1\"], doc[\"option2\"]]\n    return [doc[\"sentence\"][:idx] + opt for opt in options]\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "<function doc_to_text at 0x74bb27214900>",
+        "doc_to_choice": "<function doc_to_choice at 0x74bb27217380>",
+        "doc_to_target": "<function doc_to_target at 0x74bb27216b60>",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "sentence",
+      "metadata": {
+        "version": 1.0,
+        "config_source": "/mnt/d/Projects/tiny_llm/.venv-eval/lib/python3.12/site-packages/lm_eval/tasks/winogrande/default.yaml"
+      }
+    }
+  },
+  "versions": {
+    "winogrande": 1.0
+  },
+  "n-shot": {
+    "winogrande": 0
+  },
+  "higher_is_better": {
+    "winogrande": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "winogrande": {
+      "original": 1267,
+      "effective": 1267
+    }
+  },
+  "config": {
+    "model": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+    "model_args": null,
+    "model_num_parameters": 50919168,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": null,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 1000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 1234,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "d2b5d7b",
+  "date": 1790878814.3209262,
+  "pretty_env_info": "PyTorch version: 2.11.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 5070 Ti\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        48 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               8\nOn-line CPU(s) list:                  0-7\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 9800X3D 8-Core Processor\nCPU family:                           26\nModel:                                68\nThread(s) per core:                   2\nCore(s) per socket:                   4\nSocket(s):                            1\nStepping:                             0\nBogoMIPS:                             9399.75\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx_vnni avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm avx512_vp2intersect\nVirtualization:                       AMD-V\nHypervisor vendor:                    Microsoft\nVirtualization type:                  full\nL1d cache:                            192 KiB (4 instances)\nL1i cache:                            128 KiB (4 instances)\nL2 cache:                             4 MiB (4 instances)\nL3 cache:                             96 MiB (1 instance)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-7\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Not affected\nVulnerability Spec rstack overflow:   Not affected\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.19.0.56\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.28.9\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.11.0+cu128\n[pip3] triton==3.6.0\n[conda] Could not collect",
+  "transformers_version": "5.17.0",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 2048,
+  "tiny_llm_evaluation": {
+    "protocol": "autoregressive likelihood",
+    "smoke_only": false,
+    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod"
+  }
+}

evaluation/repro/comparison_runner/__init__.py

@@ -0,0 +1 @@
+"""Pretraining benchmarks. Importing this package never imports torch or datasets."""

evaluation/repro/comparison_runner/adapters.py

@@ -0,0 +1,214 @@
+"""HF loading and an explicitly experimental masked-model adapter.
+
+Only imported for an actual evaluation, never for planning.
+"""
+
+import torch
+from lm_eval.models.huggingface import HFLM
+from transformers import AutoTokenizer
+from tqdm import tqdm
+
+
+class UL2HFLM(HFLM):
+    """S-denoiser continuation scoring; only answer text contributes to NLL.
+
+    Source: S + context + sentinel_0 + EOS.
+    Decoder input: BOS + sentinel_0 + answer[:-1]. Controls are conditioned
+    on, never scored; the softmax still includes the complete vocabulary.
+    """
+
+    def _encode_pair(self, context, continuation):
+        # Match causal HFLM's text boundary, without tokenizer-added controls.
+        spaces = len(context) - len(context.rstrip())
+        if spaces:
+            continuation = context[-spaces:] + continuation
+            context = context[:-spaces]
+        whole = self.tok_encode(context + continuation, add_special_tokens=False)
+        prefix = self.tok_encode(context, add_special_tokens=False)
+        return prefix, whole[len(prefix):]
+
+    @torch.inference_mode()
+    def _loglikelihood_tokens(self, requests, disable_tqdm=False, override_bs=None):
+        contract = self.model.config.ul2
+        span = contract["sentinel_ids"][0]
+        bos = self.model.config.decoder_start_token_id
+        batch_size = override_bs or self.batch_size
+        results = [None] * len(requests)
+        ordered = sorted(enumerate(requests), key=lambda x: -(len(x[1][1]) + len(x[1][2])))
+        for start in tqdm(range(0, len(ordered), batch_size), disable=disable_tqdm, desc="UL2 likelihood"):
+            batch = ordered[start:start + batch_size]
+            sources, decoders = [], []
+            for _, (_, context, answer) in batch:
+                if not answer or len(answer) >= self.max_length:
+                    raise ValueError("UL2 scoring requires a nonempty answer shorter than the context limit")
+                budget = min(self.max_length + 1 - len(answer), self.model.config.max_position_embeddings - 3)
+                context = context[-budget:]
+                sources.append([contract["mode_ids"]["S"], *context, span, contract["eos_id"]])
+                decoders.append([bos, span, *answer[:-1]])
+
+            def pad(rows):
+                ids = torch.full((len(rows), max(map(len, rows))), contract["pad_id"], dtype=torch.long, device=self.device)
+                mask = torch.zeros_like(ids)
+                for i, row in enumerate(rows):
+                    ids[i, :len(row)] = torch.tensor(row, device=self.device)
+                    mask[i, :len(row)] = 1
+                return ids, mask
+
+            source, source_mask = pad(sources)
+            decoder, decoder_mask = pad(decoders)
+            hidden = self.model.model(input_ids=source, attention_mask=source_mask,
+                                      decoder_input_ids=decoder, decoder_attention_mask=decoder_mask,
+                                      use_cache=False).last_hidden_state
+            # Project only scored positions and bound temporary vocabulary logits.
+            for row, (index, (key, _, answer)) in enumerate(batch):
+                score, greedy = 0.0, True
+                for offset in range(0, len(answer), 256):
+                    target = torch.tensor(answer[offset:offset + 256], device=self.device)
+                    logits = self.model.lm_head(hidden[row, 1 + offset:1 + offset + len(target)]).float()
+                    score += torch.log_softmax(logits, -1).gather(1, target[:, None]).sum().item()
+                    greedy = greedy and bool((logits.argmax(-1) == target).all())
+                results[index] = (score, greedy)
+                if key is not None:
+                    self.cache_hook.add_partial("loglikelihood", key, results[index])
+        return results
+
+    def loglikelihood_rolling(self, requests, disable_tqdm=False):
+        raise NotImplementedError("UL2 continuation likelihood is not rolling autoregressive perplexity")
+
+    def generate_until(self, requests, disable_tqdm=False):
+        raise NotImplementedError("UL2 benchmark adapter currently supports likelihood tasks only")
+
+
+class PrefixHFLM(HFLM):
+    """Score causal answers conditioned on a fully bidirectional text prefix."""
+
+    @torch.inference_mode()
+    def _loglikelihood_tokens(self, requests, disable_tqdm=False, override_bs=None):
+        results = [None] * len(requests)
+        ordered = sorted(enumerate(requests), key=lambda x: -(len(x[1][1]) + len(x[1][2])))
+        size = override_bs or self.batch_size
+        for start in tqdm(range(0, len(ordered), size), disable=disable_tqdm, desc="Prefix likelihood"):
+            batch = ordered[start:start + size]
+            rows, prefixes = [], []
+            for _, (_, context, answer) in batch:
+                if not answer or len(answer) > self.max_length:
+                    raise ValueError("Require nonempty answer no longer than context limit")
+                context = context[-(self.max_length + 1 - len(answer)):]
+                rows.append(context + answer[:-1])
+                prefixes.append(len(context))
+            ids = torch.full((len(rows), max(map(len, rows))), self.tokenizer.pad_token_id or 0,
+                             device=self.device, dtype=torch.long)
+            mask = torch.zeros_like(ids)
+            for i, row in enumerate(rows):
+                ids[i, :len(row)] = torch.tensor(row, device=self.device)
+                mask[i, :len(row)] = 1
+            hidden = self.model.model(ids, attention_mask=mask,
+                                      prefix_lengths=torch.tensor(prefixes, device=self.device),
+                                      use_cache=False).last_hidden_state
+            for i, (index, (key, _, answer)) in enumerate(batch):
+                score, greedy = 0., True
+                for offset in range(0, len(answer), 256):
+                    target = torch.tensor(answer[offset:offset+256], device=self.device)
+                    logits = self.model.lm_head(hidden[i, prefixes[i]-1+offset:prefixes[i]-1+offset+len(target)]).float()
+                    score += torch.log_softmax(logits, -1).gather(1, target[:, None]).sum().item()
+                    greedy = greedy and bool((logits.argmax(-1) == target).all())
+                results[index] = (score, greedy)
+                if key is not None:
+                    self.cache_hook.add_partial("loglikelihood", key, results[index])
+        return results
+
+    def loglikelihood_rolling(self, requests, disable_tqdm=False):
+        raise NotImplementedError("Use an explicitly defined prefix-continuation protocol")
+
+
+class DiffusionHFLM(HFLM):
+    """Continuation PLL, with other answer tokens visible.
+
+    The bool is full-token reconstruction accuracy, NOT AR exact match.
+    Generation uses one masked next-token slot at a time, not the model's
+    unconditional parallel denoiser. Both are experimental protocols.
+    """
+
+    @torch.inference_mode()
+    def _loglikelihood_tokens(self, requests, disable_tqdm=False, override_bs=None):
+        results = []
+        for cache_key, context, continuation in tqdm(requests, disable=disable_tqdm, desc="Diffusion PLL"):
+            if not continuation:
+                results.append((0.0, True))
+                continue
+            if len(continuation) >= self.max_length:
+                raise ValueError("Diffusion PLL needs room for context and the entire continuation")
+            context = context[-(self.max_length - len(continuation)):]
+            tokens = context + continuation
+            base = torch.tensor([tokens], dtype=torch.long, device=self.device)
+            timestep = torch.tensor([1 / len(tokens)], device=self.device)
+            score, greedy = 0.0, True
+            # Independent single-mask replicas preserve the PLL definition.
+            # Project only masked positions instead of every vocabulary logit.
+            for start in range(len(context), len(tokens), 16):
+                positions = torch.arange(start, min(start + 16, len(tokens)), device=self.device)
+                rows = torch.arange(len(positions), device=self.device)
+                masked = base.expand(len(positions), -1).clone()
+                masked[rows, positions] = self.model.config.mask_token_id
+                hidden = self.model.model(input_ids=masked, timesteps=timestep.expand(len(positions)))
+                logits = self.model.lm_head(hidden[rows, positions]).float()
+                targets = base[0, positions]
+                score += torch.log_softmax(logits, dim=-1).gather(1, targets[:, None]).sum().item()
+                greedy = greedy and bool((logits.argmax(-1) == targets).all())
+            result = (score, greedy)
+            results.append(result)
+            if cache_key is not None:
+                self.cache_hook.add_partial("loglikelihood", cache_key, result)
+        return results
+
+    def loglikelihood_rolling(self, requests, disable_tqdm=False):
+        raise NotImplementedError("Diffusion PLL must not be reported as autoregressive perplexity")
+
+    @torch.inference_mode()
+    def _model_generate(self, context, max_length, stop, **generation_kwargs):
+        if context.shape[0] != 1:
+            raise ValueError("Diffusion generation requires batch size 1")
+        if generation_kwargs.get("do_sample", False):
+            raise ValueError("Diffusion adapter implements greedy generation only")
+        sequence = context
+        prompt_length = context.shape[1]
+        while sequence.shape[1] < min(max_length, self.max_length):
+            slot = torch.full((1, 1), self.model.config.mask_token_id,
+                              dtype=torch.long, device=self.device)
+            masked = torch.cat((sequence, slot), dim=1)
+            timestep = torch.tensor([1 / masked.shape[1]], device=self.device)
+            logits = self.model(input_ids=masked, timesteps=timestep).logits[:, -1]
+            predicted = logits.argmax(dim=-1, keepdim=True)
+            sequence = torch.cat((sequence, predicted), dim=1)
+            text = self.tok_decode(sequence[0, prompt_length:].tolist())
+            if predicted.item() == self.eot_token_id or any(s and s in text for s in stop):
+                break
+        return sequence
+
+
+def load_model(plan, args):
+    common = dict(batch_size=args.batch_size, device=args.device, dtype=args.dtype,
+                  max_length=args.max_length, trust_remote_code=args.trust_remote_code,
+                  revision=args.revision)
+    tokenizer = AutoTokenizer.from_pretrained(
+        plan["tokenizer"], trust_remote_code=args.trust_remote_code,
+        revision=args.tokenizer_revision or args.revision,
+    )
+    if plan["local_model"] is None:
+        return HFLM(pretrained=plan["checkpoint"], tokenizer=tokenizer, **common)
+
+    from tiny_llm.models import MODEL_REGISTRY
+
+    entry = MODEL_REGISTRY[plan["local_model"]]
+    dtype = args.dtype if args.dtype == "auto" else getattr(torch, args.dtype)
+    # Checkpoints have no auto_map or tokenizer. Never load train_state.pt.
+    config = entry["config_class"].from_pretrained(plan["checkpoint"])
+    model = entry["model_class"].from_pretrained(plan["checkpoint"], config=config, dtype=dtype)
+    model.to(args.device).eval()
+    backend = "seq2seq" if plan["kind"] == "seq2seq" else "causal"
+    adapter = DiffusionHFLM if plan["kind"] == "diffusion" else HFLM
+    if plan['local_model'] == 'prefixlm':
+        adapter = PrefixHFLM
+    if plan["kind"] == "seq2seq" and getattr(config, "ul2", None):
+        adapter = UL2HFLM
+    return adapter(pretrained=model, tokenizer=tokenizer, backend=backend, **common)

evaluation/repro/comparison_runner/catalog.py

@@ -0,0 +1,105 @@
+"""Explicit protocols for lm-eval 0.4.12, checked on 2026-09-18.
+
+These are project comparison protocols, not claims to reproduce model cards.
+"""
+
+from dataclasses import dataclass
+
+HARNESS_VERSION = "0.4.12"
+
+
+@dataclass(frozen=True)
+class Benchmark:
+    task: str
+    shots: int
+    description: str
+    unsafe_code: bool = False
+    rolling: bool = False
+    engine: str = "lm_eval"
+
+
+BENCHMARKS = {b.task: b for b in (
+    Benchmark("hellaswag", 0, "Commonsense completion; acc and acc_norm"),
+    Benchmark("arc_easy", 0, "AI2 ARC-Easy science questions; acc and acc_norm"),
+    Benchmark("arc_challenge", 0, "AI2 ARC-Challenge; acc and acc_norm"),
+    Benchmark("piqa", 0, "Physical commonsense; acc and acc_norm"),
+    Benchmark("winogrande", 0, "Pronoun resolution; acc"),
+    Benchmark("openbookqa", 0, "Science multiple choice; acc and acc_norm"),
+    Benchmark("boolq", 0, "Yes/no reading comprehension; acc"),
+    Benchmark("lambada_openai", 0, "Complete final word; acc and perplexity"),
+    Benchmark("mmlu", 5, "57-subject knowledge group; acc (not MMLU cloze)"),
+    Benchmark("mmlu_continuation", 0, "MMLU cloze: full answer continuations; acc"),
+    Benchmark("sciq", 0, "SciQ test with support passage; acc and acc_norm"),
+    Benchmark("balanced_copa", 0, "Balanced COPA mirrored 1000-item train split; acc; cRia split inferred"),
+    Benchmark("balanced_copa_test", 0, "Balanced COPA original 500-item test split; acc; separate protocol"),
+    Benchmark("arithmark3", 0, "ArithMark-3.0 train-named evaluation split; token-normalized acc_norm", engine="token_continuation"),
+    Benchmark("bananamind_base", 0, "BananaMind Base Bench 1.1 test; mean-token raw accuracy; gated", engine="token_continuation"),
+    Benchmark("commonsense_qa", 0, "CommonsenseQA; acc"),
+    Benchmark("triviaqa", 0, "Open-domain QA; exact match"),
+    Benchmark("social_iqa", 0, "Social reasoning; acc and acc_norm"),
+    Benchmark("race", 0, "Reading comprehension; acc and acc_norm"),
+    Benchmark("truthfulqa_mc2", 0, "TruthfulQA multiple choice; mc2"),
+    Benchmark("blimp", 0, "Full BLiMP group; grammatical minimal pairs"),
+    Benchmark("wikitext", 0, "WikiText-2 raw test; word/byte PPL and bits/byte", rolling=True),
+    Benchmark("gsm8k", 5, "Grade-school math; exact match"),
+    Benchmark("hendrycks_math500", 0, "MATH-500; exact match"),
+    Benchmark("bbh", 3, "Big-Bench Hard CoT group; exact match"),
+    Benchmark("humaneval", 0, "Python completion; pass@1", unsafe_code=True),
+    Benchmark("mbpp", 3, "Python programming; pass_at_1", unsafe_code=True),
+    Benchmark("mbpp_plus", 3, "MBPP with additional tests; pass_at_1", unsafe_code=True),
+    Benchmark("humaneval_instruct", 0, "Instruction-format HumanEval; pass@1", unsafe_code=True),
+    Benchmark("ifeval", 0, "Instruction following; separate from pretraining core"),
+)}
+
+SUITES = {
+    "core": ("hellaswag", "arc_easy", "arc_challenge", "piqa", "winogrande",
+             "openbookqa", "boolq", "lambada_openai"),
+    "extended": ("mmlu", "commonsense_qa", "triviaqa", "social_iqa", "race",
+                 "truthfulqa_mc2", "blimp"),
+    "perplexity": ("wikitext",),
+    "reasoning": ("gsm8k", "hendrycks_math500", "bbh"),
+    "code": ("humaneval", "mbpp", "mbpp_plus"),
+    "instruction": ("ifeval", "humaneval_instruct"),
+    "zarya": ("arc_challenge", "hellaswag", "gsm8k", "hendrycks_math500",
+              "humaneval", "humaneval_instruct", "mbpp", "mbpp_plus", "ifeval"),
+    "cria": ("arc_challenge", "arc_easy", "arithmark3", "balanced_copa",
+             "blimp", "boolq", "commonsense_qa", "hellaswag",
+             "lambada_openai", "mmlu_continuation", "openbookqa", "piqa", "sciq",
+             "truthfulqa_mc2", "winogrande", "bananamind_base"),
+    "cria_missing": ("sciq", "mmlu_continuation", "balanced_copa", "arithmark3",
+                     "bananamind_base"),
+}
+
+
+def select_tasks(suites: str, tasks: str | None = None) -> list[Benchmark]:
+    """Explicit tasks replace suites; reject typos rather than silently skip them."""
+    names = []
+    if tasks is not None:
+        names = [x.strip() for x in tasks.split(",") if x.strip()]
+    else:
+        for suite in suites.split(","):
+            suite = suite.strip()
+            if suite not in SUITES:
+                raise ValueError(f"Unknown suite {suite!r}; choose from {', '.join(SUITES)}")
+            names.extend(SUITES[suite])
+    if not names:
+        raise ValueError("Select at least one task")
+    unknown = set(names) - BENCHMARKS.keys()
+    if unknown:
+        raise ValueError(f"Unknown tasks: {', '.join(sorted(unknown))}; see --list")
+    return [BENCHMARKS[name] for name in dict.fromkeys(names)]
+
+# Index only selected official task directories, not thousands of unrelated YAMLs.
+TASK_DIRECTORIES = {
+    "hellaswag": "hellaswag", "arc_easy": "arc", "arc_challenge": "arc",
+    "piqa": "piqa", "winogrande": "winogrande", "openbookqa": "openbookqa",
+    "boolq": "super_glue/boolq", "lambada_openai": "lambada", "mmlu": "mmlu/default",
+    "commonsense_qa": "commonsense_qa", "triviaqa": "triviaqa", "social_iqa": "siqa",
+    "race": "race", "truthfulqa_mc2": "truthfulqa", "blimp": "blimp",
+    "wikitext": "wikitext", "gsm8k": "gsm8k", "hendrycks_math500": "hendrycks_math",
+    "bbh": "bbh/cot_fewshot", "humaneval": "humaneval", "humaneval_instruct": "humaneval",
+    "mbpp": "mbpp", "mbpp_plus": "mbpp", "ifeval": "ifeval",
+    "sciq": "sciq", "mmlu_continuation": "mmlu/continuation",
+}
+
+LOCAL_TASKS = {"balanced_copa", "balanced_copa_test"}

evaluation/repro/comparison_runner/continuation.py

@@ -0,0 +1,30 @@
+"""A common prefix-to-continuation NLL protocol for causal, prefix and UL2 models."""
+import torch
+from .losses import chunked_cross_entropy
+
+
+@torch.no_grad()
+def continuation_nll(model, kind, tokens, prefix_length, chunk_size=2048):
+    if not 1 <= prefix_length < tokens.shape[1]:
+        raise ValueError("Require a nonempty prefix and continuation")
+    targets = tokens[:, prefix_length:]
+    if kind in ("causal", "prefixlm", "zarya"):
+        kwargs = {}
+        if kind == "prefixlm":
+            kwargs["prefix_lengths"] = torch.full((tokens.shape[0],), prefix_length, device=tokens.device)
+        hidden = model.model(tokens, use_cache=False, **kwargs).last_hidden_state[:, prefix_length-1:-1]
+    elif kind == "seq2seq":
+        contract = getattr(model.config, "ul2", None)
+        if contract is None:
+            raise ValueError("Continuation scoring requires an UL2 checkpoint with its control-token contract")
+        mode = tokens.new_full((tokens.shape[0], 1), contract["mode_ids"]["S"])
+        span = tokens.new_full((tokens.shape[0], 1), contract["sentinel_ids"][0])
+        eos = tokens.new_full((tokens.shape[0], 1), contract["eos_id"])
+        source = torch.cat((mode, tokens[:, :prefix_length], span, eos), dim=1)
+        bos = tokens.new_full((tokens.shape[0], 1), model.config.decoder_start_token_id)
+        decoder = torch.cat((bos, span, targets[:, :-1]), dim=1)
+        hidden = model.model(input_ids=source, decoder_input_ids=decoder, use_cache=False).last_hidden_state[:, 1:]
+    else:
+        raise ValueError("Diffusion masked loss/PLL is not this autoregressive continuation PPL")
+    count = targets.numel()
+    return chunked_cross_entropy(hidden, model.lm_head, targets, chunk_size) * count, count

evaluation/repro/comparison_runner/losses.py

@@ -0,0 +1,58 @@
+"""Chunked cross-entropy for large-vocab language models.
+
+Materializing full logits (batch x seq x vocab) in fp32 dominates memory for
+vocab=32768 models: a single bs=8, seq=2048 logits tensor chain costs ~8 GiB
+before the transformer activations are even counted. Computing the loss in
+sequence chunks keeps peak memory flat and returns exactly the same value as
+the standard mean cross-entropy (sum over non-ignored tokens / token count).
+"""
+
+import torch
+import torch.nn.functional as F
+from torch.utils.checkpoint import checkpoint
+
+
+def chunked_cross_entropy(
+    hidden: torch.Tensor,
+    lm_head: torch.nn.Module,
+    labels: torch.Tensor,
+    chunk_size: int = 8192,
+    token_weights: torch.Tensor | None = None,
+    denominator: int | None = None,
+    filter_ignored: bool = False,
+) -> torch.Tensor:
+    """Mean cross-entropy computed over chunks of the flattened sequence.
+
+    Args:
+        hidden: (B, T, H) decoder hidden states, already shifted/aligned.
+        lm_head: output projection (H -> vocab).
+        labels: (B, T) aligned target token ids, -100 = ignore.
+        chunk_size: tokens per chunk; peak logits memory is
+            chunk_size x vocab x ~6 bytes.
+
+    Returns:
+        Scalar loss, identical to F.cross_entropy(..., ignore_index=-100)
+        computed on full logits.
+    """
+    h = hidden.reshape(-1, hidden.size(-1))
+    t = labels.reshape(-1)
+    total = torch.zeros((), device=h.device, dtype=torch.float32)
+    # token count stays a GPU tensor: no host sync per chunk
+    count = denominator if denominator is not None else t.ne(-100).sum().clamp(min=1)
+    weights = token_weights.reshape(-1) if token_weights is not None else torch.ones_like(t, dtype=torch.float32)
+    if filter_ignored:
+        valid = t.ne(-100)
+        h, t, weights = h[valid], t[valid], weights[valid]
+        # Keep an all-unmasked diffusion draw differentiable with zero loss.
+        total = h.float().sum() * 0
+    def chunk_loss(states, targets, weight):
+        # Checkpoint the projection AND CE: otherwise autograd retains every
+        # chunk's vocabulary-sized activations until the eventual backward.
+        losses = F.cross_entropy(lm_head(states).float(), targets,
+                                 ignore_index=-100, reduction="none")
+        return (losses * weight).sum()
+    for i in range(0, h.size(0), chunk_size):
+        inputs = (h[i:i + chunk_size], t[i:i + chunk_size], weights[i:i + chunk_size])
+        total = total + (checkpoint(chunk_loss, *inputs, use_reentrant=False)
+                         if torch.is_grad_enabled() else chunk_loss(*inputs))
+    return total / count

evaluation/repro/comparison_runner/tasks/balanced_copa.yaml

@@ -0,0 +1,20 @@
+task: balanced_copa
+dataset_path: pkavumba/balanced-copa
+dataset_kwargs:
+  revision: 813bd03cd6e07d9bd8d7333896ad5d40abb95ea9
+output_type: multiple_choice
+# The mirrored 1000-item evaluation is named train on the Hub.
+# cRia's 58.70% / stderr .0156 suggests n=1000; its exact split is unpublished.
+test_split: train
+num_fewshot: 0
+target_delimiter: " "
+doc_to_text: "{{premise.strip()[:-1]}} {{'because' if question == 'cause' else 'therefore'}}"
+doc_to_choice: "{{[choice1[0].lower() + choice1[1:], choice2[0].lower() + choice2[1:]]}}"
+doc_to_target: "{{label}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1
+  reference_protocol: cRia split and exact prompt unverified; COPA-style prompt

evaluation/repro/comparison_runner/tasks/balanced_copa_test.yaml

@@ -0,0 +1,6 @@
+include: balanced_copa.yaml
+task: balanced_copa_test
+test_split: test
+metadata:
+  version: 1
+  reference_protocol: Original 500-item test split; not the mirrored 1000-item split

evaluation/repro/comparison_runner/token_continuation.py

@@ -0,0 +1,167 @@
+"""Pinned continuation benchmarks whose normalization is by model tokens.
+
+The usual harness acc_norm divides by string length. These two benchmarks
+instead tokenize context/answers separately, omit BOS, and rank mean token
+log-probabilities. Reuse each model adapter's likelihood implementation.
+"""
+
+from collections import Counter, defaultdict
+import hashlib
+import json
+import math
+
+
+DATASETS = {
+    "arithmark3": {
+        "repo": "AxiomicLabs/Arithmark-3.0",
+        "revision": "6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746",
+        "file": "arithmark-3.jsonl", "split": "train", "count": 1000,
+        "context_field": "ctx", "choices_field": "endings",
+        "category_field": "activity_label", "context_cap": 1024,
+        "context_truncation": "per_choice",
+        "primary_metric": "acc_norm",
+    },
+    "bananamind_base": {
+        "repo": "BananaMind/BananaMind-Base-Bench-1.1",
+        "revision": "d4aade51312889e8580963e1ce960c6eaef1a450",
+        "file": "data/test.jsonl", "split": "test", "count": 350,
+        "sha256": "2f563bb46df778ca494fa20f994a8d3045d4c51fbbffeee433764e2813abea21",
+        "context_field": "context", "choices_field": "continuations",
+        "category_field": "category", "context_cap": None,
+        "context_truncation": "shared_longest_choice",
+        "primary_metric": "raw_accuracy",
+    },
+}
+
+
+def validate_examples(examples, spec):
+    if len(examples) != spec["count"]:
+        raise ValueError(f"Expected {spec['count']} examples, got {len(examples)}")
+    for index, doc in enumerate(examples):
+        context = doc[spec["context_field"]]
+        choices = doc[spec["choices_field"]]
+        if (not isinstance(context, str) or not context
+                or (spec["context_truncation"] == "per_choice" and context[-1].isspace())):
+            raise ValueError(f"Invalid context/boundary at example {index}")
+        if len(choices) != 4 or any(not isinstance(c, str) or not c or not c[0].isspace() for c in choices):
+            raise ValueError(f"Expected four whitespace-prefixed continuations at example {index}")
+        if str(doc["label"]) not in ("0", "1", "2", "3"):
+            raise ValueError(f"Invalid label at example {index}")
+        if "sha256" in spec and (doc.get("version") != "1.1" or doc.get("score_mode") != "mean_logprob"):
+            raise ValueError(f"Unexpected BananaMind version/scoring at example {index}")
+    if "sha256" in spec:
+        categories = Counter(doc["category"] for doc in examples)
+        if len(categories) != 7 or set(categories.values()) != {50}:
+            raise ValueError("Expected seven BananaMind categories of 50 items")
+        if Counter(doc["difficulty"] for doc in examples) != {"easy": 117, "medium": 117, "hard": 116}:
+            raise ValueError("Unexpected BananaMind difficulty balance")
+        if Counter(int(doc["label"]) for doc in examples) != {0: 88, 1: 88, 2: 87, 3: 87}:
+            raise ValueError("Unexpected BananaMind answer balance")
+
+
+def load_examples(task):
+    from huggingface_hub import hf_hub_download
+    from huggingface_hub.errors import GatedRepoError
+    from pathlib import Path
+
+    spec = DATASETS[task]
+    try:
+        path = hf_hub_download(spec["repo"], spec["file"], repo_type="dataset", revision=spec["revision"])
+    except GatedRepoError as exc:
+        raise RuntimeError(
+            f"Accept the evaluation-only terms at https://huggingface.co/datasets/{spec['repo']} "
+            "and authenticate this evaluation environment with hf auth login, then use a new output directory."
+        ) from exc
+    data = Path(path).read_bytes()
+    digest = hashlib.sha256(data).hexdigest()
+    if spec.get("sha256", digest) != digest:
+        raise ValueError(f"{task} dataset checksum mismatch")
+    examples = [json.loads(line) for line in data.decode("utf-8").splitlines() if line.strip()]
+    validate_examples(examples, spec)
+    return examples, digest
+
+
+def accuracy_metrics(hits, name):
+    n = len(hits)
+    mean = sum(hits) / n
+    return {f"{name},none": mean,
+            f"{name}_stderr,none": math.sqrt(mean * (1 - mean) / (n - 1)) if n > 1 else "N/A"}
+
+
+def evaluate(model, task, args):
+    spec = DATASETS[task]
+    examples, digest = load_examples(task)
+    total = len(examples)
+    examples = examples[:args.limit] if args.limit is not None else examples
+    cap = min(model.max_length, spec["context_cap"] or model.max_length)
+    if cap < 2:
+        raise ValueError("Continuation evaluation requires context length >= 2")
+    raw_hits, mean_hits, samples = [], [], []
+    categories = defaultdict(list)
+    truncated = 0
+    # Bound requests and logits through the existing adapter batch-size setting.
+    for start in range(0, len(examples), args.batch_size):
+        batch = examples[start:start + args.batch_size]
+        requests, lengths, row_truncations = [], [], []
+        for doc in batch:
+            context = model.tok_encode(doc[spec["context_field"]], add_special_tokens=False)
+            if not context:
+                if task != "bananamind_base":
+                    raise ValueError("Empty tokenized context; implicit BOS would change the protocol")
+                # Match the official runner's fallback for tokenizers that erase
+                # a nonempty context. This is not unconditional BOS insertion.
+                fallback = next((getattr(model.tokenizer, name, None)
+                                 for name in ("eos_token_id", "bos_token_id", "pad_token_id")
+                                 if getattr(model.tokenizer, name, None) is not None), 0)
+                context = [int(fallback)]
+            answers = [model.tok_encode(choice, add_special_tokens=False)
+                       for choice in doc[spec["choices_field"]]]
+            if any(not answer or len(answer) >= cap for answer in answers):
+                raise ValueError("Continuation must leave at least one context token; cannot truncate answers")
+            shared_available = cap - max(map(len, answers))
+            was_truncated = False
+            for answer in answers:
+                available = (shared_available if spec["context_truncation"] == "shared_longest_choice"
+                             else cap - len(answer))
+                was_truncated |= len(context) > available
+                requests.append((None, context[-available:], answer))
+                lengths.append(len(answer))
+            row_truncations.append(was_truncated)
+        scores = model._loglikelihood_tokens(requests, disable_tqdm=True)
+        for i, doc in enumerate(batch):
+            raw = [float(score[0]) for score in scores[4 * i:4 * i + 4]]
+            counts = lengths[4 * i:4 * i + 4]
+            if len(raw) != 4 or not all(math.isfinite(x) for x in raw):
+                raise ValueError("Expected four finite continuation likelihoods")
+            normalized = [score / count for score, count in zip(raw, counts)]
+            label = int(doc["label"])
+            raw_prediction = max(range(4), key=raw.__getitem__)
+            prediction = max(range(4), key=normalized.__getitem__)
+            raw_hits.append(int(raw_prediction == label))
+            mean_hits.append(int(prediction == label))
+            category = doc[spec["category_field"]]
+            categories[category].append(mean_hits[-1])
+            truncated += row_truncations[i]
+            if args.log_samples:
+                samples.append({"doc_id": start + i, "label": label, "category": category,
+                                "raw_prediction": raw_prediction, "prediction": prediction,
+                                "loglikelihoods": raw, "token_counts": counts,
+                                "mean_loglikelihoods": normalized, "context_truncated": row_truncations[i]})
+    metrics = accuracy_metrics(mean_hits, spec["primary_metric"])
+    if task == "arithmark3":
+        metrics.update(accuracy_metrics(raw_hits, "acc"))
+    result = {
+        "results": {task: metrics},
+        "n-samples": {task: {"original": total, "effective": len(examples)}},
+        "configs": {task: {**spec, "dataset_sha256": digest, "num_fewshot": 0,
+                           "effective_max_length": cap, "add_bos": False,
+                           "tokenization": "context and continuation separately; add_special_tokens=False",
+                           "normalization": "continuation token count; not character count",
+                           "context_truncated_examples": truncated,
+                           "runner": "tiny_llm token_continuation v2; not an official leaderboard submission"}},
+        "categories": {name: {"count": len(hits), **accuracy_metrics(hits, spec["primary_metric"])}
+                       for name, hits in sorted(categories.items())},
+    }
+    if args.log_samples:
+        result["samples"] = {task: samples}
+    return result

evaluation/repro/continuation.py

@@ -0,0 +1,83 @@
+"""Standalone fixed-prefix WikiText evaluation, separate from the sixteen-task mean."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import numpy as np
+import torch
+import torch.nn.functional as F
+from datasets import load_dataset
+from transformers import AutoModelForCausalLM, AutoModelForMaskedLM, AutoModelForSeq2SeqLM, AutoTokenizer
+from comparison_runner.continuation import continuation_nll
+
+
+@torch.inference_mode()
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", required=True, type=Path)
+    parser.add_argument("--device", default="cuda:0")
+    parser.add_argument("--dtype", choices=["bfloat16", "float32"], default="bfloat16")
+    parser.add_argument("--limit-blocks", type=int, help="Smoke only, not a full benchmark")
+    args = parser.parse_args()
+    assert not args.output.exists()
+    assert args.limit_blocks is None or args.limit_blocks > 0
+    root = Path(__file__).resolve().parents[2]
+    expected = json.loads((root / "evaluation/results.json").read_text())
+    with (root / "model.safetensors").open("rb") as stream:
+        assert hashlib.file_digest(stream, "sha256").hexdigest() == expected["checkpoint_sha256"]["model.safetensors"]
+    config = json.loads((root / "config.json").read_text())
+    kind = "diffusion" if config["model_type"] == "diffusion_lm" else "seq2seq" if config.get("ul2") else "causal"
+    cls = AutoModelForMaskedLM if kind == "diffusion" else AutoModelForSeq2SeqLM if kind == "seq2seq" else AutoModelForCausalLM
+    torch.set_num_threads(4)
+    torch.backends.cuda.matmul.allow_tf32 = False
+    torch.backends.cudnn.allow_tf32 = False
+    tokenizer = AutoTokenizer.from_pretrained(root, trust_remote_code=True)
+    tokenizer.model_max_length = 10**9
+    data = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="test")
+    text = "\n".join(data["text"])
+    ids = tokenizer.encode(text, add_special_tokens=False)
+    blocks = torch.tensor(ids[:len(ids)//1024*1024]).reshape(-1, 1024)
+    if args.limit_blocks is not None:
+        blocks = blocks[:args.limit_blocks]
+    # Historical diffusion PLL cast FP32 buffers after loading; preserve this.
+    model = cls.from_pretrained(root, trust_remote_code=True,
+        dtype=torch.float32 if kind == "diffusion" else getattr(torch, args.dtype)).to(args.device).eval()
+    if kind == "diffusion":
+        model.to(getattr(torch, args.dtype))
+    values = []
+    for index, block in enumerate(blocks):
+        tokens = block[None].to(args.device)
+        if kind == "diffusion":
+            total = 0.
+            for start in range(512, 1024, 16):
+                positions = torch.arange(start, start+16, device=args.device)
+                rows = torch.arange(16, device=args.device)
+                masked = tokens.expand(16, -1).clone()
+                masked[rows, positions] = model.config.mask_token_id
+                hidden = model.model(masked, timesteps=torch.full((16,), 1/1024, device=args.device))
+                logits = model.lm_head(hidden[rows, positions]).float()
+                total += F.cross_entropy(logits, tokens[0, positions], reduction="sum").item()
+            value = total / 512
+        else:
+            nll, count = continuation_nll(model, kind, tokens, 512, 512)
+            value = nll.item() / count
+        values.append(value)
+        if index % 50 == 0:
+            print(index, "/", len(blocks), flush=True)
+    array = np.asarray(values)
+    rng = np.random.default_rng(2026)
+    low, high = np.quantile(array[rng.integers(len(array), size=(10000, len(array)))].mean(1), [.025, .975])
+    result = dict(nll=float(array.mean()), nll_ci95=[float(low), float(high)],
+        ppl=float(np.exp(array.mean())), ppl_ci95=[float(np.exp(low)), float(np.exp(high))],
+        block_nll=values, blocks=len(blocks), scored_tokens=len(blocks)*512,
+        dropped_tail_tokens=len(ids)%1024, corpus_sha256=hashlib.sha256(text.encode()).hexdigest(),
+        checkpoint_sha256=expected["checkpoint_sha256"], kind=kind, dtype=args.dtype, device=args.device,
+        smoke_only=args.limit_blocks is not None, bootstrap_samples=10000, bootstrap_seed=2026,
+        protocol="1024-token nonoverlapping blocks; 512-token prefix and 512-token suffix. "
+        "Diffusion: single-mask continuation PLL, not autoregressive likelihood. "
+        "Other models: conditional continuation NLL. Not rolling AR or word perplexity.")
+    args.output.write_text(json.dumps(result, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    main()

evaluation/repro/requirements.txt

@@ -0,0 +1,6 @@
+torch==2.11.0
+transformers==5.17.0
+datasets==5.0.1
+accelerate==1.15.0
+lm_eval==0.4.12
+tensorboard>=2.20,<3

evaluation/repro/run.py

@@ -0,0 +1,80 @@
+"""Reproduce the full sixteen-task comparison suite using the published model code."""
+import argparse
+import hashlib
+import importlib.metadata
+import json
+from pathlib import Path
+
+import torch
+from transformers import AutoModelForCausalLM, AutoModelForMaskedLM, AutoModelForSeq2SeqLM, AutoTokenizer
+from lm_eval import evaluator, tasks as harness_tasks
+from lm_eval.models.huggingface import HFLM
+
+from comparison_runner.adapters import DiffusionHFLM, PrefixHFLM, UL2HFLM
+from comparison_runner.catalog import BENCHMARKS, HARNESS_VERSION, LOCAL_TASKS, TASK_DIRECTORIES
+from comparison_runner.token_continuation import evaluate
+
+TASKS = "hellaswag,arc_easy,arc_challenge,piqa,winogrande,openbookqa,boolq,lambada_openai,arithmark3,balanced_copa,blimp,commonsense_qa,mmlu_continuation,sciq,truthfulqa_mc2,bananamind_base"
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--device", default="cpu")
+    parser.add_argument("--dtype", default="float32", choices=["float32", "bfloat16"])
+    parser.add_argument("--batch-size", type=int, default=1)
+    parser.add_argument("--tasks", default=TASKS)
+    parser.add_argument("--limit", type=int, help="Smoke only; not a benchmark result")
+    parser.add_argument("--output", required=True, type=Path)
+    args = parser.parse_args()
+    args.log_samples = False
+    assert args.batch_size > 0 and (args.limit is None or args.limit > 0)
+    assert importlib.metadata.version("lm_eval") == HARNESS_VERSION
+    root = Path(__file__).resolve().parent
+    model_root = root.parents[1]
+    expected = json.loads((root.parent / "results.json").read_text())
+    with (model_root / "model.safetensors").open("rb") as stream:
+        assert hashlib.file_digest(stream, "sha256").hexdigest() == expected["checkpoint_sha256"]["model.safetensors"]
+    torch.set_num_threads(4)
+    torch.backends.cuda.matmul.allow_tf32 = False
+    torch.backends.cudnn.allow_tf32 = False
+    torch.manual_seed(1234)
+    config = json.loads((model_root / "config.json").read_text())
+    diffusion = config["model_type"] == "diffusion_lm"
+    seq2seq = bool(config.get("ul2"))
+    cls = AutoModelForMaskedLM if diffusion else AutoModelForSeq2SeqLM if seq2seq else AutoModelForCausalLM
+    adapter_cls = (DiffusionHFLM if diffusion else UL2HFLM if seq2seq else
+                   PrefixHFLM if config["model_type"] == "prefix_lm" else HFLM)
+    if diffusion and args.batch_size != 1:
+        parser.error("Experimental diffusion scoring requires --batch-size 1")
+    model = cls.from_pretrained(model_root, trust_remote_code=True, dtype=getattr(torch, args.dtype)).to(args.device).eval()
+    tokenizer = AutoTokenizer.from_pretrained(model_root, trust_remote_code=True)
+    adapter = adapter_cls(pretrained=model, tokenizer=tokenizer, backend="seq2seq" if seq2seq else "causal",
+                          device=args.device, dtype=args.dtype, batch_size=args.batch_size, max_length=2048)
+    names = args.tasks.split(",")
+    assert names and set(names) <= set(TASKS.split(","))
+    harness_root = Path(harness_tasks.__file__).parent
+    directories = {harness_root / TASK_DIRECTORIES[t] for t in names if t in TASK_DIRECTORIES}
+    if set(names) & LOCAL_TASKS:
+        directories.add(root / "comparison_runner/tasks")
+    manager = harness_tasks.TaskManager(include_defaults=False, include_path=sorted(directories))
+    args.output.mkdir(parents=True, exist_ok=False)
+    metadata = dict(model_root=str(model_root), weights_sha256=expected["checkpoint_sha256"]["model.safetensors"],
+                    dtype=args.dtype, device=args.device, batch_size=args.batch_size, max_length=2048,
+                    smoke_only=args.limit is not None, tasks=names, harness=HARNESS_VERSION,
+                    protocol=expected["protocol"], seed=1234, tf32=False)
+    (args.output / "manifest.json").write_text(json.dumps(metadata, indent=2))
+    for task in names:
+        if BENCHMARKS[task].engine == "token_continuation":
+            result = evaluate(adapter, task, args)
+        else:
+            result = evaluator.simple_evaluate(model=adapter, tasks=[task], num_fewshot=0,
+                limit=args.limit, bootstrap_iters=1000, log_samples=False, task_manager=manager,
+                apply_chat_template=False, gen_kwargs={"do_sample": False},
+                random_seed=1234, numpy_random_seed=1234, torch_random_seed=1234, fewshot_random_seed=1234)
+        (args.output / (task + ".json")).write_text(json.dumps(result, indent=2,
+            default=lambda value: value.item() if hasattr(value, "item") else str(value)))
+    (args.output / "status.json").write_text(json.dumps(dict(status="complete", smoke_only=args.limit is not None)))
+
+
+if __name__ == "__main__":
+    main()

evaluation/results.json

@@ -0,0 +1,1480 @@
+{
+  "label": "Q-MoD",
+  "repo": "d0rj/q-mod-51M-base",
+  "revision": null,
+  "kind": "causal",
+  "checkpoint_sha256": {
+    "config.json": "194e3135cfd3dc01fb1e01c1c2a801e19212af8a532c465448d2c40af1f64034",
+    "generation_config.json": "80da72cc13262a67c14c8b96f4f049f6e7890a027d808a02b8f0b0d48704c245",
+    "model.safetensors": "e5be4b27775d846ab2e65293a83f647d89ba2e0a606c00645fa892b21f6d5cd3"
+  },
+  "parameters": 50919168,
+  "tasks": {
+    "hellaswag": {
+      "label": "HellaSwag",
+      "dataset": "Rowan/hellaswag",
+      "config": "default",
+      "split": "validation",
+      "samples": 10042,
+      "shots": 0,
+      "primary": "acc_norm",
+      "metrics": {
+        "acc": {
+          "value": 0.27633937462656843,
+          "stderr": 0.004462727543055927,
+          "ci95": [
+            0.2676798053353836,
+            0.28516999640463647
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        },
+        "acc_norm": {
+          "value": 0.2874925313682533,
+          "stderr": 0.004516681953878919,
+          "ci95": [
+            0.27872301915216174,
+            0.29642456629450026
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/hellaswag.json",
+      "result_sha256": "70c7f8f0f310bad716da22939e4ea8651e35d8262cf7a0688da7c0ee93cba0ef",
+      "categories": {}
+    },
+    "arc_easy": {
+      "label": "ARC-Easy",
+      "dataset": "allenai/ai2_arc",
+      "config": "ARC-Easy",
+      "split": "test",
+      "samples": 2376,
+      "shots": 0,
+      "primary": "acc_norm",
+      "metrics": {
+        "acc": {
+          "value": 0.4701178451178451,
+          "stderr": 0.010241444322886225,
+          "ci95": [
+            0.45011358796458134,
+            0.49021857177921657
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        },
+        "acc_norm": {
+          "value": 0.41919191919191917,
+          "stderr": 0.010124905282491126,
+          "ci95": [
+            0.3994976736933545,
+            0.43914703998001464
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/arc_easy.json",
+      "result_sha256": "ebe51440ccfc39c2b605ecb210d1aa3d7152cc2941906b72b35594a2afb56b34",
+      "categories": {}
+    },
+    "arc_challenge": {
+      "label": "ARC-Challenge",
+      "dataset": "allenai/ai2_arc",
+      "config": "ARC-Challenge",
+      "split": "test",
+      "samples": 1172,
+      "shots": 0,
+      "primary": "acc_norm",
+      "metrics": {
+        "acc": {
+          "value": 0.18344709897610922,
+          "stderr": 0.011310170179554583,
+          "ci95": [
+            0.16233527070001194,
+            0.20662727570978537
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        },
+        "acc_norm": {
+          "value": 0.23122866894197952,
+          "stderr": 0.01232085883477234,
+          "ci95": [
+            0.2079920763450836,
+            0.2562214064556877
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/arc_challenge.json",
+      "result_sha256": "f5efc3025bb25c50c5017030cb36f77de89afaf762d8990dc82304ef99454d58",
+      "categories": {}
+    },
+    "piqa": {
+      "label": "PIQA",
+      "dataset": "baber/piqa",
+      "config": "default",
+      "split": "validation",
+      "samples": 1838,
+      "shots": 0,
+      "primary": "acc_norm",
+      "metrics": {
+        "acc": {
+          "value": 0.5924918389553863,
+          "stderr": 0.01146449062565553,
+          "ci95": [
+            0.5698576660793783,
+            0.61474019839572
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        },
+        "acc_norm": {
+          "value": 0.5875952121871599,
+          "stderr": 0.011485407152743092,
+          "ci95": [
+            0.5649303971157125,
+            0.6098946392401159
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/piqa.json",
+      "result_sha256": "c2d4b2727900015aa72ab771afd02e18a7086d31e744e85d785881c5ac132b33",
+      "categories": {}
+    },
+    "winogrande": {
+      "label": "WinoGrande",
+      "dataset": "allenai/winogrande",
+      "config": "winogrande_xl",
+      "split": "validation",
+      "samples": 1267,
+      "shots": 0,
+      "primary": "acc",
+      "metrics": {
+        "acc": {
+          "value": 0.5138121546961326,
+          "stderr": 0.014047122916440271,
+          "ci95": [
+            0.4862910104211901,
+            0.5412497970942327
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/winogrande.json",
+      "result_sha256": "00c93cb02ff2d9638d4392d90a73a968694febf6d246b53b6b5195f17bf96bf3",
+      "categories": {}
+    },
+    "openbookqa": {
+      "label": "OpenBookQA",
+      "dataset": "allenai/openbookqa",
+      "config": "main",
+      "split": "test",
+      "samples": 500,
+      "shots": 0,
+      "primary": "acc_norm",
+      "metrics": {
+        "acc": {
+          "value": 0.168,
+          "stderr": 0.01673655354154187,
+          "ci95": [
+            0.13778823297684417,
+            0.20327432908962656
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        },
+        "acc_norm": {
+          "value": 0.306,
+          "stderr": 0.020629569998345414,
+          "ci95": [
+            0.26721346373113636,
+            0.3477447803679459
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/openbookqa.json",
+      "result_sha256": "398ce770d102c1d5b33ac91c684b3090c2f5d935253b7c1ef54b01e4a05c08e4",
+      "categories": {}
+    },
+    "boolq": {
+      "label": "BoolQ",
+      "dataset": "aps/super_glue",
+      "config": "boolq",
+      "split": "validation",
+      "samples": 3270,
+      "shots": 0,
+      "primary": "acc",
+      "metrics": {
+        "acc": {
+          "value": 0.5577981651376147,
+          "stderr": 0.008686430526114761,
+          "ci95": [
+            0.5407177130610681,
+            0.5747429788369947
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/boolq.json",
+      "result_sha256": "9a3b1d284692559f2af9eb3bc1bd099e6ed785ad49c0abcaca8527c1aab4f8ff",
+      "categories": {}
+    },
+    "lambada_openai": {
+      "label": "LAMBADA OpenAI",
+      "dataset": "EleutherAI/lambada_openai",
+      "config": "default",
+      "split": "test",
+      "samples": 5153,
+      "shots": 0,
+      "primary": "acc",
+      "metrics": {
+        "acc": {
+          "value": 0.16689307199689501,
+          "stderr": 0.005194952730633079,
+          "ci95": [
+            0.1569610474392711,
+            0.17732137574286708
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        },
+        "perplexity": {
+          "value": 325.20143973341675,
+          "stderr": 15.346256502549156
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/lambada_openai.json",
+      "result_sha256": "f73b22ca335b65a93dc0c7c84fb6d624d96bab4ac8033ee3d09ff64ae1be1627",
+      "categories": {}
+    },
+    "arithmark3": {
+      "label": "ArithMark-3",
+      "dataset": "AxiomicLabs/Arithmark-3.0",
+      "config": "default",
+      "split": "train",
+      "samples": 1000,
+      "shots": 0,
+      "primary": "acc_norm",
+      "metrics": {
+        "acc": {
+          "value": 0.338,
+          "stderr": 0.014965960710224385,
+          "ci95": [
+            0.3093514451469222,
+            0.36788842460239773
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        },
+        "acc_norm": {
+          "value": 0.341,
+          "stderr": 0.01499813134840284,
+          "ci95": [
+            0.31227731829126676,
+            0.37093959090713985
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/arithmark3.json",
+      "result_sha256": "02c6c465acd3c3b7472e5f33890f841bac021be2608c7a50fb4a8007e09fefd1",
+      "categories": {
+        "elementary_school_math_continuation::addition::grades_1_2::easy": {
+          "count": 128,
+          "acc_norm,none": 0.234375,
+          "acc_norm_stderr,none": 0.037589092002846054
+        },
+        "elementary_school_math_continuation::comparison::grades_2_3::medium": {
+          "count": 44,
+          "acc_norm,none": 0.25,
+          "acc_norm_stderr,none": 0.06603381797442179
+        },
+        "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": {
+          "count": 48,
+          "acc_norm,none": 0.3125,
+          "acc_norm_stderr,none": 0.06761023307918171
+        },
+        "elementary_school_math_continuation::data::grades_2_3::easy": {
+          "count": 43,
+          "acc_norm,none": 0.13953488372093023,
+          "acc_norm_stderr,none": 0.05346670079545209
+        },
+        "elementary_school_math_continuation::division::grades_3_4::medium": {
+          "count": 54,
+          "acc_norm,none": 0.3888888888888889,
+          "acc_norm_stderr,none": 0.06696300316010247
+        },
+        "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": {
+          "count": 50,
+          "acc_norm,none": 0.22,
+          "acc_norm_stderr,none": 0.05917804336345137
+        },
+        "elementary_school_math_continuation::geometry_area::grades_4_5::medium": {
+          "count": 52,
+          "acc_norm,none": 0.5,
+          "acc_norm_stderr,none": 0.07001400420140048
+        },
+        "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": {
+          "count": 45,
+          "acc_norm,none": 0.4888888888888889,
+          "acc_norm_stderr,none": 0.0753592220347252
+        },
+        "elementary_school_math_continuation::measurement::grades_2_3::easy": {
+          "count": 76,
+          "acc_norm,none": 0.35526315789473684,
+          "acc_norm_stderr,none": 0.05526315789473684
+        },
+        "elementary_school_math_continuation::money::grades_3_4::medium": {
+          "count": 64,
+          "acc_norm,none": 0.3125,
+          "acc_norm_stderr,none": 0.058397074018894594
+        },
+        "elementary_school_math_continuation::multiplication::grades_3_4::medium": {
+          "count": 74,
+          "acc_norm,none": 0.5675675675675675,
+          "acc_norm_stderr,none": 0.057983774751431
+        },
+        "elementary_school_math_continuation::patterns::grades_3_4::medium": {
+          "count": 53,
+          "acc_norm,none": 0.2641509433962264,
+          "acc_norm_stderr,none": 0.06113906319252699
+        },
+        "elementary_school_math_continuation::subtraction::grades_1_2::easy": {
+          "count": 117,
+          "acc_norm,none": 0.21367521367521367,
+          "acc_norm_stderr,none": 0.038058271647936806
+        },
+        "elementary_school_math_continuation::time::grades_2_3::easy": {
+          "count": 55,
+          "acc_norm,none": 0.9090909090909091,
+          "acc_norm_stderr,none": 0.03912104390108503
+        },
+        "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": {
+          "count": 46,
+          "acc_norm,none": 0.21739130434782608,
+          "acc_norm_stderr,none": 0.06148754619013457
+        },
+        "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": {
+          "count": 19,
+          "acc_norm,none": 0.2631578947368421,
+          "acc_norm_stderr,none": 0.10379087338771256
+        },
+        "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": {
+          "count": 32,
+          "acc_norm,none": 0.1875,
+          "acc_norm_stderr,none": 0.07010217197868432
+        }
+      }
+    },
+    "balanced_copa": {
+      "label": "Balanced COPA",
+      "dataset": "pkavumba/balanced-copa",
+      "config": "default",
+      "split": "train",
+      "samples": 1000,
+      "shots": 0,
+      "primary": "acc",
+      "metrics": {
+        "acc": {
+          "value": 0.533,
+          "stderr": 0.015784807891138876,
+          "ci95": [
+            0.5020104989134991,
+            0.5637369350264544
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/balanced_copa.json",
+      "result_sha256": "9abed3267bb970d308a99687f73347bf6093126c89fdf40f35f8e665a6f47c7c",
+      "categories": {}
+    },
+    "blimp": {
+      "label": "BLiMP",
+      "dataset": "nyu-mll/blimp",
+      "config": "67 minimal-pair subsets",
+      "split": "train",
+      "samples": 67000,
+      "shots": 0,
+      "primary": "acc",
+      "metrics": {
+        "acc": {
+          "value": 0.7317910447761194,
+          "stderr": 0.001517376416839114
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/blimp.json",
+      "result_sha256": "75189517422b1f0dc53fb4dbcf810b32ac9f40dfe64dab96df6e524ffe9318da",
+      "categories": {}
+    },
+    "commonsense_qa": {
+      "label": "CommonsenseQA",
+      "dataset": "tau/commonsense_qa",
+      "config": "default",
+      "split": "validation",
+      "samples": 1221,
+      "shots": 0,
+      "primary": "acc",
+      "metrics": {
+        "acc": {
+          "value": 0.19492219492219492,
+          "stderr": 0.011341478090883516,
+          "ci95": [
+            0.17367347122724505,
+            0.21808454395514162
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/commonsense_qa.json",
+      "result_sha256": "5968567f6503d5ba2924a34c4e7bd017c341dd7b98de53bddfbc80e21c82720b",
+      "categories": {}
+    },
+    "mmlu_continuation": {
+      "label": "MMLU continuation",
+      "dataset": "cais/mmlu",
+      "config": "57 subjects",
+      "split": "test",
+      "samples": 14042,
+      "shots": 0,
+      "primary": "acc",
+      "metrics": {
+        "acc": {
+          "value": 0.2521720552627831,
+          "stderr": 0.0036528205915989775
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/mmlu_continuation.json",
+      "result_sha256": "b9cbea26d9c7326596e7220d5fc4b75323207a2a25eee724155d8aca7e78b249",
+      "categories": {}
+    },
+    "sciq": {
+      "label": "SciQ (with support)",
+      "dataset": "allenai/sciq",
+      "config": "default",
+      "split": "test",
+      "samples": 1000,
+      "shots": 0,
+      "primary": "acc_norm",
+      "metrics": {
+        "acc": {
+          "value": 0.724,
+          "stderr": 0.01414298497574059,
+          "ci95": [
+            0.6954767082717307,
+            0.7508089039267406
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        },
+        "acc_norm": {
+          "value": 0.612,
+          "stderr": 0.015417317979911216,
+          "ci95": [
+            0.5814239245764211,
+            0.6417188815228144
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/sciq.json",
+      "result_sha256": "7cb4cd157ad4871d5063c07feedaa3976f63fb450e5e2f2cc856860b60cd9665",
+      "categories": {}
+    },
+    "truthfulqa_mc2": {
+      "label": "TruthfulQA MC2",
+      "dataset": "truthfulqa/truthful_qa",
+      "config": "multiple_choice",
+      "split": "validation",
+      "samples": 817,
+      "shots": 0,
+      "primary": "acc",
+      "metrics": {
+        "acc": {
+          "value": 0.4417644031264386,
+          "stderr": 0.01519901667714673
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/truthfulqa_mc2.json",
+      "result_sha256": "0c85b79db33e94c60bcf454318ecfdd19641326c1bd57d146833c1857429f5e4",
+      "categories": {}
+    },
+    "bananamind_base": {
+      "label": "BananaMind Base 1.1",
+      "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
+      "config": "default",
+      "split": "test",
+      "samples": 350,
+      "shots": 0,
+      "primary": "raw_accuracy",
+      "metrics": {
+        "raw_accuracy": {
+          "value": 0.45714285714285713,
+          "stderr": 0.02666588697376804,
+          "ci95": [
+            0.40570065529637145,
+            0.5095156110507492
+          ],
+          "ci_method": "Wilson 95%; item independence approximation"
+        }
+      },
+      "result_path": "runs/eval/pending-release-20261001/q-mod/attempt-001/bananamind_base.json",
+      "result_sha256": "7d1893178aee2b2c2f9a13b345c405a5b1cb0efed70f574d31e32d449904fb43",
+      "categories": {
+        "code_completion": {
+          "count": 50,
+          "raw_accuracy,none": 0.28,
+          "raw_accuracy_stderr,none": 0.06414269805898186
+        },
+        "commonsense": {
+          "count": 50,
+          "raw_accuracy,none": 0.5,
+          "raw_accuracy_stderr,none": 0.07142857142857142
+        },
+        "context_tracking": {
+          "count": 50,
+          "raw_accuracy,none": 0.16,
+          "raw_accuracy_stderr,none": 0.05237229365663817
+        },
+        "language_completion": {
+          "count": 50,
+          "raw_accuracy,none": 0.9,
+          "raw_accuracy_stderr,none": 0.04285714285714285
+        },
+        "logical_reasoning": {
+          "count": 50,
+          "raw_accuracy,none": 0.44,
+          "raw_accuracy_stderr,none": 0.07091242083423346
+        },
+        "quantitative": {
+          "count": 50,
+          "raw_accuracy,none": 0.26,
+          "raw_accuracy_stderr,none": 0.06266203485560375
+        },
+        "world_knowledge": {
+          "count": 50,
+          "raw_accuracy,none": 0.66,
+          "raw_accuracy_stderr,none": 0.0676726816132972
+        }
+      }
+    }
+  },
+  "manifests": {
+    "runs/eval/pending-release-20261001/q-mod/attempt-001": {
+      "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/checkpoints/q-mod",
+      "local_model": "q_mod",
+      "kind": "causal",
+      "tokenizer": "tokenizer",
+      "revision": "main",
+      "tokenizer_revision": null,
+      "protocol": "autoregressive likelihood",
+      "harness_version": "0.4.12",
+      "smoke_only": false,
+      "tasks": [
+        {
+          "task": "hellaswag",
+          "shots": 0,
+          "description": "Commonsense completion; acc and acc_norm",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "arc_easy",
+          "shots": 0,
+          "description": "AI2 ARC-Easy science questions; acc and acc_norm",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "arc_challenge",
+          "shots": 0,
+          "description": "AI2 ARC-Challenge; acc and acc_norm",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "piqa",
+          "shots": 0,
+          "description": "Physical commonsense; acc and acc_norm",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "winogrande",
+          "shots": 0,
+          "description": "Pronoun resolution; acc",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "openbookqa",
+          "shots": 0,
+          "description": "Science multiple choice; acc and acc_norm",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "boolq",
+          "shots": 0,
+          "description": "Yes/no reading comprehension; acc",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "lambada_openai",
+          "shots": 0,
+          "description": "Complete final word; acc and perplexity",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "arithmark3",
+          "shots": 0,
+          "description": "ArithMark-3.0 train-named evaluation split; token-normalized acc_norm",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "token_continuation"
+        },
+        {
+          "task": "balanced_copa",
+          "shots": 0,
+          "description": "Balanced COPA mirrored 1000-item train split; acc; cRia split inferred",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "blimp",
+          "shots": 0,
+          "description": "Full BLiMP group; grammatical minimal pairs",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "commonsense_qa",
+          "shots": 0,
+          "description": "CommonsenseQA; acc",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "mmlu_continuation",
+          "shots": 0,
+          "description": "MMLU cloze: full answer continuations; acc",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "sciq",
+          "shots": 0,
+          "description": "SciQ test with support passage; acc and acc_norm",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "truthfulqa_mc2",
+          "shots": 0,
+          "description": "TruthfulQA multiple choice; mc2",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "lm_eval"
+        },
+        {
+          "task": "bananamind_base",
+          "shots": 0,
+          "description": "BananaMind Base Bench 1.1 test; mean-token raw accuracy; gated",
+          "unsafe_code": false,
+          "rolling": false,
+          "engine": "token_continuation"
+        }
+      ],
+      "options": {
+        "checkpoint": "runs/eval/pending-release-20261001/checkpoints/q-mod",
+        "model": null,
+        "tokenizer": "tokenizer",
+        "revision": "main",
+        "tokenizer_revision": null,
+        "suite": "core",
+        "tasks": "hellaswag,arc_easy,arc_challenge,piqa,winogrande,openbookqa,boolq,lambada_openai,arithmark3,balanced_copa,blimp,commonsense_qa,mmlu_continuation,sciq,truthfulqa_mc2,bananamind_base",
+        "num_fewshot": null,
+        "device": "cuda:0",
+        "dtype": "bfloat16",
+        "allow_tf32": false,
+        "batch_size": 8,
+        "max_length": 2048,
+        "max_gen_toks": null,
+        "limit": null,
+        "seed": 1234,
+        "threads": 4,
+        "bootstrap_iters": 1000,
+        "trust_remote_code": false,
+        "allow_code_execution": false,
+        "apply_chat_template": false,
+        "log_samples": false,
+        "output": "/mnt/d/Projects/tiny_llm/runs/eval/pending-release-20261001/q-mod/attempt-001",
+        "tb_dir": "runs/q-mod-total-v1/tb",
+        "training_step": 15000,
+        "list": false,
+        "dry_run": false
+      },
+      "checkpoint_stamp": {
+        "model.safetensors": [
+          203692144,
+          1790864804000000000
+        ],
+        "config.json": [
+          808,
+          1790864803000000000
+        ],
+        "generation_config.json": [
+          195,
+          1790864803000000000
+        ]
+      },
+      "checkpoint_sha256": {
+        "config.json": "194e3135cfd3dc01fb1e01c1c2a801e19212af8a532c465448d2c40af1f64034",
+        "generation_config.json": "80da72cc13262a67c14c8b96f4f049f6e7890a027d808a02b8f0b0d48704c245",
+        "model.safetensors": "e5be4b27775d846ab2e65293a83f647d89ba2e0a606c00645fa892b21f6d5cd3"
+      },
+      "source_sha256": {
+        "src/tiny_llm/__init__.py": "8c7f075d51006a53fe1d645ef5a84d65db58ff4251e7544074fe29ee7549c14c",
+        "src/tiny_llm/benchmarks.py": "5d9f461366389bd1f926c6a6468192f3e2313c87caf5b1de27f407d563a858a4",
+        "src/tiny_llm/checkpoint.py": "ac9df5f8c981a47eb1f1e59d0687ff41a1f6c38ca74ced50f71ac446a58d6ef5",
+        "src/tiny_llm/comet_sync.py": "ea802bfb58dacad66f83b9e5ac5ccb330c190204a81c90c9e14aafe44c893c37",
+        "src/tiny_llm/comet_views.py": "3e03f4ae69f8edc2eaf5f946f5cc90f373ca1b9cc91f9dd2e9adae6d6a672689",
+        "src/tiny_llm/continuation.py": "4f4143c828d4f94a851bcbf8a466d3e1b3e9eaec1027970a426208cc0ff4f4db",
+        "src/tiny_llm/data.py": "1a52e38afef9af32463779cc2f471d9b7174a7c4c73753e74f485d526d3c6733",
+        "src/tiny_llm/diffusion_diagnostics.py": "2ae667a076b1d3f057a29d20f00559068eb73a073b3f99030e1d5b4c2f478fa5",
+        "src/tiny_llm/evaluation/__init__.py": "e49e80f7be3ab6e200f0db14b028e465de9caa8f6f705430eabce0c41b8322ba",
+        "src/tiny_llm/evaluation/adapters.py": "8e64e5542bbcb0f9ad448b320e712916a9559823987e298b22321a8d8e2f727f",
+        "src/tiny_llm/evaluation/catalog.py": "2fd29084ec780539023a38bcbfd7302931fa6be48624e227875f13518a1b986f",
+        "src/tiny_llm/evaluation/cli.py": "a322e39bf58ba52132249dae524e9772c3a8e1a5ba388f5c0267f8dc1ea18882",
+        "src/tiny_llm/evaluation/tasks/balanced_copa.yaml": "f47baa6a713e15f7a5600ed257f316f8c54252bf4977bbc716a6da7aa580f05d",
+        "src/tiny_llm/evaluation/tasks/balanced_copa_test.yaml": "dce21ad70b395d8d576c584be1b87ebae846e360e59878030645215a387b28e6",
+        "src/tiny_llm/evaluation/telemetry.py": "f4dad30574753739ea10ba71ef9a7ed9d9118a89a8202dee224961be27f13a48",
+        "src/tiny_llm/evaluation/token_continuation.py": "e8d88cd4aca51ca2fda07f2b799a54612d0e2964a98e05824a135bea1a533550",
+        "src/tiny_llm/losses.py": "592a05287369f6e5077b3faca404a9392de4772cf06389fb8aa7e36924e74f86",
+        "src/tiny_llm/models/__init__.py": "b94ab167b6eb9f9b455823eea382e67b5ac2cc7f9ef5f6de183a09b9650b9718",
+        "src/tiny_llm/models/diffusion/__init__.py": "c92eaf5986d604d8aaf1894a862a217cefa32ab25359175b88110e45a71c1b25",
+        "src/tiny_llm/models/diffusion/configuration_diffusion_lm.py": "3b303aba60f0a831c966f5cfa7f5441daa2584814e335646195bd6ceecfa6d0c",
+        "src/tiny_llm/models/diffusion/modeling_diffusion_lm.py": "4fc21c2aed1d5a231afe6f7634d9d8c3682b4e2abfc8fecef27b52136de77b2f",
+        "src/tiny_llm/models/looped/__init__.py": "4b46b426021c59ed313cb2f49afc1926ad2793d771ef90117e3fef0462957cff",
+        "src/tiny_llm/models/looped/configuration_looped_lm.py": "b8d99c52b98e0a97a4cae76ca4c20affd8f4d019ec38e0a9beefd5fad864dc25",
+        "src/tiny_llm/models/looped/modeling_looped_lm.py": "9bdb3c5e3aaae663d09e98055f95df1adbc42e0a4bf5dffb7f6c39242af41b9e",
+        "src/tiny_llm/models/moe_t5/__init__.py": "18e33fe38b2049521f573512ac8796afa209e1b25c2052784e1c927a7160d66c",
+        "src/tiny_llm/models/moe_t5/configuration_alicet5_moe.py": "4647b1e9ab9ce07887bffc1abc3beb3e691962a380b4cc237aae376490bb56b7",
+        "src/tiny_llm/models/moe_t5/modeling_alicet5_moe.py": "5f899c4726b120eed6c9bf6258ad319bb1ccea734645f27e7e6ccbc5057222ff",
+        "src/tiny_llm/models/prefixlm/__init__.py": "c94c3c861acba8d2147a3a6c46bccfc71ac8adc8d70757bede009598029d31ea",
+        "src/tiny_llm/models/prefixlm/configuration_prefix_lm.py": "20b3a62ae3628b5a64566ef19d2947c90f10c6159daad00dd13d18fc5910c42c",
+        "src/tiny_llm/models/prefixlm/modeling_prefix_lm.py": "f62a38211df175696f64945173d1e41d9f7fcf6152a7648020da8c91788e98fd",
+        "src/tiny_llm/models/q50m/__init__.py": "9d0dfec6fa0cfa6967e2f17a73d8fbeca69ba93b4a93651eda4d95626d5f5eb5",
+        "src/tiny_llm/models/q50m/configuration_q50m.py": "796806b2d5d982b633796bc7de53c3073b1b6e52d259f44ef798b63f55eaea92",
+        "src/tiny_llm/models/q50m/modeling_q50m.py": "b68abce1f19a36deb39c4ac4e520ffaab0c879d8a7cc2abef35f014bacc011f2",
+        "src/tiny_llm/models/qmod/__init__.py": "e8c3e10710ac1cceac72ea329db480e680e8bbaae2a92ec26c36c92a778552ad",
+        "src/tiny_llm/models/qmod/configuration_qmod.py": "0dfc9a4540697a9b15e423b9592df9fc435dcfc437ddad20dc6b6377ebebea40",
+        "src/tiny_llm/models/qmod/modeling_qmod.py": "3831e34a86309b91a7a0787b9471eaceb91aa4c267f8030eb2e6722a0fcab6f8",
+        "src/tiny_llm/models/qmoe/__init__.py": "cfb2bb24bc2f3d5b72b3890a393b0386cb25f127e375f5454255478a36dec32f",
+        "src/tiny_llm/models/qmoe/configuration_qmoe.py": "bba7c9ac16f444146a78c99e7a26b4a11864b215d2b673bf658d9930f2f4de2f",
+        "src/tiny_llm/models/qmoe/modeling_qmoe.py": "1d713e5e287f46dbf13b7cf9c562b4539efbf6231f7ceaa261f8cea1acd2768c",
+        "src/tiny_llm/models/qt5/__init__.py": "df943b42a9f339e006687bcd30a666838c1dec15c79751b944e5f7a2bc69186c",
+        "src/tiny_llm/models/qt5/configuration_qt5.py": "59709fe2c9fe08ccecea6633b29b3b47170fd30abcbee37952507d676446cd44",
+        "src/tiny_llm/models/qt5/modeling_qt5.py": "6269829cca6631824622df034ff7b544dbbe47790556c60b26e356b5279cceff",
+        "src/tiny_llm/models/t5/__init__.py": "e4d8838253800edbcac1f9a9b6dc875bcb3ffa2962a0c9044d066448ea10488e",
+        "src/tiny_llm/models/t5/configuration_alicet5.py": "81b1f594139f201af9afb25ba58df862432c15dedf57f6e340ee2af80ed0d44f",
+        "src/tiny_llm/models/t5/modeling_alicet5.py": "bd52ce0022668f2defb57770df229f148e070ea73becca9a2e681325114bfc20",
+        "src/tiny_llm/models/zarya/__init__.py": "651b19cdba422680ff16bf2b2f63be0c3e9045c44c410fb17dbf51df2a23bd55",
+        "src/tiny_llm/models/zarya/configuration_zarya_lm.py": "42a3e7a1b11b687bac34e00f9cd3cb59d1cd573eec7492e1c55834662d37ed3f",
+        "src/tiny_llm/models/zarya/modeling_zarya_lm.py": "73ab033860288f7367d5b912c79de908fd81a4e88f369a658b96dd2bcfcd7bfe",
+        "src/tiny_llm/telemetry.py": "7ee8f14f4545bfa0e5ac632440d49c7f86fd6812a16429f862ebe3cfe09e3545",
+        "src/tiny_llm/ul2.py": "f4d948d89d5e106965612971dd473a54b57719f811761f18457c6221d763b876",
+        "src/tiny_llm/zarya.py": "8b5084ab5086c75745115d3a9ba4f2fa9cfda8ad713f65d3f8a22e77db8c0941"
+      },
+      "packages": {
+        "lm_eval": "0.4.12",
+        "torch": "2.11.0+cu128",
+        "transformers": "5.17.0",
+        "datasets": "5.0.1",
+        "accelerate": "1.15.0"
+      },
+      "git_commit": "d2b5d7b5cbb913e4bed99b37a2fdcd9f0ae98dbb",
+      "git_dirty": true
+    }
+  },
+  "continuation": {
+    "result": {
+      "model": "q-mod",
+      "checkpoint_sha256": {
+        "model.safetensors": "e5be4b27775d846ab2e65293a83f647d89ba2e0a606c00645fa892b21f6d5cd3"
+      },
+      "corpus_sha256": "aca2f46735043bcfd0a44eca981d04627b9cdf74c4c9a04bf0856d04066f58fc",
+      "tokenizer_sha256": "fc71a154536f6c84a5bf729d06324032126603ef5183c0b4979407e4fad68a8d",
+      "blocks": 291,
+      "scored_tokens": 148992,
+      "dropped_tail_tokens": 335,
+      "protocol": "WikiText-2 raw test, newline-joined; tokenizer-specific nonoverlapping 1024-token blocks; 512 prefix + 512 scored suffix; full vocabulary. GPU BF16 and FP32 separately, TF32 off. Block bootstrap, 10000 resamples, seed 2026. PPL is not rolling or word PPL; different tokenizer results are not directly comparable.",
+      "measurements": {
+        "torch.bfloat16": {
+          "nll": 3.6892377994314502,
+          "nll_ci95": [
+            3.6496566288659666,
+            3.7284908163793307
+          ],
+          "ppl": 40.014336381234685,
+          "ppl_ci95": [
+            38.46145722721316,
+            41.616254177697236
+          ],
+          "block_nll": [
+            3.0334739685058594,
+            3.740220069885254,
+            3.628894805908203,
+            3.6683175563812256,
+            4.014724254608154,
+            3.891292095184326,
+            4.447037696838379,
+            4.0522780418396,
+            3.8349380493164062,
+            3.823822021484375,
+            3.633845329284668,
+            3.2401468753814697,
+            3.266814708709717,
+            3.5836172103881836,
+            3.8599278926849365,
+            3.666652202606201,
+            3.956968307495117,
+            3.8685550689697266,
+            3.7003097534179688,
+            3.9863815307617188,
+            3.7272982597351074,
+            3.4920151233673096,
+            4.134518623352051,
+            3.097790241241455,
+            3.377092123031616,
+            3.659186363220215,
+            3.691166877746582,
+            3.5041279792785645,
+            3.5620369911193848,
+            3.4219396114349365,
+            3.945758819580078,
+            4.074984073638916,
+            3.4408555030822754,
+            3.3952572345733643,
+            3.482008695602417,
+            4.128378868103027,
+            3.7565321922302246,
+            3.6840639114379883,
+            3.3337631225585938,
+            3.7233495712280273,
+            3.358358144760132,
+            3.8522372245788574,
+            3.4598348140716553,
+            3.858750343322754,
+            3.539834976196289,
+            3.7883243560791016,
+            3.972238302230835,
+            3.525768518447876,
+            3.943371295928955,
+            3.6065564155578613,
+            3.502178192138672,
+            3.6186389923095703,
+            3.925199270248413,
+            4.104352951049805,
+            3.556373357772827,
+            4.266332626342773,
+            3.2788915634155273,
+            3.275848150253296,
+            3.6967127323150635,
+            3.381783962249756,
+            3.3477587699890137,
+            3.2133116722106934,
+            3.3617265224456787,
+            2.6435787677764893,
+            3.3020172119140625,
+            3.4059677124023438,
+            3.1757893562316895,
+            3.1442549228668213,
+            3.4717979431152344,
+            3.4961135387420654,
+            3.559631824493408,
+            3.3489818572998047,
+            3.8820650577545166,
+            3.555074691772461,
+            3.8889122009277344,
+            3.5896503925323486,
+            3.769655227661133,
+            3.8484110832214355,
+            3.7232885360717773,
+            3.7955780029296875,
+            3.5623092651367188,
+            3.676947832107544,
+            3.684424877166748,
+            4.132134437561035,
+            4.085684776306152,
+            4.260656833648682,
+            3.877305507659912,
+            3.9520206451416016,
+            3.867370128631592,
+            4.114029884338379,
+            3.622154951095581,
+            3.7936620712280273,
+            3.7642085552215576,
+            3.8050858974456787,
+            3.628535270690918,
+            3.818166494369507,
+            3.595874309539795,
+            3.5458860397338867,
+            3.472426652908325,
+            4.054523468017578,
+            3.888446092605591,
+            3.6914849281311035,
+            3.0026016235351562,
+            3.6276140213012695,
+            3.5678868293762207,
+            3.2021985054016113,
+            3.4582393169403076,
+            3.1557910442352295,
+            3.4210400581359863,
+            3.4203357696533203,
+            3.337662696838379,
+            3.530256986618042,
+            2.94315767288208,
+            3.107532501220703,
+            3.6951563358306885,
+            3.2887697219848633,
+            3.3305134773254395,
+            3.8404202461242676,
+            3.448058605194092,
+            3.5578668117523193,
+            3.8797223567962646,
+            3.9302845001220703,
+            3.7769031524658203,
+            3.735970973968506,
+            4.003993511199951,
+            3.8857574462890625,
+            3.718710422515869,
+            3.696420669555664,
+            3.990670680999756,
+            3.836711883544922,
+            3.9563257694244385,
+            3.8588461875915527,
+            3.8682708740234375,
+            3.552096366882324,
+            4.007606506347656,
+            3.6597299575805664,
+            3.721909284591675,
+            3.916825532913208,
+            3.618872880935669,
+            4.049559593200684,
+            3.5828661918640137,
+            3.300783395767212,
+            4.254912376403809,
+            3.9667701721191406,
+            4.053187370300293,
+            4.266885757446289,
+            4.450619220733643,
+            4.25779914855957,
+            3.4460010528564453,
+            3.620326519012451,
+            3.8935182094573975,
+            3.7257468700408936,
+            3.1817684173583984,
+            3.5945611000061035,
+            3.14451265335083,
+            3.5998497009277344,
+            3.827526092529297,
+            3.8065614700317383,
+            4.318783760070801,
+            3.381446361541748,
+            3.701540470123291,
+            3.774550437927246,
+            3.0764577388763428,
+            3.0016181468963623,
+            2.8728439807891846,
+            3.309659242630005,
+            3.2308201789855957,
+            3.528296709060669,
+            3.1466598510742188,
+            4.0538530349731445,
+            3.837440252304077,
+            4.182062149047852,
+            4.07778263092041,
+            4.6475090980529785,
+            4.582226753234863,
+            3.9743824005126953,
+            4.288257598876953,
+            3.4931979179382324,
+            3.476074695587158,
+            4.683747291564941,
+            3.927859306335449,
+            4.299893379211426,
+            4.255132675170898,
+            4.288380146026611,
+            4.4493513107299805,
+            4.2332000732421875,
+            4.063198566436768,
+            4.357582092285156,
+            4.2598466873168945,
+            4.221273422241211,
+            4.55598258972168,
+            4.1570329666137695,
+            3.369076728820801,
+            4.477698802947998,
+            4.309594631195068,
+            3.729815721511841,
+            3.8138747215270996,
+            3.4916563034057617,
+            3.8918566703796387,
+            3.7358622550964355,
+            3.576416254043579,
+            3.2475414276123047,
+            3.634760856628418,
+            3.4506139755249023,
+            3.6861281394958496,
+            3.718350887298584,
+            3.434321880340576,
+            3.793008804321289,
+            3.359323501586914,
+            3.665397882461548,
+            3.6199021339416504,
+            3.3855361938476562,
+            3.4763875007629395,
+            3.5085606575012207,
+            3.3470630645751953,
+            3.6974496841430664,
+            3.754701614379883,
+            3.4645261764526367,
+            3.109065532684326,
+            3.254002094268799,
+            3.4032177925109863,
+            3.6319453716278076,
+            3.323887825012207,
+            3.2882790565490723,
+            3.724388837814331,
+            3.4665753841400146,
+            3.174009323120117,
+            3.791433095932007,
+            4.040876388549805,
+            3.4488320350646973,
+            3.8699331283569336,
+            3.6910126209259033,
+            3.5286316871643066,
+            3.798151731491089,
+            3.882476806640625,
+            3.844326972961426,
+            4.06152868270874,
+            3.8325114250183105,
+            3.337923526763916,
+            3.7876949310302734,
+            3.5918965339660645,
+            3.615245819091797,
+            3.2705817222595215,
+            3.1231560707092285,
+            3.385634422302246,
+            3.420309066772461,
+            3.39042329788208,
+            3.7529053688049316,
+            4.090065956115723,
+            3.57220196723938,
+            4.007654190063477,
+            4.212186813354492,
+            3.671332359313965,
+            3.585268974304199,
+            3.951240062713623,
+            3.822371006011963,
+            3.564332962036133,
+            3.556788921356201,
+            3.841912031173706,
+            3.7584228515625,
+            3.6320042610168457,
+            3.6407132148742676,
+            3.273137092590332,
+            3.4374094009399414,
+            3.547950506210327,
+            3.7734694480895996,
+            3.434028148651123,
+            3.8533287048339844,
+            3.884617567062378,
+            3.1127676963806152,
+            3.9401354789733887,
+            3.767160177230835,
+            3.10684871673584,
+            3.317154884338379,
+            3.160114288330078,
+            2.7552032470703125,
+            3.654399871826172,
+            3.568580389022827,
+            3.8528037071228027,
+            3.8489787578582764,
+            3.773097515106201,
+            3.8523926734924316,
+            3.409104108810425,
+            3.7001962661743164,
+            3.8040661811828613,
+            3.4476029872894287,
+            3.7821860313415527,
+            3.9857730865478516,
+            4.512575149536133,
+            4.251725673675537,
+            3.6333508491516113
+          ]
+        },
+        "torch.float32": {
+          "nll": 3.6888507895453277,
+          "nll_ci95": [
+            3.649182625079073,
+            3.728063138858559
+          ],
+          "ppl": 39.99885343368865,
+          "ppl_ci95": [
+            38.44323067090763,
+            41.598459646718396
+          ],
+          "block_nll": [
+            3.027667760848999,
+            3.738102912902832,
+            3.6281819343566895,
+            3.6695809364318848,
+            4.015194892883301,
+            3.894198417663574,
+            4.446779251098633,
+            4.050177574157715,
+            3.8330178260803223,
+            3.821993112564087,
+            3.6341664791107178,
+            3.2412233352661133,
+            3.2660343647003174,
+            3.5816807746887207,
+            3.8598198890686035,
+            3.66544771194458,
+            3.954576015472412,
+            3.8663859367370605,
+            3.700993299484253,
+            3.98309588432312,
+            3.726853370666504,
+            3.4925546646118164,
+            4.133160591125488,
+            3.097299098968506,
+            3.3766369819641113,
+            3.6585729122161865,
+            3.6912124156951904,
+            3.5034966468811035,
+            3.560659408569336,
+            3.421860456466675,
+            3.944514751434326,
+            4.076074123382568,
+            3.4416584968566895,
+            3.3954355716705322,
+            3.483755111694336,
+            4.1278228759765625,
+            3.755769968032837,
+            3.682060718536377,
+            3.33280086517334,
+            3.7252397537231445,
+            3.360107183456421,
+            3.8527755737304688,
+            3.460200786590576,
+            3.855809211730957,
+            3.5398995876312256,
+            3.7887706756591797,
+            3.971407890319824,
+            3.5239522457122803,
+            3.942335844039917,
+            3.6075119972229004,
+            3.4993295669555664,
+            3.6204614639282227,
+            3.9251532554626465,
+            4.106016159057617,
+            3.555708408355713,
+            4.265496730804443,
+            3.2797160148620605,
+            3.276913642883301,
+            3.6964378356933594,
+            3.384550094604492,
+            3.348212242126465,
+            3.213447332382202,
+            3.362593173980713,
+            2.643364429473877,
+            3.302009105682373,
+            3.4066994190216064,
+            3.1740009784698486,
+            3.146824836730957,
+            3.4738929271698,
+            3.4947752952575684,
+            3.5605530738830566,
+            3.346153497695923,
+            3.879917860031128,
+            3.5560824871063232,
+            3.8901753425598145,
+            3.5909934043884277,
+            3.768007516860962,
+            3.8438334465026855,
+            3.71993088722229,
+            3.7953481674194336,
+            3.5641112327575684,
+            3.6742324829101562,
+            3.6839911937713623,
+            4.1331987380981445,
+            4.086156845092773,
+            4.263089179992676,
+            3.8775670528411865,
+            3.950777530670166,
+            3.86856746673584,
+            4.112517833709717,
+            3.620612144470215,
+            3.7948694229125977,
+            3.7628049850463867,
+            3.8032991886138916,
+            3.6272482872009277,
+            3.817592144012451,
+            3.594144105911255,
+            3.547070264816284,
+            3.4730372428894043,
+            4.054729461669922,
+            3.8879971504211426,
+            3.6934356689453125,
+            3.000913381576538,
+            3.625133991241455,
+            3.568268060684204,
+            3.2034006118774414,
+            3.4547390937805176,
+            3.1572463512420654,
+            3.4198713302612305,
+            3.4206738471984863,
+            3.3378567695617676,
+            3.5324578285217285,
+            2.9415979385375977,
+            3.1075520515441895,
+            3.6945276260375977,
+            3.289034366607666,
+            3.333094596862793,
+            3.8393657207489014,
+            3.4496171474456787,
+            3.5584096908569336,
+            3.878268241882324,
+            3.929405689239502,
+            3.7761900424957275,
+            3.736541271209717,
+            3.9996914863586426,
+            3.8869969844818115,
+            3.717820167541504,
+            3.6949455738067627,
+            3.9903812408447266,
+            3.836249828338623,
+            3.959465980529785,
+            3.858759880065918,
+            3.868380308151245,
+            3.550398826599121,
+            4.00911808013916,
+            3.6598963737487793,
+            3.722276210784912,
+            3.916846990585327,
+            3.616698980331421,
+            4.050436019897461,
+            3.5778284072875977,
+            3.2959275245666504,
+            4.254424571990967,
+            3.9653892517089844,
+            4.0521240234375,
+            4.270989418029785,
+            4.4489850997924805,
+            4.254066467285156,
+            3.445030689239502,
+            3.61728572845459,
+            3.894674777984619,
+            3.7256832122802734,
+            3.181084632873535,
+            3.5941505432128906,
+            3.144176483154297,
+            3.5967330932617188,
+            3.829509735107422,
+            3.805874824523926,
+            4.318291664123535,
+            3.3804588317871094,
+            3.7016525268554688,
+            3.7745108604431152,
+            3.076606273651123,
+            2.9988205432891846,
+            2.872002601623535,
+            3.307823896408081,
+            3.2268385887145996,
+            3.525599956512451,
+            3.147146224975586,
+            4.053147315979004,
+            3.8385915756225586,
+            4.182605743408203,
+            4.073055267333984,
+            4.646452903747559,
+            4.581335544586182,
+            3.976539134979248,
+            4.287915229797363,
+            3.492595911026001,
+            3.4744696617126465,
+            4.682605743408203,
+            3.9282143115997314,
+            4.29866886138916,
+            4.253904342651367,
+            4.2884039878845215,
+            4.448787689208984,
+            4.234960556030273,
+            4.059430122375488,
+            4.357001781463623,
+            4.257026672363281,
+            4.221226692199707,
+            4.557419776916504,
+            4.155788898468018,
+            3.3654861450195312,
+            4.480966567993164,
+            4.309287071228027,
+            3.730501174926758,
+            3.8162360191345215,
+            3.4917783737182617,
+            3.889458656311035,
+            3.737387180328369,
+            3.574091911315918,
+            3.245443820953369,
+            3.635066032409668,
+            3.4508020877838135,
+            3.685305595397949,
+            3.7202224731445312,
+            3.436110019683838,
+            3.793421745300293,
+            3.3612964153289795,
+            3.664384603500366,
+            3.6176421642303467,
+            3.384169578552246,
+            3.4765501022338867,
+            3.50846004486084,
+            3.3470969200134277,
+            3.69547700881958,
+            3.752682685852051,
+            3.464268445968628,
+            3.1101222038269043,
+            3.250185489654541,
+            3.4053843021392822,
+            3.6328437328338623,
+            3.3243253231048584,
+            3.287609100341797,
+            3.7263553142547607,
+            3.465762138366699,
+            3.1718688011169434,
+            3.7949929237365723,
+            4.041664123535156,
+            3.4488463401794434,
+            3.8704917430877686,
+            3.690990924835205,
+            3.5286307334899902,
+            3.799253463745117,
+            3.8777198791503906,
+            3.8435721397399902,
+            4.059334754943848,
+            3.8340835571289062,
+            3.3388919830322266,
+            3.788696050643921,
+            3.5927886962890625,
+            3.6131863594055176,
+            3.2687082290649414,
+            3.120223045349121,
+            3.3863675594329834,
+            3.4221267700195312,
+            3.3879547119140625,
+            3.7518320083618164,
+            4.08969783782959,
+            3.574854850769043,
+            4.006902694702148,
+            4.210338592529297,
+            3.671088695526123,
+            3.586747407913208,
+            3.94930362701416,
+            3.821291923522949,
+            3.565288782119751,
+            3.5583267211914062,
+            3.840996742248535,
+            3.755756378173828,
+            3.632883071899414,
+            3.638172149658203,
+            3.272749185562134,
+            3.437190294265747,
+            3.547724723815918,
+            3.7728371620178223,
+            3.4354710578918457,
+            3.8511710166931152,
+            3.8856048583984375,
+            3.1140756607055664,
+            3.939563751220703,
+            3.765720844268799,
+            3.105600118637085,
+            3.3165178298950195,
+            3.1615777015686035,
+            2.7535500526428223,
+            3.6520907878875732,
+            3.569089889526367,
+            3.8519628047943115,
+            3.850273609161377,
+            3.775264263153076,
+            3.8534600734710693,
+            3.4068784713745117,
+            3.702049732208252,
+            3.802506923675537,
+            3.449129581451416,
+            3.781672954559326,
+            3.984006881713867,
+            4.511618137359619,
+            4.252767562866211,
+            3.6308560371398926
+          ]
+        }
+      }
+    },
+    "path": "runs/eval/pending-release-20261001/q-mod/continuation.json",
+    "sha256": "3b31fcaa4f71b0f317100a1b37b94481bb3773046676f909dec2f545c54ef40b"
+  },
+  "mean_16": 0.41461276742189673,
+  "mean_16_percent": 41.461276742189675,
+  "tensorboard_verified_points": 1244,
+  "training_history": {
+    "train/grad_clipped_fraction": {
+      "points": 750,
+      "first_step": 20,
+      "last_step": 15000
+    },
+    "train/grad_norm_pre_clip": {
+      "points": 750,
+      "first_step": 20,
+      "last_step": 15000
+    },
+    "train/loss": {
+      "points": 750,
+      "first_step": 20,
+      "last_step": 15000
+    },
+    "train/lr": {
+      "points": 750,
+      "first_step": 20,
+      "last_step": 15000
+    },
+    "train/mod_router_bce_raw": {
+      "points": 750,
+      "first_step": 20,
+      "last_step": 15000
+    },
+    "train/mod_router_bce_weighted": {
+      "points": 750,
+      "first_step": 20,
+      "last_step": 15000
+    },
+    "train/token_ce": {
+      "points": 750,
+      "first_step": 20,
+      "last_step": 15000
+    }
+  },
+  "tb": "runs/q-mod-total-v1/tb",
+  "external_reference": false,
+  "protocol": "Full selected splits; lm-eval 0.4.12; seed 1234; BF16 on RTX 5070 Ti;\ncontext cap 2048 (ArithMark 1024), TF32 disabled on new runs (historical core flag unrecorded), no chat template and no added\nfew-shot examples. TruthfulQA retains the harness's fixed six-QA preamble.\nArithMark and BananaMind normalize by continuation token count; ordinary harness\nacc_norm uses its own length normalization. BananaMind is raw accuracy, not Elo.\nSciQ includes the support passage. Balanced COPA uses the mirrored 1000-item\ntrain-named evaluation split; cRia's split was inferred, not confirmed.\nMMLU scores full answer continuations across 57 subjects, weighted by item count;\nBLiMP averages 67 equal-sized minimal-pair subsets. Standard errors are retained\nfrom each evaluator. Wilson intervals are reported only where the runner logged\nbinary item accuracy; MC2 is probability mass, not binary accuracy. These\nintervals do not model dependence between paired/templated examples or training\nseed variation. UL2, PrefixLM and experimental diffusion PLL use their documented\nconditional scoring protocols; PLL exposes the other answer tokens and is not\nautoregressive likelihood. cRia's published scores used a different precision\nand benchmark-adapted checkpoint; this completes our comparison coverage, not\nan independent reproduction of cRia or an official leaderboard submission."
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
+  "do_sample": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5be4b27775d846ab2e65293a83f647d89ba2e0a606c00645fa892b21f6d5cd3
+size 203692144

modeling_q50m.py

@@ -0,0 +1,453 @@
+"""Modeling for the Q-50M architecture (replica of q-project/Q-50M-Base).
+
+Self-contained implementation of the exact architecture described in the
+model card: Mistral-shaped 512-wide 10-layer decoder, GQA 8q/2kv, per-head
+QK-Norm, RoPE in every layer (configurable NoPE), scalar content-dependent
+gates (gate = multiplier * sigmoid(Wx)) on attention and MLP residual
+branches, SwiGLU MLP, tied input/output embeddings, final RMSNorm.
+
+Follows the transformers v5 modeling pattern (DynamicCache, create_causal_mask,
+GradientCheckpointingLayer, GenerationMixin).
+"""
+
+from collections.abc import Callable
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.generation import GenerationMixin
+from transformers.masking_utils import create_causal_mask
+from transformers.modeling_layers import GradientCheckpointingLayer
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from transformers.processing_utils import Unpack
+from transformers.utils import TransformersKwargs
+
+try:
+    from .configuration_q50m import Q50MConfig
+except ImportError:  # standalone use inside an HF model directory
+    from configuration_q50m import Q50MConfig
+
+
+def _rotate_half(x: torch.Tensor) -> torch.Tensor:
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def _apply_rotary_pos_emb(q, k, cos, sin):
+    # cos/sin: (batch, seq, head_dim)
+    cos = cos.unsqueeze(1)
+    sin = sin.unsqueeze(1)
+    return q * cos + _rotate_half(q) * sin, k * cos + _rotate_half(k) * sin
+
+
+class Q50MRMSNorm(nn.Module):
+    def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return (self.weight * hidden_states.to(input_dtype)).to(input_dtype)
+
+    def extra_repr(self):
+        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+class Q50MRotaryEmbedding(nn.Module):
+    def __init__(self, config: Q50MConfig, device=None):
+        super().__init__()
+        self.config = config
+        inv_freq = 1.0 / (
+            config.rope_theta
+            ** (torch.arange(0, config.head_dim, 2, dtype=torch.float32, device=device) / config.head_dim)
+        )
+        self.inv_freq = nn.Buffer(inv_freq, persistent=True)
+
+    @torch.no_grad()
+    def forward(self, x, position_ids):
+        inv_freq_expanded = (
+            self.inv_freq[None, :, None].expand(position_ids.shape[0], -1, 1)
+            .to(dtype=torch.float32, device=x.device)
+        )
+        position_ids_expanded = position_ids[:, None, :].float()
+        freqs = (inv_freq_expanded @ position_ids_expanded).transpose(1, 2)
+        emb = torch.cat((freqs, freqs), dim=-1)
+        return emb.cos().to(dtype=x.dtype), emb.sin().to(dtype=x.dtype)
+
+
+class Q50MScalarGate(nn.Module):
+    """Content-dependent scalar gate: output = branch * multiplier * sigmoid(W x)."""
+
+    def __init__(self, hidden_size: int, multiplier: float = 2.0):
+        super().__init__()
+        self.projection = nn.Linear(hidden_size, 1, bias=False)
+        self.multiplier = multiplier
+
+    def forward(self, branch: torch.Tensor, residual_input: torch.Tensor) -> torch.Tensor:
+        return branch * self.multiplier * torch.sigmoid(self.projection(residual_input))
+
+    def reset_parameters(self):
+        nn.init.zeros_(self.projection.weight)
+
+
+def _repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+def _eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    scaling: float,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    key_states = _repeat_kv(key, module.num_key_value_groups)
+    value_states = _repeat_kv(value, module.num_key_value_groups)
+    attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
+    if attention_mask is not None:
+        attn_weights = attn_weights + attention_mask
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+    attn_output = torch.matmul(attn_weights, value_states)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+    return attn_output, attn_weights
+
+
+def _sdpa_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    scaling: float,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    """Local SDPA wrapper. HF's generic SDPA interface falls back to the math
+    backend for GQA shapes (q heads != kv heads), which materializes
+    (batch, heads, seq, seq) fp32 attention scores. Expanding kv first keeps
+    the fused flash / memory-efficient kernels eligible."""
+    n_rep = getattr(module, "num_key_value_groups", 1)
+    if n_rep > 1:
+        key = _repeat_kv(key, n_rep)
+        value = _repeat_kv(value, n_rep)
+    is_causal = (
+        attention_mask is None
+        and getattr(module, "is_causal", False)
+        and query.shape[2] > 1
+    )
+    attn_output = nn.functional.scaled_dot_product_attention(
+        query,
+        key,
+        value,
+        attn_mask=attention_mask,
+        dropout_p=dropout,
+        is_causal=is_causal,
+        scale=scaling,
+    )
+    return attn_output.transpose(1, 2).contiguous(), None
+
+
+class Q50MAttention(nn.Module):
+    def __init__(self, config: Q50MConfig, layer_idx: int):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.head_dim = config.head_dim
+        self.num_heads = config.num_attention_heads
+        self.num_kv_heads = config.num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_kv_heads
+        self.scaling = self.head_dim**-0.5
+        self.attention_dropout = config.attention_dropout
+        self.is_causal = True
+        self.use_rope = layer_idx not in config.nope_layers
+
+        inner = self.num_heads * self.head_dim
+        kv_inner = self.num_kv_heads * self.head_dim
+        self.q_proj = nn.Linear(config.hidden_size, inner, bias=False)
+        self.k_proj = nn.Linear(config.hidden_size, kv_inner, bias=False)
+        self.v_proj = nn.Linear(config.hidden_size, kv_inner, bias=False)
+        self.o_proj = nn.Linear(inner, config.hidden_size, bias=False)
+        self.q_norm = Q50MRMSNorm(self.head_dim, config.rms_norm_eps) if config.qk_norm else None
+        self.k_norm = Q50MRMSNorm(self.head_dim, config.rms_norm_eps) if config.qk_norm else None
+        self.output_gate = (
+            Q50MScalarGate(config.hidden_size, config.gate_multiplier)
+            if config.attention_scalar_gate
+            else None
+        )
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
+        attention_mask: torch.Tensor | None = None,
+        past_key_values: Cache | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+        q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        if self.q_norm is not None:
+            q = self.q_norm(q)
+            k = self.k_norm(k)
+        if self.use_rope:
+            cos, sin = position_embeddings
+            q, k = _apply_rotary_pos_emb(q, k, cos, sin)
+
+        if past_key_values is not None:
+            k, v = past_key_values.update(k, v, self.layer_idx)
+
+        if self.config._attn_implementation == "sdpa":
+            attention_interface: Callable = _sdpa_attention_forward
+        else:
+            attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface(
+                self.config._attn_implementation, _eager_attention_forward
+            )
+        attn_output, attn_weights = attention_interface(
+            self,
+            q, k, v,
+            attention_mask,
+            dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scaling,
+            **kwargs,
+        )
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        attn_output = self.o_proj(attn_output)
+        if self.output_gate is not None:
+            attn_output = self.output_gate(attn_output, hidden_states)
+        return attn_output, attn_weights
+
+
+class Q50MMLP(nn.Module):
+    def __init__(self, config: Q50MConfig):
+        super().__init__()
+        self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
+        self.output_gate = (
+            Q50MScalarGate(config.hidden_size, config.gate_multiplier)
+            if config.mlp_scalar_gate
+            else None
+        )
+
+    def forward(self, hidden_states):
+        output = self.down_proj(F.silu(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))
+        if self.output_gate is not None:
+            output = self.output_gate(output, hidden_states)
+        return output
+
+
+class Q50MDecoderLayer(GradientCheckpointingLayer):
+    def __init__(self, config: Q50MConfig, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.self_attn = Q50MAttention(config, layer_idx)
+        self.mlp = Q50MMLP(config)
+        self.input_layernorm = Q50MRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = Q50MRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        use_cache: bool | None = False,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> torch.Tensor:
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+        hidden_states, _ = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            position_embeddings=position_embeddings,
+            **kwargs,
+        )
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+        return hidden_states
+
+
+class Q50MPreTrainedModel(PreTrainedModel):
+    config_class = Q50MConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Q50MDecoderLayer"]
+    _skip_keys_device_placement = ["past_key_values"]
+    _supports_sdpa = True
+    _supports_flash_attn = False
+    _supports_flex_attn = False
+
+
+class Q50MModel(Q50MPreTrainedModel):
+    def __init__(self, config: Q50MConfig):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList(
+            [Q50MDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self.norm = Q50MRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.rotary_emb = Q50MRotaryEmbedding(config=config)
+        self.gradient_checkpointing = False
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        use_cache: bool | None = None,
+        rotary_position_ids: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutputWithPast:
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache(config=self.config)
+
+        standard_positions = position_ids is None
+        if position_ids is None:
+            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+            position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
+            position_ids = position_ids.unsqueeze(0)
+
+        # Unpadded, uncached SDPA can use its native causal flag, including
+        # under torch.compile where HF otherwise materializes a dense mask.
+        causal_mask = None if (
+            attention_mask is None and past_key_values is None and standard_positions
+            and self.config._attn_implementation == "sdpa"
+        ) else create_causal_mask(
+            config=self.config,
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            position_ids=position_ids,
+        )
+
+        hidden_states = inputs_embeds
+        # Slotted Zarya keeps source positions for RoPE but attends causally in
+        # the permuted sequence order. Never let HF infer packed-sequence masks
+        # from those nonmonotonic rotary positions.
+        position_embeddings = self.rotary_emb(
+            hidden_states, position_ids=position_ids if rotary_position_ids is None else rotary_position_ids)
+
+        for decoder_layer in self.layers[: self.config.num_hidden_layers]:
+            if self.gradient_checkpointing and self.training:
+                hidden_states = self._gradient_checkpointing_func(
+                    decoder_layer.forward,
+                    hidden_states,
+                    causal_mask,
+                    position_ids,
+                    past_key_values,
+                    use_cache,
+                    position_embeddings,
+                )
+            else:
+                hidden_states = decoder_layer(
+                    hidden_states,
+                    attention_mask=causal_mask,
+                    position_embeddings=position_embeddings,
+                    position_ids=position_ids,
+                    past_key_values=past_key_values,
+                    use_cache=use_cache,
+                    **kwargs,
+                )
+
+        hidden_states = self.norm(hidden_states)
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=past_key_values,
+        )
+
+
+class Q50MForCausalLM(Q50MPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
+
+    def __init__(self, config: Q50MConfig):
+        super().__init__(config)
+        self.model = Q50MModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        use_cache: bool | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> CausalLMOutputWithPast:
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            **kwargs,
+        )
+        hidden_states = outputs.last_hidden_state
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+        loss = None
+        if labels is not None:
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss = F.cross_entropy(
+                shift_logits.view(-1, shift_logits.size(-1)),
+                shift_labels.view(-1),
+                ignore_index=-100,
+            )
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+        )

modeling_qmod.py

@@ -0,0 +1,259 @@
+"""Sparse whole-block routing with top-k training and causal threshold inference.
+
+Raposo et al., arXiv:2404.02258, sections 3.4/3.5 (same-router BCE option).
+Only selected tokens enter attention AND SwiGLU. Original RoPE positions and
+sequence order survive packing. Eval never uses sequence-wide top-k decisions.
+"""
+import copy
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+from transformers.cache_utils import DynamicCache
+from transformers.modeling_outputs import BaseModelOutputWithPast
+
+from .modeling_q50m import (
+    Q50MDecoderLayer, Q50MForCausalLM, Q50MPreTrainedModel,
+    Q50MRMSNorm, Q50MRotaryEmbedding,
+)
+from .configuration_qmod import QMoDConfig
+
+
+class QMoDCache(DynamicCache):
+    """Packed per-layer KV with source positions, validity and a global clock.
+
+Padding between batch members/chunks can remain in storage; masked entries
+never become keys. The clock counts source tokens, not sparse KV entries.
+"""
+    def __init__(self, config):
+        super().__init__(config=config)
+        self.seen_tokens = 0
+        self.token_positions = [None] * config.num_hidden_layers
+        self.valid_tokens = [None] * config.num_hidden_layers
+
+    def get_seq_length(self, layer_idx=0):
+        return self.seen_tokens
+
+    def append_metadata(self, layer_idx, positions, valid):
+        old = self.token_positions[layer_idx]
+        self.token_positions[layer_idx] = positions if old is None else torch.cat((old, positions), -1)
+        old_valid = self.valid_tokens[layer_idx]
+        self.valid_tokens[layer_idx] = valid if old_valid is None else torch.cat((old_valid, valid), -1)
+        return self.token_positions[layer_idx], self.valid_tokens[layer_idx]
+
+    def reorder_cache(self, beam_idx):
+        super().reorder_cache(beam_idx)
+        for i, positions in enumerate(self.token_positions):
+            if positions is not None:
+                self.token_positions[i] = positions.index_select(0, beam_idx.to(positions.device))
+                self.valid_tokens[i] = self.valid_tokens[i].index_select(0, beam_idx.to(positions.device))
+
+    def batch_repeat_interleave(self, repeats):
+        super().batch_repeat_interleave(repeats)
+        for i, positions in enumerate(self.token_positions):
+            if positions is not None:
+                self.token_positions[i] = positions.repeat_interleave(repeats, dim=0)
+                self.valid_tokens[i] = self.valid_tokens[i].repeat_interleave(repeats, dim=0)
+
+    def batch_select_indices(self, indices):
+        self.reorder_cache(indices)
+
+    def crop(self, max_length):
+        raise NotImplementedError("QMoD packed cache does not support assisted-decoding rollback")
+
+    def reset(self):
+        # DynamicLayer.reset retains storage, which would desynchronize metadata.
+        for layer in self.layers:
+            if layer.is_initialized:
+                layer.keys = layer.keys[..., :0, :]
+                layer.values = layer.values[..., :0, :]
+        self.seen_tokens = 0
+        self.token_positions = [None] * len(self.layers)
+        self.valid_tokens = [None] * len(self.layers)
+
+
+def gather_tokens(x, indices):
+    return x.gather(1, indices[..., None].expand(-1, -1, x.shape[-1]))
+
+
+class QMoDDecoderLayer(Q50MDecoderLayer):
+    def __init__(self, config, layer_idx):
+        block_config = copy.deepcopy(config)
+        self.routed = layer_idx in config.mod_layers
+        if self.routed:
+            block_config.intermediate_size = config.mod_intermediate_size
+        super().__init__(block_config, layer_idx)
+        self.config = config
+        self.layer_idx = layer_idx
+        self.router = nn.Linear(config.hidden_size, 1, bias=False) if self.routed else None
+
+    def route(self, hidden_states, valid):
+        # FP32 router and BCE, also when the block runs in BF16 autocast.
+        with torch.autocast(device_type=hidden_states.device.type, enabled=False):
+            logits = F.linear(hidden_states.float(), self.router.weight.float()).squeeze(-1)
+        if self.training:
+            # Deterministic tie breaking preserves checkpoint replay. Capacity is
+            # per valid sequence, never per batch and never allocated to padding.
+            counts = torch.ceil(valid.sum(-1) * self.config.mod_capacity).long()
+            order = logits.detach().masked_fill(~valid, -torch.inf).argsort(dim=-1, descending=True, stable=True)
+            ranks = torch.empty_like(order).scatter(1, order, torch.arange(order.shape[1], device=order.device).expand_as(order))
+            selected = (ranks < counts[:, None]) & valid
+            bce = F.binary_cross_entropy_with_logits(logits, selected.float(), reduction="none")
+            aux = (bce * valid).sum() / valid.sum().clamp_min(1)
+        else:
+            # No top-k, capacity clipping or sequence normalization in eval:
+            # changing future tokens must not alter any prefix routing decision.
+            selected = (logits > 0) & valid
+            aux = logits.new_zeros(())
+        return logits, selected, aux
+
+    def forward(self, hidden_states, valid, token_positions, position_embeddings, past_key_values=None):
+        batch, seq, _ = hidden_states.shape
+        zero = hidden_states.new_zeros((), dtype=torch.float32)
+        if self.routed:
+            logits, selected, aux = self.route(hidden_states, valid)
+            count = selected.sum(-1)
+            width = int(count.max().item())
+            stats = torch.stack((selected.sum() / valid.sum().clamp_min(1),
+                                 (((logits > 0) != selected) & valid).sum() / valid.sum().clamp_min(1))).detach()
+            if width == 0:
+                return hidden_states, aux, stats
+            # Sort by ORIGINAL time, not router score. Surplus entries are dummy
+            # slots for shorter batch members, excluded from keys and scatter.
+            order = torch.arange(seq, device=hidden_states.device).expand(batch, -1)
+            indices = order.masked_fill(~selected, seq).sort(-1).values[:, :width].clamp_max(seq - 1)
+            packed_valid = torch.arange(width, device=hidden_states.device)[None, :] < count[:, None]
+            packed = gather_tokens(hidden_states, indices)
+            positions = token_positions.gather(1, indices)
+            rotary = tuple(gather_tokens(x, indices) for x in position_embeddings)
+        else:
+            aux, stats = zero, torch.stack((zero, zero))
+            packed, packed_valid, positions, rotary = hidden_states, valid, token_positions, position_embeddings
+
+        key_positions, key_valid = positions, packed_valid
+        if past_key_values is not None:
+            key_positions, key_valid = past_key_values.append_metadata(self.layer_idx, positions, packed_valid)
+        # The common unpadded training path can use fused SDPA's causal flag.
+        mask = None
+        if not (past_key_values is None and self.config._attn_implementation == "sdpa"
+                and bool(packed_valid.all())):
+            allowed = (key_positions[:, None, :] <= positions[:, :, None]) & key_valid[:, None, :]
+            mask = packed.new_zeros((batch, 1, packed.shape[1], key_positions.shape[1]))
+            mask.masked_fill_(~allowed[:, None], torch.finfo(packed.dtype).min)
+        transformed = super().forward(packed, attention_mask=mask,
+                                      position_embeddings=rotary, past_key_values=past_key_values)
+        if not self.routed:
+            return torch.where(valid[..., None], transformed, hidden_states), aux, stats
+        # Weight the residual UPDATE, never the identity/skip path. Raw scalar
+        # weights follow equation (1); sigmoid is only the BCE interpretation.
+        update = (transformed - packed) * logits.gather(1, indices)[..., None].to(packed.dtype)
+        update = update * packed_valid[..., None]
+        result = hidden_states.scatter_add(1, indices[..., None].expand_as(update), update)
+        return result, aux, stats
+
+
+class QMoDModel(Q50MPreTrainedModel):
+    config_class = QMoDConfig
+    _no_split_modules = ["QMoDDecoderLayer"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
+        self.layers = nn.ModuleList([QMoDDecoderLayer(config, i) for i in range(config.num_hidden_layers)])
+        self.norm = Q50MRMSNorm(config.hidden_size, config.rms_norm_eps)
+        self.rotary_emb = Q50MRotaryEmbedding(config)
+        self.gradient_checkpointing = False
+        self.router_loss = None
+        self.last_router_metrics = {}
+        self.post_init()
+
+    def forward(self, input_ids=None, attention_mask=None, position_ids=None,
+                past_key_values=None, inputs_embeds=None, use_cache=None, **kwargs):
+        if (input_ids is None) == (inputs_embeds is None):
+            raise ValueError("Specify exactly one of input_ids or inputs_embeds")
+        if self.training and (use_cache or past_key_values is not None):
+            raise ValueError("MoD top-k training does not support a KV cache; use model.eval() for inference")
+        if past_key_values is not None and not isinstance(past_key_values, QMoDCache):
+            raise ValueError("QMoD requires its own packed QMoDCache")
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+        batch, seq, _ = inputs_embeds.shape
+        if seq == 0:
+            raise ValueError("Input sequence must not be empty")
+        if use_cache and past_key_values is None:
+            past_key_values = QMoDCache(self.config)
+        past = 0 if past_key_values is None else past_key_values.get_seq_length()
+        positions = torch.arange(past, past + seq, device=inputs_embeds.device).expand(batch, -1)
+        valid = torch.ones((batch, seq), dtype=torch.bool, device=inputs_embeds.device)
+        if attention_mask is not None:
+            if attention_mask.ndim != 2 or attention_mask.shape != (batch, past + seq):
+                raise ValueError("QMoD expects a 2D attention_mask covering the full cached + current sequence")
+            valid = attention_mask[:, -seq:].to(device=inputs_embeds.device, dtype=torch.bool)
+        if position_ids is None:
+            position_ids = positions if attention_mask is None else attention_mask.long().cumsum(-1)[:, -seq:] - 1
+            position_ids = position_ids.clamp_min(0)
+        position_ids = position_ids.to(inputs_embeds.device).expand(batch, seq)
+        rotary = self.rotary_emb(inputs_embeds, position_ids)
+        hidden = inputs_embeds
+        losses = []
+        self.router_loss = None
+        self.last_router_metrics = {}
+        for i, layer in enumerate(self.layers):
+            if self.gradient_checkpointing and self.training:
+                hidden, aux, stats = self._gradient_checkpointing_func(
+                    layer.forward, hidden, valid, positions, rotary)
+            else:
+                hidden, aux, stats = layer(hidden, valid, positions, rotary, past_key_values=past_key_values)
+            if layer.routed:
+                losses.append(aux)
+                self.last_router_metrics[f"mod/layer_{i}/selected_fraction"] = stats[0]
+                self.last_router_metrics[f"mod/layer_{i}/threshold_disagreement"] = stats[1]
+        if self.training:
+            self.router_loss = torch.stack(losses).mean() if losses else hidden.sum() * 0
+        if past_key_values is not None:
+            past_key_values.seen_tokens += seq
+        return BaseModelOutputWithPast(last_hidden_state=self.norm(hidden), past_key_values=past_key_values)
+
+
+class QMoDForCausalLM(Q50MForCausalLM):
+    config_class = QMoDConfig
+    _no_split_modules = ["QMoDDecoderLayer"]
+
+    def __init__(self, config):
+        Q50MPreTrainedModel.__init__(self, config)
+        self.model = QMoDModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.post_init()
+
+    @classmethod
+    def _supports_default_dynamic_cache(cls):
+        return False
+
+    def forward(self, *args, **kwargs):
+        out = super().forward(*args, **kwargs)
+        if out.loss is not None and self.training:
+            out.loss = out.loss + self.consume_router_loss()
+        return out
+
+    def consume_router_loss(self, metrics=None):
+        aux = self.model.router_loss
+        if aux is None:
+            raise RuntimeError("Consume router loss once after a training forward")
+        self.model.router_loss = None
+        if metrics is not None:
+            metrics["train/mod_router_bce_raw"] = aux.detach()
+            metrics["train/mod_router_bce_weighted"] = aux.detach() * self.config.router_aux_coef
+        return aux * self.config.router_aux_coef
+
+    def router_metrics(self):
+        return self.model.last_router_metrics.copy()
+
+    def parameter_budget(self):
+        total = sum(p.numel() for p in self.parameters())
+        conditional = sum(p.numel() for layer in self.model.layers if layer.routed
+                          for name, p in layer.named_parameters() if not name.startswith("router."))
+        return {"total": total, "always_active": total - conditional,
+                "expected_active_training": total - conditional + self.config.mod_capacity * conditional,
+                "active_min": total - conditional, "active_max": total,
+                "capacity": self.config.mod_capacity}

release-manifest.json

@@ -0,0 +1,157 @@
+{
+  "repo": "d0rj/q-mod-51M-base",
+  "source_run": "q-mod-total-v1",
+  "weights_sha256": "e5be4b27775d846ab2e65293a83f647d89ba2e0a606c00645fa892b21f6d5cd3",
+  "files_sha256": {
+    "README.md": "1ca5a8137f256573c7e04ade31e0b6b142606f9643e134703ae2f2eaef21c49f",
+    "config.json": "bac171e1314be5dd05458a4ee532f6999f8f82145b184278ddae286ed4e92d96",
+    "configuration_q50m.py": "796806b2d5d982b633796bc7de53c3073b1b6e52d259f44ef798b63f55eaea92",
+    "configuration_qmod.py": "82dd452f9c193d85f3e2a001075c5728297f523aa9314aad8095ebda64498db9",
+    "evaluation/continuation.json": "3b31fcaa4f71b0f317100a1b37b94481bb3773046676f909dec2f545c54ef40b",
+    "evaluation/raw/arc_challenge.json": "f5efc3025bb25c50c5017030cb36f77de89afaf762d8990dc82304ef99454d58",
+    "evaluation/raw/arc_easy.json": "ebe51440ccfc39c2b605ecb210d1aa3d7152cc2941906b72b35594a2afb56b34",
+    "evaluation/raw/arithmark3.json": "02c6c465acd3c3b7472e5f33890f841bac021be2608c7a50fb4a8007e09fefd1",
+    "evaluation/raw/balanced_copa.json": "9abed3267bb970d308a99687f73347bf6093126c89fdf40f35f8e665a6f47c7c",
+    "evaluation/raw/bananamind_base.json": "7d1893178aee2b2c2f9a13b345c405a5b1cb0efed70f574d31e32d449904fb43",
+    "evaluation/raw/blimp.json": "75189517422b1f0dc53fb4dbcf810b32ac9f40dfe64dab96df6e524ffe9318da",
+    "evaluation/raw/boolq.json": "9a3b1d284692559f2af9eb3bc1bd099e6ed785ad49c0abcaca8527c1aab4f8ff",
+    "evaluation/raw/commonsense_qa.json": "5968567f6503d5ba2924a34c4e7bd017c341dd7b98de53bddfbc80e21c82720b",
+    "evaluation/raw/hellaswag.json": "70c7f8f0f310bad716da22939e4ea8651e35d8262cf7a0688da7c0ee93cba0ef",
+    "evaluation/raw/lambada_openai.json": "f73b22ca335b65a93dc0c7c84fb6d624d96bab4ac8033ee3d09ff64ae1be1627",
+    "evaluation/raw/mmlu_continuation.json": "b9cbea26d9c7326596e7220d5fc4b75323207a2a25eee724155d8aca7e78b249",
+    "evaluation/raw/openbookqa.json": "398ce770d102c1d5b33ac91c684b3090c2f5d935253b7c1ef54b01e4a05c08e4",
+    "evaluation/raw/piqa.json": "c2d4b2727900015aa72ab771afd02e18a7086d31e744e85d785881c5ac132b33",
+    "evaluation/raw/sciq.json": "7cb4cd157ad4871d5063c07feedaa3976f63fb450e5e2f2cc856860b60cd9665",
+    "evaluation/raw/truthfulqa_mc2.json": "0c85b79db33e94c60bcf454318ecfdd19641326c1bd57d146833c1857429f5e4",
+    "evaluation/raw/winogrande.json": "00c93cb02ff2d9638d4392d90a73a968694febf6d246b53b6b5195f17bf96bf3",
+    "evaluation/repro/comparison_runner/__init__.py": "e49e80f7be3ab6e200f0db14b028e465de9caa8f6f705430eabce0c41b8322ba",
+    "evaluation/repro/comparison_runner/adapters.py": "8e64e5542bbcb0f9ad448b320e712916a9559823987e298b22321a8d8e2f727f",
+    "evaluation/repro/comparison_runner/catalog.py": "2fd29084ec780539023a38bcbfd7302931fa6be48624e227875f13518a1b986f",
+    "evaluation/repro/comparison_runner/continuation.py": "4f4143c828d4f94a851bcbf8a466d3e1b3e9eaec1027970a426208cc0ff4f4db",
+    "evaluation/repro/comparison_runner/losses.py": "592a05287369f6e5077b3faca404a9392de4772cf06389fb8aa7e36924e74f86",
+    "evaluation/repro/comparison_runner/tasks/balanced_copa.yaml": "f47baa6a713e15f7a5600ed257f316f8c54252bf4977bbc716a6da7aa580f05d",
+    "evaluation/repro/comparison_runner/tasks/balanced_copa_test.yaml": "dce21ad70b395d8d576c584be1b87ebae846e360e59878030645215a387b28e6",
+    "evaluation/repro/comparison_runner/token_continuation.py": "e8d88cd4aca51ca2fda07f2b799a54612d0e2964a98e05824a135bea1a533550",
+    "evaluation/repro/continuation.py": "6d3e1743047002e6eb33f3149f1e505ed5f97d202622ba7e2f2df1e7f0f02ee9",
+    "evaluation/repro/requirements.txt": "2d5c31a7bf4bca6fd361ae94be46f5e2cf52dd1d13ce039a8ed5080dfa5b6cf3",
+    "evaluation/repro/run.py": "04eef40e03f24ce432c8062b41fb3604059016a9ea72ac18194eacf6ba05ebaf",
+    "evaluation/results.json": "dcda27cae5a1e4437f1d4d634fe57cf6ad609603e33d524fe209792157254309",
+    "generation_config.json": "a9dd7c763300c1e5dcc5341ee772e2c4f35abfc11da7ec1aec986477f207f5c8",
+    "model.safetensors": "e5be4b27775d846ab2e65293a83f647d89ba2e0a606c00645fa892b21f6d5cd3",
+    "modeling_q50m.py": "b68abce1f19a36deb39c4ac4e520ffaab0c879d8a7cc2abef35f014bacc011f2",
+    "modeling_qmod.py": "eceb59328374894893d5c0798708fd1e9575d67bf9a3d2c7a1b53f780d801e37",
+    "requirements.txt": "a2b109a8c01d0c30dae771b2232700311d3ed6c280f1d5b3ae880d02adca09fa",
+    "tensorboard/events.out.tfevents.1790785789.DESKTOP-OU3A33R.14755.0": "76005dc07e72c936f1e6728c11596f6a3bf4e7b1aaf6e48454bf0406cce61ea9",
+    "tensorboard/events.out.tfevents.1790874945.DESKTOP-OU3A33R.25655.2.continuation-release-20261001": "e6e871852bd7198579fa7c42d86fb8070d87251c5cd7d54af223a406a30de825",
+    "tensorboard/events.out.tfevents.1790878703.DESKTOP-OU3A33R.32581.0.eval": "ee3978fb6340c4d6b62e4c4a25d9d6c984f20414e473a2de096925ac785ba1ca",
+    "tensorboard/events.out.tfevents.1790878760.DESKTOP-OU3A33R.32581.1.eval": "652a6f405bfdd73bfa84c9ee90dd9925ef9b6d1aec3cfb7fb91285581179229d",
+    "tensorboard/events.out.tfevents.1790878791.DESKTOP-OU3A33R.32581.2.eval": "bbdb8af7f01195dc4b85404d513de905c0931c09c0cd9332e11d8b893a07b361",
+    "tensorboard/events.out.tfevents.1790878814.DESKTOP-OU3A33R.32581.3.eval": "88005a681436a2b357d42fd67e4e44262c0cfe65e0f9de789e41cc03dc0e210c",
+    "tensorboard/events.out.tfevents.1790878826.DESKTOP-OU3A33R.32581.4.eval": "8000c7f98338021622d91eab8549927b8da85657c434449d5a43ffeb4319160f",
+    "tensorboard/events.out.tfevents.1790878836.DESKTOP-OU3A33R.32581.5.eval": "91fe5a391f512fedaf1228b202aa5a206e1e37dc79ed719774c1701fda38fd59",
+    "tensorboard/events.out.tfevents.1790878859.DESKTOP-OU3A33R.32581.6.eval": "9b07cdcd9c667ceee6b5d9ec7f6c1a26c93b513cdcc635f0fd2eb7de9ace325d",
+    "tensorboard/events.out.tfevents.1790878893.DESKTOP-OU3A33R.32581.7.eval": "fda9cc55438813ffe8a6ff0e50426b20ea7b883244dff8afcca2068738a61850",
+    "tensorboard/events.out.tfevents.1790878916.DESKTOP-OU3A33R.32581.8.eval": "3da90b5bac05951a954bd7900f455ff09f545e0d1652b85e7d060b05070e23a4",
+    "tensorboard/events.out.tfevents.1790878934.DESKTOP-OU3A33R.32581.9.eval": "df5cb6a3c6760dbd4e689dc072c4640e2eb94eea40be38ef453a4dcf13ec1818",
+    "tensorboard/events.out.tfevents.1790879411.DESKTOP-OU3A33R.32581.10.eval": "af81181e46c14ec75fe556f24a8e332df8d47419ead5ab5f65d234168094ddca",
+    "tensorboard/events.out.tfevents.1790879418.DESKTOP-OU3A33R.32581.11.eval": "93d757cf5534204e23b4af7e755ff32e2d13dc7f55acbbd1980b3744ccf49600",
+    "tensorboard/events.out.tfevents.1790879558.DESKTOP-OU3A33R.32581.12.eval": "8ce66c7b838c47bf28380f6266a36550ce5d4427ce368c24e7814e270a8bc6b7",
+    "tensorboard/events.out.tfevents.1790879569.DESKTOP-OU3A33R.32581.13.eval": "59cd365bc84b14dae49ec6bff8e9399b0af465775f1e470227b0724109a3f0cf",
+    "tensorboard/events.out.tfevents.1790879590.DESKTOP-OU3A33R.32581.14.eval": "3f80079f9bcab184bcddc4c0d34c0ea53e8e80ef41a33bc55426ad7936a328e0",
+    "tensorboard/events.out.tfevents.1790879593.DESKTOP-OU3A33R.32581.15.eval": "84a001d26d212b48301e439d9219a352b79536be1b0d1044633ffa6796e881d8",
+    "tokenizer.json": "6155a4212832ddd656acd1cd45ab4acb327dbb080e3acdbec84f086adf17414e",
+    "tokenizer_config.json": "cccfc292e0846e8ab8aa04dafae32483a831bcf1df6f15100f82d306b4a9f4c6",
+    "training_config.json": "693c8b3ad22d53ea736ea44a6b9a5fc7b1c6062d98be8bec392dce8879f3f826"
+  },
+  "validation": {
+    "logits_bitwise_equal": true,
+    "controls": 3,
+    "tokenizer_vocabulary_identical": true,
+    "parameters": 50919168,
+    "greedy_cache_parity": true,
+    "generated_ids": [
+      [
+        [
+          463,
+          2934,
+          283,
+          4226,
+          314,
+          265,
+          2046,
+          283,
+          6270,
+          17,
+          366,
+          2046,
+          283
+        ]
+      ],
+      [
+        [
+          9632,
+          7295,
+          786,
+          314,
+          261,
+          1035,
+          904,
+          292,
+          853,
+          265,
+          683,
+          576
+        ]
+      ],
+      [
+        [
+          463,
+          7371,
+          353,
+          8896,
+          287,
+          265,
+          4071,
+          17,
+          1023,
+          353,
+          433,
+          8896,
+          287,
+          265,
+          1031,
+          4071
+        ]
+      ]
+    ],
+    "weights_sha256": "e5be4b27775d846ab2e65293a83f647d89ba2e0a606c00645fa892b21f6d5cd3",
+    "output": "hf_exports/pending-20261001/q-mod/q_mod",
+    "source_sha256": {
+      "configuration_q50m.py": "796806b2d5d982b633796bc7de53c3073b1b6e52d259f44ef798b63f55eaea92",
+      "configuration_qmod.py": "82dd452f9c193d85f3e2a001075c5728297f523aa9314aad8095ebda64498db9",
+      "modeling_q50m.py": "b68abce1f19a36deb39c4ac4e520ffaab0c879d8a7cc2abef35f014bacc011f2",
+      "modeling_qmod.py": "eceb59328374894893d5c0798708fd1e9575d67bf9a3d2c7a1b53f780d801e37"
+    }
+  },
+  "tasks": [
+    "hellaswag",
+    "arc_easy",
+    "arc_challenge",
+    "piqa",
+    "winogrande",
+    "openbookqa",
+    "boolq",
+    "lambada_openai",
+    "arithmark3",
+    "balanced_copa",
+    "blimp",
+    "commonsense_qa",
+    "mmlu_continuation",
+    "sciq",
+    "truthfulqa_mc2",
+    "bananamind_base"
+  ],
+  "parameters": 50919168,
+  "source_tokens": 3932160000,
+  "optimizer_step": 15000
+}

requirements.txt

@@ -0,0 +1,4 @@
+torch==2.11.0
+transformers==5.17.0
+tokenizers>=0.22
+safetensors>=0.4

tensorboard/events.out.tfevents.1790785789.DESKTOP-OU3A33R.14755.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76005dc07e72c936f1e6728c11596f6a3bf4e7b1aaf6e48454bf0406cce61ea9
+size 1591094

tensorboard/events.out.tfevents.1790874945.DESKTOP-OU3A33R.25655.2.continuation-release-20261001

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6e871852bd7198579fa7c42d86fb8070d87251c5cd7d54af223a406a30de825
+size 1562

tensorboard/events.out.tfevents.1790878703.DESKTOP-OU3A33R.32581.0.eval

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee3978fb6340c4d6b62e4c4a25d9d6c984f20414e473a2de096925ac785ba1ca
+size 3759

tensorboard/events.out.tfevents.1790878760.DESKTOP-OU3A33R.32581.1.eval

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:652a6f405bfdd73bfa84c9ee90dd9925ef9b6d1aec3cfb7fb91285581179229d
+size 3457

tensorboard/events.out.tfevents.1790878791.DESKTOP-OU3A33R.32581.2.eval

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbdb8af7f01195dc4b85404d513de905c0931c09c0cd9332e11d8b893a07b361
+size 3529

tensorboard/events.out.tfevents.1790878814.DESKTOP-OU3A33R.32581.3.eval

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88005a681436a2b357d42fd67e4e44262c0cfe65e0f9de789e41cc03dc0e210c
+size 3236

tensorboard/events.out.tfevents.1790878826.DESKTOP-OU3A33R.32581.4.eval

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8000c7f98338021622d91eab8549927b8da85657c434449d5a43ffeb4319160f
+size 3567

tensorboard/events.out.tfevents.1790878836.DESKTOP-OU3A33R.32581.5.eval

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91fe5a391f512fedaf1228b202aa5a206e1e37dc79ed719774c1701fda38fd59
+size 3452

tensorboard/events.out.tfevents.1790878859.DESKTOP-OU3A33R.32581.6.eval

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b07cdcd9c667ceee6b5d9ec7f6c1a26c93b513cdcc635f0fd2eb7de9ace325d
+size 3059

tensorboard/events.out.tfevents.1790878893.DESKTOP-OU3A33R.32581.7.eval

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fda9cc55438813ffe8a6ff0e50426b20ea7b883244dff8afcca2068738a61850
+size 3332

tensorboard/events.out.tfevents.1790878916.DESKTOP-OU3A33R.32581.8.eval

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3da90b5bac05951a954bd7900f455ff09f545e0d1652b85e7d060b05070e23a4
+size 3020