language:
- en
library_name: transformers
pipeline_tag: text-generation
tags:
- pretrained
- from-scratch
- tiny-llm-ablation
- custom_code
- tensorboard
- causal-lm
datasets:
- HuggingFaceFW/fineweb-edu
model-index:
- name: q-mod-51M-base
results:
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: Rowan/hellaswag
name: HellaSwag
config: default
split: validation
metrics:
- type: acc_norm
name: acc_norm (fraction; full comparison protocol)
value: 0.2874925313682533
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.004516681953878919
evaluation_date: '2026-10-01'
ci95_low: 0.27872301915216174
ci95_high: 0.29642456629450026
ci_method: Wilson 95%; item independence approximation
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: allenai/ai2_arc
name: ARC-Easy
config: ARC-Easy
split: test
metrics:
- type: acc_norm
name: acc_norm (fraction; full comparison protocol)
value: 0.41919191919191917
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.010124905282491126
evaluation_date: '2026-10-01'
ci95_low: 0.3994976736933545
ci95_high: 0.43914703998001464
ci_method: Wilson 95%; item independence approximation
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: allenai/ai2_arc
name: ARC-Challenge
config: ARC-Challenge
split: test
metrics:
- type: acc_norm
name: acc_norm (fraction; full comparison protocol)
value: 0.23122866894197952
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.01232085883477234
evaluation_date: '2026-10-01'
ci95_low: 0.2079920763450836
ci95_high: 0.2562214064556877
ci_method: Wilson 95%; item independence approximation
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: baber/piqa
name: PIQA
config: default
split: validation
metrics:
- type: acc_norm
name: acc_norm (fraction; full comparison protocol)
value: 0.5875952121871599
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.011485407152743092
evaluation_date: '2026-10-01'
ci95_low: 0.5649303971157125
ci95_high: 0.6098946392401159
ci_method: Wilson 95%; item independence approximation
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: allenai/winogrande
name: WinoGrande
config: winogrande_xl
split: validation
metrics:
- type: acc
name: acc (fraction; full comparison protocol)
value: 0.5138121546961326
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.014047122916440271
evaluation_date: '2026-10-01'
ci95_low: 0.4862910104211901
ci95_high: 0.5412497970942327
ci_method: Wilson 95%; item independence approximation
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: allenai/openbookqa
name: OpenBookQA
config: main
split: test
metrics:
- type: acc_norm
name: acc_norm (fraction; full comparison protocol)
value: 0.306
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.020629569998345414
evaluation_date: '2026-10-01'
ci95_low: 0.26721346373113636
ci95_high: 0.3477447803679459
ci_method: Wilson 95%; item independence approximation
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: aps/super_glue
name: BoolQ
config: boolq
split: validation
metrics:
- type: acc
name: acc (fraction; full comparison protocol)
value: 0.5577981651376147
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.008686430526114761
evaluation_date: '2026-10-01'
ci95_low: 0.5407177130610681
ci95_high: 0.5747429788369947
ci_method: Wilson 95%; item independence approximation
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: EleutherAI/lambada_openai
name: LAMBADA OpenAI
config: default
split: test
metrics:
- type: acc
name: acc (fraction; full comparison protocol)
value: 0.16689307199689501
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.005194952730633079
evaluation_date: '2026-10-01'
ci95_low: 0.1569610474392711
ci95_high: 0.17732137574286708
ci_method: Wilson 95%; item independence approximation
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: AxiomicLabs/Arithmark-3.0
name: ArithMark-3
config: default
split: train
metrics:
- type: acc_norm
name: acc_norm (fraction; full comparison protocol)
value: 0.341
args:
dtype: bfloat16
num_few_shot: 0
max_length: 1024
standard_error: 0.01499813134840284
evaluation_date: '2026-10-01'
ci95_low: 0.31227731829126676
ci95_high: 0.37093959090713985
ci_method: Wilson 95%; item independence approximation
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: pkavumba/balanced-copa
name: Balanced COPA
config: default
split: train
metrics:
- type: acc
name: acc (fraction; full comparison protocol)
value: 0.533
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.015784807891138876
evaluation_date: '2026-10-01'
ci95_low: 0.5020104989134991
ci95_high: 0.5637369350264544
ci_method: Wilson 95%; item independence approximation
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: nyu-mll/blimp
name: BLiMP
config: 67 minimal-pair subsets
split: train
metrics:
- type: acc
name: acc (fraction; full comparison protocol)
value: 0.7317910447761194
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.001517376416839114
evaluation_date: '2026-10-01'
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: tau/commonsense_qa
name: CommonsenseQA
config: default
split: validation
metrics:
- type: acc
name: acc (fraction; full comparison protocol)
value: 0.19492219492219492
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.011341478090883516
evaluation_date: '2026-10-01'
ci95_low: 0.17367347122724505
ci95_high: 0.21808454395514162
ci_method: Wilson 95%; item independence approximation
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: cais/mmlu
name: MMLU continuation
config: 57 subjects
split: test
metrics:
- type: acc
name: acc (fraction; full comparison protocol)
value: 0.2521720552627831
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.0036528205915989775
evaluation_date: '2026-10-01'
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: allenai/sciq
name: SciQ (with support)
config: default
split: test
metrics:
- type: acc_norm
name: acc_norm (fraction; full comparison protocol)
value: 0.612
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.015417317979911216
evaluation_date: '2026-10-01'
ci95_low: 0.5814239245764211
ci95_high: 0.6417188815228144
ci_method: Wilson 95%; item independence approximation
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: truthfulqa/truthful_qa
name: TruthfulQA MC2
config: multiple_choice
split: validation
metrics:
- type: acc
name: acc (fraction; full comparison protocol)
value: 0.4417644031264386
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.01519901667714673
evaluation_date: '2026-10-01'
- task:
type: text-generation
name: Continuation likelihood
dataset:
type: BananaMind/BananaMind-Base-Bench-1.1
name: BananaMind Base 1.1
config: default
split: test
metrics:
- type: raw_accuracy
name: raw_accuracy (fraction; full comparison protocol)
value: 0.45714285714285713
args:
dtype: bfloat16
num_few_shot: 0
max_length: 2048
standard_error: 0.02666588697376804
evaluation_date: '2026-10-01'
ci95_low: 0.40570065529637145
ci95_high: 0.5095156110507492
ci_method: Wilson 95%; item independence approximation
Q-MoD
A 50,919,168-parameter English base model from the Tiny llm ablation experiment. Trained from random initialization for 15,000 optimizer steps, processing 3,932,160,000 source tokens. These are processed input tokens, not unique text or supervised target-token counts.
Architecture
10 decoder layers, width 512, 8 query / 2 KV heads, head dimension 64, RoPE, QK normalization, gated residuals and tied 32,768-token embeddings. Five alternating whole blocks (zero-based layers 1, 3, 5, 7, 9) use Mixture-of-Depths token routing. Training selects the top 12.5% of valid tokens per sequence; evaluation selects each token causally with router logit > 0, without future-dependent top-k. Routed FFN width 1797 (ordinary blocks 1792) makes the total exactly 50,919,168 parameters. This routes tokens through blocks, not experts. Inference compute depends on selected tokens.
Tokenizer foundation: q-project/Q-50M-Base. This model is trained from scratch, not fine-tuned from the reference weights. Exact trained configuration and custom model code are included.
Training
- Data: FineWeb-Edu,
sample-10BT, local Parquet shards; shuffle buffer 100,000. - Objective: Causal next-token cross-entropy plus router binary classification auxiliary loss, coefficient 0.01.
- Final recorded batch: 4 sequences × 32 gradient accumulation × 2048 tokens = 262,144 source tokens per update.
- Fused AdamW, peak LR 0.001, betas (0.9, 0.95), weight decay 0.1 excluding bias/norm/1D parameters, gradient clipping 1.0. Warmup 150 steps, cosine decay to 10% of peak LR.
- BF16 compute, one RTX 5070 Ti, seed 2026; released checkpoint weights remain FP32. Training configuration.
Evaluation
Full selected splits, lm-eval 0.4.12, no added few-shot examples or chat template, BF16 on RTX 5070 Ti, context cap 2048 (ArithMark 1024). New evaluations explicitly disable TF32; historical core manifests predate the explicit TF32 flag. Scores below are percentages; ± is one standard error, and the separate interval column is 95% CI where available.
| Dataset | Split | Examples | Metric | Score ± SE (%) | 95% CI (%) |
|---|---|---|---|---|---|
| HellaSwag | validation | 10,042 | acc_norm |
28.75 ± 0.45 | [27.87, 29.64] |
| ARC-Easy | test | 2,376 | acc_norm |
41.92 ± 1.01 | [39.95, 43.91] |
| ARC-Challenge | test | 1,172 | acc_norm |
23.12 ± 1.23 | [20.80, 25.62] |
| PIQA | validation | 1,838 | acc_norm |
58.76 ± 1.15 | [56.49, 60.99] |
| WinoGrande | validation | 1,267 | acc |
51.38 ± 1.40 | [48.63, 54.12] |
| OpenBookQA | test | 500 | acc_norm |
30.60 ± 2.06 | [26.72, 34.77] |
| BoolQ | validation | 3,270 | acc |
55.78 ± 0.87 | [54.07, 57.47] |
| LAMBADA OpenAI | test | 5,153 | acc |
16.69 ± 0.52 | [15.70, 17.73] |
| ArithMark-3 | train | 1,000 | acc_norm |
34.10 ± 1.50 | [31.23, 37.09] |
| Balanced COPA | train | 1,000 | acc |
53.30 ± 1.58 | [50.20, 56.37] |
| BLiMP | train | 67,000 | acc |
73.18 ± 0.15 | — |
| CommonsenseQA | validation | 1,221 | acc |
19.49 ± 1.13 | [17.37, 21.81] |
| MMLU continuation | test | 14,042 | acc |
25.22 ± 0.37 | — |
| SciQ (with support) | test | 1,000 | acc_norm |
61.20 ± 1.54 | [58.14, 64.17] |
| TruthfulQA MC2 | validation | 817 | acc |
44.18 ± 1.52 | — |
| BananaMind Base 1.1 | test | 350 | raw_accuracy |
45.71 ± 2.67 | [40.57, 50.95] |
LAMBADA requires all final-word tokens to match. The descriptive macro-average is 41.4613%: one primary metric per each of the 16 tasks, with equal task weight. PPL, standard errors, CI bounds and MMLU/BLiMP leaf scores are excluded. It is not an official leaderboard score or a statistical ranking test.
Full selected splits; lm-eval 0.4.12; seed 1234; BF16 on RTX 5070 Ti; context cap 2048 (ArithMark 1024), TF32 disabled on new runs (historical core flag unrecorded), no chat template and no added few-shot examples. TruthfulQA retains the harness's fixed six-QA preamble. ArithMark and BananaMind normalize by continuation token count; ordinary harness acc_norm uses its own length normalization. BananaMind is raw accuracy, not Elo. SciQ includes the support passage. Balanced COPA uses the mirrored 1000-item train-named evaluation split; cRia's split was inferred, not confirmed. MMLU scores full answer continuations across 57 subjects, weighted by item count; BLiMP averages 67 equal-sized minimal-pair subsets. Standard errors are retained from each evaluator. Wilson intervals are reported only where the runner logged binary item accuracy; MC2 is probability mass, not binary accuracy. These intervals do not model dependence between paired/templated examples or training seed variation. UL2, PrefixLM and experimental diffusion PLL use their documented conditional scoring protocols; PLL exposes the other answer tokens and is not autoregressive likelihood. cRia's published scores used a different precision and benchmark-adapted checkpoint; this completes our comparison coverage, not an independent reproduction of cRia or an official leaderboard submission.
WikiText-2 raw test continuation, reported separately: 291 nonoverlapping blocks of 1024 tokens, 512-token prefix plus 512-token scored suffix; 148,992 scored tokens, 335 tail tokens excluded. GPU evaluation; percentile block bootstrap with 10,000 resamples and seed 2026. PPL intervals exponentiate NLL endpoints. This is not rolling or word perplexity.
torch.bfloat16: NLL 3.689238, 95% CI [3.649657, 3.728491]; PPL 40.014, 95% CI [38.461, 41.616].torch.float32: NLL 3.688851, 95% CI [3.649183, 3.728063]; PPL 39.999, 95% CI [38.443, 41.598].
Structured results and provenance, raw task outputs and per-block continuation scores. HF metadata contains author-reported measurements; no verified benchmark badge is claimed. Historical and new measurements refer to identical weight hashes; individual run manifests retain their original dates and source hashes.
Usage
Install requirements.txt; tested with PyTorch 2.11.0 and Transformers 5.17.0. Loading custom code requires trust_remote_code=True. Example on CPU:
from transformers import AutoTokenizer, AutoModelForCausalLM
repo = "d0rj/q-mod-51M-base"
tokenizer = AutoTokenizer.from_pretrained(repo)
model = AutoModelForCausalLM.from_pretrained(repo, trust_remote_code=True).eval()
inputs = tokenizer("The capital of France is", return_tensors="pt")
output = model.generate(**inputs, max_new_tokens=32, do_sample=False)
print(tokenizer.decode(output[0], skip_special_tokens=True))
After downloading this repository, install the evaluation dependencies and authenticate for the gated BananaMind dataset after accepting its terms:
pip install -r evaluation/repro/requirements.txt
hf auth login
python evaluation/repro/run.py --device cuda:0 --dtype bfloat16 --batch-size 8 --output comparison-rerun
python evaluation/repro/continuation.py --device cuda:0 --dtype bfloat16 --output continuation-bf16.json
python evaluation/repro/continuation.py --device cuda:0 --dtype float32 --output continuation-fp32.json
--limit / --limit-blocks are smoke checks only. Dataset examples are not redistributed. The bundled runners use this published model and tokenizer.
Training logs and limitations
TensorBoard event files contain available training telemetry and the full evaluation at optimizer step 15,000. train/loss has 750 visible points spanning steps 20–15,000. Resume/purge records are retained. Comet experiment. Evaluation exports retain numeric metrics, uncertainty and task/subtask breakdowns.
Equal source-token budgets do not imply equal target supervision, parameter count or FLOPs. Training dataset file revisions and overlap with benchmarks were not independently audited. Results come from one training seed; uncertainty intervals do not capture training-seed variability or dependence between templated examples. These are small base models, not instruction-tuned assistants.