{
  "name": "hypernix.3-mini",
  "params": 48706048,
  "config": {
    "vocab_size": 32000,
    "n_layers": 8,
    "n_heads": 8,
    "n_kv_heads": 2,
    "d_model": 512,
    "d_ff": 2203,
    "max_seq_len": 512,
    "rope_theta": 100000.0,
    "norm_eps": 1e-05,
    "dropout": 0.0,
    "tie_embeddings": true,
    "use_sliding_window": false,
    "sliding_window_size": 512,
    "attention_type": "gqa",
    "name": "hypernix.3-mini"
  },
  "tokens_trained": 873660416,
  "steps": 18464,
  "total_steps": 18464,
  "best_val_loss": 7.839838576316834,
  "best_val_ppl": 2539.7948177006024,
  "elapsed_seconds": 1047820.9095537663,
  "elapsed_human": "12d03h03m",
  "deadline_hours": 9999.0,
  "within_deadline": true,
  "final_ladder": {
    "micro_batch": 8,
    "accum": 8,
    "ce_chunk": 2048,
    "grad_ckpt": false,
    "rung": 1
  },
  "dataset_bytes": 2422007338,
  "optimizer": "pcv6",
  "sources": [
    {
      "repo": "HuggingFaceFW/fineweb-edu",
      "config": "sample-10BT",
      "split": "train",
      "weight": 1.0,
      "text_key": "text"
    }
  ]
}