run_name: camera-ready-ablation-onset-35pct-total-l2-held-33pct-b3-sequential-nld-eng-fineweb-100m
hf_user: Beetle-FineWeb-100M
model:
  architecture: pico
  n_layers: 14
  d_model: 768
  vocab_size: 50000
  attention_n_heads: 12
  attention_n_kv_heads: 1
  max_seq_len: 512
  batch_size: 64
  position_emb_theta: 10000.0
  activation_hidden_dim: 3072
  norm_eps: 1.0e-05
  dropout: 0.1
data:
  data_mix: B3
  lang_sources:
  - Beetle-Data/nl-100M-pretok
  l2_streams:
  - Beetle-Data/en-for-nl-100M-pretok
  pretokenized: true
  pretokenized_cache_dir: ./pretok_cache/humanscale-100m/nld_L1-eng_L2
tokenizer:
  pretrained_tokenizer_path: Beetle-Data/tokenizer-nl-en
  dataset_source: fineweb
  vocab_size: 50000
  vocab_sharing: shared
  type: bpe
  target_tokens: 100000000
  output_dir: ./tokenizers/fineweb_humanscale_nld_L1-eng_L2_shared
  overwrite: false
  push_to_hub: true
training:
  fabric:
    accelerator: cuda
    devices: 1
    strategy: ddp
    precision: bf16-mixed
  optimization:
    optimizer: adamw
    lr: 0.0005
    lr_warmup_steps: 500
    lr_scheduler: linear_with_warmup
  max_steps: 10000
checkpointing:
  runs_dir: ./runs
  run_name: camera-ready-ablation-onset-35pct-total-l2-held-33pct-b3-sequential-nld-eng-fineweb-100m
  checkpoints_dir: checkpoints
  fabric_checkpoint_dir: fabric_checkpoint
  fabric_checkpoint_filename: checkpoint.pt
  logs_dir: logs
  save_every_n_steps: 1000
  keep_local_checkpoints: 1
  save_to_hf: true
  learning_dynamics: {}
  compute_gradients: false
curriculum:
  type: smooth
  mode: tokens
  schedule:
  - kind: sigmoid_ramp
    from_weights:
      l1: 1.0
      l2: 0.0
    to_weights:
      l1: 0.4846
      l2: 0.5154
    midpoint: 0.35
    steepness: 12.0
ewc:
  enabled: false
maml:
  enabled: false
generative_replay:
  enabled: false
polca:
  enabled: false
  every_n_steps: 500
  k_eigenvectors: 32
  basis_rank: 32
  holdout_batch_size: 8
  holdout_seq_len: 512
  artifact_dir: polca
  push_to_hf: false
  precision: fp32
monitoring:
  logging:
    log_level: INFO
    log_every_n_steps: 100
  save_to_wandb: false
  wandb:
    project: ''
    entity: ''