run_name: beetle-bilingual-l2-50-simultaneous-b2-fineweb-spa-eng
hf_user: Beetle-FineWeb2-24B
model:
  architecture: pico
  n_layers: 14
  d_model: 768
  vocab_size: 50000
  attention_n_heads: 12
  attention_n_kv_heads: 1
  max_seq_len: 512
  batch_size: 64
  position_emb_theta: 10000.0
  activation_hidden_dim: 3072
  norm_eps: 1.0e-05
  dropout: 0.1
data:
  data_mix: B2
  lang_sources:
  - Beetle-Data/es-28B
  l2_streams:
  - Beetle-Data/en-for-es-28B
  pretokenized: true
  pretokenized_cache_dir: ./pretok_cache/beetle-bilingual-l2-50-simultaneous-b2-fineweb-spa-eng
  batch_size: 64
tokenizer:
  pretrained_tokenizer_path: Beetle-Data/tokenizer-es-en
  dataset_source: fineweb
  vocab_size: 50000
  vocab_sharing: shared
  type: bpe
  target_tokens: 1000000000
  output_dir: ./tokenizers/fineweb_spa_L1-eng_L2_shared
  overwrite: false
  push_to_hub: true
training:
  fabric:
    accelerator: cuda
    devices: 1
    strategy: ddp
    precision: bf16-mixed
  optimization:
    optimizer: adamw
    lr: 0.0005
    lr_warmup_steps: 625
    lr_scheduler: linear_with_warmup
  max_steps: 91552
checkpointing:
  runs_dir: ./runs
  run_name: beetle-bilingual-l2-50-simultaneous-b2-fineweb-spa-eng
  checkpoints_dir: checkpoints
  fabric_checkpoint_dir: fabric_checkpoint
  fabric_checkpoint_filename: checkpoint.pt
  logs_dir: logs
  save_every_n_steps: 1000
  keep_local_checkpoints: 1
  save_to_hf: true
  learning_dynamics: {}
  compute_gradients: false
curriculum:
  type: smooth
  condition: B2
  mode: tokens
ewc:
  enabled: false
maml:
  enabled: false
generative_replay:
  enabled: false
monitoring:
  logging:
    log_level: INFO
    log_every_n_steps: 100
  save_to_wandb: false
  wandb:
    project: ''
    entity: ''