Buckets:
YangyiH/depth-bench-ckpts / pretrain-mhc-300M-backbone-L42-lr2e3-b200-8gpu-final-eval /step7600 /config.json
| {"model": {"d_model": 768, "vocab_size": 50280, "n_layers": 42, "block": {"sequence_mixer": {"name": "default", "n_heads": 16, "n_kv_heads": 16, "head_dim": 48, "bias": false, "rope": {"name": "default", "theta": 10000, "full_precision": true, "no_global_rope": false, "_CLASS_": "olmo_core.nn.rope.RoPEConfig"}, "dtype": "bfloat16", "_CLASS_": "olmo_core.nn.attention.AttentionConfig", "type": "attention"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "bfloat16", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "feed_forward": {"hidden_size": 2048, "name": "default", "bias": false, "dtype": "bfloat16", "activation": "silu", "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"}, "name": "mhc", "hyper_connections": {"kind": "liger_mhc", "num_residual_streams": 4, "tanh": false, "gating_factor_init": 0.01, "sinkhorn_iters": 20, "sinkhorn_tau": 1.0, "disable_static_weight_decay": true, "scale_output_init_by_sqrt_n": true, "liger_phi_dtype": "bfloat16", "liger_allow_fp32": false, "liger_rms_eps": 1e-06, "liger_pre_eps": 0.0, "liger_sinkhorn_eps": 1e-06, "liger_post_mult": 2.0, "collapse": "auto", "_CLASS_": "olmo_core.nn.transformer.config.HyperConnectionsConfig"}, "moda_skip_ffn_kv": false, "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "bfloat16", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "bfloat16", "loss_implementation": "default", "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"}, "name": "default", "dtype": "bfloat16", "init_method": "normal", "init_seed": 0, "init_std": 0.02, "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 50280, "eos_token_id": 50279, "pad_token_id": 1, "identifier": "pretrain/OLMo-core/src/olmo_core/data/tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "paths": ["/lustre/fast/fast/wliu/yy/DepthBench_workspace/DepthBench/data/fineweb-edu/pre-tokenize/train/*.npy"], "expand_glob": true, "include_instance_metadata": true, "work_dir": "/lustre/fast/fast/wliu/yy/DepthBench_workspace/DepthBench/ckpt/depthbench/pretrain-mhc-300M-backbone-L42-lr2e3-b200-8gpu-final-eval/dataset-cache", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "generate_doc_lengths": false, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyFSLDatasetConfig"}, "data_loader": {"global_batch_size": 1048576, "seed": 42, "num_workers": 4, "ignore_fingerprint_mismatch": false, "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig", "type": "numpy"}, "train_module": {"rank_microbatch_size": 32768, "max_sequence_length": 2048, "optim": {"group_overrides": [{"params": ["blocks.*.attention_hyper_connection.b", "blocks.*.attention_hyper_connection.alpha_pre", "blocks.*.attention_hyper_connection.alpha_post", "blocks.*.attention_hyper_connection.alpha_res", "blocks.*.feed_forward_hyper_connection.b", "blocks.*.feed_forward_hyper_connection.alpha_pre", "blocks.*.feed_forward_hyper_connection.alpha_post", "blocks.*.feed_forward_hyper_connection.alpha_res"], "opts": {"weight_decay": 0.0}, "_CLASS_": "olmo_core.optim.config.OptimGroupOverride"}], "compile": false, "fixed_fields": ["initial_lr"], "lr": 0.002, "betas": [0.9, 0.95], "eps": 1e-08, "weight_decay": 0.1, "_CLASS_": "olmo_core.optim.adamw.AdamWConfig", "type": "adamw"}, "max_grad_norm": 1.0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "units": "steps", "warmup": 760, "alpha_f": 0.1, "warmup_min_lr": 0.0, "_CLASS_": "olmo_core.optim.scheduler.CosWithWarmup", "type": "cos_with_warmup"}, "compile_model": false, "dp_config": {"name": "ddp", "param_dtype": "bfloat16", "reduce_dtype": "float32", "wrapping_strategy": "full", "prefetch_factor": 0, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig"}, "autocast_precision": "bfloat16", "label_ignore_index": -100, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerTrainModuleConfig"}, "trainer": {"save_folder": "/lustre/fast/fast/wliu/yy/DepthBench_workspace/DepthBench/ckpt/depthbench/pretrain-mhc-300M-backbone-L42-lr2e3-b200-8gpu-final-eval", "load_strategy": "if_available", "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "olmo_core.train.checkpoint.CheckpointerConfig"}, "save_overwrite": true, "max_duration": {"value": 7600, "unit": "steps", "_CLASS_": "olmo_core.train.common.Duration"}, "cancel_check_interval": 10, "metrics_collect_interval": 10, "callbacks": {"gpu_monitor": {"_CLASS_": "olmo_core.train.callbacks.gpu_memory_monitor.GPUMemoryMonitorCallback"}, "layer_stats": {"enabled": false, "interval": 1, "_CLASS_": "olmo_core.train.callbacks.layer_stats_monitor.LayerStatsMonitorCallback"}, "checkpointer": {"save_interval": 3000, "save_async": true, "remove": "ephemeral_only", "enabled": true, "_CLASS_": "olmo_core.train.callbacks.checkpointer.CheckpointerCallback"}, "wandb": {"enabled": false, "name": "pretrain-mhc-300M-backbone-L42-lr2e3-b200-8gpu-final-eval", "project": "", "entity": "wang-keyu-2002-max-planck-society", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "_CLASS_": "olmo_core.train.callbacks.wandb.WandBCallback"}, "config_saver": {"fname": "config.json", "_CLASS_": "olmo_core.train.callbacks.config_saver.ConfigSaverCallback"}, "lm_evaluator": {"eval_dataset": {"tokenizer": {"vocab_size": 50280, "eos_token_id": 50279, "pad_token_id": 1, "identifier": "pretrain/OLMo-core/src/olmo_core/data/tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "paths": ["/lustre/fast/fast/wliu/yy/DepthBench_workspace/DepthBench/data/fineweb-edu/pre-tokenize/eval/*.npy"], "expand_glob": true, "metadata": [{"label": "fineweb-edu-eval"}], "include_instance_metadata": true, "work_dir": "/lustre/fast/fast/wliu/yy/DepthBench_workspace/DepthBench/ckpt/depthbench/pretrain-mhc-300M-backbone-L42-lr2e3-b200-8gpu-final-eval/dataset-cache", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyPaddedFSLDatasetConfig"}, "eval_interval": 7600, "eval_on_startup": false, "eval_on_finish": false, "cancel_after_first_eval": false, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "deterministic": true, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.LMEvaluatorCallbackConfig"}}, "bookkeeping_soft_timeout": 120, "no_checkpoints": false, "no_evals": false, "_CLASS_": "olmo_core.train.config.TrainerConfig"}, "init_seed": 42, "load_trainer_state": false, "_CLASS_": "pretrain_llama_base.ExperimentConfig"} |
Xet Storage Details
- Size:
- 6.83 kB
- Xet hash:
- ec771a5f69ba2073dbf1a033ae1b667eff8e867fe4c3a203824115cac6319eae
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.