update weights from bf16 to fp32

config.json

@@ -117,7 +117,7 @@
     }
   },
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float32",
   "transformers_version": "4.52.3",
   "use_cache": true,
   "vit_config": {

model-00001-of-00004.safetensors → model-00001-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0bc665d48e0f6675b8aac77c66e9431b84148890e4d6ab543a7c458971ca6142
-size 4878581216
+oid sha256:d29a7f1839d126a0411170ffbc64d44f1beb5eef4bc81801734e15243ab8bf0a
+size 4978520816

model-00002-of-00004.safetensors → model-00002-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a825abfe7f8d124a314052b4e3aee8d854fb061a841d95d495724ecfdcb89119
-size 4932745864
+oid sha256:98ee8f980eed8c7b4af75e3ee83f8da931f6a035244e6a079092a6dd374220e1
+size 4778633920

model-00003-of-00004.safetensors → model-00003-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7955e52696c59a1e4b49a22dfe7eef2a1dc902f00307ffe4eddbe52609bacb8b
-size 4994552920
+oid sha256:718c8bf8cba41303819daafc1111fa568b42d6e199306048717a37efba44b8f7
+size 4661160168

model-00004-of-00004.safetensors → model-00004-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88f9e18a24cc86ca87a247792595032e4a9d4811291855fafdc2dadc6ae4ce70
-size 1433042592
+oid sha256:c8b9447005d27a357a5db12441f0b24f4110d2c33ab6c8b6867009159fdd5468
+size 4661160192

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2c10a4c73bd01ab05b1b4a53d54cd26c732304c401335b2538e7521db04fa71
+size 4661160192

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d531688caab11726c2b1f4a768ddc90c97e992eef9f4e1b0484ed74988a9e748
+size 4997750712

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47a1d7f43a1521c4b5511b20221832c7925b9cdb328ba463e036e80849cc041f
+size 3739371680

model.yaml

@@ -0,0 +1,195 @@
+model_name: molmo
+llm:
+  d_model: 3584
+  n_heads: 28
+  n_kv_heads: 4
+  head_dim: null
+  qkv_bias: true
+  clip_qkv: null
+  n_layers: 28
+  mlp_ratio: 4
+  mlp_hidden_size: 37888
+  activation_type: swiglu
+  block_type: sequential
+  rope: true
+  rope_full_precision: true
+  rope_theta: 1000000.0
+  rope_type: default
+  rope_factor: null
+  rope_high_freq_factor: null
+  rope_low_freq_factor: null
+  rope_original_max_position_embeddings: null
+  attention_type: sdpa
+  float32_attention: true
+  attention_dropout: 0.0
+  attention_layer_norm: false
+  attention_layer_norm_type: olmo
+  residual_dropout: 0.1
+  response_residual_dropout: 0.0
+  layer_norm_type: rms
+  layer_norm_with_affine: true
+  layer_norm_eps: 1.0e-06
+  attention_layer_norm_with_affine: true
+  max_sequence_length: 4096
+  max_position_embeddings: null
+  include_bias: false
+  bias_for_layer_norm: null
+  norm_after: false
+  moe_num_experts: 8
+  moe_top_k: 2
+  moe_mlp_impl: sparse
+  moe_log_expert_assignment: false
+  moe_shared_expert: false
+  moe_lbl_in_fp32: false
+  moe_interleave: false
+  moe_loss_weight: 0.1
+  moe_zloss_weight: null
+  moe_dropless: true
+  moe_capacity_factor: 1.25
+  embedding_dropout: 0.0
+  scale_logits: false
+  vocab_size: 152064
+  additional_vocab_size: 128
+  weight_tying: false
+  embedding_size: 152064
+  use_position_ids: true
+  tokenizer:
+    identifier: Qwen/Qwen2.5-7B
+    tokenizer_dir: null
+    depth_tokens: true
+  init_path: gs://mm-olmo/pretrained_llms/qwen2.5-7b.pt
+  init_incremental: null
+  new_embedding_init_range: 0.02
+  initializer_range: 0.02
+  normalize_input_embeds: false
+  activation_checkpoint: whole_layer
+  compile: blocks
+  fix_pad_tokenizer: false
+  resize_vocab: false
+  init_std: 0.02
+  init_fn: normal
+  init_cutoff_factor: null
+vision_backbone:
+  vit:
+    image_model_type: siglip
+    image_default_input_size:
+    - 378
+    - 378
+    image_patch_size: 14
+    image_pos_patch_size: 14
+    image_emb_dim: 1152
+    image_num_heads: 16
+    image_num_key_value_heads: 16
+    image_num_layers: 27
+    image_head_dim: 72
+    image_mlp_dim: 4304
+    image_mlp_activations: gelu_pytorch_tanh
+    image_dropout_rate: 0.0
+    image_num_pos: 729
+    image_norm_eps: 1.0e-06
+    attention_dropout: 0.0
+    residual_dropout: 0.0
+    initializer_range: 0.02
+    float32_attention: true
+    attention_type: sdpa
+    activation_checkpointing: true
+    init_path: gs://mm-olmo/pretrained_image_encoders/siglip2-so400m-14-384.pt
+    resize_mode: siglip
+    pad_value: 0.0
+    normalize: siglip
+  image_pooling_2d: attention_meanq
+  pooling_attention_mask: false
+  image_projector: mlp
+  image_padding_embed: null
+  vit_layers:
+  - -3
+  - -9
+  skip_unused_layers: true
+  image_feature_dropout: 0.0
+  connector_activation_checkpointing: true
+  compile_vit: blocks
+data_formatter:
+  prompt_templates: uber_model
+  message_format: role
+  system_prompt: demo_or_style
+  always_start_with_space: false
+  default_inference_len: 65
+  select_answer: best
+  debug: false
+  image_last: false
+  format_message_list: null
+  p_one_message: 0.0
+mm_preprocessor:
+  crop_mode: overlap-and-resize-c2
+  max_crops: 8
+  max_images: 2
+  max_multi_image_crops: 8
+  pooling_w: 2
+  pooling_h: 2
+  overlap_margins:
+  - 4
+  - 4
+  use_col_tokens: true
+  loss_token_weighting: root_subsegments
+  legacy_image_mask: false
+  max_answer_len: null
+  img_aug: false
+bi_directional_attn: null
+lora_enable: false
+lora_rank: 64
+lora_alpha: 16
+lora_dropout: 0.05
+lora_bias: none
+n_action_bins: 256
+norm_stats:
+  molmoact:
+    action:
+      mean:
+      - 0.0005706787342205644
+      - 0.0002448957529850304
+      - -3.5987635783385485e-05
+      - 0.00021597897284664214
+      - -0.0004896928439848125
+      - -0.000241481073317118
+      - 0.5570635199546814
+      std:
+      - 0.005207270849496126
+      - 0.007506529800593853
+      - 0.006415561307221651
+      - 0.013248044066131115
+      - 0.010928540490567684
+      - 0.014873150736093521
+      - 0.49715080857276917
+      min:
+      - -0.07434078305959702
+      - -0.07339745759963989
+      - -0.06539416313171387
+      - -0.1688285619020462
+      - -0.10289879888296127
+      - -0.2667275667190552
+      - 0.0
+      max:
+      - 0.06042003631591797
+      - 0.09417290985584259
+      - 0.07019275426864624
+      - 0.2616892158985138
+      - 0.11751057207584381
+      - 0.16968433558940887
+      - 1.0
+      q01:
+      - -0.01538565568625927
+      - -0.021047022193670273
+      - -0.01688069850206375
+      - -0.044314172118902206
+      - -0.03890235349535942
+      - -0.04788423702120781
+      - 0.0
+      q99:
+      - 0.014661382883787155
+      - 0.026515591889619827
+      - 0.021398313343524933
+      - 0.04216696694493294
+      - 0.03401297703385353
+      - 0.04957397282123566
+      - 1.0
+    num_entries: 1560068