e-zorzi/Qwen2.5-VL-7B-sft-lora-noreasoning

Finetuned using axolotl, with the following configuration

base_model: Qwen/Qwen2.5-VL-7B-Instruct
processor_type: AutoProcessor

# these 3 lines are needed for now to handle vision chat templates w images
skip_prepare_dataset: true
remove_unused_columns: false
sample_packing: false

chat_template: qwen2_vl
datasets:
  - path: e-zorzi/reasoning_distractors_choice_chat_only
    type: chat_template
    split: train
    revision: small

test_datasets:
  - path: e-zorzi/reasoning_distractors_choice_chat_only
    type: chat_template
    split: val_seen[:5%]
    revision: small
  - path: e-zorzi/reasoning_distractors_choice_chat_only
    type: chat_template
    split: val_unseen[:5%]
    revision: small
  
load_in_8bit: True
adapter: lora
lora_model_dir:

sequence_len: 2048 #8192
pad_to_sequence_len: false

lora_r: 128
lora_alpha: 16
lora_dropout: 0.05
lora_target_modules: 'model.language_model.layers.[\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj'

gradient_accumulation_steps: 4
micro_batch_size: 16
num_epochs: 15
optimizer: adamw_bnb_8bit
lr_scheduler: cosine
learning_rate: 0.001

bf16: true
fp16:
tf32: true

gradient_checkpointing: true
logging_steps: 1
flash_attention: true
eager_attention:

warmup_ratio: 0.0067
evals_per_epoch: 3
saves_per_epoch: 1
save_strategy: epoch
weight_decay: 0.0

# save_first_step: true  # uncomment this to validate checkpoint saving works with your config