Training in progress, step 26840, checkpoint

last-checkpoint/global_step26840/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6352feb7c350a62238622e0e788c88e5be55b9750c1d51f8302cd35c9553dde
+size 2312371472

last-checkpoint/global_step26840/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d56362d42c54a0d2bb62d2e97838d22ecd01464a52ae7c75fc8fc1e6332ad0b
+size 2312371472

last-checkpoint/global_step26840/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08ba8602308ec40a45363a91a1b8852fe1667dbde98ee7a820352bd0d362fb64
+size 2312371472

last-checkpoint/global_step26840/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cf488272b732bcdee7cdbe6f749c49a2a75f7a04db9854f32ca40e6a84cbfbf
+size 2312368912

last-checkpoint/global_step26840/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3171d7599d1cb0bcf24ce6742e8cfea0dd60af0b23741ec2101bcb89755bd1d5
+size 2312373776

last-checkpoint/global_step26840/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a0dd81cb4997639e0a3c94957b34e3017bfa238110fea417571ac6865f1a1d7
+size 2312374928

last-checkpoint/global_step26840/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e82f3a417fdc51e897f5ef589501668deab4dddb98090196a4bd85d553cd9098
+size 2312374608

last-checkpoint/global_step26840/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12d2df4faf88886659ba484719ea6153a394e5bf686c86c21a6e450ede3be099
+size 2312374352

last-checkpoint/global_step26840/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbffd1b2694b080e9059f0575316a7fa0e33c6c85eb9126c34f540ea81f6d8ba
+size 3091157252

last-checkpoint/latest

@@ -1 +1 @@
-global_step23485
+global_step26840

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:714bbedeea9887f4a4175fd6ca7185f327bfb632f864ad8e2be90ecc389d5a56
+oid sha256:0994eab9f09245b2604408956fb81c5bb1bbba22b4386ceb38dc21867b83b3d2
 size 3219909288

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77d8ab6905cd3a4e24d75f95e0ee11957f3affc8737a6ab6794d643c09553c81
+oid sha256:55e464ee5687df2468f6598f5fbaf5ecca79e676c224afe9b2b9c93e7de313e3
 size 15984

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:110c4f76ce40322613c082b35e2f64980aead421fe536dfcb89def5e8adf624e
+oid sha256:09c2768a057a5737f0dd811a2de2e72b92803061644a587506be3a1b78938712
 size 15984

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:80e1f4315f94ccb9c260de1cd47266e51224bfc98c4655b866c79b57e5cf6db9
+oid sha256:77e4ad39a8765593ef6b70913f6cfd1e72a7a7481a92bc4a9779fe9c7952cf13
 size 15984

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:516f55b71bbd475c8c83b214d527ab1a6fc8c9cbf0f2471d183b7809c99f1dbf
+oid sha256:a45b7061361563da1cab22f3338e147ac5b6aaf172e88e9248f4f95e493cef24
 size 15984

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d449ea9890c8ccd230b1934042d174f2679448dd3ab3b039c7a39b6d4b5e0e2a
+oid sha256:db243ba40667e19f840cf467486fdee8150837809162868348f5b0a5a63faf87
 size 15984

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f2d10c0b913462f38a1335aacdd7e667fbf36ea7e7efe0e6678d7d62ea77b2ba
+oid sha256:648d594d4dbe02334ab30eec0ae10841e3ae65746846a43bb94d60b5366ef4c9
 size 15984

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7759888fa3895c36ed13c6ffe7b753a06b450fe3e08446c2d9d4e800a625cdd7
+oid sha256:da7b5be279fb61e58c292b03764c230adec8a20a83b7cbe5e8f29d20187238b2
 size 15984

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c5eb6791e3c9c99e1287dfb1f51164085acc9747dbddb8dfeac1903f3fd1ab3
+oid sha256:1d0f3cc458c5e8588ba44862c062dee2ec0f7322cdb2054e8c69057d17238686
 size 15984

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": 23485,
   "best_metric": 0.2517834662190516,
   "best_model_checkpoint": "./whisper-large-v3-ar-test/checkpoint-23485",
-  "epoch": 3.4989570917759236,
+  "epoch": 3.99880810488677,
   "eval_steps": 3355,
-  "global_step": 23485,
+  "global_step": 26840,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1790,6 +1790,280 @@
       "eval_mgb2_wer": 9.467092337917485,
       "eval_mgb2_wer_ortho": 13.79169224473092,
       "step": 23485
+    },
+    {
+      "epoch": 3.50119189511323,
+      "grad_norm": 0.37351053953170776,
+      "learning_rate": 2.9153628371330657e-06,
+      "loss": 0.0262,
+      "step": 23500
+    },
+    {
+      "epoch": 3.5160905840286056,
+      "grad_norm": 1.6208069324493408,
+      "learning_rate": 2.9029454130035267e-06,
+      "loss": 0.024,
+      "step": 23600
+    },
+    {
+      "epoch": 3.530989272943981,
+      "grad_norm": 0.6350374817848206,
+      "learning_rate": 2.8905279888739885e-06,
+      "loss": 0.0234,
+      "step": 23700
+    },
+    {
+      "epoch": 3.545887961859356,
+      "grad_norm": 1.354862928390503,
+      "learning_rate": 2.8781105647444494e-06,
+      "loss": 0.025,
+      "step": 23800
+    },
+    {
+      "epoch": 3.5607866507747317,
+      "grad_norm": 0.6370978355407715,
+      "learning_rate": 2.8656931406149112e-06,
+      "loss": 0.0245,
+      "step": 23900
+    },
+    {
+      "epoch": 3.575685339690107,
+      "grad_norm": 0.7340075373649597,
+      "learning_rate": 2.8532757164853726e-06,
+      "loss": 0.0269,
+      "step": 24000
+    },
+    {
+      "epoch": 3.5905840286054826,
+      "grad_norm": 0.6038511395454407,
+      "learning_rate": 2.8408582923558344e-06,
+      "loss": 0.025,
+      "step": 24100
+    },
+    {
+      "epoch": 3.605482717520858,
+      "grad_norm": 0.6267861127853394,
+      "learning_rate": 2.8284408682262953e-06,
+      "loss": 0.0221,
+      "step": 24200
+    },
+    {
+      "epoch": 3.6203814064362336,
+      "grad_norm": 1.2206661701202393,
+      "learning_rate": 2.8160234440967567e-06,
+      "loss": 0.0251,
+      "step": 24300
+    },
+    {
+      "epoch": 3.635280095351609,
+      "grad_norm": 0.6847667098045349,
+      "learning_rate": 2.803606019967218e-06,
+      "loss": 0.024,
+      "step": 24400
+    },
+    {
+      "epoch": 3.6501787842669846,
+      "grad_norm": 0.8292557597160339,
+      "learning_rate": 2.7911885958376795e-06,
+      "loss": 0.0246,
+      "step": 24500
+    },
+    {
+      "epoch": 3.66507747318236,
+      "grad_norm": 0.6944147944450378,
+      "learning_rate": 2.7787711717081413e-06,
+      "loss": 0.0228,
+      "step": 24600
+    },
+    {
+      "epoch": 3.679976162097735,
+      "grad_norm": 0.5313336849212646,
+      "learning_rate": 2.766353747578602e-06,
+      "loss": 0.0249,
+      "step": 24700
+    },
+    {
+      "epoch": 3.694874851013111,
+      "grad_norm": 0.47161412239074707,
+      "learning_rate": 2.753936323449064e-06,
+      "loss": 0.026,
+      "step": 24800
+    },
+    {
+      "epoch": 3.709773539928486,
+      "grad_norm": 0.9280988574028015,
+      "learning_rate": 2.7415188993195254e-06,
+      "loss": 0.0224,
+      "step": 24900
+    },
+    {
+      "epoch": 3.7246722288438616,
+      "grad_norm": 0.9222899675369263,
+      "learning_rate": 2.7291014751899867e-06,
+      "loss": 0.0229,
+      "step": 25000
+    },
+    {
+      "epoch": 3.739570917759237,
+      "grad_norm": 0.6520107388496399,
+      "learning_rate": 2.716684051060448e-06,
+      "loss": 0.0215,
+      "step": 25100
+    },
+    {
+      "epoch": 3.7544696066746126,
+      "grad_norm": 0.5594351291656494,
+      "learning_rate": 2.70426662693091e-06,
+      "loss": 0.0227,
+      "step": 25200
+    },
+    {
+      "epoch": 3.769368295589988,
+      "grad_norm": 0.8105785846710205,
+      "learning_rate": 2.691849202801371e-06,
+      "loss": 0.0231,
+      "step": 25300
+    },
+    {
+      "epoch": 3.7842669845053636,
+      "grad_norm": 0.8062728643417358,
+      "learning_rate": 2.6794317786718326e-06,
+      "loss": 0.0212,
+      "step": 25400
+    },
+    {
+      "epoch": 3.799165673420739,
+      "grad_norm": 0.6356725692749023,
+      "learning_rate": 2.667014354542294e-06,
+      "loss": 0.022,
+      "step": 25500
+    },
+    {
+      "epoch": 3.8140643623361146,
+      "grad_norm": 0.8131894469261169,
+      "learning_rate": 2.6545969304127554e-06,
+      "loss": 0.0223,
+      "step": 25600
+    },
+    {
+      "epoch": 3.82896305125149,
+      "grad_norm": 0.34240245819091797,
+      "learning_rate": 2.6421795062832168e-06,
+      "loss": 0.0211,
+      "step": 25700
+    },
+    {
+      "epoch": 3.843861740166865,
+      "grad_norm": 0.7727413177490234,
+      "learning_rate": 2.6297620821536786e-06,
+      "loss": 0.0229,
+      "step": 25800
+    },
+    {
+      "epoch": 3.858760429082241,
+      "grad_norm": 1.0136847496032715,
+      "learning_rate": 2.6173446580241395e-06,
+      "loss": 0.0225,
+      "step": 25900
+    },
+    {
+      "epoch": 3.873659117997616,
+      "grad_norm": 0.603969931602478,
+      "learning_rate": 2.6049272338946013e-06,
+      "loss": 0.0221,
+      "step": 26000
+    },
+    {
+      "epoch": 3.8885578069129916,
+      "grad_norm": 0.7807964086532593,
+      "learning_rate": 2.5925098097650627e-06,
+      "loss": 0.0228,
+      "step": 26100
+    },
+    {
+      "epoch": 3.903456495828367,
+      "grad_norm": 0.5105961561203003,
+      "learning_rate": 2.580092385635524e-06,
+      "loss": 0.0194,
+      "step": 26200
+    },
+    {
+      "epoch": 3.9183551847437426,
+      "grad_norm": 0.6721608638763428,
+      "learning_rate": 2.5676749615059854e-06,
+      "loss": 0.0221,
+      "step": 26300
+    },
+    {
+      "epoch": 3.933253873659118,
+      "grad_norm": 1.0928480625152588,
+      "learning_rate": 2.555257537376447e-06,
+      "loss": 0.023,
+      "step": 26400
+    },
+    {
+      "epoch": 3.9481525625744935,
+      "grad_norm": 0.8083678483963013,
+      "learning_rate": 2.542840113246908e-06,
+      "loss": 0.0235,
+      "step": 26500
+    },
+    {
+      "epoch": 3.963051251489869,
+      "grad_norm": 0.6882435083389282,
+      "learning_rate": 2.5304226891173695e-06,
+      "loss": 0.0211,
+      "step": 26600
+    },
+    {
+      "epoch": 3.977949940405244,
+      "grad_norm": 0.6654950380325317,
+      "learning_rate": 2.5180052649878313e-06,
+      "loss": 0.0202,
+      "step": 26700
+    },
+    {
+      "epoch": 3.99284862932062,
+      "grad_norm": 0.31399238109588623,
+      "learning_rate": 2.5055878408582923e-06,
+      "loss": 0.0199,
+      "step": 26800
+    },
+    {
+      "epoch": 3.99880810488677,
+      "eval_quran_cer": 0.559225892559226,
+      "eval_quran_cer_ortho": 0.5632298965632299,
+      "eval_quran_loss": 0.006387302652001381,
+      "eval_quran_runtime": 884.8053,
+      "eval_quran_samples_per_second": 1.13,
+      "eval_quran_steps_per_second": 0.018,
+      "eval_quran_wer": 0.8742481465939292,
+      "eval_quran_wer_ortho": 0.8952301021121835,
+      "step": 26840
+    },
+    {
+      "epoch": 3.99880810488677,
+      "eval_hadith_cer": 3.8342385316344125,
+      "eval_hadith_cer_ortho": 3.9314555329436014,
+      "eval_hadith_loss": 0.06706906855106354,
+      "eval_hadith_runtime": 859.7812,
+      "eval_hadith_samples_per_second": 1.163,
+      "eval_hadith_steps_per_second": 0.019,
+      "eval_hadith_wer": 6.03901938133744,
+      "eval_hadith_wer_ortho": 6.417661404184314,
+      "step": 26840
+    },
+    {
+      "epoch": 3.99880810488677,
+      "eval_mgb2_cer": 2.902677893823859,
+      "eval_mgb2_cer_ortho": 3.678463284022172,
+      "eval_mgb2_loss": 0.292144775390625,
+      "eval_mgb2_runtime": 460.4502,
+      "eval_mgb2_samples_per_second": 1.073,
+      "eval_mgb2_steps_per_second": 0.017,
+      "eval_mgb2_wer": 10.019646365422396,
+      "eval_mgb2_wer_ortho": 13.971761817065683,
+      "step": 26840
     }
   ],
   "logging_steps": 100,
@@ -1804,7 +2078,7 @@
         "early_stopping_threshold": 0.0
       },
       "attributes": {
-        "early_stopping_patience_counter": 0
+        "early_stopping_patience_counter": 1
       }
     },
     "TrainerControl": {
@@ -1818,7 +2092,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.5532878087120287e+21,
+  "total_flos": 2.9180432098191653e+21,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null