Upload from GitHub Actions: results: refresh model_health.json snapshot

results/model_health.json

@@ -1,13 +1,13 @@
 [
   {
-    "model":"deepseek\/deepseek-v3.2-speciale-20251201",
+    "model":"baidu\/ernie-4.5-21b-a3b",
     "total":41,
     "failed":41,
     "score_nonfailed":null,
     "failed_pct":100.0
   },
   {
-    "model":"baidu\/ernie-4.5-21b-a3b",
+    "model":"deepseek\/deepseek-v3.2-speciale-20251201",
     "total":41,
     "failed":41,
     "score_nonfailed":null,
@@ -55,13 +55,6 @@
     "score_nonfailed":0.8238520798,
     "failed_pct":83.1669993347
   },
-  {
-    "model":"deepcogito\/cogito-v2.1-671b-20251118",
-    "total":15030,
-    "failed":12478,
-    "score_nonfailed":0.5737762836,
-    "failed_pct":83.0206254158
-  },
   {
     "model":"z-ai\/glm-4.7-flash-20260119",
     "total":15030,
@@ -90,6 +83,20 @@
     "score_nonfailed":0.3930247453,
     "failed_pct":75.0099800399
   },
+  {
+    "model":"mistralai\/mistral-medium-3.5-20260430",
+    "total":15078,
+    "failed":10929,
+    "score_nonfailed":0.8846018241,
+    "failed_pct":72.4830879427
+  },
+  {
+    "model":"deepcogito\/cogito-v2.1-671b-20251118",
+    "total":15080,
+    "failed":10563,
+    "score_nonfailed":0.5708075958,
+    "failed_pct":70.0464190981
+  },
   {
     "model":"nousresearch\/hermes-4-405b",
     "total":15030,
@@ -153,13 +160,6 @@
     "score_nonfailed":0.6658006801,
     "failed_pct":54.6773120426
   },
-  {
-    "model":"deepseek\/deepseek-v3.1-terminus",
-    "total":15030,
-    "failed":8071,
-    "score_nonfailed":0.6103529696,
-    "failed_pct":53.6992681304
-  },
   {
     "model":"qwen\/qwen3.6-35b-a3b-20260415",
     "total":15030,
@@ -209,13 +209,6 @@
     "score_nonfailed":0.5493210221,
     "failed_pct":40.4191616766
   },
-  {
-    "model":"ai21\/jamba-large-1.7",
-    "total":15080,
-    "failed":5789,
-    "score_nonfailed":0.4810803786,
-    "failed_pct":38.3885941645
-  },
   {
     "model":"prime-intellect\/intellect-3-20251126",
     "total":15030,
@@ -230,6 +223,13 @@
     "score_nonfailed":0.3154198804,
     "failed_pct":37.0192947438
   },
+  {
+    "model":"google\/gemini-3.5-flash-20260519",
+    "total":15078,
+    "failed":5308,
+    "score_nonfailed":0.6580770752,
+    "failed_pct":35.2036079056
+  },
   {
     "model":"z-ai\/glm-4.5-air",
     "total":15030,
@@ -245,18 +245,25 @@
     "failed_pct":31.7073170732
   },
   {
-    "model":"arcee-ai\/trinity-mini-20251201",
+    "model":"ai21\/jamba-large-1.7",
     "total":15080,
-    "failed":4602,
-    "score_nonfailed":0.3631567738,
-    "failed_pct":30.5172413793
+    "failed":4464,
+    "score_nonfailed":0.4806710493,
+    "failed_pct":29.6021220159
   },
   {
-    "model":"ibm-granite\/granite-4.0-h-micro",
+    "model":"deepseek\/deepseek-v3.1-terminus",
     "total":15080,
-    "failed":4246,
-    "score_nonfailed":0.3396400581,
-    "failed_pct":28.1564986737
+    "failed":4324,
+    "score_nonfailed":0.6049424824,
+    "failed_pct":28.6737400531
+  },
+  {
+    "model":"arcee-ai\/trinity-mini-20251201",
+    "total":15080,
+    "failed":4149,
+    "score_nonfailed":0.3666731261,
+    "failed_pct":27.5132625995
   },
   {
     "model":"nousresearch\/hermes-4-70b",
@@ -265,13 +272,6 @@
     "score_nonfailed":0.2473206013,
     "failed_pct":25.4111405836
   },
-  {
-    "model":"amazon\/nova-2-lite-v1",
-    "total":15080,
-    "failed":3710,
-    "score_nonfailed":0.535897764,
-    "failed_pct":24.6021220159
-  },
   {
     "model":"deepseek\/deepseek-chat-v3.1",
     "total":15080,
@@ -280,11 +280,18 @@
     "failed_pct":21.8766578249
   },
   {
-    "model":"liquid\/lfm-2-24b-a2b-20260224",
+    "model":"amazon\/nova-2-lite-v1",
     "total":15080,
-    "failed":2808,
-    "score_nonfailed":0.349189204,
-    "failed_pct":18.6206896552
+    "failed":3197,
+    "score_nonfailed":0.5306743634,
+    "failed_pct":21.200265252
+  },
+  {
+    "model":"deepseek\/deepseek-v4-pro",
+    "total":15078,
+    "failed":2829,
+    "score_nonfailed":0.6879473714,
+    "failed_pct":18.7624353363
   },
   {
     "model":"bytedance-seed\/seed-1.6-flash-20250625",
@@ -300,6 +307,13 @@
     "score_nonfailed":0.4370955661,
     "failed_pct":18.0412371134
   },
+  {
+    "model":"ibm-granite\/granite-4.0-h-micro",
+    "total":15080,
+    "failed":2667,
+    "score_nonfailed":0.3411225636,
+    "failed_pct":17.6856763926
+  },
   {
     "model":"deepseek\/deepseek-r1-0528",
     "total":41,
@@ -314,20 +328,6 @@
     "score_nonfailed":0.4648651834,
     "failed_pct":16.9461077844
   },
-  {
-    "model":"microsoft\/phi-4-mini-instruct",
-    "total":15080,
-    "failed":2517,
-    "score_nonfailed":0.2620524844,
-    "failed_pct":16.6909814324
-  },
-  {
-    "model":"google\/gemma-4-31b-it-20260402",
-    "total":15080,
-    "failed":2507,
-    "score_nonfailed":0.6070484846,
-    "failed_pct":16.624668435
-  },
   {
     "model":"z-ai\/glm-4-32b-0414",
     "total":15030,
@@ -336,11 +336,18 @@
     "failed_pct":16.5801729874
   },
   {
-    "model":"google\/gemini-3.1-flash-lite",
+    "model":"microsoft\/phi-4-mini-instruct",
     "total":15080,
-    "failed":1827,
-    "score_nonfailed":0.6534476235,
-    "failed_pct":12.1153846154
+    "failed":2282,
+    "score_nonfailed":0.2616162964,
+    "failed_pct":15.1326259947
+  },
+  {
+    "model":"google\/gemma-4-31b-it-20260402",
+    "total":15080,
+    "failed":2177,
+    "score_nonfailed":0.6035083328,
+    "failed_pct":14.4363395225
   },
   {
     "model":"qwen\/qwen3-30b-a3b",
@@ -356,6 +363,13 @@
     "score_nonfailed":0.5317395427,
     "failed_pct":10.8289124668
   },
+  {
+    "model":"liquid\/lfm-2-24b-a2b-20260224",
+    "total":15080,
+    "failed":1422,
+    "score_nonfailed":0.3444065389,
+    "failed_pct":9.4297082228
+  },
   {
     "model":"qwen\/qwen3-30b-a3b-instruct-2507",
     "total":15030,
@@ -408,9 +422,9 @@
   {
     "model":"anthropic\/claude-sonnet-4.6",
     "total":15080,
-    "failed":873,
-    "score_nonfailed":0.6504398418,
-    "failed_pct":5.7891246684
+    "failed":767,
+    "score_nonfailed":0.6487447649,
+    "failed_pct":5.0862068966
   },
   {
     "model":"mistralai\/ministral-8b-2512",
@@ -426,13 +440,6 @@
     "score_nonfailed":0.6238057314,
     "failed_pct":4.0848806366
   },
-  {
-    "model":"mistralai\/ministral-14b-2512",
-    "total":15080,
-    "failed":612,
-    "score_nonfailed":0.4586929167,
-    "failed_pct":4.0583554377
-  },
   {
     "model":"qwen\/qwen3.6-flash",
     "total":15030,
@@ -440,13 +447,6 @@
     "score_nonfailed":0.5823627104,
     "failed_pct":3.6859614105
   },
-  {
-    "model":"ibm-granite\/granite-4.1-8b-20260429",
-    "total":15080,
-    "failed":553,
-    "score_nonfailed":0.4250007907,
-    "failed_pct":3.6671087533
-  },
   {
     "model":"anthropic\/claude-opus-4.6",
     "total":15030,
@@ -454,12 +454,26 @@
     "score_nonfailed":0.6793265387,
     "failed_pct":3.5662009315
   },
+  {
+    "model":"mistralai\/ministral-14b-2512",
+    "total":15080,
+    "failed":489,
+    "score_nonfailed":0.4569203903,
+    "failed_pct":3.2427055703
+  },
+  {
+    "model":"google\/gemini-3.1-flash-lite",
+    "total":15080,
+    "failed":445,
+    "score_nonfailed":0.6527671954,
+    "failed_pct":2.950928382
+  },
   {
     "model":"deepseek\/deepseek-v3.2-exp",
     "total":15080,
-    "failed":504,
-    "score_nonfailed":0.5810977119,
-    "failed_pct":3.3421750663
+    "failed":411,
+    "score_nonfailed":0.5804285596,
+    "failed_pct":2.725464191
   },
   {
     "model":"openai\/gpt-5-nano",
@@ -468,6 +482,13 @@
     "score_nonfailed":0.5014027303,
     "failed_pct":2.374005305
   },
+  {
+    "model":"ibm-granite\/granite-4.1-8b-20260429",
+    "total":15080,
+    "failed":270,
+    "score_nonfailed":0.4243063326,
+    "failed_pct":1.7904509284
+  },
   {
     "model":"bytedance-seed\/seed-2.0-mini-20260224",
     "total":15080,
@@ -475,13 +496,6 @@
     "score_nonfailed":0.5152390813,
     "failed_pct":1.2533156499
   },
-  {
-    "model":"google\/gemini-3.1-pro-preview",
-    "total":15080,
-    "failed":188,
-    "score_nonfailed":0.6897643923,
-    "failed_pct":1.2466843501
-  },
   {
     "model":"bytedance-seed\/seed-1.6-20250625",
     "total":15080,
@@ -489,6 +503,13 @@
     "score_nonfailed":0.5004845457,
     "failed_pct":1.1737400531
   },
+  {
+    "model":"google\/gemini-3.1-pro-preview",
+    "total":15080,
+    "failed":173,
+    "score_nonfailed":0.6900094808,
+    "failed_pct":1.1472148541
+  },
   {
     "model":"openai\/gpt-5.5",
     "total":15030,
@@ -510,19 +531,12 @@
     "score_nonfailed":0.6309648172,
     "failed_pct":0.998003992
   },
-  {
-    "model":"mistralai\/mistral-small-3.2-24b-instruct",
-    "total":15080,
-    "failed":139,
-    "score_nonfailed":0.4631506534,
-    "failed_pct":0.9217506631
-  },
   {
     "model":"anthropic\/claude-opus-4.8",
     "total":15080,
-    "failed":128,
-    "score_nonfailed":0.661031115,
-    "failed_pct":0.848806366
+    "failed":111,
+    "score_nonfailed":0.6609524402,
+    "failed_pct":0.7360742706
   },
   {
     "model":"openai\/gpt-5.3-chat-20260303",
@@ -531,12 +545,19 @@
     "score_nonfailed":0.6358423209,
     "failed_pct":0.7252162342
   },
+  {
+    "model":"mistralai\/mistral-small-3.2-24b-instruct",
+    "total":15080,
+    "failed":104,
+    "score_nonfailed":0.4624688777,
+    "failed_pct":0.6896551724
+  },
   {
     "model":"anthropic\/claude-opus-4.7",
     "total":15080,
-    "failed":108,
-    "score_nonfailed":0.6685617849,
-    "failed_pct":0.7161803714
+    "failed":100,
+    "score_nonfailed":0.6684399203,
+    "failed_pct":0.6631299735
   },
   {
     "model":"x-ai\/grok-4.20",
@@ -555,9 +576,9 @@
   {
     "model":"bytedance-seed\/seed-2.0-lite-20260309",
     "total":15080,
-    "failed":75,
-    "score_nonfailed":0.5853661867,
-    "failed_pct":0.4973474801
+    "failed":73,
+    "score_nonfailed":0.5854214454,
+    "failed_pct":0.4840848806
   },
   {
     "model":"openai\/gpt-5-mini",
@@ -609,10 +630,10 @@
     "failed_pct":0.0
   },
   {
-    "model":"alpindale\/goliath-120b",
-    "total":170,
+    "model":"baidu\/ernie-4.5-300b-a47b",
+    "total":15030,
     "failed":0,
-    "score_nonfailed":0.3700368906,
+    "score_nonfailed":0.5751447662,
     "failed_pct":0.0
   },
   {
@@ -623,17 +644,17 @@
     "failed_pct":0.0
   },
   {
-    "model":"openai\/o4-mini-2025-04-16",
-    "total":167,
+    "model":"qwen\/qwen3-8b-04-28",
+    "total":113,
     "failed":0,
-    "score_nonfailed":0.646647725,
+    "score_nonfailed":0.6392222245,
     "failed_pct":0.0
   },
   {
-    "model":"x-ai\/grok-3",
+    "model":"alpindale\/goliath-120b",
     "total":170,
     "failed":0,
-    "score_nonfailed":0.6330974229,
+    "score_nonfailed":0.3700368906,
     "failed_pct":0.0
   },
   {
@@ -643,6 +664,13 @@
     "score_nonfailed":0.5469999688,
     "failed_pct":0.0
   },
+  {
+    "model":"qwen\/qwen3-vl-30b-a3b-thinking",
+    "total":17,
+    "failed":0,
+    "score_nonfailed":0.5794774559,
+    "failed_pct":0.0
+  },
   {
     "model":"x-ai\/grok-4",
     "total":3475,
@@ -651,10 +679,17 @@
     "failed_pct":0.0
   },
   {
-    "model":"qwen\/qwen3-8b-04-28",
-    "total":113,
+    "model":"anthropic\/claude-3.5-sonnet",
+    "total":14288,
     "failed":0,
-    "score_nonfailed":0.6392222245,
+    "score_nonfailed":0.6279979071,
+    "failed_pct":0.0
+  },
+  {
+    "model":"x-ai\/grok-3",
+    "total":170,
+    "failed":0,
+    "score_nonfailed":0.6330974229,
     "failed_pct":0.0
   },
   {
@@ -679,10 +714,17 @@
     "failed_pct":0.0
   },
   {
-    "model":"anthropic\/claude-3.5-sonnet",
-    "total":14288,
+    "model":"openai\/o4-mini-2025-04-16",
+    "total":167,
     "failed":0,
-    "score_nonfailed":0.6279979071,
+    "score_nonfailed":0.646647725,
+    "failed_pct":0.0
+  },
+  {
+    "model":"anthropic\/claude-haiku-4.5",
+    "total":15080,
+    "failed":0,
+    "score_nonfailed":0.5380086427,
     "failed_pct":0.0
   },
   {
@@ -693,38 +735,45 @@
     "failed_pct":0.0
   },
   {
-    "model":"qwen\/qwen3-vl-30b-a3b-thinking",
+    "model":"google\/gemini-2.0-flash-lite-001",
     "total":17,
     "failed":0,
-    "score_nonfailed":0.5794774559,
+    "score_nonfailed":0.7041712433,
     "failed_pct":0.0
   },
   {
-    "model":"anthropic\/claude-haiku-4.5",
+    "model":"openai\/gpt-4.1",
     "total":15080,
     "failed":0,
-    "score_nonfailed":0.5380086427,
+    "score_nonfailed":0.5577101349,
     "failed_pct":0.0
   },
   {
-    "model":"openai\/gpt-3.5-turbo-0613",
+    "model":"openai\/gpt-5-codex",
     "total":170,
     "failed":0,
-    "score_nonfailed":0.5504296912,
+    "score_nonfailed":0.7040281767,
     "failed_pct":0.0
   },
   {
-    "model":"openai\/gpt-5-codex",
-    "total":170,
+    "model":"meta-llama\/llama-3.3-70b-instruct",
+    "total":15080,
     "failed":0,
-    "score_nonfailed":0.7040281767,
+    "score_nonfailed":0.484181525,
     "failed_pct":0.0
   },
   {
-    "model":"openai\/gpt-5-chat-2025-08-07",
-    "total":170,
+    "model":"mistralai\/mistral-medium-3.1",
+    "total":15080,
     "failed":0,
-    "score_nonfailed":0.639942047,
+    "score_nonfailed":0.5110552261,
+    "failed_pct":0.0
+  },
+  {
+    "model":"mistralai\/magistral-small-2506",
+    "total":120,
+    "failed":0,
+    "score_nonfailed":0.3314969313,
     "failed_pct":0.0
   },
   {
@@ -756,17 +805,17 @@
     "failed_pct":0.0
   },
   {
-    "model":"meta-llama\/llama-3.3-70b-instruct",
+    "model":"meta-llama\/llama-3.1-70b-instruct",
     "total":15080,
     "failed":0,
-    "score_nonfailed":0.484181525,
+    "score_nonfailed":0.4854957629,
     "failed_pct":0.0
   },
   {
-    "model":"meta-llama\/llama-3.1-70b-instruct",
-    "total":15080,
+    "model":"openai\/gpt-5-chat-2025-08-07",
+    "total":170,
     "failed":0,
-    "score_nonfailed":0.4854957629,
+    "score_nonfailed":0.639942047,
     "failed_pct":0.0
   },
   {
@@ -776,13 +825,6 @@
     "score_nonfailed":0.467426787,
     "failed_pct":0.0
   },
-  {
-    "model":"cohere\/command-r-plus-08-2024",
-    "total":6369,
-    "failed":0,
-    "score_nonfailed":0.3479812964,
-    "failed_pct":0.0
-  },
   {
     "model":"google\/gemma-3-27b-it",
     "total":15080,
@@ -819,150 +861,129 @@
     "failed_pct":0.0
   },
   {
-    "model":"google\/gemini-2.5-flash",
+    "model":"mistralai\/mistral-nemo",
     "total":15080,
     "failed":0,
-    "score_nonfailed":0.4238758246,
+    "score_nonfailed":0.3398077816,
     "failed_pct":0.0
   },
   {
-    "model":"google\/gemini-2.0-flash-lite-001",
+    "model":"mistralai\/mistral-small-3.1-24b-instruct",
     "total":17,
     "failed":0,
-    "score_nonfailed":0.7041712433,
+    "score_nonfailed":0.70419565,
     "failed_pct":0.0
   },
   {
-    "model":"cohere\/command-r-08-2024",
+    "model":"mistralai\/mixtral-8x7b-instruct",
     "total":170,
     "failed":0,
-    "score_nonfailed":0.5222153257,
-    "failed_pct":0.0
-  },
-  {
-    "model":"mistralai\/magistral-small-2506",
-    "total":120,
-    "failed":0,
-    "score_nonfailed":0.3314969313,
+    "score_nonfailed":0.4240249599,
     "failed_pct":0.0
   },
   {
-    "model":"cohere\/command-a",
+    "model":"moonshotai\/kimi-k2",
     "total":15080,
     "failed":0,
-    "score_nonfailed":0.5047155086,
+    "score_nonfailed":0.5523425103,
     "failed_pct":0.0
   },
   {
-    "model":"openai\/gpt-3.5-turbo",
-    "total":15080,
+    "model":"cohere\/command-r-plus-08-2024",
+    "total":6369,
     "failed":0,
-    "score_nonfailed":0.3503358897,
+    "score_nonfailed":0.3479812964,
     "failed_pct":0.0
   },
   {
-    "model":"openai\/gpt-4o-mini",
+    "model":"neversleep\/noromaid-20b",
     "total":170,
     "failed":0,
-    "score_nonfailed":0.6541716124,
-    "failed_pct":0.0
-  },
-  {
-    "model":"openai\/gpt-4o-2024-11-20",
-    "total":17,
-    "failed":0,
-    "score_nonfailed":0.742557214,
+    "score_nonfailed":0.2124753832,
     "failed_pct":0.0
   },
   {
-    "model":"openai\/gpt-4o",
-    "total":15080,
+    "model":"nousresearch\/hermes-3-llama-3.1-405b",
+    "total":170,
     "failed":0,
-    "score_nonfailed":0.5872301625,
+    "score_nonfailed":0.6629729711,
     "failed_pct":0.0
   },
   {
-    "model":"openai\/gpt-4.1-nano",
+    "model":"cohere\/command-r-08-2024",
     "total":170,
     "failed":0,
-    "score_nonfailed":0.6064425983,
+    "score_nonfailed":0.5222153257,
     "failed_pct":0.0
   },
   {
-    "model":"openai\/gpt-4.1-mini",
-    "total":170,
+    "model":"cohere\/command-a",
+    "total":15080,
     "failed":0,
-    "score_nonfailed":0.6424898144,
+    "score_nonfailed":0.5047155086,
     "failed_pct":0.0
   },
   {
-    "model":"openai\/gpt-4.1",
+    "model":"openai\/gpt-3.5-turbo",
     "total":15080,
     "failed":0,
-    "score_nonfailed":0.5577101349,
+    "score_nonfailed":0.3503358897,
     "failed_pct":0.0
   },
   {
-    "model":"baidu\/ernie-4.5-300b-a47b",
-    "total":15030,
+    "model":"openai\/gpt-3.5-turbo-0613",
+    "total":170,
     "failed":0,
-    "score_nonfailed":0.5751447662,
+    "score_nonfailed":0.5504296912,
     "failed_pct":0.0
   },
   {
-    "model":"mistralai\/mistral-medium-3.1",
+    "model":"google\/gemini-2.5-flash",
     "total":15080,
     "failed":0,
-    "score_nonfailed":0.5110552261,
+    "score_nonfailed":0.4238758246,
     "failed_pct":0.0
   },
   {
-    "model":"nousresearch\/hermes-3-llama-3.1-405b",
+    "model":"openai\/gpt-4.1-mini",
     "total":170,
     "failed":0,
-    "score_nonfailed":0.6629729711,
+    "score_nonfailed":0.6424898144,
     "failed_pct":0.0
   },
   {
-    "model":"neversleep\/noromaid-20b",
+    "model":"openai\/gpt-4.1-nano",
     "total":170,
     "failed":0,
-    "score_nonfailed":0.2124753832,
+    "score_nonfailed":0.6064425983,
     "failed_pct":0.0
   },
   {
-    "model":"moonshotai\/kimi-k2",
+    "model":"openai\/gpt-4o",
     "total":15080,
     "failed":0,
-    "score_nonfailed":0.5523425103,
+    "score_nonfailed":0.5872301625,
     "failed_pct":0.0
   },
   {
-    "model":"mistralai\/mixtral-8x7b-instruct",
-    "total":170,
+    "model":"openai\/gpt-4o-2024-11-20",
+    "total":17,
     "failed":0,
-    "score_nonfailed":0.4240249599,
+    "score_nonfailed":0.742557214,
     "failed_pct":0.0
   },
   {
-    "model":"mistralai\/mistral-saba",
-    "total":15080,
+    "model":"openai\/gpt-4o-mini",
+    "total":170,
     "failed":0,
-    "score_nonfailed":0.4559403599,
+    "score_nonfailed":0.6541716124,
     "failed_pct":0.0
   },
   {
-    "model":"mistralai\/mistral-nemo",
+    "model":"mistralai\/mistral-saba",
     "total":15080,
     "failed":0,
-    "score_nonfailed":0.3398077816,
-    "failed_pct":0.0
-  },
-  {
-    "model":"mistralai\/mistral-small-3.1-24b-instruct",
-    "total":17,
-    "failed":0,
-    "score_nonfailed":0.70419565,
+    "score_nonfailed":0.4559403599,
     "failed_pct":0.0
   }
 ]