| { |
| "results_sha256": "db3d7cb81972ad2909a75aca659b97015d9e96a770f04620275b79b9fbdbd80d", |
| "source_commit": "ee6bd37d34ae35d2e69ebb0c4b0957a10c727357", |
| "freeze_sha256": "eb163f63b0eb3a706adaaf18d20ebe3acaef4ddb0690005149eef2cbc750d2a6", |
| "observed": 3277, |
| "full_panel_complete": true, |
| "timing_exclusions": { |
| "latency_exclusion_only": true, |
| "quality_rows_retained": true, |
| "prior_uncertain_ranges": { |
| "1491": [ |
| 1496, |
| "first foreign-GPU watchdog stop; per-case timestamps unavailable" |
| ], |
| "2651": [ |
| 2660, |
| "RustDesk watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| "new_overlap_samples": 0, |
| "excluded_cases": [ |
| { |
| "index": 1491, |
| "panel": "typed-decisions", |
| "id": "agent_trace_observability_000042/risk", |
| "reasons": [ |
| "first foreign-GPU watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 1492, |
| "panel": "typed-decisions", |
| "id": "agent_trace_observability_000042/urgency", |
| "reasons": [ |
| "first foreign-GPU watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 1493, |
| "panel": "typed-decisions", |
| "id": "agent_trace_observability_000043/action", |
| "reasons": [ |
| "first foreign-GPU watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 1494, |
| "panel": "typed-decisions", |
| "id": "agent_trace_observability_000043/needs_review", |
| "reasons": [ |
| "first foreign-GPU watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 1495, |
| "panel": "typed-decisions", |
| "id": "agent_trace_observability_000043/outcome", |
| "reasons": [ |
| "first foreign-GPU watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 1496, |
| "panel": "typed-decisions", |
| "id": "agent_trace_observability_000043/risk", |
| "reasons": [ |
| "first foreign-GPU watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 2651, |
| "panel": "typed-decisions", |
| "id": "invoice_processing_000074/matches_order", |
| "reasons": [ |
| "RustDesk watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 2652, |
| "panel": "typed-decisions", |
| "id": "invoice_processing_000074/urgency", |
| "reasons": [ |
| "RustDesk watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 2653, |
| "panel": "typed-decisions", |
| "id": "invoice_processing_000075/discrepancy_severity", |
| "reasons": [ |
| "RustDesk watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 2654, |
| "panel": "typed-decisions", |
| "id": "invoice_processing_000075/disposition", |
| "reasons": [ |
| "RustDesk watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 2655, |
| "panel": "typed-decisions", |
| "id": "invoice_processing_000075/duplicate", |
| "reasons": [ |
| "RustDesk watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 2656, |
| "panel": "typed-decisions", |
| "id": "invoice_processing_000075/matches_order", |
| "reasons": [ |
| "RustDesk watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 2657, |
| "panel": "typed-decisions", |
| "id": "invoice_processing_000075/urgency", |
| "reasons": [ |
| "RustDesk watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 2658, |
| "panel": "typed-decisions", |
| "id": "invoice_processing_000076/discrepancy_severity", |
| "reasons": [ |
| "RustDesk watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 2659, |
| "panel": "typed-decisions", |
| "id": "invoice_processing_000076/disposition", |
| "reasons": [ |
| "RustDesk watchdog stop; per-case timestamps unavailable" |
| ] |
| }, |
| { |
| "index": 2660, |
| "panel": "typed-decisions", |
| "id": "invoice_processing_000076/duplicate", |
| "reasons": [ |
| "RustDesk watchdog stop; per-case timestamps unavailable" |
| ] |
| } |
| ] |
| }, |
| "panels": { |
| "jevbench-public": { |
| "observed": 231, |
| "expected": 231, |
| "complete": true, |
| "valid": 231, |
| "semantics": "verified label accuracy", |
| "errors": [], |
| "fallbacks": [], |
| "direct": { |
| "n": 231, |
| "top1": 175, |
| "nll": 0.6548821957788397, |
| "brier": 0.3384088069425174, |
| "score_expected_level_mae": 0.18163427398620083 |
| }, |
| "v2": { |
| "n": 231, |
| "top1": 195, |
| "nll": 0.5102332515038959, |
| "brier": 0.24715639752392515, |
| "score_expected_level_mae": 0.1169276614338399 |
| }, |
| "v3": { |
| "n": 231, |
| "top1": 202, |
| "nll": 0.4439350676441406, |
| "brier": 0.21173286180768633, |
| "score_expected_level_mae": 0.08733389277501005 |
| }, |
| "historical": { |
| "n": 231, |
| "top1": 203, |
| "nll": 0.4433104727261114, |
| "brier": 0.211353951167008, |
| "score_expected_level_mae": 0.08733389277501005 |
| }, |
| "direct_to_v3": { |
| "n": 231, |
| "fixes": 35, |
| "regressions": 8, |
| "winner_changes": 54, |
| "max_probability_delta": 0.4938297051869622 |
| }, |
| "v2_to_v3": { |
| "n": 231, |
| "fixes": 15, |
| "regressions": 8, |
| "winner_changes": 26, |
| "max_probability_delta": 0.49954774239515043 |
| }, |
| "historical_to_v3": { |
| "n": 231, |
| "fixes": 0, |
| "regressions": 1, |
| "winner_changes": 1, |
| "max_probability_delta": 0.07599484280024105 |
| }, |
| "routed": 131, |
| "completed_blends": 131, |
| "max_direct_delta": 0.0, |
| "historical_text_equal": 126, |
| "historical_text_compared": 131, |
| "generation_output_tokens": { |
| "sum": 123713, |
| "mean": 944.3740458015267, |
| "max": 2952, |
| "distribution": { |
| "n": 131, |
| "sum": 123713, |
| "mean": 944.3740458015267, |
| "median": 856, |
| "p95": 1936.5, |
| "p99": 2826.3999999999987, |
| "max": 2952 |
| } |
| }, |
| "pipeline_seconds_all_cost": { |
| "n": 231, |
| "sum": 313.31913769757375, |
| "mean": 1.3563599034527003, |
| "median": 0.8873364869505167, |
| "p95": 4.099002962524537, |
| "p99": 6.056786250858566, |
| "max": 7.182334668934345 |
| }, |
| "pipeline_seconds_clean": { |
| "n": 231, |
| "sum": 313.31913769757375, |
| "mean": 1.3563599034527003, |
| "median": 0.8873364869505167, |
| "p95": 4.099002962524537, |
| "p99": 6.056786250858566, |
| "max": 7.182334668934345 |
| }, |
| "timing_excluded_count": 0, |
| "v2_selective_seconds_clean_matched": { |
| "n": 231, |
| "sum": 314.0389325831784, |
| "mean": 1.3594758986284778, |
| "median": 0.8249348889803514, |
| "p95": 4.52698246500222, |
| "p99": 5.536772494891189, |
| "max": 5.958748230012134 |
| }, |
| "max_native_backend_allocated_bytes": 6601572352, |
| "natural_eos_output_tokens": { |
| "n": 131, |
| "sum": 123713, |
| "mean": 944.3740458015267, |
| "median": 856, |
| "p95": 1936.5, |
| "p99": 2826.3999999999987, |
| "max": 2952 |
| }, |
| "clean_phase_seconds": { |
| "/v1/chat/completions": { |
| "n": 231, |
| "sum": 283.491011307342, |
| "mean": 1.2272338151832987, |
| "median": 0.8251801070291549, |
| "p95": 3.6716601609950885, |
| "p99": 5.589920414669901, |
| "max": 6.879954903037287 |
| }, |
| "/v1/systemone": { |
| "n": 231, |
| "sum": 28.06992222147528, |
| "mean": 0.1215148148115813, |
| "median": 0.05774561199359596, |
| "p95": 0.4264363810652867, |
| "p99": 0.5270983364898711, |
| "max": 0.5750212969724089 |
| }, |
| "/v1/winnow/inspect": { |
| "n": 231, |
| "sum": 1.6974414952564985, |
| "mean": 0.007348231581196963, |
| "median": 0.004081002902239561, |
| "p95": 0.02844280900899321, |
| "p99": 0.0330876518972218, |
| "max": 0.039218239951878786 |
| } |
| }, |
| "clean_chat_metrics": { |
| "n": 131, |
| "output_tokens": 123713, |
| "decode_seconds": 272.717043, |
| "aggregate_decode_tokens_per_second": 453.63134859158765, |
| "draft_proposed": 152252, |
| "draft_accepted": 85751, |
| "draft_acceptance": 0.563217560360455, |
| "prefill_ms": { |
| "n": 131, |
| "sum": 10241.078, |
| "mean": 78.1761679389313, |
| "median": 46.572, |
| "p95": 210.508, |
| "p99": 227.93679999999998, |
| "max": 248.51 |
| }, |
| "decode_ms": { |
| "n": 131, |
| "sum": 272717.043, |
| "mean": 2081.8094885496184, |
| "median": 1831.607, |
| "p95": 4055.5175, |
| "p99": 5768.822199999997, |
| "max": 6808.545 |
| } |
| }, |
| "clean_chat_metrics_natural_eos": { |
| "n": 131, |
| "output_tokens": 123713, |
| "decode_seconds": 272.717043, |
| "aggregate_decode_tokens_per_second": 453.63134859158765, |
| "draft_proposed": 152252, |
| "draft_accepted": 85751, |
| "draft_acceptance": 0.563217560360455, |
| "prefill_ms": { |
| "n": 131, |
| "sum": 10241.078, |
| "mean": 78.1761679389313, |
| "median": 46.572, |
| "p95": 210.508, |
| "p99": 227.93679999999998, |
| "max": 248.51 |
| }, |
| "decode_ms": { |
| "n": 131, |
| "sum": 272717.043, |
| "mean": 2081.8094885496184, |
| "median": 1831.607, |
| "p95": 4055.5175, |
| "p99": 5768.822199999997, |
| "max": 6808.545 |
| } |
| }, |
| "by_state": { |
| "dict": { |
| "direct": { |
| "n": 35, |
| "top1": 24, |
| "nll": 0.8098638610329222, |
| "brier": 0.4492148897620551, |
| "score_expected_level_mae": null |
| }, |
| "v2": { |
| "n": 35, |
| "top1": 30, |
| "nll": 0.5730443227047298, |
| "brier": 0.2778388271622036, |
| "score_expected_level_mae": null |
| }, |
| "v3": { |
| "n": 35, |
| "top1": 28, |
| "nll": 0.5986073814983259, |
| "brier": 0.2874113653226452, |
| "score_expected_level_mae": null |
| }, |
| "historical": { |
| "n": 35, |
| "top1": 29, |
| "nll": 0.5952986408668535, |
| "brier": 0.2851932314045603, |
| "score_expected_level_mae": null |
| }, |
| "direct_to_v3": { |
| "n": 35, |
| "fixes": 7, |
| "regressions": 3, |
| "winner_changes": 12, |
| "max_probability_delta": 0.4847284009193595 |
| }, |
| "v2_to_v3": { |
| "n": 35, |
| "fixes": 1, |
| "regressions": 3, |
| "winner_changes": 5, |
| "max_probability_delta": 0.4927516672740554 |
| }, |
| "historical_to_v3": { |
| "n": 35, |
| "fixes": 0, |
| "regressions": 1, |
| "winner_changes": 1, |
| "max_probability_delta": 0.07599484280024105 |
| } |
| }, |
| "str": { |
| "direct": { |
| "n": 196, |
| "top1": 151, |
| "nll": 0.6272068984120392, |
| "brier": 0.31862200643902855, |
| "score_expected_level_mae": 0.18163427398620083 |
| }, |
| "v2": { |
| "n": 196, |
| "top1": 165, |
| "nll": 0.49901698878946127, |
| "brier": 0.24167739223137544, |
| "score_expected_level_mae": 0.1169276614338399 |
| }, |
| "v3": { |
| "n": 196, |
| "top1": 174, |
| "nll": 0.41631501159875034, |
| "brier": 0.19821884332287223, |
| "score_expected_level_mae": 0.08733389277501005 |
| }, |
| "historical": { |
| "n": 196, |
| "top1": 174, |
| "nll": 0.4161697284152646, |
| "brier": 0.19816836541030225, |
| "score_expected_level_mae": 0.08733389277501005 |
| }, |
| "direct_to_v3": { |
| "n": 196, |
| "fixes": 28, |
| "regressions": 5, |
| "winner_changes": 42, |
| "max_probability_delta": 0.4938297051869622 |
| }, |
| "v2_to_v3": { |
| "n": 196, |
| "fixes": 14, |
| "regressions": 5, |
| "winner_changes": 21, |
| "max_probability_delta": 0.49954774239515043 |
| }, |
| "historical_to_v3": { |
| "n": 196, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 0, |
| "max_probability_delta": 0.010968767130365098 |
| } |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "direct": { |
| "n": 139, |
| "top1": 100, |
| "nll": 0.8075315805861729, |
| "brier": 0.39522492072947685, |
| "score_expected_level_mae": null |
| }, |
| "v2": { |
| "n": 139, |
| "top1": 115, |
| "nll": 0.6044044303940057, |
| "brier": 0.2790112001195667, |
| "score_expected_level_mae": null |
| }, |
| "v3": { |
| "n": 139, |
| "top1": 119, |
| "nll": 0.5397349946166345, |
| "brier": 0.24901431746883707, |
| "score_expected_level_mae": null |
| }, |
| "historical": { |
| "n": 139, |
| "top1": 119, |
| "nll": 0.5396015160770918, |
| "brier": 0.24895584375003005, |
| "score_expected_level_mae": null |
| }, |
| "direct_to_v3": { |
| "n": 139, |
| "fixes": 24, |
| "regressions": 5, |
| "winner_changes": 39, |
| "max_probability_delta": 0.4938297051869622 |
| }, |
| "v2_to_v3": { |
| "n": 139, |
| "fixes": 9, |
| "regressions": 5, |
| "winner_changes": 17, |
| "max_probability_delta": 0.49954774239515043 |
| }, |
| "historical_to_v3": { |
| "n": 139, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 0, |
| "max_probability_delta": 0.010968767130365098 |
| } |
| }, |
| "noul": { |
| "direct": { |
| "n": 74, |
| "top1": 61, |
| "nll": 0.43519757844670776, |
| "brier": 0.26491784130498336, |
| "score_expected_level_mae": null |
| }, |
| "v2": { |
| "n": 74, |
| "top1": 64, |
| "nll": 0.3773254061123309, |
| "brier": 0.207808720831387, |
| "score_expected_level_mae": null |
| }, |
| "v3": { |
| "n": 74, |
| "top1": 66, |
| "nll": 0.3184973321031578, |
| "brier": 0.16806665325527254, |
| "score_expected_level_mae": null |
| }, |
| "historical": { |
| "n": 74, |
| "top1": 67, |
| "nll": 0.31679830630493755, |
| "brier": 0.1669936733757791, |
| "score_expected_level_mae": null |
| }, |
| "direct_to_v3": { |
| "n": 74, |
| "fixes": 8, |
| "regressions": 3, |
| "winner_changes": 11, |
| "max_probability_delta": 0.4496872566776117 |
| }, |
| "v2_to_v3": { |
| "n": 74, |
| "fixes": 5, |
| "regressions": 3, |
| "winner_changes": 8, |
| "max_probability_delta": 0.4990190336545404 |
| }, |
| "historical_to_v3": { |
| "n": 74, |
| "fixes": 0, |
| "regressions": 1, |
| "winner_changes": 1, |
| "max_probability_delta": 0.07599484280024105 |
| } |
| }, |
| "score": { |
| "direct": { |
| "n": 18, |
| "top1": 14, |
| "nll": 0.3792375954654199, |
| "brier": 0.20179167587530403, |
| "score_expected_level_mae": 0.18163427398620083 |
| }, |
| "v2": { |
| "n": 18, |
| "top1": 16, |
| "nll": 0.32942140112892654, |
| "brier": 0.16292920388246132, |
| "score_expected_level_mae": 0.1169276614338399 |
| }, |
| "v3": { |
| "n": 18, |
| "top1": 17, |
| "nll": 0.21983521102503334, |
| "brier": 0.10335381158427896, |
| "score_expected_level_mae": 0.08733389277501005 |
| }, |
| "historical": { |
| "n": 18, |
| "top1": 17, |
| "nll": 0.21983521102503334, |
| "brier": 0.10335381158427896, |
| "score_expected_level_mae": 0.08733389277501005 |
| }, |
| "direct_to_v3": { |
| "n": 18, |
| "fixes": 3, |
| "regressions": 0, |
| "winner_changes": 4, |
| "max_probability_delta": 0.4173095607519327 |
| }, |
| "v2_to_v3": { |
| "n": 18, |
| "fixes": 1, |
| "regressions": 0, |
| "winner_changes": 1, |
| "max_probability_delta": 0.49626072038671637 |
| }, |
| "historical_to_v3": { |
| "n": 18, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 0, |
| "max_probability_delta": 0.0 |
| } |
| } |
| } |
| }, |
| "kev-v9-clean": { |
| "observed": 1046, |
| "expected": 1046, |
| "complete": true, |
| "valid": 1046, |
| "semantics": "verified label accuracy", |
| "errors": [], |
| "fallbacks": [ |
| { |
| "id": "mmlu_pro/test/9247/answer", |
| "reason": "generation_context_limit_or_unverified" |
| }, |
| { |
| "id": "unknowable_control/warranty_claim/v5-test-20260920-warranty_claim-0004/a/decision", |
| "reason": "generation_context_limit_or_unverified" |
| } |
| ], |
| "direct": { |
| "n": 1046, |
| "top1": 728, |
| "nll": 0.9841937846927966, |
| "brier": 0.44164586585817817, |
| "score_expected_level_mae": 0.29737852879492405 |
| }, |
| "v2": { |
| "n": 1046, |
| "top1": 845, |
| "nll": 0.7616620243944804, |
| "brier": 0.31723865346147595, |
| "score_expected_level_mae": 0.16328748801957052 |
| }, |
| "v3": { |
| "n": 1046, |
| "top1": 851, |
| "nll": 0.743563618944474, |
| "brier": 0.3110523200923067, |
| "score_expected_level_mae": 0.16352370646939757 |
| }, |
| "historical": { |
| "n": 1046, |
| "top1": 851, |
| "nll": 0.7439440288222531, |
| "brier": 0.3113648971845476, |
| "score_expected_level_mae": 0.1636530071709598 |
| }, |
| "direct_to_v3": { |
| "n": 1046, |
| "fixes": 173, |
| "regressions": 50, |
| "winner_changes": 264, |
| "max_probability_delta": 0.49877661297158915 |
| }, |
| "v2_to_v3": { |
| "n": 1046, |
| "fixes": 38, |
| "regressions": 32, |
| "winner_changes": 100, |
| "max_probability_delta": 0.4998410532110379 |
| }, |
| "historical_to_v3": { |
| "n": 1046, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 2, |
| "max_probability_delta": 0.49223936331112117 |
| }, |
| "routed": 676, |
| "completed_blends": 674, |
| "max_direct_delta": 0.0, |
| "historical_text_equal": 674, |
| "historical_text_compared": 676, |
| "generation_output_tokens": { |
| "sum": 508239, |
| "mean": 751.8328402366864, |
| "max": 8012, |
| "distribution": { |
| "n": 676, |
| "sum": 508239, |
| "mean": 751.8328402366864, |
| "median": 565.5, |
| "p95": 1889.75, |
| "p99": 3618.25, |
| "max": 8012 |
| } |
| }, |
| "pipeline_seconds_all_cost": { |
| "n": 1046, |
| "sum": 1267.75761802739, |
| "mean": 1.212005370963088, |
| "median": 1.028721651993692, |
| "p95": 3.5224499475152697, |
| "p99": 6.895163247355956, |
| "max": 12.624659975990653 |
| }, |
| "pipeline_seconds_clean": { |
| "n": 1046, |
| "sum": 1267.75761802739, |
| "mean": 1.212005370963088, |
| "median": 1.028721651993692, |
| "p95": 3.5224499475152697, |
| "p99": 6.895163247355956, |
| "max": 12.624659975990653 |
| }, |
| "timing_excluded_count": 0, |
| "v2_selective_seconds_clean_matched": { |
| "n": 1046, |
| "sum": 1300.9493182514561, |
| "mean": 1.2437373979459427, |
| "median": 0.9990504610468633, |
| "p95": 3.6775082352978643, |
| "p99": 6.746468279679535, |
| "max": 9.355459655052982 |
| }, |
| "max_native_backend_allocated_bytes": 6603538432, |
| "natural_eos_output_tokens": { |
| "n": 674, |
| "sum": 492665, |
| "mean": 730.9569732937686, |
| "median": 563.5, |
| "p95": 1851.4500000000003, |
| "p99": 3412.4799999999996, |
| "max": 5334 |
| }, |
| "clean_phase_seconds": { |
| "/v1/chat/completions": { |
| "n": 1046, |
| "sum": 1196.6142937389668, |
| "mean": 1.1439907205917466, |
| "median": 0.9535917384782806, |
| "p95": 3.3786256242892705, |
| "p99": 6.638901050179261, |
| "max": 12.581271308939904 |
| }, |
| "/v1/systemone": { |
| "n": 1046, |
| "sum": 66.71202236169484, |
| "mean": 0.06377822405515758, |
| "median": 0.06213975552236661, |
| "p95": 0.13457796376314946, |
| "p99": 0.23082962960470413, |
| "max": 0.39296685194130987 |
| }, |
| "/v1/winnow/inspect": { |
| "n": 1046, |
| "sum": 4.167626591864973, |
| "mean": 0.003984346646142422, |
| "median": 0.004179205046966672, |
| "p95": 0.008609114302089438, |
| "p99": 0.010857289913110432, |
| "max": 0.01358645805157721 |
| } |
| }, |
| "clean_chat_metrics": { |
| "n": 676, |
| "output_tokens": 508239, |
| "decode_seconds": 1172.2648219999999, |
| "aggregate_decode_tokens_per_second": 433.55305939565255, |
| "draft_proposed": 666445, |
| "draft_accepted": 342220, |
| "draft_acceptance": 0.513500738995716, |
| "prefill_ms": { |
| "n": 676, |
| "sum": 22825.388, |
| "mean": 33.76536686390533, |
| "median": 32.602500000000006, |
| "p95": 41.06625, |
| "p99": 48.360749999999996, |
| "max": 80.819 |
| }, |
| "decode_ms": { |
| "n": 676, |
| "sum": 1172264.822, |
| "mean": 1734.1195591715975, |
| "median": 1317.3195, |
| "p95": 4071.9035000000003, |
| "p99": 8340.009, |
| "max": 12524.258 |
| } |
| }, |
| "clean_chat_metrics_natural_eos": { |
| "n": 674, |
| "output_tokens": 492665, |
| "decode_seconds": 1147.630398, |
| "aggregate_decode_tokens_per_second": 429.28890769935845, |
| "draft_proposed": 653084, |
| "draft_accepted": 329989, |
| "draft_acceptance": 0.5052780346785406, |
| "prefill_ms": { |
| "n": 674, |
| "sum": 22739.867, |
| "mean": 33.73867507418397, |
| "median": 32.602500000000006, |
| "p95": 41.03915, |
| "p99": 46.812639999999995, |
| "max": 80.819 |
| }, |
| "decode_ms": { |
| "n": 674, |
| "sum": 1147630.398, |
| "mean": 1702.7157240356082, |
| "median": 1315.1734999999999, |
| "p95": 4007.4803000000006, |
| "p99": 7377.133189999987, |
| "max": 11452.211 |
| } |
| }, |
| "by_state": { |
| "dict": { |
| "direct": { |
| "n": 757, |
| "top1": 506, |
| "nll": 1.0263358490810022, |
| "brier": 0.4660603526231764, |
| "score_expected_level_mae": 0.29737852879492405 |
| }, |
| "v2": { |
| "n": 757, |
| "top1": 621, |
| "nll": 0.7171781019173493, |
| "brier": 0.3007661792699979, |
| "score_expected_level_mae": 0.16328748801957052 |
| }, |
| "v3": { |
| "n": 757, |
| "top1": 619, |
| "nll": 0.7081713179511532, |
| "brier": 0.30210716310944896, |
| "score_expected_level_mae": 0.16352370646939757 |
| }, |
| "historical": { |
| "n": 757, |
| "top1": 619, |
| "nll": 0.7087038104127751, |
| "brier": 0.3025430916237622, |
| "score_expected_level_mae": 0.1636530071709598 |
| }, |
| "direct_to_v3": { |
| "n": 757, |
| "fixes": 150, |
| "regressions": 37, |
| "winner_changes": 225, |
| "max_probability_delta": 0.49877661297158915 |
| }, |
| "v2_to_v3": { |
| "n": 757, |
| "fixes": 24, |
| "regressions": 26, |
| "winner_changes": 78, |
| "max_probability_delta": 0.4998410532110379 |
| }, |
| "historical_to_v3": { |
| "n": 757, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 2, |
| "max_probability_delta": 0.49223936331112117 |
| } |
| }, |
| "list": { |
| "direct": { |
| "n": 11, |
| "top1": 9, |
| "nll": 0.45089530046520027, |
| "brier": 0.2974764690463965, |
| "score_expected_level_mae": null |
| }, |
| "v2": { |
| "n": 11, |
| "top1": 10, |
| "nll": 0.1983185854747978, |
| "brier": 0.12111648754883751, |
| "score_expected_level_mae": null |
| }, |
| "v3": { |
| "n": 11, |
| "top1": 11, |
| "nll": 0.1368364542171407, |
| "brier": 0.07533518108769578, |
| "score_expected_level_mae": null |
| }, |
| "historical": { |
| "n": 11, |
| "top1": 11, |
| "nll": 0.13636481228161182, |
| "brier": 0.07505861319212954, |
| "score_expected_level_mae": null |
| }, |
| "direct_to_v3": { |
| "n": 11, |
| "fixes": 2, |
| "regressions": 0, |
| "winner_changes": 2, |
| "max_probability_delta": 0.4603535862018596 |
| }, |
| "v2_to_v3": { |
| "n": 11, |
| "fixes": 1, |
| "regressions": 0, |
| "winner_changes": 1, |
| "max_probability_delta": 0.48803378909777917 |
| }, |
| "historical_to_v3": { |
| "n": 11, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 0, |
| "max_probability_delta": 0.0013950736901103267 |
| } |
| }, |
| "str": { |
| "direct": { |
| "n": 278, |
| "top1": 213, |
| "nll": 0.8905417724072999, |
| "brier": 0.38086923594388294, |
| "score_expected_level_mae": null |
| }, |
| "v2": { |
| "n": 278, |
| "top1": 214, |
| "nll": 0.9050832731114038, |
| "brier": 0.3698537857923678, |
| "score_expected_level_mae": null |
| }, |
| "v3": { |
| "n": 278, |
| "top1": 221, |
| "nll": 0.8639448083831234, |
| "brier": 0.34473711277242924, |
| "score_expected_level_mae": null |
| }, |
| "historical": { |
| "n": 278, |
| "top1": 221, |
| "nll": 0.8639448083831234, |
| "brier": 0.34473711277242924, |
| "score_expected_level_mae": null |
| }, |
| "direct_to_v3": { |
| "n": 278, |
| "fixes": 21, |
| "regressions": 13, |
| "winner_changes": 37, |
| "max_probability_delta": 0.4906821240628119 |
| }, |
| "v2_to_v3": { |
| "n": 278, |
| "fixes": 13, |
| "regressions": 6, |
| "winner_changes": 21, |
| "max_probability_delta": 0.498794986167734 |
| }, |
| "historical_to_v3": { |
| "n": 278, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 0, |
| "max_probability_delta": 0.0 |
| } |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "direct": { |
| "n": 576, |
| "top1": 343, |
| "nll": 1.3449669496263348, |
| "brier": 0.5692832548094673, |
| "score_expected_level_mae": null |
| }, |
| "v2": { |
| "n": 576, |
| "top1": 430, |
| "nll": 1.0454033035132229, |
| "brier": 0.4124806597366923, |
| "score_expected_level_mae": null |
| }, |
| "v3": { |
| "n": 576, |
| "top1": 433, |
| "nll": 1.0170622984634021, |
| "brier": 0.40463624557307726, |
| "score_expected_level_mae": null |
| }, |
| "historical": { |
| "n": 576, |
| "top1": 433, |
| "nll": 1.018041954435723, |
| "brier": 0.405391147268223, |
| "score_expected_level_mae": null |
| }, |
| "direct_to_v3": { |
| "n": 576, |
| "fixes": 120, |
| "regressions": 30, |
| "winner_changes": 191, |
| "max_probability_delta": 0.49877661297158915 |
| }, |
| "v2_to_v3": { |
| "n": 576, |
| "fixes": 27, |
| "regressions": 24, |
| "winner_changes": 81, |
| "max_probability_delta": 0.4998410532110379 |
| }, |
| "historical_to_v3": { |
| "n": 576, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 2, |
| "max_probability_delta": 0.49223936331112117 |
| } |
| }, |
| "noul": { |
| "direct": { |
| "n": 370, |
| "top1": 311, |
| "nll": 0.5029575433062489, |
| "brier": 0.2629504138741171, |
| "score_expected_level_mae": null |
| }, |
| "v2": { |
| "n": 370, |
| "top1": 318, |
| "nll": 0.45439529360118935, |
| "brier": 0.22222817979439785, |
| "score_expected_level_mae": null |
| }, |
| "v3": { |
| "n": 370, |
| "top1": 321, |
| "nll": 0.4465732399814046, |
| "brier": 0.2164619524787992, |
| "score_expected_level_mae": null |
| }, |
| "historical": { |
| "n": 370, |
| "top1": 321, |
| "nll": 0.44608433300775163, |
| "brier": 0.21615934481882615, |
| "score_expected_level_mae": null |
| }, |
| "direct_to_v3": { |
| "n": 370, |
| "fixes": 28, |
| "regressions": 18, |
| "winner_changes": 46, |
| "max_probability_delta": 0.4901768441417294 |
| }, |
| "v2_to_v3": { |
| "n": 370, |
| "fixes": 11, |
| "regressions": 8, |
| "winner_changes": 19, |
| "max_probability_delta": 0.49871298353542987 |
| }, |
| "historical_to_v3": { |
| "n": 370, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 0, |
| "max_probability_delta": 0.028342231557091452 |
| } |
| }, |
| "score": { |
| "direct": { |
| "n": 100, |
| "top1": 74, |
| "nll": 0.6867144478058433, |
| "brier": 0.36762767783977907, |
| "score_expected_level_mae": 0.29737852879492405 |
| }, |
| "v2": { |
| "n": 100, |
| "top1": 97, |
| "nll": 0.2641991606057006, |
| "brier": 0.12018344988441894, |
| "score_expected_level_mae": 0.16328748801957052 |
| }, |
| "v3": { |
| "n": 100, |
| "top1": 97, |
| "nll": 0.26707562707880517, |
| "brier": 0.12199326949304623, |
| "score_expected_level_mae": 0.16352370646939757 |
| }, |
| "historical": { |
| "n": 100, |
| "top1": 97, |
| "nll": 0.2672208518023219, |
| "brier": 0.12203424045574633, |
| "score_expected_level_mae": 0.1636530071709598 |
| }, |
| "direct_to_v3": { |
| "n": 100, |
| "fixes": 25, |
| "regressions": 2, |
| "winner_changes": 27, |
| "max_probability_delta": 0.4908280003843208 |
| }, |
| "v2_to_v3": { |
| "n": 100, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 0, |
| "max_probability_delta": 0.2012603587773023 |
| }, |
| "historical_to_v3": { |
| "n": 100, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 0, |
| "max_probability_delta": 0.0027665416307858237 |
| } |
| } |
| } |
| }, |
| "typed-decisions": { |
| "observed": 2000, |
| "expected": 2000, |
| "complete": true, |
| "valid": 2000, |
| "semantics": "synthetic teacher agreement", |
| "errors": [], |
| "fallbacks": [], |
| "direct": { |
| "n": 2000, |
| "top1": 1237, |
| "nll": 0.98012896186682, |
| "brier": 0.5366594455578602, |
| "score_expected_level_mae": 0.5718142114825938 |
| }, |
| "v2": { |
| "n": 2000, |
| "top1": 1342, |
| "nll": 0.9213980450857948, |
| "brier": 0.48476748880056186, |
| "score_expected_level_mae": 0.5018408163629516 |
| }, |
| "v3": { |
| "n": 2000, |
| "top1": 1366, |
| "nll": 0.8821571197982664, |
| "brier": 0.4700707747544247, |
| "score_expected_level_mae": 0.48623367185770394 |
| }, |
| "historical": { |
| "n": 2000, |
| "top1": 1365, |
| "nll": 0.8821582687252656, |
| "brier": 0.4699970802405581, |
| "score_expected_level_mae": 0.48607930491573853 |
| }, |
| "direct_to_v3": { |
| "n": 2000, |
| "fixes": 334, |
| "regressions": 205, |
| "winner_changes": 632, |
| "max_probability_delta": 0.49734083274590535 |
| }, |
| "v2_to_v3": { |
| "n": 2000, |
| "fixes": 173, |
| "regressions": 149, |
| "winner_changes": 388, |
| "max_probability_delta": 0.49990235636218977 |
| }, |
| "historical_to_v3": { |
| "n": 2000, |
| "fixes": 1, |
| "regressions": 0, |
| "winner_changes": 1, |
| "max_probability_delta": 0.08040670116881293 |
| }, |
| "routed": 1777, |
| "completed_blends": 1777, |
| "max_direct_delta": 0.0, |
| "historical_text_equal": 1777, |
| "historical_text_compared": 1777, |
| "generation_output_tokens": { |
| "sum": 1367542, |
| "mean": 769.5790658413056, |
| "max": 2712, |
| "distribution": { |
| "n": 1777, |
| "sum": 1367542, |
| "mean": 769.5790658413056, |
| "median": 762, |
| "p95": 1051.1999999999998, |
| "p99": 1254.3600000000001, |
| "max": 2712 |
| } |
| }, |
| "pipeline_seconds_all_cost": { |
| "n": 2000, |
| "sum": 3901.123203223222, |
| "mean": 1.950561601611611, |
| "median": 2.093935512995813, |
| "p95": 3.0059627732262015, |
| "p99": 3.7421011833066586, |
| "max": 8.422256662975997 |
| }, |
| "pipeline_seconds_clean": { |
| "n": 1984, |
| "sum": 3871.4878648194717, |
| "mean": 1.9513547705743306, |
| "median": 2.0951888249837793, |
| "p95": 3.0047559435595756, |
| "p99": 3.743849764257903, |
| "max": 8.422256662975997 |
| }, |
| "timing_excluded_count": 16, |
| "v2_selective_seconds_clean_matched": { |
| "n": 1984, |
| "sum": 4021.9517563040135, |
| "mean": 2.0271934255564585, |
| "median": 2.178467755962629, |
| "p95": 3.0571437860082367, |
| "p99": 4.1093565620703165, |
| "max": 8.320580558967777 |
| }, |
| "max_native_backend_allocated_bytes": 6753550336, |
| "natural_eos_output_tokens": { |
| "n": 1777, |
| "sum": 1367542, |
| "mean": 769.5790658413056, |
| "median": 762, |
| "p95": 1051.1999999999998, |
| "p99": 1254.3600000000001, |
| "max": 2712 |
| }, |
| "clean_phase_seconds": { |
| "/v1/chat/completions": { |
| "n": 1984, |
| "sum": 3670.5011939010583, |
| "mean": 1.8500510049904528, |
| "median": 1.9909222425194457, |
| "p95": 2.8783525532402563, |
| "p99": 3.5913489963649803, |
| "max": 8.18496916803997 |
| }, |
| "/v1/systemone": { |
| "n": 1984, |
| "sum": 187.3238072542008, |
| "mean": 0.09441724155957702, |
| "median": 0.09884031204273924, |
| "p95": 0.12675707904854788, |
| "p99": 0.14250179579830732, |
| "max": 0.2218943398911506 |
| }, |
| "/v1/winnow/inspect": { |
| "n": 1984, |
| "sum": 12.908032948966138, |
| "mean": 0.006506064994438577, |
| "median": 0.006854985433164984, |
| "p95": 0.008897835278185084, |
| "p99": 0.01095620173728094, |
| "max": 0.016994947916828096 |
| } |
| }, |
| "clean_chat_metrics": { |
| "n": 1763, |
| "output_tokens": 1356905, |
| "decode_seconds": 3597.4189079999996, |
| "aggregate_decode_tokens_per_second": 377.18848838607374, |
| "draft_proposed": 2043980, |
| "draft_accepted": 847677, |
| "draft_acceptance": 0.4147188328652922, |
| "prefill_ms": { |
| "n": 1763, |
| "sum": 68639.242, |
| "mean": 38.933205899035734, |
| "median": 39.707, |
| "p95": 45.585, |
| "p99": 51.141939999999984, |
| "max": 55.325 |
| }, |
| "decode_ms": { |
| "n": 1763, |
| "sum": 3597418.908, |
| "mean": 2040.5098740782757, |
| "median": 2028.45, |
| "p95": 2874.6335999999988, |
| "p99": 3605.586639999997, |
| "max": 8142.273 |
| } |
| }, |
| "clean_chat_metrics_natural_eos": { |
| "n": 1763, |
| "output_tokens": 1356905, |
| "decode_seconds": 3597.4189079999996, |
| "aggregate_decode_tokens_per_second": 377.18848838607374, |
| "draft_proposed": 2043980, |
| "draft_accepted": 847677, |
| "draft_acceptance": 0.4147188328652922, |
| "prefill_ms": { |
| "n": 1763, |
| "sum": 68639.242, |
| "mean": 38.933205899035734, |
| "median": 39.707, |
| "p95": 45.585, |
| "p99": 51.141939999999984, |
| "max": 55.325 |
| }, |
| "decode_ms": { |
| "n": 1763, |
| "sum": 3597418.908, |
| "mean": 2040.5098740782757, |
| "median": 2028.45, |
| "p95": 2874.6335999999988, |
| "p99": 3605.586639999997, |
| "max": 8142.273 |
| } |
| }, |
| "by_state": { |
| "dict": { |
| "direct": { |
| "n": 2000, |
| "top1": 1237, |
| "nll": 0.98012896186682, |
| "brier": 0.5366594455578602, |
| "score_expected_level_mae": 0.5718142114825938 |
| }, |
| "v2": { |
| "n": 2000, |
| "top1": 1342, |
| "nll": 0.9213980450857948, |
| "brier": 0.48476748880056186, |
| "score_expected_level_mae": 0.5018408163629516 |
| }, |
| "v3": { |
| "n": 2000, |
| "top1": 1366, |
| "nll": 0.8821571197982664, |
| "brier": 0.4700707747544247, |
| "score_expected_level_mae": 0.48623367185770394 |
| }, |
| "historical": { |
| "n": 2000, |
| "top1": 1365, |
| "nll": 0.8821582687252656, |
| "brier": 0.4699970802405581, |
| "score_expected_level_mae": 0.48607930491573853 |
| }, |
| "direct_to_v3": { |
| "n": 2000, |
| "fixes": 334, |
| "regressions": 205, |
| "winner_changes": 632, |
| "max_probability_delta": 0.49734083274590535 |
| }, |
| "v2_to_v3": { |
| "n": 2000, |
| "fixes": 173, |
| "regressions": 149, |
| "winner_changes": 388, |
| "max_probability_delta": 0.49990235636218977 |
| }, |
| "historical_to_v3": { |
| "n": 2000, |
| "fixes": 1, |
| "regressions": 0, |
| "winner_changes": 1, |
| "max_probability_delta": 0.08040670116881293 |
| } |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "direct": { |
| "n": 600, |
| "top1": 358, |
| "nll": 1.0383454926500515, |
| "brier": 0.5461734111343014, |
| "score_expected_level_mae": null |
| }, |
| "v2": { |
| "n": 600, |
| "top1": 390, |
| "nll": 0.9958149854968003, |
| "brier": 0.5063973319925225, |
| "score_expected_level_mae": null |
| }, |
| "v3": { |
| "n": 600, |
| "top1": 393, |
| "nll": 0.9563012200742745, |
| "brier": 0.4976482188673111, |
| "score_expected_level_mae": null |
| }, |
| "historical": { |
| "n": 600, |
| "top1": 393, |
| "nll": 0.9558431778730259, |
| "brier": 0.4973719628971465, |
| "score_expected_level_mae": null |
| }, |
| "direct_to_v3": { |
| "n": 600, |
| "fixes": 93, |
| "regressions": 58, |
| "winner_changes": 190, |
| "max_probability_delta": 0.49734083274590535 |
| }, |
| "v2_to_v3": { |
| "n": 600, |
| "fixes": 50, |
| "regressions": 47, |
| "winner_changes": 123, |
| "max_probability_delta": 0.49990235636218977 |
| }, |
| "historical_to_v3": { |
| "n": 600, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 0, |
| "max_probability_delta": 0.02832484742590488 |
| } |
| }, |
| "noul": { |
| "direct": { |
| "n": 600, |
| "top1": 438, |
| "nll": 0.781763055105711, |
| "brier": 0.4553453509151658, |
| "score_expected_level_mae": null |
| }, |
| "v2": { |
| "n": 600, |
| "top1": 470, |
| "nll": 0.7073079185383874, |
| "brier": 0.3751118454262236, |
| "score_expected_level_mae": null |
| }, |
| "v3": { |
| "n": 600, |
| "top1": 477, |
| "nll": 0.6776685742578799, |
| "brier": 0.36021695730742676, |
| "score_expected_level_mae": null |
| }, |
| "historical": { |
| "n": 600, |
| "top1": 476, |
| "nll": 0.6782093598018603, |
| "brier": 0.360367007607702, |
| "score_expected_level_mae": null |
| }, |
| "direct_to_v3": { |
| "n": 600, |
| "fixes": 73, |
| "regressions": 34, |
| "winner_changes": 107, |
| "max_probability_delta": 0.49051290118046414 |
| }, |
| "v2_to_v3": { |
| "n": 600, |
| "fixes": 31, |
| "regressions": 24, |
| "winner_changes": 55, |
| "max_probability_delta": 0.49906662495041354 |
| }, |
| "historical_to_v3": { |
| "n": 600, |
| "fixes": 1, |
| "regressions": 0, |
| "winner_changes": 1, |
| "max_probability_delta": 0.010967217955198949 |
| } |
| }, |
| "score": { |
| "direct": { |
| "n": 800, |
| "top1": 441, |
| "nll": 1.085240993850228, |
| "brier": 0.59050954235755, |
| "score_expected_level_mae": 0.5718142114825938 |
| }, |
| "v2": { |
| "n": 800, |
| "top1": 482, |
| "nll": 1.0261529346880962, |
| "brier": 0.550786838937345, |
| "score_expected_level_mae": 0.5018408163629516 |
| }, |
| "v3": { |
| "n": 800, |
| "top1": 496, |
| "nll": 0.9799154537465503, |
| "brier": 0.5317780547550084, |
| "score_expected_level_mae": 0.48623367185770394 |
| }, |
| "historical": { |
| "n": 800, |
| "top1": 496, |
| "nll": 0.9798562685569995, |
| "brier": 0.5316884727227588, |
| "score_expected_level_mae": 0.48607930491573853 |
| }, |
| "direct_to_v3": { |
| "n": 800, |
| "fixes": 168, |
| "regressions": 113, |
| "winner_changes": 335, |
| "max_probability_delta": 0.49632280685398744 |
| }, |
| "v2_to_v3": { |
| "n": 800, |
| "fixes": 92, |
| "regressions": 78, |
| "winner_changes": 210, |
| "max_probability_delta": 0.49985733751079203 |
| }, |
| "historical_to_v3": { |
| "n": 800, |
| "fixes": 0, |
| "regressions": 0, |
| "winner_changes": 0, |
| "max_probability_delta": 0.08040670116881293 |
| } |
| } |
| } |
| } |
| } |
| } |
|
|
