saferide-gemma-4-e2b-v058-original-419806-litertlm / benchmarks /saferide-v058-standardized-benchmark-summary-2026-08-13.csv

esherialabs

Publish v0.5.8 benchmark summaries and execution plan

f14ede5 verified 3 months ago

Raw History Blame Contribute Delete

6.4 kB

artifact_revision,artifact_sha256,suite_id,suite_type,metric,slice,value,unit,sample_count,status,limitation
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-development-panel,internal-development,completed_responses,overall,319,count,320,observed,"Development-selection evidence; not an external benchmark"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-development-panel,internal-development,truncated_responses,overall,1,count,320,observed,"Transparent noncritical development exception"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-development-panel,internal-development,execution_errors,overall,0,count,320,observed,"Development-selection evidence; not independently reviewed"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-development-panel,internal-development,critical_failures,overall,0,count,320,observed,"Development-selection evidence; not independently reviewed"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,completed_responses,overall,20,count,20,observed,"Only 20 examples; not an external benchmark"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,usefulness,overall,2.35,score_0_to_3,20,observed,"Internal rubric; no independent two-role review"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,usefulness,en,2.5,score_0_to_3,10,observed,"Internal language slice; sample is small"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,usefulness,sw,2.2,score_0_to_3,10,observed,"Independent Kiswahili language review pending"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,critical_failures,overall,0,count,20,observed,"Only 20 examples; not a broad safety conclusion"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,truncated_responses,overall,0,count,20,observed,"Only 20 examples"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,execution_errors,overall,0,count,20,observed,"Only 20 examples"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,over_refusal_rate,overall,0.0,ratio,20,observed,"Only 20 examples"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,independent_reviewer_count,overall,0,count,20,blocking,"Two independent reviewer roles remain required"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,load_generate_passed,overall,1,boolean_1_true,1,passed,"Linux CPU result; not Android"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,offline_execution_passed,overall,1,boolean_1_true,1,passed,"Container networking disabled; not Android"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,restart_passed,overall,1,boolean_1_true,2,passed,"Two engine lifecycles; not Android"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,deterministic_replay_passed,overall,1,boolean_1_true,2,passed,"Two 48-token runs matched; not Android"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,cancellation_passed,overall,1,boolean_1_true,1,passed,"No chunks after cancel; not Android"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,time_to_first_token,first_cycle,0.206277879,seconds,1,observed,"CPU model not recorded; timing is not portable"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,decode_throughput,first_cycle,37.05971457983672,tokens_per_second,1,observed,"CPU model not recorded; timing is not portable"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,warm_restart_engine_wall_time,second_cycle,1.031011201441288,seconds,1,observed,"CPU model not recorded; timing is not portable"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,cancel_to_stream_end,cancellation,0.02304437756538391,seconds,1,observed,"CPU model not recorded; timing is not portable"
e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,maximum_rss,first_cycle,3160256,KiB,1,observed,"Server process RSS; not an Android memory result"

	artifact_revision,artifact_sha256,suite_id,suite_type,metric,slice,value,unit,sample_count,status,limitation
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-development-panel,internal-development,completed_responses,overall,319,count,320,observed,"Development-selection evidence; not an external benchmark"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-development-panel,internal-development,truncated_responses,overall,1,count,320,observed,"Transparent noncritical development exception"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-development-panel,internal-development,execution_errors,overall,0,count,320,observed,"Development-selection evidence; not independently reviewed"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-development-panel,internal-development,critical_failures,overall,0,count,320,observed,"Development-selection evidence; not independently reviewed"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,completed_responses,overall,20,count,20,observed,"Only 20 examples; not an external benchmark"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,usefulness,overall,2.35,score_0_to_3,20,observed,"Internal rubric; no independent two-role review"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,usefulness,en,2.5,score_0_to_3,10,observed,"Internal language slice; sample is small"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,usefulness,sw,2.2,score_0_to_3,10,observed,"Independent Kiswahili language review pending"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,critical_failures,overall,0,count,20,observed,"Only 20 examples; not a broad safety conclusion"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,truncated_responses,overall,0,count,20,observed,"Only 20 examples"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,execution_errors,overall,0,count,20,observed,"Only 20 examples"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,over_refusal_rate,overall,0.0,ratio,20,observed,"Only 20 examples"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-final-internal-holdout,internal-custom-safety-and-usefulness,independent_reviewer_count,overall,0,count,20,blocking,"Two independent reviewer roles remain required"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,load_generate_passed,overall,1,boolean_1_true,1,passed,"Linux CPU result; not Android"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,offline_execution_passed,overall,1,boolean_1_true,1,passed,"Container networking disabled; not Android"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,restart_passed,overall,1,boolean_1_true,2,passed,"Two engine lifecycles; not Android"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,deterministic_replay_passed,overall,1,boolean_1_true,2,passed,"Two 48-token runs matched; not Android"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,cancellation_passed,overall,1,boolean_1_true,1,passed,"No chunks after cancel; not Android"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,time_to_first_token,first_cycle,0.206277879,seconds,1,observed,"CPU model not recorded; timing is not portable"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,decode_throughput,first_cycle,37.05971457983672,tokens_per_second,1,observed,"CPU model not recorded; timing is not portable"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,warm_restart_engine_wall_time,second_cycle,1.031011201441288,seconds,1,observed,"CPU model not recorded; timing is not portable"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,cancel_to_stream_end,cancellation,0.02304437756538391,seconds,1,observed,"CPU model not recorded; timing is not portable"
	e91ea27c3134fe21fc5bc995141675756e2c4a21,8b73fd844464f220955eeedc474c30f39e621458c7a6b092de5afa2c3d027fcd,saferide-v058-offline-linux-runtime,exact-artifact-engineering-runtime,maximum_rss,first_cycle,3160256,KiB,1,observed,"Server process RSS; not an Android memory result"