{"record_id": "jevbench:original-policy-01-0", "group_id": "jevbench:original-policy-01", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.9820848107337952, "true": 0.017915144562721252}, "correct": true}
{"record_id": "jevbench:original-policy-01-1", "group_id": "jevbench:original-policy-01", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.9784570932388306, "true": 0.02154294028878212}, "correct": true}
{"record_id": "jevbench:original-policy-02-0", "group_id": "jevbench:original-policy-02", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.34094181656837463, "true": 0.6590582132339478}, "correct": true}
{"record_id": "jevbench:original-policy-02-1", "group_id": "jevbench:original-policy-02", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.69185471534729, "true": 0.30814534425735474}, "correct": false}
{"record_id": "jevbench:original-policy-03-0", "group_id": "jevbench:original-policy-03", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.33038008213043213, "true": 0.6696198582649231}, "correct": true}
{"record_id": "jevbench:original-policy-03-1", "group_id": "jevbench:original-policy-03", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.9413718581199646, "true": 0.05862807855010033}, "correct": false}
{"record_id": "jevbench:original-policy-04-0", "group_id": "jevbench:original-policy-04", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.6078335642814636, "true": 0.39216646552085876}, "correct": true}
{"record_id": "jevbench:original-policy-04-1", "group_id": "jevbench:original-policy-04", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.9599436521530151, "true": 0.040056340396404266}, "correct": true}
{"record_id": "jevbench:original-policy-05-0", "group_id": "jevbench:original-policy-05", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.5599515438079834, "true": 0.4400485157966614}, "correct": true}
{"record_id": "jevbench:original-policy-05-1", "group_id": "jevbench:original-policy-05", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.8501847386360168, "true": 0.14981529116630554}, "correct": true}
{"record_id": "jevbench:original-policy-06-0", "group_id": "jevbench:original-policy-06", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.1529567986726761, "true": 0.8470432162284851}, "correct": true}
{"record_id": "jevbench:original-policy-06-1", "group_id": "jevbench:original-policy-06", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.18743525445461273, "true": 0.8125647306442261}, "correct": true}
{"record_id": "jevbench:original-intent-01-0", "group_id": "jevbench:original-intent-01", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "label": "refund", "prediction": "refund", "probabilities": {"cancel": 0.3644479513168335, "change_address": 0.0052671185694634914, "other": 0.18913643062114716, "refund": 0.43468838930130005, "status": 0.006460071075707674}, "correct": true}
{"record_id": "jevbench:original-intent-01-1", "group_id": "jevbench:original-intent-01", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "label": "refund", "prediction": "other", "probabilities": {"cancel": 0.06150296702980995, "change_address": 0.005042537115514278, "other": 0.5047032833099365, "refund": 0.4231768548488617, "status": 0.005574310664087534}, "correct": false}
{"record_id": "jevbench:original-intent-02-0", "group_id": "jevbench:original-intent-02", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "label": "status", "prediction": "other", "probabilities": {"cancel": 0.004139136988669634, "change_address": 0.08017218858003616, "other": 0.879235029220581, "refund": 0.0045811813324689865, "status": 0.031872499734163284}, "correct": false}
{"record_id": "jevbench:original-intent-02-1", "group_id": "jevbench:original-intent-02", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "label": "status", "prediction": "status", "probabilities": {"cancel": 0.006674482487142086, "change_address": 0.01924518495798111, "other": 0.1106015294790268, "refund": 0.007292765658348799, "status": 0.8561861515045166}, "correct": true}
{"record_id": "jevbench:original-intent-03-0", "group_id": "jevbench:original-intent-03", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "label": "change_address", "prediction": "change_address", "probabilities": {"cancel": 0.006889356765896082, "change_address": 0.9395559430122375, "other": 0.03101896494626999, "refund": 0.0063799102790653706, "status": 0.016155794262886047}, "correct": true}
{"record_id": "jevbench:original-intent-03-1", "group_id": "jevbench:original-intent-03", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "label": "change_address", "prediction": "change_address", "probabilities": {"cancel": 0.004706079140305519, "change_address": 0.9637589454650879, "other": 0.020158428698778152, "refund": 0.0052562374621629715, "status": 0.006120260804891586}, "correct": true}
{"record_id": "jevbench:original-intent-04-0", "group_id": "jevbench:original-intent-04", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "label": "cancel", "prediction": "cancel", "probabilities": {"cancel": 0.86097651720047, "change_address": 0.005098074208945036, "other": 0.111374631524086, "refund": 0.01644974574446678, "status": 0.006101028993725777}, "correct": true}
{"record_id": "jevbench:original-intent-04-1", "group_id": "jevbench:original-intent-04", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "label": "cancel", "prediction": "cancel", "probabilities": {"cancel": 0.9010534882545471, "change_address": 0.0049303313717246056, "other": 0.0710391029715538, "refund": 0.017404600977897644, "status": 0.005572488531470299}, "correct": true}
{"record_id": "jevbench:original-intent-05-0", "group_id": "jevbench:original-intent-05", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "label": "other", "prediction": "other", "probabilities": {"cancel": 0.011166281998157501, "change_address": 0.00661383755505085, "other": 0.6169039011001587, "refund": 0.35698869824409485, "status": 0.00832726527005434}, "correct": true}
{"record_id": "jevbench:original-intent-05-1", "group_id": "jevbench:original-intent-05", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "label": "other", "prediction": "other", "probabilities": {"cancel": 0.011350086890161037, "change_address": 0.0067932698875665665, "other": 0.5838204026222229, "refund": 0.39073365926742554, "status": 0.007302628364413977}, "correct": true}
{"record_id": "jevbench:original-intent-06-0", "group_id": "jevbench:original-intent-06", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "label": "status", "prediction": "status", "probabilities": {"cancel": 0.008124235086143017, "change_address": 0.005023099482059479, "other": 0.06437826156616211, "refund": 0.005695158615708351, "status": 0.9167792201042175}, "correct": true}
{"record_id": "jevbench:original-intent-06-1", "group_id": "jevbench:original-intent-06", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "label": "status", "prediction": "other", "probabilities": {"cancel": 0.024292532354593277, "change_address": 0.006526053883135319, "other": 0.5796230435371399, "refund": 0.011555868200957775, "status": 0.3780025243759155}, "correct": false}
{"record_id": "jevbench:original-ordinal-01-0", "group_id": "jevbench:original-ordinal-01", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "label": "0", "prediction": "0", "probabilities": {"0": 0.7966601848602295, "1": 0.14949285984039307, "2": 0.0347670316696167, "3": 0.019079916179180145}, "correct": true}
{"record_id": "jevbench:original-ordinal-01-1", "group_id": "jevbench:original-ordinal-01", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "label": "0", "prediction": "0", "probabilities": {"0": 0.8894200325012207, "1": 0.08357778191566467, "2": 0.017300773411989212, "3": 0.00970135722309351}, "correct": true}
{"record_id": "jevbench:original-ordinal-02-0", "group_id": "jevbench:original-ordinal-02", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "label": "1", "prediction": "1", "probabilities": {"0": 0.129643976688385, "1": 0.7044506072998047, "2": 0.15204192698001862, "3": 0.013863401487469673}, "correct": true}
{"record_id": "jevbench:original-ordinal-02-1", "group_id": "jevbench:original-ordinal-02", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "label": "1", "prediction": "1", "probabilities": {"0": 0.14323140680789948, "1": 0.7756932377815247, "2": 0.05339828506112099, "3": 0.027677098289132118}, "correct": true}
{"record_id": "jevbench:original-ordinal-03-0", "group_id": "jevbench:original-ordinal-03", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "label": "2", "prediction": "2", "probabilities": {"0": 0.05866279453039169, "1": 0.06714099645614624, "2": 0.8500614762306213, "3": 0.024134760722517967}, "correct": true}
{"record_id": "jevbench:original-ordinal-03-1", "group_id": "jevbench:original-ordinal-03", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "label": "2", "prediction": "2", "probabilities": {"0": 0.08978110551834106, "1": 0.08696084469556808, "2": 0.7950498461723328, "3": 0.028208177536725998}, "correct": true}
{"record_id": "jevbench:original-ordinal-04-0", "group_id": "jevbench:original-ordinal-04", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "label": "3", "prediction": "3", "probabilities": {"0": 0.07108540832996368, "1": 0.040017932653427124, "2": 0.028015123680233955, "3": 0.8608815670013428}, "correct": true}
{"record_id": "jevbench:original-ordinal-04-1", "group_id": "jevbench:original-ordinal-04", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "label": "3", "prediction": "3", "probabilities": {"0": 0.05620380491018295, "1": 0.04848242178559303, "2": 0.04197282716631889, "3": 0.8533409833908081}, "correct": true}
{"record_id": "jevbench:original-ordinal-05-0", "group_id": "jevbench:original-ordinal-05", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "label": "0", "prediction": "0", "probabilities": {"0": 0.7533204555511475, "1": 0.07781760394573212, "2": 0.1587381213903427, "3": 0.010123832151293755}, "correct": true}
{"record_id": "jevbench:original-ordinal-05-1", "group_id": "jevbench:original-ordinal-05", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "label": "0", "prediction": "0", "probabilities": {"0": 0.8063844442367554, "1": 0.07062622159719467, "2": 0.09733320772647858, "3": 0.02565610036253929}, "correct": true}
{"record_id": "jevbench:original-ordinal-06-0", "group_id": "jevbench:original-ordinal-06", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "label": "2", "prediction": "2", "probabilities": {"0": 0.12796221673488617, "1": 0.17462517321109772, "2": 0.6878089308738708, "3": 0.009603685699403286}, "correct": true}
{"record_id": "jevbench:original-ordinal-06-1", "group_id": "jevbench:original-ordinal-06", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "label": "2", "prediction": "2", "probabilities": {"0": 0.05836427956819534, "1": 0.08754994720220566, "2": 0.8373844027519226, "3": 0.01670132949948311}, "correct": true}
{"record_id": "jevbench:original-extraction-01-0", "group_id": "jevbench:original-extraction-01", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "label": "pickup", "prediction": "pickup", "probabilities": {"courier": 0.15170365571975708, "pickup": 0.7861368060112, "post": 0.013022752478718758, "unknown": 0.049136701971292496}, "correct": true}
{"record_id": "jevbench:original-extraction-01-1", "group_id": "jevbench:original-extraction-01", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "label": "pickup", "prediction": "pickup", "probabilities": {"courier": 0.04365920647978783, "pickup": 0.7771766185760498, "post": 0.032695285975933075, "unknown": 0.1464688926935196}, "correct": true}
{"record_id": "jevbench:original-extraction-02-0", "group_id": "jevbench:original-extraction-02", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "label": "unknown", "prediction": "unknown", "probabilities": {"courier": 0.018771041184663773, "pickup": 0.012700573541224003, "post": 0.07328638434410095, "unknown": 0.8952419757843018}, "correct": true}
{"record_id": "jevbench:original-extraction-02-1", "group_id": "jevbench:original-extraction-02", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "label": "unknown", "prediction": "unknown", "probabilities": {"courier": 0.014903866685926914, "pickup": 0.006793852895498276, "post": 0.07229368388652802, "unknown": 0.9060086607933044}, "correct": true}
{"record_id": "jevbench:original-extraction-03-0", "group_id": "jevbench:original-extraction-03", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "label": "courier", "prediction": "courier", "probabilities": {"courier": 0.8133912682533264, "pickup": 0.08454808592796326, "post": 0.029490264132618904, "unknown": 0.07257035374641418}, "correct": true}
{"record_id": "jevbench:original-extraction-03-1", "group_id": "jevbench:original-extraction-03", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "label": "courier", "prediction": "courier", "probabilities": {"courier": 0.9471832513809204, "pickup": 0.011383387260138988, "post": 0.017323071137070656, "unknown": 0.024110279977321625}, "correct": true}
{"record_id": "jevbench:original-extraction-04-0", "group_id": "jevbench:original-extraction-04", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "label": "post", "prediction": "post", "probabilities": {"courier": 0.020070437341928482, "pickup": 0.009202828630805016, "post": 0.9401825070381165, "unknown": 0.0305442214012146}, "correct": true}
{"record_id": "jevbench:original-extraction-04-1", "group_id": "jevbench:original-extraction-04", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "label": "post", "prediction": "post", "probabilities": {"courier": 0.0174594447016716, "pickup": 0.007786744274199009, "post": 0.9604358673095703, "unknown": 0.014317966066300869}, "correct": true}
{"record_id": "jevbench:original-extraction-05-0", "group_id": "jevbench:original-extraction-05", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "label": "unknown", "prediction": "unknown", "probabilities": {"courier": 0.020473206415772438, "pickup": 0.012499884702265263, "post": 0.010473898611962795, "unknown": 0.9565531015396118}, "correct": true}
{"record_id": "jevbench:original-extraction-05-1", "group_id": "jevbench:original-extraction-05", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "label": "unknown", "prediction": "unknown", "probabilities": {"courier": 0.06829206645488739, "pickup": 0.014146750792860985, "post": 0.017522597685456276, "unknown": 0.9000385999679565}, "correct": true}
{"record_id": "jevbench:original-extraction-06-0", "group_id": "jevbench:original-extraction-06", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "label": "courier", "prediction": "courier", "probabilities": {"courier": 0.9117640852928162, "pickup": 0.007031403481960297, "post": 0.06952229142189026, "unknown": 0.011682182550430298}, "correct": true}
{"record_id": "jevbench:original-extraction-06-1", "group_id": "jevbench:original-extraction-06", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "label": "courier", "prediction": "courier", "probabilities": {"courier": 0.9395111203193665, "pickup": 0.009402698837220669, "post": 0.019907066598534584, "unknown": 0.031179146841168404}, "correct": true}
{"record_id": "jevbench:original-adequacy-01-0", "group_id": "jevbench:original-adequacy-01", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.30193427205085754, "true": 0.6980656981468201}, "correct": true}
{"record_id": "jevbench:original-adequacy-01-1", "group_id": "jevbench:original-adequacy-01", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.6118249297142029, "true": 0.3881750702857971}, "correct": false}
{"record_id": "jevbench:original-adequacy-02-0", "group_id": "jevbench:original-adequacy-02", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.25436681509017944, "true": 0.7456331849098206}, "correct": false}
{"record_id": "jevbench:original-adequacy-02-1", "group_id": "jevbench:original-adequacy-02", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.869661808013916, "true": 0.13033826649188995}, "correct": true}
{"record_id": "jevbench:original-adequacy-03-0", "group_id": "jevbench:original-adequacy-03", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.1074480190873146, "true": 0.8925520181655884}, "correct": false}
{"record_id": "jevbench:original-adequacy-03-1", "group_id": "jevbench:original-adequacy-03", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.34779831767082214, "true": 0.6522017121315002}, "correct": false}
{"record_id": "jevbench:original-adequacy-04-0", "group_id": "jevbench:original-adequacy-04", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.1608610302209854, "true": 0.8391389846801758}, "correct": true}
{"record_id": "jevbench:original-adequacy-04-1", "group_id": "jevbench:original-adequacy-04", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.4702622890472412, "true": 0.5297377109527588}, "correct": true}
{"record_id": "jevbench:original-adequacy-05-0", "group_id": "jevbench:original-adequacy-05", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.6476667523384094, "true": 0.35233327746391296}, "correct": true}
{"record_id": "jevbench:original-adequacy-05-1", "group_id": "jevbench:original-adequacy-05", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.6054099202156067, "true": 0.3945901095867157}, "correct": true}
{"record_id": "jevbench:original-adequacy-06-0", "group_id": "jevbench:original-adequacy-06", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.5385457873344421, "true": 0.4614541828632355}, "correct": false}
{"record_id": "jevbench:original-adequacy-06-1", "group_id": "jevbench:original-adequacy-06", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.36792606115341187, "true": 0.6320738792419434}, "correct": true}
{"record_id": "jevbench:original-routing-01-0", "group_id": "jevbench:original-routing-01", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "label": "math", "prediction": "math", "probabilities": {"coding": 0.049350492656230927, "coding_agent": 0.02929607778787613, "document": 0.009837327525019646, "general": 0.029763495549559593, "math": 0.8714296817779541, "tools": 0.01032282505184412}, "correct": true}
{"record_id": "jevbench:original-routing-01-1", "group_id": "jevbench:original-routing-01", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "label": "math", "prediction": "math", "probabilities": {"coding": 0.04346639662981033, "coding_agent": 0.038384810090065, "document": 0.015851275995373726, "general": 0.04466238245368004, "math": 0.8416165709495544, "tools": 0.016018468886613846}, "correct": true}
{"record_id": "jevbench:original-routing-02-0", "group_id": "jevbench:original-routing-02", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "label": "coding", "prediction": "coding", "probabilities": {"coding": 0.9394723773002625, "coding_agent": 0.015048202127218246, "document": 0.006478498689830303, "general": 0.026889052242040634, "math": 0.006863461807370186, "tools": 0.005248400382697582}, "correct": true}
{"record_id": "jevbench:original-routing-02-1", "group_id": "jevbench:original-routing-02", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "label": "coding", "prediction": "coding", "probabilities": {"coding": 0.8767480850219727, "coding_agent": 0.06496869027614594, "document": 0.007581401150673628, "general": 0.03778612241148949, "math": 0.006265046540647745, "tools": 0.006650720257312059}, "correct": true}
{"record_id": "jevbench:original-routing-03-0", "group_id": "jevbench:original-routing-03", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "label": "coding_agent", "prediction": "coding_agent", "probabilities": {"coding": 0.019650820642709732, "coding_agent": 0.9149630069732666, "document": 0.008381291292607784, "general": 0.0415160246193409, "math": 0.0034604878164827824, "tools": 0.012028327211737633}, "correct": true}
{"record_id": "jevbench:original-routing-03-1", "group_id": "jevbench:original-routing-03", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "label": "coding_agent", "prediction": "coding_agent", "probabilities": {"coding": 0.02412232756614685, "coding_agent": 0.9183186888694763, "document": 0.012833165936172009, "general": 0.02893947809934616, "math": 0.003271940629929304, "tools": 0.012514404021203518}, "correct": true}
{"record_id": "jevbench:original-routing-04-0", "group_id": "jevbench:original-routing-04", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "label": "document", "prediction": "document", "probabilities": {"coding": 0.016015075147151947, "coding_agent": 0.06886952370405197, "document": 0.8374806046485901, "general": 0.05954901874065399, "math": 0.00618733512237668, "tools": 0.011898408643901348}, "correct": true}
{"record_id": "jevbench:original-routing-04-1", "group_id": "jevbench:original-routing-04", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "label": "document", "prediction": "document", "probabilities": {"coding": 0.026437461376190186, "coding_agent": 0.09617802500724792, "document": 0.7348860502243042, "general": 0.06745143979787827, "math": 0.010818945243954659, "tools": 0.06422804296016693}, "correct": true}
{"record_id": "jevbench:original-routing-05-0", "group_id": "jevbench:original-routing-05", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "label": "tools", "prediction": "tools", "probabilities": {"coding": 0.023472242057323456, "coding_agent": 0.03891479969024658, "document": 0.02011990360915661, "general": 0.1345909982919693, "math": 0.006094112526625395, "tools": 0.7768079042434692}, "correct": true}
{"record_id": "jevbench:original-routing-05-1", "group_id": "jevbench:original-routing-05", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "label": "tools", "prediction": "general", "probabilities": {"coding": 0.06872521340847015, "coding_agent": 0.1521887630224228, "document": 0.07046601176261902, "general": 0.565834105014801, "math": 0.007334951311349869, "tools": 0.13545091450214386}, "correct": false}
{"record_id": "jevbench:original-routing-06-0", "group_id": "jevbench:original-routing-06", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "label": "general", "prediction": "general", "probabilities": {"coding": 0.08055929094552994, "coding_agent": 0.26992344856262207, "document": 0.06413951516151428, "general": 0.5156599283218384, "math": 0.01616707071661949, "tools": 0.05355079844594002}, "correct": true}
{"record_id": "jevbench:original-routing-06-1", "group_id": "jevbench:original-routing-06", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "label": "general", "prediction": "general", "probabilities": {"coding": 0.08126389980316162, "coding_agent": 0.2651814818382263, "document": 0.035502344369888306, "general": 0.5594341158866882, "math": 0.011759407818317413, "tools": 0.04685871675610542}, "correct": true}
{"record_id": "jevbench:easy-intent-00", "group_id": "jevbench:easy-intent-00", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "label": "track_order", "prediction": "track_order", "probabilities": {"billing_question": 0.008227434009313583, "cancel_order": 0.014150082133710384, "change_address": 0.008860406465828419, "report_damage": 0.012232329696416855, "track_order": 0.9565298557281494}, "correct": true}
{"record_id": "jevbench:easy-intent-01", "group_id": "jevbench:easy-intent-01", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "label": "cancel_order", "prediction": "cancel_order", "probabilities": {"billing_question": 0.0039894492365419865, "cancel_order": 0.9836753010749817, "change_address": 0.003520919708535075, "report_damage": 0.004241629969328642, "track_order": 0.004572656936943531}, "correct": true}
{"record_id": "jevbench:easy-intent-02", "group_id": "jevbench:easy-intent-02", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "label": "change_address", "prediction": "change_address", "probabilities": {"billing_question": 0.006589591968804598, "cancel_order": 0.009798109531402588, "change_address": 0.9687209129333496, "report_damage": 0.005566462874412537, "track_order": 0.009324857033789158}, "correct": true}
{"record_id": "jevbench:easy-intent-03", "group_id": "jevbench:easy-intent-03", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "label": "report_damage", "prediction": "report_damage", "probabilities": {"billing_question": 0.005394429434090853, "cancel_order": 0.012258443050086498, "change_address": 0.007145735435187817, "report_damage": 0.9551488161087036, "track_order": 0.020052563399076462}, "correct": true}
{"record_id": "jevbench:easy-intent-04", "group_id": "jevbench:easy-intent-04", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "label": "billing_question", "prediction": "billing_question", "probabilities": {"billing_question": 0.952257513999939, "cancel_order": 0.010284856893122196, "change_address": 0.007079657632857561, "report_damage": 0.012585453689098358, "track_order": 0.01779250055551529}, "correct": true}
{"record_id": "jevbench:easy-intent-05", "group_id": "jevbench:easy-intent-05", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "label": "track_order", "prediction": "track_order", "probabilities": {"billing_question": 0.009568559005856514, "cancel_order": 0.01811406761407852, "change_address": 0.009442316368222237, "report_damage": 0.018256042152643204, "track_order": 0.9446189403533936}, "correct": true}
{"record_id": "jevbench:easy-intent-06", "group_id": "jevbench:easy-intent-06", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "label": "set_alarm", "prediction": "set_alarm", "probabilities": {"play_music": 0.00602305680513382, "send_message": 0.03962305188179016, "set_alarm": 0.9411261081695557, "turn_off_lights": 0.0056799910962581635, "weather": 0.007547800429165363}, "correct": true}
{"record_id": "jevbench:easy-intent-07", "group_id": "jevbench:easy-intent-07", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "label": "play_music", "prediction": "play_music", "probabilities": {"play_music": 0.9716208577156067, "send_message": 0.011253845877945423, "set_alarm": 0.006188366562128067, "turn_off_lights": 0.005537590943276882, "weather": 0.005399344954639673}, "correct": true}
{"record_id": "jevbench:easy-intent-08", "group_id": "jevbench:easy-intent-08", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "label": "weather", "prediction": "weather", "probabilities": {"play_music": 0.0038171252235770226, "send_message": 0.008897913619875908, "set_alarm": 0.005018141120672226, "turn_off_lights": 0.003714184043928981, "weather": 0.9785526394844055}, "correct": true}
{"record_id": "jevbench:easy-intent-09", "group_id": "jevbench:easy-intent-09", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "label": "send_message", "prediction": "send_message", "probabilities": {"play_music": 0.006885786075145006, "send_message": 0.9481940269470215, "set_alarm": 0.03287616744637489, "turn_off_lights": 0.0058937701396644115, "weather": 0.006150274071842432}, "correct": true}
{"record_id": "jevbench:easy-intent-10", "group_id": "jevbench:easy-intent-10", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "label": "turn_off_lights", "prediction": "turn_off_lights", "probabilities": {"play_music": 0.007721615023910999, "send_message": 0.009536572732031345, "set_alarm": 0.00997417513281107, "turn_off_lights": 0.9656651020050049, "weather": 0.00710255466401577}, "correct": true}
{"record_id": "jevbench:easy-intent-11", "group_id": "jevbench:easy-intent-11", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "label": "set_alarm", "prediction": "set_alarm", "probabilities": {"play_music": 0.0032055152114480734, "send_message": 0.0036875519435852766, "set_alarm": 0.9874633550643921, "turn_off_lights": 0.0030506968032568693, "weather": 0.0025929140392690897}, "correct": true}
{"record_id": "jevbench:easy-fact-00", "group_id": "jevbench:easy-fact-00", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.012932289391756058, "true": 0.9870677590370178}, "correct": true}
{"record_id": "jevbench:easy-fact-01", "group_id": "jevbench:easy-fact-01", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.9796827435493469, "true": 0.020317258313298225}, "correct": true}
{"record_id": "jevbench:easy-fact-02", "group_id": "jevbench:easy-fact-02", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.02318018302321434, "true": 0.9768198132514954}, "correct": true}
{"record_id": "jevbench:easy-fact-03", "group_id": "jevbench:easy-fact-03", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.9658337235450745, "true": 0.03416621312499046}, "correct": true}
{"record_id": "jevbench:easy-fact-04", "group_id": "jevbench:easy-fact-04", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.044848568737506866, "true": 0.9551514387130737}, "correct": true}
{"record_id": "jevbench:easy-fact-05", "group_id": "jevbench:easy-fact-05", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.9635937809944153, "true": 0.03640623018145561}, "correct": true}
{"record_id": "jevbench:easy-fact-06", "group_id": "jevbench:easy-fact-06", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.013674684800207615, "true": 0.9863253235816956}, "correct": true}
{"record_id": "jevbench:easy-fact-07", "group_id": "jevbench:easy-fact-07", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.9766446352005005, "true": 0.023355349898338318}, "correct": true}
{"record_id": "jevbench:easy-fact-08", "group_id": "jevbench:easy-fact-08", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.009176891297101974, "true": 0.9908231496810913}, "correct": true}
{"record_id": "jevbench:easy-fact-09", "group_id": "jevbench:easy-fact-09", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.9837906956672668, "true": 0.016209334135055542}, "correct": true}
{"record_id": "jevbench:easy-fact-10", "group_id": "jevbench:easy-fact-10", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.1514800786972046, "true": 0.8485199213027954}, "correct": true}
{"record_id": "jevbench:easy-fact-11", "group_id": "jevbench:easy-fact-11", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.9748988747596741, "true": 0.025101151317358017}, "correct": true}
{"record_id": "jevbench:easy-extraction-00", "group_id": "jevbench:easy-extraction-00", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "label": "paypal", "prediction": "paypal", "probabilities": {"bank_transfer": 0.005500088911503553, "cash": 0.00394475320354104, "credit_card": 0.004976535215973854, "paypal": 0.9855786561965942}, "correct": true}
{"record_id": "jevbench:easy-extraction-01", "group_id": "jevbench:easy-extraction-01", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "label": "cash", "prediction": "cash", "probabilities": {"bank_transfer": 0.006811955012381077, "cash": 0.980492353439331, "credit_card": 0.0072834608145058155, "paypal": 0.005412247497588396}, "correct": true}
{"record_id": "jevbench:easy-extraction-02", "group_id": "jevbench:easy-extraction-02", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "label": "bank_transfer", "prediction": "bank_transfer", "probabilities": {"bank_transfer": 0.9928638339042664, "cash": 0.0020378001499921083, "credit_card": 0.0026593490038067102, "paypal": 0.0024389787577092648}, "correct": true}
{"record_id": "jevbench:easy-extraction-03", "group_id": "jevbench:easy-extraction-03", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "label": "high", "prediction": "high", "probabilities": {"high": 0.9720640182495117, "low": 0.006866577081382275, "medium": 0.006821716204285622, "urgent": 0.014247783459722996}, "correct": true}
{"record_id": "jevbench:easy-extraction-04", "group_id": "jevbench:easy-extraction-04", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "label": "low", "prediction": "low", "probabilities": {"high": 0.006235568784177303, "low": 0.9777210354804993, "medium": 0.008188914507627487, "urgent": 0.00785438809543848}, "correct": true}
{"record_id": "jevbench:easy-extraction-05", "group_id": "jevbench:easy-extraction-05", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "label": "urgent", "prediction": "urgent", "probabilities": {"high": 0.011344445869326591, "low": 0.005709180608391762, "medium": 0.005590240005403757, "urgent": 0.9773560762405396}, "correct": true}
{"record_id": "jevbench:easy-extraction-06", "group_id": "jevbench:easy-extraction-06", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "label": "M", "prediction": "M", "probabilities": {"L": 0.007664407603442669, "M": 0.9786677956581116, "S": 0.009900864213705063, "XL": 0.003766964888200164}, "correct": true}
{"record_id": "jevbench:easy-extraction-07", "group_id": "jevbench:easy-extraction-07", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "label": "XL", "prediction": "XL", "probabilities": {"L": 0.01396183855831623, "M": 0.007828670553863049, "S": 0.005892825312912464, "XL": 0.9723167419433594}, "correct": true}
{"record_id": "jevbench:easy-extraction-08", "group_id": "jevbench:easy-extraction-08", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "label": "thursday", "prediction": "thursday", "probabilities": {"friday": 0.002005296526476741, "monday": 0.001988849835470319, "thursday": 0.9930902123451233, "tuesday": 0.0014833069872111082, "wednesday": 0.001432368066161871}, "correct": true}
{"record_id": "jevbench:easy-extraction-09", "group_id": "jevbench:easy-extraction-09", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "label": "monday", "prediction": "monday", "probabilities": {"friday": 0.009800161235034466, "monday": 0.9593384861946106, "thursday": 0.009840553626418114, "tuesday": 0.010571075603365898, "wednesday": 0.010449821129441261}, "correct": true}
{"record_id": "jevbench:easy-extraction-10", "group_id": "jevbench:easy-extraction-10", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "label": "friday", "prediction": "friday", "probabilities": {"friday": 0.9716123938560486, "monday": 0.005426777061074972, "thursday": 0.009203674271702766, "tuesday": 0.00634704390540719, "wednesday": 0.007409980054944754}, "correct": true}
{"record_id": "jevbench:easy-extraction-11", "group_id": "jevbench:easy-extraction-11", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "label": "credit_card", "prediction": "credit_card", "probabilities": {"bank_transfer": 0.008288341574370861, "cash": 0.005794787313789129, "credit_card": 0.9784814715385437, "paypal": 0.007435354869812727}, "correct": true}
{"record_id": "jevbench:easy-tool_selection-00", "group_id": "jevbench:easy-tool_selection-00", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "label": "get_weather", "prediction": "get_weather", "probabilities": {"convert_currency": 0.005185950081795454, "create_calendar_event": 0.005408921744674444, "get_weather": 0.9753163456916809, "send_email": 0.006556361448019743, "translate_text": 0.007532488089054823}, "correct": true}
{"record_id": "jevbench:easy-tool_selection-01", "group_id": "jevbench:easy-tool_selection-01", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "label": "send_email", "prediction": "send_email", "probabilities": {"convert_currency": 0.004465907346457243, "create_calendar_event": 0.007098136004060507, "get_weather": 0.004629234783351421, "send_email": 0.9780726432800293, "translate_text": 0.005734182428568602}, "correct": true}
{"record_id": "jevbench:easy-tool_selection-02", "group_id": "jevbench:easy-tool_selection-02", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "label": "create_calendar_event", "prediction": "create_calendar_event", "probabilities": {"convert_currency": 0.004625329747796059, "create_calendar_event": 0.9756351113319397, "get_weather": 0.00469388859346509, "send_email": 0.009546590968966484, "translate_text": 0.005499066784977913}, "correct": true}
{"record_id": "jevbench:easy-tool_selection-03", "group_id": "jevbench:easy-tool_selection-03", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "label": "convert_currency", "prediction": "convert_currency", "probabilities": {"convert_currency": 0.9827401638031006, "create_calendar_event": 0.002510045887902379, "get_weather": 0.0026591792702674866, "send_email": 0.0031512558925896883, "translate_text": 0.008939318358898163}, "correct": true}
{"record_id": "jevbench:easy-tool_selection-04", "group_id": "jevbench:easy-tool_selection-04", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "label": "translate_text", "prediction": "translate_text", "probabilities": {"convert_currency": 0.00487972516566515, "create_calendar_event": 0.006588929798454046, "get_weather": 0.004927889443933964, "send_email": 0.008195830509066582, "translate_text": 0.9754077196121216}, "correct": true}
{"record_id": "jevbench:easy-tool_selection-05", "group_id": "jevbench:easy-tool_selection-05", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "label": "create_calendar_event", "prediction": "create_calendar_event", "probabilities": {"convert_currency": 0.0037901587784290314, "create_calendar_event": 0.9790431261062622, "get_weather": 0.004132942296564579, "send_email": 0.009043588303029537, "translate_text": 0.00399018544703722}, "correct": true}
{"record_id": "jevbench:easy-tool_selection-06", "group_id": "jevbench:easy-tool_selection-06", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "label": "search_flights", "prediction": "search_flights", "probabilities": {"book_restaurant": 0.0045584081672132015, "call_taxi": 0.004934142809361219, "get_stock_price": 0.005339907947927713, "search_flights": 0.9784199595451355, "set_reminder": 0.006747524719685316}, "correct": true}
{"record_id": "jevbench:easy-tool_selection-07", "group_id": "jevbench:easy-tool_selection-07", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "label": "book_restaurant", "prediction": "book_restaurant", "probabilities": {"book_restaurant": 0.9755457043647766, "call_taxi": 0.0054708984680473804, "get_stock_price": 0.004045098088681698, "search_flights": 0.004644297994673252, "set_reminder": 0.010294102132320404}, "correct": true}
{"record_id": "jevbench:easy-tool_selection-08", "group_id": "jevbench:easy-tool_selection-08", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "label": "set_reminder", "prediction": "set_reminder", "probabilities": {"book_restaurant": 0.006881235167384148, "call_taxi": 0.00983588956296444, "get_stock_price": 0.00563881965354085, "search_flights": 0.006437710952013731, "set_reminder": 0.9712064266204834}, "correct": true}
{"record_id": "jevbench:easy-tool_selection-09", "group_id": "jevbench:easy-tool_selection-09", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "label": "get_stock_price", "prediction": "get_stock_price", "probabilities": {"book_restaurant": 0.004044337663799524, "call_taxi": 0.003910707775503397, "get_stock_price": 0.9839402437210083, "search_flights": 0.003865782869979739, "set_reminder": 0.004238942638039589}, "correct": true}
{"record_id": "jevbench:easy-tool_selection-10", "group_id": "jevbench:easy-tool_selection-10", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "label": "call_taxi", "prediction": "call_taxi", "probabilities": {"book_restaurant": 0.004687710199505091, "call_taxi": 0.9775480031967163, "get_stock_price": 0.00462234765291214, "search_flights": 0.005352507345378399, "set_reminder": 0.007789478171616793}, "correct": true}
{"record_id": "jevbench:easy-tool_selection-11", "group_id": "jevbench:easy-tool_selection-11", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "label": "set_reminder", "prediction": "set_reminder", "probabilities": {"book_restaurant": 0.006919192150235176, "call_taxi": 0.011194237507879734, "get_stock_price": 0.0054591745138168335, "search_flights": 0.006472369655966759, "set_reminder": 0.9699550867080688}, "correct": true}
{"record_id": "jevbench:hard-opus-a-long_policy-01", "group_id": "jevbench:hard-opus-a-long_policy-01", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "label": "pay_subject_to_15000_sublimit", "prediction": "pay_subject_to_15000_sublimit", "probabilities": {"deny_repeated_seepage": 0.07930454611778259, "deny_vacancy_exclusion": 0.037347063422203064, "pay_full_estimate_less_deductible": 0.1276741623878479, "pay_subject_to_10000_sublimit": 0.2621476948261261, "pay_subject_to_15000_sublimit": 0.4935266077518463}, "correct": true}
{"record_id": "jevbench:hard-opus-a-long_policy-04", "group_id": "jevbench:hard-opus-a-long_policy-04", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "label": "cfo", "prediction": "vp_and_finance_director", "probabilities": {"budget_holder": 0.015077067539095879, "cfo": 0.022811267524957657, "department_head": 0.3274816870689392, "executive_committee": 0.011428584344685078, "vp_and_finance_director": 0.6232013702392578}, "correct": false}
{"record_id": "jevbench:hard-opus-a-long_policy-08", "group_id": "jevbench:hard-opus-a-long_policy-08", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "label": "parts_covered_labour_charged", "prediction": "parts_covered_labour_charged", "probabilities": {"full_repair_no_charge": 0.12344089150428772, "not_covered": 0.2971112132072449, "parts_covered_labour_charged": 0.5163977146148682, "pro_rata_replacement_credit": 0.06305016577243805}, "correct": true}
{"record_id": "jevbench:hard-opus-a-long_policy-09", "group_id": "jevbench:hard-opus-a-long_policy-09", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "label": "license_required", "prediction": "no_license_required", "probabilities": {"license_exception_lvs": 0.26742470264434814, "license_required": 0.24532534182071686, "no_license_required": 0.4585026800632477, "shipment_prohibited": 0.0287473201751709}, "correct": false}
{"record_id": "jevbench:hard-opus-a-long_policy-11", "group_id": "jevbench:hard-opus-a-long_policy-11", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.6293221116065979, "true": 0.3706778585910797}, "correct": false}
{"record_id": "jevbench:hard-opus-a-long_policy-13", "group_id": "jevbench:hard-opus-a-long_policy-13", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.847236156463623, "true": 0.15276381373405457}, "correct": false}
{"record_id": "jevbench:hard-opus-a-long_policy-17", "group_id": "jevbench:hard-opus-a-long_policy-17", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "label": "database_oncall", "prediction": "vendor_support", "probabilities": {"database_oncall": 0.0651538297533989, "no_page_ticket_only": 0.11933223158121109, "platform_oncall": 0.054516442120075226, "security_incident_response": 0.07611004263162613, "vendor_support": 0.6848874688148499}, "correct": false}
{"record_id": "jevbench:hard-opus-a-long_policy-19", "group_id": "jevbench:hard-opus-a-long_policy-19", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.7843074798583984, "true": 0.21569253504276276}, "correct": true}
{"record_id": "jevbench:hard-opus-a-probability-03", "group_id": "jevbench:hard-opus-a-probability-03", "question_id": "decision", "source": "jevbench/hard/probability", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.1824229657649994, "true": 0.8175770044326782}, "correct": true}
{"record_id": "jevbench:hard-opus-a-probability-04", "group_id": "jevbench:hard-opus-a-probability-04", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "label": "on_time", "prediction": "on_time", "probabilities": {"early": 0.1019730269908905, "late": 0.4413301646709442, "on_time": 0.4566968083381653}, "correct": true}
{"record_id": "jevbench:hard-opus-a-probability-07", "group_id": "jevbench:hard-opus-a-probability-07", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "label": "escalated", "prediction": "escalated", "probabilities": {"escalated": 0.5911614298820496, "reopened": 0.3593387305736542, "resolved_first_contact": 0.049499839544296265}, "correct": true}
{"record_id": "jevbench:hard-opus-a-probability-08", "group_id": "jevbench:hard-opus-a-probability-08", "question_id": "decision", "source": "jevbench/hard/probability", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.6994302272796631, "true": 0.3005697727203369}, "correct": true}
{"record_id": "jevbench:hard-opus-a-temporal_numeric-03", "group_id": "jevbench:hard-opus-a-temporal_numeric-03", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.8192733526229858, "true": 0.18072661757469177}, "correct": false}
{"record_id": "jevbench:hard-opus-a-temporal_numeric-06", "group_id": "jevbench:hard-opus-a-temporal_numeric-06", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.06483927369117737, "true": 0.935160756111145}, "correct": false}
{"record_id": "jevbench:hard-opus-a-temporal_numeric-07", "group_id": "jevbench:hard-opus-a-temporal_numeric-07", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "label": "sep_26", "prediction": "sep_24", "probabilities": {"sep_24": 0.45704013109207153, "sep_25": 0.085177943110466, "sep_26": 0.07494095712900162, "sep_27": 0.12282450497150421, "sep_28": 0.26001638174057007}, "correct": false}
{"record_id": "jevbench:hard-opus-a-temporal_numeric-09", "group_id": "jevbench:hard-opus-a-temporal_numeric-09", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.4081712067127228, "true": 0.5918287634849548}, "correct": false}
{"record_id": "jevbench:hard-opus-a-temporal_numeric-12", "group_id": "jevbench:hard-opus-a-temporal_numeric-12", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "score", "label": "1", "prediction": "3", "probabilities": {"0": 0.05091855674982071, "1": 0.07585848122835159, "2": 0.11809603869915009, "3": 0.7551268935203552}, "correct": false}
{"record_id": "jevbench:hard-opus-b-ambiguous-02", "group_id": "jevbench:hard-opus-b-ambiguous-02", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "label": "real_incident", "prediction": "real_incident", "probabilities": {"cannot_determine": 0.2476509064435959, "false_positive": 0.3253965377807617, "real_incident": 0.4269525408744812}, "correct": true}
{"record_id": "jevbench:hard-opus-b-ambiguous-03", "group_id": "jevbench:hard-opus-b-ambiguous-03", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "label": "time_barred", "prediction": "within_limitation", "probabilities": {"insufficient_information": 0.38902854919433594, "time_barred": 0.2080778032541275, "within_limitation": 0.40289363265037537}, "correct": false}
{"record_id": "jevbench:hard-opus-b-ambiguous-07", "group_id": "jevbench:hard-opus-b-ambiguous-07", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "label": "within_policy", "prediction": "insufficient_information", "probabilities": {"insufficient_information": 0.6277457475662231, "over_limit": 0.3325669765472412, "within_policy": 0.039687324315309525}, "correct": false}
{"record_id": "jevbench:hard-opus-b-ambiguous-09", "group_id": "jevbench:hard-opus-b-ambiguous-09", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "label": "legitimate", "prediction": "legitimate", "probabilities": {"compromised": 0.07804165780544281, "inconclusive": 0.05964567884802818, "legitimate": 0.8623126745223999}, "correct": true}
{"record_id": "jevbench:hard-opus-b-ambiguous-10", "group_id": "jevbench:hard-opus-b-ambiguous-10", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "label": "ask_user_which_alex", "prediction": "send_to_alex_moreno", "probabilities": {"ask_user_which_alex": 0.020207751542329788, "send_to_alex_moreno": 0.9681217074394226, "send_to_alexandra_lind": 0.011670606210827827}, "correct": false}
{"record_id": "jevbench:hard-opus-b-ambiguous-11", "group_id": "jevbench:hard-opus-b-ambiguous-11", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "label": "cannot_determine", "prediction": "cannot_determine", "probabilities": {"cannot_determine": 0.6708739399909973, "grade_correct": 0.30107301473617554, "grade_should_be_higher": 0.02805302105844021}, "correct": true}
{"record_id": "jevbench:hard-opus-b-ambiguous-13", "group_id": "jevbench:hard-opus-b-ambiguous-13", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "label": "cannot_determine", "prediction": "cannot_determine", "probabilities": {"cannot_determine": 0.7156879305839539, "charge_data_team": 0.11879543960094452, "charge_platform_team": 0.027736177667975426, "split_50_50": 0.13778047263622284}, "correct": true}
{"record_id": "jevbench:hard-opus-b-multi_hop-03", "group_id": "jevbench:hard-opus-b-multi_hop-03", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "tier3_cfo", "prediction": "tier3_cfo", "probabilities": {"tier1_team_lead": 0.20146885514259338, "tier2_department_head": 0.15598875284194946, "tier3_cfo": 0.5299906134605408, "tier4_board": 0.11255175620317459}, "correct": true}
{"record_id": "jevbench:hard-opus-b-multi_hop-04", "group_id": "jevbench:hard-opus-b-multi_hop-04", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "bjorn", "prediction": "dmitri", "probabilities": {"ana": 0.0577656552195549, "bjorn": 0.20463034510612488, "chen": 0.024307046085596085, "dmitri": 0.541808545589447, "esra": 0.10070095956325531, "no_page_ticket_only": 0.0707874745130539}, "correct": false}
{"record_id": "jevbench:hard-opus-b-multi_hop-05", "group_id": "jevbench:hard-opus-b-multi_hop-05", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "score", "label": "2", "prediction": "2", "probabilities": {"0": 0.04417707398533821, "1": 0.05739416182041168, "2": 0.5470930337905884, "3": 0.20994944870471954, "4": 0.14138628542423248}, "correct": true}
{"record_id": "jevbench:hard-opus-b-multi_hop-07", "group_id": "jevbench:hard-opus-b-multi_hop-07", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "credit_20_percent", "prediction": "credit_50_percent", "probabilities": {"credit_10_percent": 0.12009267508983612, "credit_20_percent": 0.08985991775989532, "credit_25_percent": 0.18581297993659973, "credit_50_percent": 0.37378862500190735, "credit_full_month": 0.1991141140460968, "no_credit": 0.031331658363342285}, "correct": false}
{"record_id": "jevbench:hard-opus-b-multi_hop-08", "group_id": "jevbench:hard-opus-b-multi_hop-08", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.5649198889732361, "true": 0.4350801110267639}, "correct": true}
{"record_id": "jevbench:hard-opus-b-probability-01", "group_id": "jevbench:hard-opus-b-probability-01", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "label": "merchant_wins", "prediction": "merchant_wins", "probabilities": {"customer_wins": 0.3439856469631195, "customer_withdraws": 0.035501956939697266, "merchant_wins": 0.6205124855041504}, "correct": true}
{"record_id": "jevbench:hard-opus-b-probability-02", "group_id": "jevbench:hard-opus-b-probability-02", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "label": "upstream_provider", "prediction": "upstream_provider", "probabilities": {"bad_push": 0.30470404028892517, "database_hardware": 0.19553327560424805, "upstream_provider": 0.4997626543045044}, "correct": true}
{"record_id": "jevbench:hard-opus-b-probability-03", "group_id": "jevbench:hard-opus-b-probability-03", "question_id": "decision", "source": "jevbench/hard/probability", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.31205448508262634, "true": 0.6879454851150513}, "correct": true}
{"record_id": "jevbench:hard-opus-b-probability-04", "group_id": "jevbench:hard-opus-b-probability-04", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "label": "attended", "prediction": "attended", "probabilities": {"attended": 0.5470876693725586, "late_cancellation": 0.11369941383600235, "no_show": 0.33921298384666443}, "correct": true}
{"record_id": "jevbench:hard-opus-b-probability-06", "group_id": "jevbench:hard-opus-b-probability-06", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "label": "upheld", "prediction": "overturned", "probabilities": {"modified": 0.07938676327466965, "overturned": 0.8976755142211914, "upheld": 0.022937702015042305}, "correct": false}
{"record_id": "jevbench:hard-opus-b-tradeoff-01", "group_id": "jevbench:hard-opus-b-tradeoff-01", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "choice", "label": "escalate_to_engineering", "prediction": "escalate_to_security", "probabilities": {"escalate_to_engineering": 0.22105878591537476, "escalate_to_security": 0.647321879863739, "page_oncall": 0.04227851331233978, "process_refund": 0.04901821166276932, "send_kb_article": 0.04032260552048683}, "correct": false}
{"record_id": "jevbench:hard-opus-b-tradeoff-03", "group_id": "jevbench:hard-opus-b-tradeoff-03", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "choice", "label": "order_b", "prediction": "order_b", "probabilities": {"order_a": 0.026882750913500786, "order_b": 0.7411143779754639, "order_c": 0.04790101945400238, "order_d": 0.18410176038742065}, "correct": true}
{"record_id": "jevbench:hard-opus-b-tradeoff-06", "group_id": "jevbench:hard-opus-b-tradeoff-06", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.8048258423805237, "true": 0.19517411291599274}, "correct": false}
{"record_id": "jevbench:hard-opus-b-tradeoff-07", "group_id": "jevbench:hard-opus-b-tradeoff-07", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "choice", "label": "p0_fix_24h", "prediction": "p2_fix_30d", "probabilities": {"out_of_scope": 0.02644774690270424, "p0_fix_24h": 0.07393048703670502, "p1_fix_7d": 0.29789242148399353, "p2_fix_30d": 0.5749289393424988, "p3_backlog": 0.026800381019711494}, "correct": false}
{"record_id": "jevbench:hard-opus-b-tradeoff-08", "group_id": "jevbench:hard-opus-b-tradeoff-08", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.6761663556098938, "true": 0.3238336145877838}, "correct": false}
{"record_id": "jevbench:hard-opus-b-tradeoff-12", "group_id": "jevbench:hard-opus-b-tradeoff-12", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "choice", "label": "refund_now", "prediction": "refund_now", "probabilities": {"deny_refund": 0.08222435414791107, "escalate_to_fraud_team": 0.010203023441135883, "offer_store_credit": 0.0479554608464241, "refund_now": 0.44789645075798035, "wait_for_inspection": 0.41172075271606445}, "correct": true}
{"record_id": "jevbench:hard-opus-c-long_policy-02", "group_id": "jevbench:hard-opus-c-long_policy-02", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "label": "erase_but_retain_transaction_records", "prediction": "erase_but_retain_transaction_records", "probabilities": {"erase_all": 0.08157192915678024, "erase_but_retain_transaction_records": 0.8089725971221924, "erase_marketing_only": 0.03990164026618004, "reject_unverified_request": 0.01850787363946438, "restrict_all_pending_legal_hold": 0.0510459840297699}, "correct": true}
{"record_id": "jevbench:hard-opus-c-long_policy-03", "group_id": "jevbench:hard-opus-c-long_policy-03", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "label": "eur_400", "prediction": "eur_0", "probabilities": {"eur_0": 0.287776917219162, "eur_150": 0.106198750436306, "eur_200": 0.13694162666797638, "eur_300": 0.13917456567287445, "eur_400": 0.19953863322734833, "eur_600": 0.13036948442459106}, "correct": false}
{"record_id": "jevbench:hard-opus-c-long_policy-04", "group_id": "jevbench:hard-opus-c-long_policy-04", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.6594837307929993, "true": 0.34051623940467834}, "correct": false}
{"record_id": "jevbench:hard-opus-c-long_policy-05", "group_id": "jevbench:hard-opus-c-long_policy-05", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "score", "label": "1", "prediction": "2", "probabilities": {"0": 0.1764277070760727, "1": 0.23852360248565674, "2": 0.36544546484947205, "3": 0.21960313618183136}, "correct": false}
{"record_id": "jevbench:hard-opus-c-long_policy-08", "group_id": "jevbench:hard-opus-c-long_policy-08", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "score", "label": "1", "prediction": "1", "probabilities": {"0": 0.0070486534386873245, "1": 0.6957435607910156, "2": 0.16174204647541046, "3": 0.13546575605869293}, "correct": true}
{"record_id": "jevbench:hard-opus-c-long_policy-10", "group_id": "jevbench:hard-opus-c-long_policy-10", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "label": "credit_minus_restocking_fee", "prediction": "full_credit", "probabilities": {"credit_minus_restocking_fee": 0.14705803990364075, "full_credit": 0.7305264472961426, "reject_return": 0.04698280617594719, "repair_or_replace_only": 0.07543270289897919}, "correct": false}
{"record_id": "jevbench:hard-opus-c-long_policy-11", "group_id": "jevbench:hard-opus-c-long_policy-11", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.7507756948471069, "true": 0.24922434985637665}, "correct": true}
{"record_id": "jevbench:hard-opus-c-probability-03", "group_id": "jevbench:hard-opus-c-probability-03", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "label": "kuznets_technik", "prediction": "ortega_seal_works", "probabilities": {"halvorsen_polymer": 0.037095364183187485, "kuznets_technik": 0.3268915116786957, "ortega_seal_works": 0.6360131502151489}, "correct": false}
{"record_id": "jevbench:hard-opus-c-temporal_numeric-02", "group_id": "jevbench:hard-opus-c-temporal_numeric-02", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.6276508569717407, "true": 0.3723491430282593}, "correct": false}
{"record_id": "jevbench:hard-opus-c-temporal_numeric-03", "group_id": "jevbench:hard-opus-c-temporal_numeric-03", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "label": "eur_692_00", "prediction": "eur_693_90", "probabilities": {"eur_617_00": 0.17735980451107025, "eur_688_00": 0.22998873889446259, "eur_692_00": 0.21757884323596954, "eur_693_90": 0.30480146408081055, "eur_812_00": 0.07027118653059006}, "correct": false}
{"record_id": "jevbench:hard-opus-c-temporal_numeric-04", "group_id": "jevbench:hard-opus-c-temporal_numeric-04", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "label": "usd_2335_00", "prediction": "usd_2303_01", "probabilities": {"usd_2222_50": 0.1922774761915207, "usd_2303_01": 0.2328849732875824, "usd_2303_33": 0.20734180510044098, "usd_2335_00": 0.2153155356645584, "usd_2340_00": 0.15218020975589752}, "correct": false}
{"record_id": "jevbench:hard-opus-c-temporal_numeric-06", "group_id": "jevbench:hard-opus-c-temporal_numeric-06", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "label": "sep_18_book_set", "prediction": "none_declined", "probabilities": {"none_declined": 0.6305046081542969, "sep_15_office_chair": 0.09083997458219528, "sep_16_taxi": 0.052508413791656494, "sep_17_parking": 0.05737852677702904, "sep_18_book_set": 0.0712629035115242, "sep_21_client_dinner": 0.0975055992603302}, "correct": false}
{"record_id": "jevbench:hard-opus-c-temporal_numeric-08", "group_id": "jevbench:hard-opus-c-temporal_numeric-08", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "label": "eur_498_81", "prediction": "eur_498_81", "probabilities": {"eur_497_40": 0.2607588469982147, "eur_498_81": 0.2904238998889923, "eur_506_06": 0.16178898513317108, "eur_513_69": 0.12518742680549622, "eur_532_01": 0.1618407964706421}, "correct": true}
{"record_id": "jevbench:hard-opus-c-temporal_numeric-12", "group_id": "jevbench:hard-opus-c-temporal_numeric-12", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "label": "expired_30_month_cap", "prediction": "expired_30_month_cap", "probabilities": {"covered": 0.16353420913219452, "expired_24_month_term": 0.35102781653404236, "expired_30_month_cap": 0.48543795943260193}, "correct": true}
{"record_id": "jevbench:hard-sol-a-adversarial-01", "group_id": "jevbench:hard-sol-a-adversarial-01", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "choice", "label": "deny_missing_approval", "prediction": "deny_missing_approval", "probabilities": {"deny_inactive_user": 0.009100019931793213, "deny_missing_approval": 0.6213160157203674, "grant_access": 0.02419205568730831, "grant_owner_exception": 0.3453919291496277}, "correct": true}
{"record_id": "jevbench:hard-sol-a-adversarial-06", "group_id": "jevbench:hard-sol-a-adversarial-06", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.9291604161262512, "true": 0.07083962857723236}, "correct": true}
{"record_id": "jevbench:hard-sol-a-adversarial-07", "group_id": "jevbench:hard-sol-a-adversarial-07", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "choice", "label": "retain_until_2031", "prediction": "retain_until_2031", "probabilities": {"delete_now": 0.0304971132427454, "needs_signature_review": 0.014544100500643253, "retain_indefinitely": 0.03171730414032936, "retain_until_2031": 0.9232414960861206}, "correct": true}
{"record_id": "jevbench:hard-sol-a-adversarial-08", "group_id": "jevbench:hard-sol-a-adversarial-08", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.03648354858160019, "true": 0.9635164141654968}, "correct": true}
{"record_id": "jevbench:hard-sol-a-adversarial-09", "group_id": "jevbench:hard-sol-a-adversarial-09", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "score", "label": "2", "prediction": "2", "probabilities": {"0": 0.054662562906742096, "1": 0.10786087810993195, "2": 0.605410099029541, "3": 0.23206649720668793}, "correct": true}
{"record_id": "jevbench:hard-sol-a-adversarial-11", "group_id": "jevbench:hard-sol-a-adversarial-11", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "choice", "label": "standard_research", "prediction": "standard_research", "probabilities": {"ethics_review": 0.016717970371246338, "major_grants": 0.016967525705695152, "manual_hold": 0.09926779568195343, "standard_research": 0.8670467734336853}, "correct": true}
{"record_id": "jevbench:hard-sol-a-multi_hop-01", "group_id": "jevbench:hard-sol-a-multi_hop-01", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "accept_s2_dusk", "prediction": "reject_cutoff", "probabilities": {"accept_s2_dusk": 0.3725583851337433, "reject_cutoff": 0.4426102042198181, "reject_tier": 0.1126924604177475, "reroute_s3": 0.07213897258043289}, "correct": false}
{"record_id": "jevbench:hard-sol-a-multi_hop-02", "group_id": "jevbench:hard-sol-a-multi_hop-02", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "security_primary", "prediction": "security_primary", "probabilities": {"executive_bridge": 0.12401773035526276, "heron_q2": 0.33090680837631226, "insufficient_information": 0.10420987010002136, "security_primary": 0.4408656060695648}, "correct": true}
{"record_id": "jevbench:hard-sol-a-multi_hop-05", "group_id": "jevbench:hard-sol-a-multi_hop-05", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "require_director", "prediction": "require_director", "probabilities": {"pay_manager_approved": 0.07043515890836716, "reject_category": 0.3321118652820587, "require_director": 0.5498997569084167, "require_finance_board": 0.047553207725286484}, "correct": true}
{"record_id": "jevbench:hard-sol-a-multi_hop-07", "group_id": "jevbench:hard-sol-a-multi_hop-07", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "admit", "prediction": "manual_equivalency", "probabilities": {"admit": 0.03955405205488205, "deny_prerequisite": 0.34229907393455505, "deny_score": 0.216408371925354, "manual_equivalency": 0.4017385244369507}, "correct": false}
{"record_id": "jevbench:hard-sol-a-multi_hop-08", "group_id": "jevbench:hard-sol-a-multi_hop-08", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "central_engineering", "prediction": "central_engineering", "probabilities": {"central_engineering": 0.5219905376434326, "insufficient_log": 0.12164927273988724, "local_maintenance": 0.1063808724284172, "safety_shutdown": 0.24997933208942413}, "correct": true}
{"record_id": "jevbench:hard-sol-a-multi_hop-09", "group_id": "jevbench:hard-sol-a-multi_hop-09", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "require_steward", "prediction": "require_steward", "probabilities": {"confirm_l3": 0.031020650640130043, "reject_capacity": 0.048368215560913086, "require_steward": 0.7082996368408203, "switch_l4": 0.2123115062713623}, "correct": true}
{"record_id": "jevbench:hard-sol-a-multi_hop-10", "group_id": "jevbench:hard-sol-a-multi_hop-10", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "approve_45_days", "prediction": "reduce_to_30_days", "probabilities": {"approve_45_days": 0.10196077078580856, "deny_sensitive": 0.3362988233566284, "needs_field_review": 0.1946527510881424, "reduce_to_30_days": 0.36708760261535645}, "correct": false}
{"record_id": "jevbench:hard-sol-a-multi_hop-12", "group_id": "jevbench:hard-sol-a-multi_hop-12", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "needs_authentication", "prediction": "needs_authentication", "probabilities": {"covered_recurrence": 0.22140616178512573, "expired": 0.2588846981525421, "needs_authentication": 0.34173932671546936, "wrong_issue_class": 0.1779697835445404}, "correct": true}
{"record_id": "jevbench:hard-sol-a-trap-02", "group_id": "jevbench:hard-sol-a-trap-02", "question_id": "decision", "source": "jevbench/hard/trap", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.9391950964927673, "true": 0.06080486997961998}, "correct": true}
{"record_id": "jevbench:hard-sol-a-trap-04", "group_id": "jevbench:hard-sol-a-trap-04", "question_id": "decision", "source": "jevbench/hard/trap", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.8216222524642944, "true": 0.17837773263454437}, "correct": true}
{"record_id": "jevbench:hard-sol-a-trap-05", "group_id": "jevbench:hard-sol-a-trap-05", "question_id": "decision", "source": "jevbench/hard/trap", "type": "score", "label": "0", "prediction": "0", "probabilities": {"0": 0.4580850899219513, "1": 0.2006053775548935, "2": 0.21469822525978088, "3": 0.12661129236221313}, "correct": true}
{"record_id": "jevbench:hard-sol-a-trap-06", "group_id": "jevbench:hard-sol-a-trap-06", "question_id": "decision", "source": "jevbench/hard/trap", "type": "choice", "label": "keep_open_unresolved", "prediction": "keep_open_unresolved", "probabilities": {"await_internal_metrics": 0.20002800226211548, "close_agent_verified": 0.062226951122283936, "close_confirmed": 0.040135566145181656, "keep_open_unresolved": 0.6976094841957092}, "correct": true}
{"record_id": "jevbench:hard-sol-a-trap-08", "group_id": "jevbench:hard-sol-a-trap-08", "question_id": "decision", "source": "jevbench/hard/trap", "type": "choice", "label": "late", "prediction": "on_time", "probabilities": {"canceled": 0.01418087724596262, "indeterminate": 0.05711594596505165, "late": 0.14182649552822113, "on_time": 0.7868767380714417}, "correct": false}
{"record_id": "jevbench:hard-sol-a-trap-10", "group_id": "jevbench:hard-sol-a-trap-10", "question_id": "decision", "source": "jevbench/hard/trap", "type": "choice", "label": "blocked_second_review", "prediction": "blocked_second_review", "probabilities": {"blocked_failed_test": 0.048584241420030594, "blocked_second_review": 0.8530124425888062, "ready_one_review": 0.02104303613305092, "ready_two_reviews": 0.07736024260520935}, "correct": true}
{"record_id": "jevbench:hard-sol-a-trap-13", "group_id": "jevbench:hard-sol-a-trap-13", "question_id": "decision", "source": "jevbench/hard/trap", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.8001976013183594, "true": 0.19980239868164062}, "correct": true}
{"record_id": "jevbench:hard-sol-a-trap-15", "group_id": "jevbench:hard-sol-a-trap-15", "question_id": "decision", "source": "jevbench/hard/trap", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.36253827810287476, "true": 0.6374617218971252}, "correct": false}
{"record_id": "jevbench:hard-sol-b-judge_hard-01", "group_id": "jevbench:hard-sol-b-judge_hard-01", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.2506294250488281, "true": 0.7493705749511719}, "correct": true}
{"record_id": "jevbench:hard-sol-b-judge_hard-02", "group_id": "jevbench:hard-sol-b-judge_hard-02", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.3051733374595642, "true": 0.6948266625404358}, "correct": false}
{"record_id": "jevbench:hard-sol-b-judge_hard-05", "group_id": "jevbench:hard-sol-b-judge_hard-05", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.21710701286792755, "true": 0.7828930020332336}, "correct": true}
{"record_id": "jevbench:hard-sol-b-judge_hard-08", "group_id": "jevbench:hard-sol-b-judge_hard-08", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.19036948680877686, "true": 0.8096305727958679}, "correct": false}
{"record_id": "jevbench:hard-sol-b-judge_hard-10", "group_id": "jevbench:hard-sol-b-judge_hard-10", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.5321475267410278, "true": 0.46785247325897217}, "correct": true}
{"record_id": "jevbench:hard-sol-b-judge_hard-14", "group_id": "jevbench:hard-sol-b-judge_hard-14", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.36279261112213135, "true": 0.6372073292732239}, "correct": false}
{"record_id": "jevbench:hard-sol-b-judge_hard-15", "group_id": "jevbench:hard-sol-b-judge_hard-15", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.472262978553772, "true": 0.527737021446228}, "correct": true}
{"record_id": "jevbench:hard-sol-b-judge_hard-18", "group_id": "jevbench:hard-sol-b-judge_hard-18", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.35453587770462036, "true": 0.6454641222953796}, "correct": false}
{"record_id": "jevbench:hard-sol-b-long_policy-01", "group_id": "jevbench:hard-sol-b-long_policy-01", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "label": "reject_entire_claim", "prediction": "approve_lodging_only", "probabilities": {"approve_all": 0.2195461541414261, "approve_lodging_only": 0.5525206327438354, "approve_meals_only": 0.07036943733692169, "reject_entire_claim": 0.15756374597549438}, "correct": false}
{"record_id": "jevbench:hard-sol-b-long_policy-02", "group_id": "jevbench:hard-sol-b-long_policy-02", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "label": "deny_current_export", "prediction": "privacy_and_security", "probabilities": {"analytics_owner_only": 0.2962091863155365, "deny_current_export": 0.21680057048797607, "privacy_and_security": 0.34808868169784546, "privacy_only": 0.1389015167951584}, "correct": false}
{"record_id": "jevbench:hard-sol-b-long_policy-05", "group_id": "jevbench:hard-sol-b-long_policy-05", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "label": "delete_profile_omar_april_and_table", "prediction": "delete_only_profile", "probabilities": {"delete_everything": 0.05439995974302292, "delete_nothing": 0.14005494117736816, "delete_only_profile": 0.643703281879425, "delete_profile_omar_april_and_table": 0.16184180974960327}, "correct": false}
{"record_id": "jevbench:hard-sol-b-long_policy-06", "group_id": "jevbench:hard-sol-b-long_policy-06", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "label": "east_ward", "prediction": "east_ward", "probabilities": {"east_ward": 0.4030003845691681, "harbor_neighbors": 0.20460054278373718, "no_award": 0.1443929374217987, "river_arts": 0.24800613522529602}, "correct": true}
{"record_id": "jevbench:hard-sol-b-routing_hard-01", "group_id": "jevbench:hard-sol-b-routing_hard-01", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "label": "office_file_repair", "prediction": "office_file_repair", "probabilities": {"data_pipeline": 0.006780586205422878, "desktop_support": 0.010867821052670479, "finance_review": 0.009302408434450626, "office_file_repair": 0.927355945110321, "spreadsheet_formula": 0.045693207532167435}, "correct": true}
{"record_id": "jevbench:hard-sol-b-routing_hard-02", "group_id": "jevbench:hard-sol-b-routing_hard-02", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "label": "ledger_reconciliation", "prediction": "ledger_reconciliation", "probabilities": {"customer_support": 0.005622476805001497, "fraud_review": 0.0030721521470695734, "ledger_reconciliation": 0.9758802652359009, "mobile_app": 0.006609315983951092, "payment_disputes": 0.008815896697342396}, "correct": true}
{"record_id": "jevbench:hard-sol-b-routing_hard-03", "group_id": "jevbench:hard-sol-b-routing_hard-03", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "label": "network_dns", "prediction": "network_dns", "probabilities": {"api_platform": 0.0421319305896759, "authoritative_dns": 0.30494779348373413, "endpoint_devices": 0.014534590765833855, "network_dns": 0.6283749341964722, "tls_certificates": 0.010010804049670696}, "correct": true}
{"record_id": "jevbench:hard-sol-b-routing_hard-07", "group_id": "jevbench:hard-sol-b-routing_hard-07", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "label": "release_notarization", "prediction": "code_signing", "probabilities": {"ci_infrastructure": 0.11375533789396286, "code_signing": 0.41204026341438293, "endpoint_security": 0.019950861111283302, "macos_runtime": 0.1799687296152115, "release_notarization": 0.27428483963012695}, "correct": false}
{"record_id": "jevbench:hard-sol-b-routing_hard-09", "group_id": "jevbench:hard-sol-b-routing_hard-09", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "label": "source_access", "prediction": "source_access", "probabilities": {"git_client": 0.045928582549095154, "identity_login": 0.027846548706293106, "repository_admin": 0.014928936958312988, "source_access": 0.657455563545227, "vpn_network": 0.2538404166698456}, "correct": true}
{"record_id": "jevbench:hard-sol-b-temporal_numeric-01", "group_id": "jevbench:hard-sol-b-temporal_numeric-01", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "label": "late_by_under_2h", "prediction": "within_window", "probabilities": {"before_open": 0.12202752381563187, "late_by_2h_or_more": 0.26034775376319885, "late_by_under_2h": 0.1189715564250946, "within_window": 0.4986531734466553}, "correct": false}
{"record_id": "jevbench:hard-sol-b-temporal_numeric-02", "group_id": "jevbench:hard-sol-b-temporal_numeric-02", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "label": "over_limit", "prediction": "over_limit", "probabilities": {"at_least_100gb_below": 0.2299937754869461, "exactly_equal": 0.04111615940928459, "over_limit": 0.5994839072227478, "under_100gb_below": 0.1294061243534088}, "correct": true}
{"record_id": "jevbench:hard-sol-b-temporal_numeric-03", "group_id": "jevbench:hard-sol-b-temporal_numeric-03", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "label": "9_credits", "prediction": "9_credits", "probabilities": {"12_credits": 0.28052958846092224, "15_credits": 0.2033003270626068, "6_credits": 0.20880404114723206, "9_credits": 0.3073660731315613}, "correct": true}
{"record_id": "jevbench:hard-sol-b-temporal_numeric-04", "group_id": "jevbench:hard-sol-b-temporal_numeric-04", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "label": "pass", "prediction": "clear_fail", "probabilities": {"clear_fail": 0.35486143827438354, "narrow_fail": 0.3542650640010834, "pass": 0.2908734679222107}, "correct": false}
{"record_id": "jevbench:hard-sol-c-judge_hard-03", "group_id": "jevbench:hard-sol-c-judge_hard-03", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.5843634009361267, "true": 0.4156365990638733}, "correct": true}
{"record_id": "jevbench:hard-sol-c-judge_hard-05", "group_id": "jevbench:hard-sol-c-judge_hard-05", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.5731537938117981, "true": 0.4268461763858795}, "correct": false}
{"record_id": "jevbench:hard-sol-c-judge_hard-07", "group_id": "jevbench:hard-sol-c-judge_hard-07", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.2519032061100006, "true": 0.748096764087677}, "correct": false}
{"record_id": "jevbench:hard-sol-c-judge_hard-08", "group_id": "jevbench:hard-sol-c-judge_hard-08", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.3512498140335083, "true": 0.6487501859664917}, "correct": false}
{"record_id": "jevbench:hard-sol-c-judge_hard-09", "group_id": "jevbench:hard-sol-c-judge_hard-09", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.3796171247959137, "true": 0.6203829050064087}, "correct": true}
{"record_id": "jevbench:hard-sol-c-judge_hard-10", "group_id": "jevbench:hard-sol-c-judge_hard-10", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "true", "prediction": "true", "probabilities": {"false": 0.45133325457572937, "true": 0.5486667156219482}, "correct": true}
{"record_id": "jevbench:hard-sol-c-judge_hard-11", "group_id": "jevbench:hard-sol-c-judge_hard-11", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "false", "prediction": "false", "probabilities": {"false": 0.5684359669685364, "true": 0.4315640330314636}, "correct": true}
{"record_id": "jevbench:hard-sol-c-judge_hard-13", "group_id": "jevbench:hard-sol-c-judge_hard-13", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "false", "prediction": "true", "probabilities": {"false": 0.497303307056427, "true": 0.502696692943573}, "correct": false}
{"record_id": "jevbench:hard-sol-c-judge_hard-15", "group_id": "jevbench:hard-sol-c-judge_hard-15", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "label": "true", "prediction": "false", "probabilities": {"false": 0.5728118419647217, "true": 0.4271881878376007}, "correct": false}
{"record_id": "jevbench:hard-sol-c-multi_hop-07", "group_id": "jevbench:hard-sol-c-multi_hop-07", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "programme_review", "prediction": "programme_review", "probabilities": {"partial_release": 0.11607901006937027, "programme_review": 0.5964341163635254, "reject_draw": 0.08382941037416458, "release_draw": 0.20365744829177856}, "correct": true}
{"record_id": "jevbench:hard-sol-c-multi_hop-09", "group_id": "jevbench:hard-sol-c-multi_hop-09", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "bus_substitution", "prediction": "bus_substitution", "probabilities": {"bus_substitution": 0.6290871500968933, "call_fen_cross": 0.102838896214962, "pass_fen_cross": 0.10045100003480911, "terminate_oakmere": 0.16762295365333557}, "correct": true}
{"record_id": "jevbench:hard-sol-c-multi_hop-11", "group_id": "jevbench:hard-sol-c-multi_hop-11", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "coverage_excluded", "prediction": "documentation_needed", "probabilities": {"adjuster_referral": 0.12701432406902313, "coverage_confirmed": 0.13149182498455048, "coverage_excluded": 0.34033238887786865, "documentation_needed": 0.40116143226623535}, "correct": false}
{"record_id": "jevbench:hard-sol-c-multi_hop-12", "group_id": "jevbench:hard-sol-c-multi_hop-12", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "conservation_hold", "prediction": "digital_copy_only", "probabilities": {"conservation_hold": 0.10441930592060089, "deny_request": 0.305535227060318, "digital_copy_only": 0.37308570742607117, "supervised_delivery": 0.21695972979068756}, "correct": false}
{"record_id": "jevbench:hard-sol-c-multi_hop-13", "group_id": "jevbench:hard-sol-c-multi_hop-13", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "label": "normal_release", "prediction": "normal_release", "probabilities": {"engineering_review": 0.16476695239543915, "normal_release": 0.6624904274940491, "rework": 0.10748638212680817, "scrap_lot": 0.06525624543428421}, "correct": true}