{ "repo": "FineEnvs/Qwen3.5-2B-opencode-standalone-RL", "revision": "main", "checkpoint_step": 1000, "article": "https://huggingface.co/spaces/AdithyaSK/multi-harness-rl", "notes": [ "Training curves: trailing 50-step arithmetic mean, minimum 10 observations.", "Evaluation points are measured checkpoints, not smoothed. Hollow points have incomplete coverage.", "Tool savings use varying task/harness cohorts solved by both base and checkpoint. Qwen instead shows absolute calls.", "SFT has only two post-training evaluation checkpoints." ], "panels": [ { "title": "Training reward", "desc": "50-step trailing mean \u00b7 higher is better", "series": [ { "label": "Training reward", "x": [ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000 ], "y": [ 0.73719, 0.7307818181818182, 0.6698833333333334, 0.6183538461538463, 0.5741857142857143, 0.5359066666666668, 0.5180375, 0.5006352941176472, 0.4728222222222223, 0.44793684210526324, 0.4380400000000001, 0.445752380952381, 0.43847727272727277, 0.45046956521739134, 0.45947916666666666, 0.47609999999999997, 0.48576153846153847, 0.5048074074074074, 0.5127535714285714, 0.5209344827586206, 0.5119033333333333, 0.5138225806451613, 0.529015625, 0.5173151515151515, 0.5021, 0.50204, 0.5089277777777778, 0.5067567567567568, 0.4934210526315789, 0.49615384615384617, 0.50875, 0.5207317073170732, 0.5321428571428571, 0.539146511627907, 0.5405295454545456, 0.5285177777777779, 0.5170282608695653, 0.5060276595744682, 0.49965208333333333, 0.4894551020408163, 0.484666, 0.487666, 0.490166, 0.486166, 0.485332, 0.485332, 0.49961800000000006, 0.504062, 0.489062, 0.469062, 0.460728, 0.45354800000000006, 0.45354800000000006, 0.45354799999999995, 0.47354799999999997, 0.489548, 0.504548, 0.520104, 0.520104, 0.520104, 0.515104, 0.509104, 0.52339, 0.5254679999999999, 0.512134, 0.494634, 0.480088, 0.460088, 0.44554199999999994, 0.43054200000000004, 0.440542, 0.434114, 0.41855800000000004, 0.41570000000000007, 0.429546, 0.42288000000000003, 0.40787999999999996, 0.39930800000000005, 0.40130800000000005, 0.40359400000000006, 0.40359400000000006, 0.398594, 0.378594, 0.361928, 0.349928, 0.352428, 0.36842800000000003, 0.37842800000000004, 0.374428, 0.377286, 0.392286, 0.39728600000000003, 0.402286, 0.406286, 0.408786, 0.391644, 0.37164400000000003, 0.351644, 0.346644, 0.346644, 0.346644, 0.347156, 0.347156, 0.347156, 0.327156, 0.32258400000000004, 0.30925, 0.28925, 0.28925000000000006, 0.28925, 0.28925000000000006, 0.28325, 0.26325000000000004, 0.24688600000000002, 0.24688600000000002, 0.26022, 0.27522, 0.27522, 0.27521999999999996, 0.27521999999999996, 0.26022, 0.260934, 0.265062, 0.275062, 0.266216, 0.262882, 0.26288200000000006, 0.262882, 0.260882, 0.24659599999999998, 0.233262, 0.21826199999999996, 0.218262, 0.218262, 0.218262, 0.215762, 0.199762, 0.189762, 0.18976200000000001, 0.18690400000000001, 0.171904, 0.151904, 0.131904, 0.11590400000000001, 0.10090400000000001, 0.11137999999999998, 0.12137999999999999, 0.138522, 0.146522, 0.14652199999999999, 0.14652199999999999, 0.148744, 0.168744, 0.178744, 0.17874400000000001, 0.167316, 0.16065, 0.16065000000000002, 0.16731599999999996, 0.167316, 0.167316, 0.167316, 0.167316, 0.167316, 0.167316, 0.15398199999999998, 0.138982, 0.14787, 0.16787, 0.18644199999999997, 0.206442, 0.20517200000000002, 0.19660000000000002, 0.20409999999999998, 0.21052800000000002, 0.217194, 0.217194, 0.217194, 0.217194, 0.232194, 0.24052800000000002, 0.24052799999999996, 0.24052800000000002, 0.24052800000000002, 0.24052800000000002, 0.24052799999999996, 0.24052800000000002, 0.24052800000000002, 0.24052800000000002, 0.24052799999999996, 0.23552800000000002, 0.23552800000000002, 0.23552800000000002, 0.23152799999999998, 0.23152799999999998, 0.21819400000000003, 0.20819400000000002, 0.191052, 0.183052, 0.183052, 0.183052, 0.17916400000000002, 0.159164, 0.149164, 0.149164, 0.14916400000000002, 0.149164, 0.15583, 0.14916400000000002, 0.149164, 0.164164, 0.17416399999999999, 0.194164, 0.214164, 0.214164, 0.214164, 0.21816399999999997, 0.209276, 0.20927600000000002, 0.210704, 0.202132, 0.217688, 0.23268800000000003, 0.21518800000000002, 0.20947400000000002, 0.20280800000000002, 0.202808, 0.202808, 0.222808, 0.207808, 0.212808, 0.227094, 0.24709399999999998, 0.267094, 0.287094, 0.30709400000000003, 0.30709400000000003, 0.30709400000000003, 0.30709400000000003, 0.30709400000000003, 0.31042800000000004, 0.330428, 0.350428, 0.36242800000000003, 0.36242800000000003, 0.36242800000000003, 0.36242800000000003, 0.36242800000000003, 0.36242800000000003, 0.37742800000000004, 0.38242800000000005, 0.380762, 0.380762, 0.380762, 0.380762, 0.380762, 0.38742800000000005, 0.380762, 0.390762, 0.390762, 0.375762, 0.365762, 0.345762, 0.325762, 0.340762, 0.348262, 0.344262, 0.359262, 0.349262, 0.342596, 0.33950199999999997, 0.319502, 0.30450200000000005, 0.30450200000000005, 0.298788, 0.298788, 0.298788, 0.298788, 0.298788, 0.303788, 0.283788, 0.28750200000000004, 0.267502, 0.24750200000000003, 0.22750199999999998, 0.207502, 0.207502, 0.22083600000000003, 0.23583600000000002, 0.245836, 0.242502, 0.222502, 0.202502, 0.200502, 0.21916800000000003, 0.229168, 0.229168, 0.229168, 0.229168, 0.21416800000000003, 0.20916800000000002, 0.20583400000000002, 0.20583400000000002, 0.21183400000000002, 0.21183400000000002, 0.211834, 0.205168, 0.205168, 0.195168, 0.210584, 0.210584, 0.210584, 0.210584, 0.215584, 0.20058399999999998, 0.19808399999999998, 0.20208399999999999, 0.18708399999999997, 0.17708400000000002, 0.16375, 0.175416, 0.190416, 0.20241599999999998, 0.202416, 0.202416, 0.20241599999999998, 0.20241599999999998, 0.21491600000000002, 0.21491600000000002, 0.22241599999999997, 0.22241599999999997, 0.20441600000000001, 0.20441600000000001, 0.20441600000000001, 0.20441600000000001, 0.20775000000000002, 0.22108400000000003, 0.20775000000000002, 0.19275000000000003, 0.18775000000000003, 0.18774999999999997, 0.20024999999999998, 0.203584, 0.19358400000000003, 0.174918, 0.164918, 0.16491799999999998, 0.164918, 0.174918, 0.17491800000000002, 0.174918, 0.17491800000000002, 0.174918, 0.168918, 0.168918, 0.168918, 0.16891800000000004, 0.183918, 0.203918, 0.20600200000000002, 0.20600200000000002, 0.20600200000000002, 0.216002, 0.216002, 0.216002, 0.21100200000000002, 0.207002, 0.20700200000000002, 0.20700200000000002, 0.21366800000000002, 0.19366799999999998, 0.17866800000000002, 0.166668, 0.166668, 0.16666799999999998, 0.166668, 0.166668, 0.154168, 0.13416799999999998, 0.124168, 0.134168, 0.149168, 0.149168, 0.149168, 0.164168, 0.1775, 0.16416599999999998, 0.16416600000000003, 0.16416600000000003, 0.159166, 0.159166, 0.14666600000000002, 0.143332, 0.151666, 0.16166599999999998, 0.16666600000000004, 0.16666600000000004, 0.16666600000000004, 0.156666, 0.16166600000000003, 0.16666600000000004, 0.18666600000000003, 0.20666600000000002, 0.20666600000000002, 0.20916600000000002, 0.21416600000000002, 0.21916600000000003, 0.20416600000000001, 0.18416600000000002, 0.16666600000000004, 0.16666600000000004, 0.18166600000000002, 0.19166600000000003, 0.20000000000000004, 0.20000000000000004, 0.21, 0.216666, 0.22, 0.22000000000000003, 0.228334, 0.228334, 0.228334, 0.248334, 0.268334, 0.268334, 0.268334, 0.268334, 0.28333400000000003, 0.293334, 0.29083400000000004, 0.28083400000000003, 0.265834, 0.265834, 0.265834, 0.250834, 0.23750200000000002, 0.25000200000000006, 0.25000200000000006, 0.270002, 0.29000200000000004, 0.29000200000000004, 0.29000200000000004, 0.290002, 0.29166800000000004, 0.29166800000000004, 0.28666800000000003, 0.28666800000000003, 0.28666800000000003, 0.29166800000000004, 0.30166800000000005, 0.29666800000000004, 0.28333400000000003, 0.263334, 0.263334, 0.26583399999999996, 0.265834, 0.28083400000000003, 0.285, 0.285, 0.285, 0.285, 0.28, 0.265, 0.251666, 0.251666, 0.249666, 0.251, 0.247666, 0.247666, 0.232666, 0.252666, 0.252666, 0.24066600000000002, 0.23566600000000001, 0.23566600000000001, 0.24150000000000002, 0.2615, 0.2615, 0.2515, 0.2515, 0.2515, 0.2515, 0.2515, 0.269834, 0.284834, 0.2815, 0.269, 0.26899999999999996, 0.249, 0.22899999999999998, 0.234, 0.244, 0.249714, 0.23971399999999998, 0.229714, 0.229714, 0.229714, 0.229714, 0.22871399999999997, 0.213714, 0.21371400000000002, 0.20704799999999998, 0.20704799999999998, 0.20704799999999998, 0.207762, 0.205262, 0.18669, 0.18252400000000002, 0.197524, 0.21252400000000002, 0.21252400000000002, 0.202524, 0.197524, 0.197524, 0.197524, 0.18952399999999997, 0.181524, 0.181524, 0.181524, 0.181524, 0.161524, 0.161524, 0.153524, 0.138524, 0.14352399999999998, 0.15269, 0.15269000000000002, 0.14769000000000002, 0.14769000000000002, 0.14769, 0.14769000000000002, 0.14769000000000002, 0.14769000000000002, 0.129356, 0.11435600000000001, 0.126856, 0.130856, 0.145856, 0.157856, 0.157856, 0.152856, 0.142856, 0.137142, 0.140476, 0.140476, 0.14047600000000002, 0.14381, 0.14381, 0.13981, 0.13981000000000002, 0.13981, 0.153144, 0.16064399999999998, 0.16464399999999998, 0.15893000000000002, 0.16143000000000002, 0.16500199999999998, 0.166336, 0.16733599999999998, 0.155972, 0.155972, 0.155972, 0.155972, 0.155972, 0.160972, 0.16383, 0.16383, 0.16383, 0.16383000000000003, 0.16683, 0.16683, 0.16683, 0.16683, 0.16683, 0.16183, 0.16683, 0.14849600000000002, 0.138496, 0.138496, 0.13849599999999998, 0.138496, 0.13849599999999998, 0.138496, 0.13849599999999998, 0.138496, 0.125996, 0.122996, 0.107996, 0.09599600000000001, 0.09599599999999998, 0.09599600000000001, 0.09599599999999998, 0.09599600000000001, 0.11266200000000001, 0.112662, 0.11266200000000001, 0.10932800000000001, 0.119328, 0.122662, 0.13866199999999998, 0.138662, 0.125328, 0.12282800000000002, 0.138828, 0.138828, 0.133828, 0.14216199999999998, 0.155828, 0.139828, 0.13619199999999998, 0.13619199999999998, 0.13619199999999998, 0.13619199999999998, 0.13619199999999998, 0.131192, 0.128334, 0.143334, 0.16333399999999998, 0.16515199999999997, 0.162152, 0.16369, 0.17369, 0.19369, 0.20369, 0.20869, 0.18869, 0.18702399999999997, 0.187024, 0.20702399999999999, 0.21202400000000002, 0.21202400000000002, 0.21202400000000002, 0.23202399999999998, 0.24702400000000002, 0.267024, 0.277024, 0.28002399999999994, 0.290024, 0.31002399999999997, 0.33002400000000004, 0.35002399999999995, 0.36502399999999996, 0.36502399999999996, 0.345024, 0.345024, 0.34788199999999997, 0.347882, 0.337882, 0.334548, 0.318548, 0.32354799999999995, 0.33188199999999995, 0.32688199999999995, 0.313548, 0.320214, 0.330214, 0.32188, 0.32188, 0.32588, 0.32588, 0.32588, 0.32587999999999995, 0.33588, 0.34838, 0.35638, 0.3563799999999999, 0.34137999999999996, 0.32138, 0.329562, 0.34956199999999993, 0.36802399999999996, 0.37802399999999997, 0.3780240000000001, 0.37802399999999997, 0.37302399999999997, 0.37802399999999997, 0.38302400000000003, 0.383024, 0.363024, 0.36602399999999996, 0.366024, 0.366024, 0.34602400000000005, 0.33102400000000004, 0.31102399999999997, 0.301024, 0.314166, 0.318452, 0.306452, 0.28645200000000004, 0.272166, 0.26859400000000005, 0.268594, 0.268594, 0.268594, 0.265736, 0.280736, 0.285736, 0.285736, 0.28573600000000005, 0.280736, 0.27240200000000003, 0.292402, 0.305736, 0.31907, 0.31306999999999996, 0.30807, 0.29307, 0.28907, 0.28906999999999994, 0.28907, 0.297404, 0.295738, 0.28323800000000005, 0.275238, 0.28323800000000005, 0.283238, 0.286572, 0.296572, 0.291572, 0.271572, 0.251572, 0.239072, 0.229072, 0.232406, 0.244906, 0.244906, 0.244906, 0.24490599999999996, 0.23690600000000003, 0.23690599999999995, 0.236906, 0.236906, 0.23690600000000003, 0.24690599999999996, 0.266906, 0.261764, 0.247478, 0.256978, 0.27297800000000005, 0.26726400000000006, 0.25583600000000006, 0.25583600000000006, 0.25583600000000006, 0.25583600000000006, 0.25583600000000006, 0.24583600000000005, 0.25083600000000006, 0.270836, 0.28283600000000003, 0.28283600000000003, 0.28950200000000004, 0.289502, 0.28950200000000004, 0.28505800000000003, 0.286772, 0.286772, 0.286772, 0.28677199999999997, 0.29877200000000004, 0.315438, 0.307104, 0.29877000000000004, 0.29877000000000004, 0.29877000000000004, 0.29077, 0.29077000000000003, 0.287436, 0.267436, 0.252436, 0.252436, 0.252436, 0.24493600000000001, 0.259936, 0.276602, 0.279102, 0.290102, 0.29343600000000003, 0.29343600000000003, 0.29343600000000003, 0.29343600000000003, 0.29343600000000003, 0.29343600000000003, 0.304864, 0.299864, 0.27986400000000006, 0.267864, 0.287864, 0.290364, 0.27436400000000005, 0.274364, 0.294364, 0.314364, 0.314364, 0.334364, 0.35436400000000007, 0.349364, 0.33936400000000005, 0.319364, 0.32236400000000004, 0.34236400000000006, 0.345698, 0.333698, 0.333698, 0.338142, 0.352428, 0.36842800000000003, 0.36842800000000003, 0.379856, 0.38785600000000003, 0.38719000000000003, 0.40719, 0.41385600000000006, 0.41385600000000006, 0.41385599999999995, 0.41385600000000006, 0.41385599999999995, 0.41385599999999995, 0.43099799999999994, 0.450998, 0.458998, 0.46399799999999997, 0.46399799999999997, 0.46614, 0.46614, 0.45414, 0.43814, 0.44980599999999993, 0.46480600000000005, 0.46480600000000005, 0.46480600000000005, 0.46480599999999994, 0.46480600000000005, 0.453378, 0.448378, 0.452378, 0.462378, 0.445236, 0.425236, 0.425236, 0.425236, 0.40523600000000004, 0.385236, 0.39094999999999996, 0.37845, 0.365116, 0.385116, 0.39602600000000004, 0.409872, 0.402372, 0.382372, 0.37237200000000004, 0.36437200000000003, 0.350086, 0.330086, 0.31008600000000003, 0.294086, 0.30908599999999997, 0.317658, 0.30051599999999995, 0.28451599999999994, 0.264516, 0.263564, 0.26356399999999996, 0.263564, 0.263564, 0.263564, 0.263564, 0.246422, 0.23142200000000002, 0.22342199999999998, 0.22730999999999998, 0.24730999999999997, 0.230168, 0.222168, 0.214168, 0.214168, 0.21631, 0.22131, 0.224946, 0.224946, 0.23828, 0.25542200000000004, 0.275422, 0.295422, 0.30342199999999997, 0.31342200000000003, 0.315564, 0.315564, 0.32681399999999994, 0.33081399999999994, 0.33081399999999994, 0.330814, 0.32510000000000006, 0.32093399999999994, 0.314268, 0.294268, 0.283358, 0.269512, 0.262012, 0.262012, 0.262012, 0.262012, 0.256298, 0.26296400000000003, 0.282964, 0.292964, 0.292964, 0.290464, 0.28760600000000003, 0.297606, 0.313162, 0.307448, 0.307448, 0.31602, 0.31602, 0.33602, 0.35602000000000006, 0.35602000000000006, 0.35102000000000005, 0.35101999999999994, 0.342132, 0.32463200000000003, 0.32907600000000004, 0.31707599999999997, 0.317076, 0.317076, 0.29993400000000003, 0.279934, 0.276298, 0.27629800000000004, 0.262964, 0.250822, 0.230822, 0.215822, 0.20382199999999998, 0.18382199999999999, 0.17882199999999998, 0.18548799999999996, 0.18673799999999996, 0.20273799999999997, 0.222738, 0.22273800000000002, 0.22273800000000002, 0.21940400000000002, 0.21940399999999996, 0.21940400000000002, 0.21940400000000002, 0.21940400000000002, 0.21940399999999996, 0.21940400000000002, 0.21940400000000002, 0.21940400000000002, 0.23324999999999996, 0.22824999999999998, 0.20825, 0.19824999999999998, 0.19824999999999998, 0.20075000000000004, 0.20074999999999998, 0.19075, 0.17519400000000002, 0.17519400000000002, 0.17519400000000002, 0.166622, 0.166622, 0.155194, 0.144528, 0.14452800000000002, 0.144528, 0.144528, 0.144528, 0.142028, 0.13758399999999998, 0.137584, 0.13758399999999998, 0.15758399999999997, 0.177584, 0.17758400000000002, 0.17758400000000002, 0.177584, 0.177584, 0.19258399999999998, 0.21258400000000002, 0.207584, 0.207584, 0.21078399999999997, 0.21078400000000003, 0.204118, 0.19161799999999998, 0.171618 ], "color": "#7552dd", "complete": [ true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true ], "training": true } ] }, { "title": "Training tool calls", "desc": "Calls per rollout \u00b7 50-step mean \u00b7 lower is better", "series": [ { "label": "Training tool calls", "x": [ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000 ], "y": [ 6.5042100000000005, 6.377563636363636, 6.429433333333333, 6.196400000000001, 6.137728571428572, 5.945213333333333, 5.8236375, 5.729435294117647, 5.885883333333334, 5.733994736842106, 5.647295000000001, 5.902185714285714, 6.133904545454545, 5.991434782608695, 5.940866666666667, 5.948232000000001, 5.925746153846154, 5.979, 6.138839285714285, 6.010489655172413, 6.076806666666666, 6.106587096774193, 6.009506249999999, 6.0222060606060595, 6.208717647058823, 6.274182857142856, 6.176288888888887, 6.105886486486486, 6.055731578947367, 6.069687179487178, 5.999194999999999, 5.926043902439024, 5.862328571428571, 5.8035139534883715, 5.771615909090909, 5.832246666666666, 5.766328260869566, 5.71988085106383, 5.7632166666666675, 5.706824489795919, 5.657688, 5.625688, 5.510688, 5.489688, 5.432188, 5.418854, 5.244567999999999, 5.24068, 5.219846, 5.37918, 5.34418, 5.369649999999999, 5.3256499999999996, 5.52265, 5.488484, 5.559484, 5.539484, 5.549325999999999, 5.495082, 5.751748, 5.795384, 5.849384, 5.735098000000001, 5.8688660000000015, 5.8533100000000005, 5.92004, 6.097768000000001, 6.225496000000001, 6.171405999999999, 6.379072, 6.359072, 6.449072, 6.482406, 6.527168, 6.367588, 6.3725879999999995, 6.403302000000001, 6.503302, 6.605301999999999, 6.676159999999999, 6.767826, 6.837826, 6.848382, 6.919898000000001, 7.158564, 7.281063999999999, 7.281064, 7.356898, 7.332564, 7.532564, 7.532564, 7.571314, 7.577314, 7.589314000000002, 7.516813999999999, 7.588242, 7.7032419999999995, 7.782242, 7.869742, 7.7406939999999995, 7.884028, 7.8996699999999995, 8.031669999999998, 7.950669999999999, 8.082336, 8.060622, 8.060622, 8.035426000000001, 7.982760000000001, 7.802759999999999, 7.964124000000002, 7.886124000000001, 8.072410000000001, 8.009500000000001, 7.984500000000001, 7.853604000000001, 7.628604, 7.513604, 7.491938, 7.304825999999999, 7.241967999999999, 7.203396000000001, 7.338634, 7.431966000000001, 7.544274000000001, 7.592608, 7.7368939999999995, 7.748324, 7.735658, 7.7728, 7.792800000000001, 7.966799999999999, 8.099243999999999, 8.139062, 8.023824, 7.848465999999999, 7.980465999999999, 8.047965999999999, 8.0938, 7.9437999999999995, 8.088799999999999, 8.258049999999999, 8.294908, 8.314908, 8.394908000000001, 8.34348, 8.281813999999999, 8.314955999999999, 8.394956, 8.41817, 8.260392000000001, 8.232614000000002, 8.076042, 7.9920420000000005, 7.927041999999999, 7.968756, 7.9487559999999995, 7.9663319999999995, 8.112332, 8.08138, 7.88438, 7.83838, 7.66138, 7.66938, 7.745807999999999, 7.830807999999998, 7.849269999999999, 7.762603999999999, 7.6942699999999995, 7.709667999999999, 7.767525999999999, 7.671654, 7.623082, 7.466415999999999, 7.360701999999999, 7.241811999999999, 7.276811999999999, 7.176454, 7.103119999999999, 6.98812, 6.886453999999999, 6.719596, 6.754929999999999, 6.67943, 6.555501999999999, 6.62927, 6.723770000000001, 6.722103999999999, 6.787103999999999, 6.847103999999999, 6.8337699999999995, 6.83577, 6.957912, 6.968578, 6.922864, 6.917864, 6.92453, 6.914888000000001, 6.8815539999999995, 7.0173879999999995, 7.061831999999999, 7.016276, 7.0562759999999995, 7.022942, 6.9762759999999995, 7.1136099999999995, 7.2764679999999995, 7.3898019999999995, 7.403802, 7.558087999999999, 7.535088000000001, 7.465088000000001, 7.410088000000001, 7.336945999999999, 7.280518000000001, 7.285756, 7.339294000000001, 7.520960000000001, 7.51596, 7.491674000000001, 7.499532, 7.578976, 7.663976, 7.631754, 7.771754, 7.753975999999999, 7.748976, 7.961476, 7.961476, 8.153142, 8.169254, 8.160684000000002, 7.983064000000001, 7.946278000000001, 7.9237779999999995, 7.794868, 7.614868, 7.558199999999999, 7.4297, 7.549699999999999, 7.610534, 7.406534000000001, 7.251534, 7.204867999999999, 7.213915999999999, 7.348916, 7.403915999999999, 7.4364159999999995, 7.386892, 7.2510580000000004, 7.2060580000000005, 7.286058000000001, 7.44463, 7.6519639999999995, 7.7646299999999995, 7.562724, 7.5265320000000004, 7.523198, 7.409198, 7.246976, 7.279754, 7.329753999999999, 7.4897540000000005, 7.736895999999999, 7.786896, 7.730824, 7.793824, 7.708824, 7.828824, 7.88049, 7.860966000000001, 7.800966000000001, 7.7599659999999995, 7.9279020000000004, 7.951949999999999, 8.160283999999999, 8.190284, 8.111284, 8.119283999999999, 7.945117999999999, 8.094006, 8.083434, 8.156054, 8.364625999999998, 8.436767999999999, 8.669768, 8.692267999999999, 8.625601999999999, 8.576602, 8.446602, 8.477101999999999, 8.661101999999998, 8.886102, 9.008602000000002, 8.959933999999999, 8.9016, 8.923266000000002, 8.9266, 8.921123999999999, 8.845624, 8.830624, 8.883482, 8.721482, 8.545482, 8.485316000000001, 8.521388, 8.542221999999999, 8.58389, 8.751224, 8.797612, 8.841834, 8.824334, 8.821834, 8.641834, 8.658834, 8.661334000000002, 8.586334, 8.659668, 8.668556, 8.87189, 8.934556, 8.939556000000001, 8.811556000000001, 8.677508000000001, 8.677508, 8.690008, 8.545842, 8.398176, 8.363509999999998, 8.37351, 8.21851, 8.31651, 8.283176, 8.17889, 8.141748, 8.056748, 8.100914, 8.113414, 8.206748, 8.326748, 8.155414, 7.995414, 7.857914000000001, 7.898748, 8.073248, 8.044082000000001, 7.99075, 8.019916, 8.030750000000001, 8.19875, 8.32375, 8.345892, 8.477891999999999, 8.536558, 8.508224, 8.560724, 8.678224, 8.55489, 8.340055999999999, 8.34089, 8.27389, 8.339724, 8.297224, 8.305558, 8.263558, 8.266558, 8.231558, 8.08989, 7.985168000000001, 7.784334000000001, 7.675834, 7.735834, 7.923834, 8.018834, 8.0285, 7.980999999999999, 7.9451659999999995, 7.9851659999999995, 8.036831999999999, 7.969332, 8.028497999999999, 7.959998000000001, 8.078332, 8.200332000000001, 8.160332, 8.114498, 8.170332, 8.208666, 8.205332, 8.191332, 8.144666, 8.306666, 8.4385, 8.4535, 8.314334, 8.271833999999998, 8.198500000000001, 8.058499999999999, 8.061, 8.0735, 8.1585, 8.033499999999998, 7.944927999999999, 7.870761999999998, 7.815761999999999, 7.735761999999999, 7.570761999999999, 7.735761999999999, 7.803262, 7.754096, 7.771596, 7.661595999999999, 7.546595999999999, 7.504928, 7.636928, 7.6939280000000005, 7.828927999999999, 7.938096000000001, 8.03893, 8.074764, 8.105264, 8.14193, 7.93793, 7.841264, 7.720763999999999, 7.670764, 7.824763999999999, 7.846429999999999, 7.86643, 7.95143, 8.047264, 8.093098, 8.098098, 8.032764, 8.157764, 8.133598000000001, 8.006098, 8.089264, 8.004264, 7.813264, 7.9392640000000005, 7.943264, 7.773929999999999, 7.655596, 7.585596000000001, 7.615596, 7.665596000000001, 7.703096, 7.598096000000001, 7.420596000000001, 7.213096, 7.299762, 7.425476, 7.61881, 7.757144, 7.865478, 7.830477999999999, 7.662978000000001, 7.727978, 7.847978, 7.853812, 7.937978, 8.057978, 8.13274, 8.174074, 8.191074, 8.115074, 8.142574, 8.077574, 8.051573999999999, 8.041074, 8.079407999999999, 8.209408000000002, 8.276907999999999, 8.239408, 8.147742, 7.948741999999999, 7.913742, 7.935408, 7.97398, 7.97398, 7.905646000000001, 7.801312, 7.64298, 7.54798, 7.47998, 7.49748, 7.488480000000001, 7.612765999999999, 7.7877659999999995, 7.776766, 7.675765999999999, 7.707194, 7.743194, 7.750694000000001, 7.73736, 7.734502, 7.771168, 7.863168000000001, 7.970168000000001, 8.067668, 8.007668, 7.808859999999999, 7.593026, 7.557906, 7.5670720000000005, 7.619214, 7.7010000000000005, 7.570999999999999, 7.430166000000001, 7.442944, 7.440444, 7.50311, 7.607238000000001, 7.559618, 7.627618, 7.754451999999999, 7.773896, 7.881038, 7.832037999999999, 7.839538, 7.687872, 7.7637540000000005, 7.740254, 7.7302539999999995, 7.680254, 7.675254, 7.665254, 7.778588, 7.778766000000001, 7.773766, 7.7987660000000005, 7.964766, 8.132796, 8.182796, 8.310796, 8.335796, 8.185796, 8.096844, 8.113510000000002, 8.04851, 7.971844, 7.936416, 7.970416, 8.067916, 8.253601999999999, 8.246459999999999, 8.259794, 8.299222, 8.367222, 8.365222, 8.355222, 8.399388000000002, 8.489388, 8.570340000000002, 8.52284, 8.623198, 8.656912, 8.691912, 8.862366, 8.848054, 8.798054, 8.630388, 8.437332, 8.281618000000002, 8.21019, 8.13869, 8.034911999999998, 7.90377, 7.937104000000001, 7.970437999999999, 8.035991999999998, 7.81011, 7.8911099999999985, 8.062221999999998, 8.063888, 8.147222, 8.314722, 8.218722, 8.185972, 8.010972, 7.920972000000001, 7.710972000000001, 7.560322000000001, 7.485322000000001, 7.4319880000000005, 7.530988000000001, 7.727805999999999, 7.707472, 7.540806, 7.480806, 7.497471999999999, 7.604806000000001, 7.484806, 7.435806000000001, 7.238454000000001, 7.335120000000001, 7.21512, 7.0703580000000015, 6.932358000000002, 6.921024000000001, 6.879358, 6.9038580000000005, 6.836358000000001, 6.659692, 6.574692, 6.447192, 6.434192, 6.399192, 6.269738000000001, 6.357938000000001, 6.472938, 6.682937999999999, 6.8104379999999995, 6.938437999999999, 7.014866, 6.938032, 6.859032, 6.877578, 6.893860000000001, 6.970909999999999, 6.8620220000000005, 6.872021999999999, 6.737022, 6.550909999999999, 6.6109100000000005, 6.557575999999999, 6.517575999999999, 6.398575999999999, 6.382576, 6.392575999999999, 6.422575999999999, 6.4175759999999995, 6.408528000000001, 6.418528, 6.366862000000001, 6.315861999999999, 6.254043999999999, 6.239044, 6.211044000000002, 6.271044, 6.316044, 6.249852, 6.409852, 6.378851999999999, 6.419566, 6.4379, 6.5329, 6.671233999999999, 6.684234, 6.592568, 6.584234, 6.532234000000001, 6.569734, 6.609734, 6.589734000000001, 6.524734, 6.379734000000001, 6.486734000000001, 6.635734000000001, 6.625734, 6.480734, 6.275734, 6.155734, 6.0857339999999995, 5.900734, 5.838234, 5.886568, 5.954522, 5.897906000000001, 5.742521999999999, 5.742522, 5.732522, 5.697522, 5.712522, 5.680856, 5.660856, 5.611689999999999, 5.76269, 5.76669, 5.78169, 5.8050239999999995, 5.846024, 5.8813580000000005, 5.971358, 6.101358, 6.046215999999999, 5.95193, 5.99793, 5.9959299999999995, 5.973071999999999, 6.033072, 5.985930000000001, 5.8309299999999995, 5.8309299999999995, 5.910216, 5.790216, 5.680215999999999, 5.550216000000001, 5.515216000000001, 5.51355, 5.64755, 5.61255, 5.574772, 5.527272, 5.642272, 5.84013, 5.923464, 5.996464, 5.834321999999999, 5.825036, 5.940036, 6.036702, 6.046702, 6.134536, 6.250536, 6.358035999999999, 6.388036, 6.265536, 6.272536, 6.332536, 6.472536, 6.482536, 6.592536, 6.722536, 6.684202, 6.709202, 6.630868, 6.509867999999999, 6.5698680000000005, 6.596868000000001, 6.639534000000001, 6.628534000000001, 6.586534, 6.501534, 6.356534, 6.291676, 6.295961999999999, 6.284962, 6.274962, 6.23782, 6.109820000000001, 6.094820000000001, 6.20982, 6.261962, 6.192962, 6.1879620000000015, 6.297962, 6.282962, 6.262962, 6.249628, 6.115627999999999, 6.120628, 6.118406, 6.122572, 6.002572, 5.809713999999999, 5.744379999999999, 5.559379999999999, 5.548522, 5.509474, 5.50114, 5.57876, 5.73076, 5.804926, 5.783925999999999, 5.714204, 5.647535999999999, 5.647536, 5.632536, 5.602536, 5.482536, 5.482536, 5.392536, 5.262536, 5.302536, 5.271536, 5.263202, 5.248202, 5.120201999999999, 5.078202000000001, 5.022202, 5.08863, 5.121488, 5.126488, 5.256487999999999, 5.442488, 5.412488, 5.387488000000001, 5.390821999999999, 5.462822000000001, 5.4588220000000005, 5.457156, 5.3604899999999995, 5.283348, 5.236235999999999, 5.276236, 5.3412359999999985, 5.341236, 5.341236, 5.341236, 5.479569999999999, 5.70057, 5.7155700000000005, 5.763904, 5.763904, 5.773904, 5.908403999999999, 5.94269, 5.935689999999999, 5.903023999999999, 5.835447999999999, 5.711162000000001, 5.619161999999999, 5.429162, 5.350161999999999, 5.3557179999999995, 5.544051999999999, 5.578337999999999, 5.578337999999999, 5.548338, 5.5383379999999995, 5.550838, 5.53441, 5.49941, 5.47541, 5.479409999999999, 5.467743999999999, 5.487744, 5.5260299999999996, 5.71603, 5.750315999999999, 5.835553999999999, 5.866982, 5.876268, 5.774267999999999, 5.590267999999998, 5.578840000000001, 5.638840000000001, 5.7045960000000004, 5.676596, 5.708596, 5.675262, 5.664786, 5.857286, 5.925064, 5.865064, 5.730518000000001, 5.764364, 5.876863999999999, 6.084864, 6.093198, 6.043864000000001, 6.06315, 6.01815, 6.038150000000001, 6.044816000000001, 5.887316000000001, 5.86103, 5.836030000000001, 5.879808, 5.919605999999999, 5.969606, 5.919606000000001, 5.924606000000001, 5.9743200000000005, 6.033844, 5.894844000000001, 6.024558000000001, 6.089558, 6.129557999999999, 6.155114, 6.137614000000001, 6.165708, 6.276708, 6.388708, 6.544041999999999, 6.564756000000001, 6.524756, 6.475924, 6.307036, 6.314972000000001, 6.126878000000001, 6.077592000000001, 6.053306, 6.013306, 6.0090200000000005, 6.0404480000000005, 6.113781999999999, 6.044692, 6.008692, 5.983692, 6.033691999999999, 6.054834, 5.919, 5.897334000000001, 5.953334000000001, 5.93288, 5.921892000000001, 5.839392, 5.704725999999999, 5.566392, 5.465726000000001, 5.618106, 5.70144, 5.73144, 5.784774, 5.779774, 5.769274, 5.864274, 5.786496, 5.728720000000001, 5.638720000000001, 5.805690000000001, 5.789262000000001, 5.766882000000001, 5.672023999999999, 5.6160239999999995, 5.462023999999999, 5.432024, 5.576467999999998, 5.640911999999999, 5.650912, 5.6475800000000005, 5.579579999999998, 5.4630079999999985, 5.330786, 5.329072, 5.384456, 5.475002000000001, 5.44989, 5.394334, 5.440761999999999, 5.475762, 5.527762, 5.5797620000000006, 5.579047999999999, 5.651356000000001, 5.491356000000001, 5.503856000000001, 5.503856, 5.543856, 5.553856000000001, 5.6198559999999995, 5.771189999999999, 5.9424399999999995, 5.93544, 6.119606, 6.188326, 6.381183999999999, 6.315849999999999, 6.417214, 6.356213999999999, 6.175702, 6.185702, 6.135702, 6.077923999999999, 6.072924, 6.155423999999999, 6.087090000000001, 6.127089999999999, 6.272228, 6.288382, 6.1294119999999985, 6.36084, 6.577506, 6.580933999999998, 6.677599999999999, 6.710675999999999, 6.902614, 6.760392, 6.756392000000001, 6.791391999999999, 7.0027099999999995, 7.0237099999999995, 7.233661999999999, 7.208704, 7.242704, 7.202319999999999, 7.158628, 7.379808000000001, 7.6142959999999995, 7.581796000000001, 7.531795999999999, 7.4988860000000015, 7.668886, 7.773486, 7.661178, 7.749844, 7.870678000000001, 8.085678 ], "color": "#169b89", "complete": [ true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true ], "training": true } ] }, { "title": "Held-out pass@1 (%)", "desc": "250 tasks \u00d7 4 harnesses \u00b7 one graded attempt per cell", "series": [ { "label": "OpenCode", "x": [ 0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000 ], "y": [ 12.8, 19.6, 17.2, 20.8, 20.4, 18.0, 16.8, 18.4, 20.0, 18.8, 20.4 ], "color": "#7552dd", "complete": [ true, true, true, true, true, true, true, true, true, true, true ], "training": false }, { "label": "Claude Code", "x": [ 0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000 ], "y": [ 16.8, 20.4, 24.0, 29.2, 32.4, 26.4, 32.4, 29.6, 32.4, 34.0, 33.2 ], "color": "#169b89", "complete": [ true, true, true, true, true, true, true, true, true, true, true ], "training": false }, { "label": "Codex", "x": [ 0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000 ], "y": [ 15.2, 18.0, 25.6, 16.8, 25.6, 18.8, 18.4, 14.4, 18.0, 14.4, 29.6 ], "color": "#267bc1", "complete": [ true, true, true, true, true, true, true, true, true, true, true ], "training": false }, { "label": "Mini-SWE-Agent", "x": [ 0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000 ], "y": [ 18.8, 20.8, 21.6, 19.6, 27.2, 29.2, 32.8, 30.4, 32.0, 34.0, 36.0 ], "color": "#df8940", "complete": [ true, true, true, true, true, true, true, true, true, true, true ], "training": false }, { "label": "Overall", "x": [ 0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000 ], "y": [ 15.9, 19.7, 22.1, 21.6, 26.4, 23.1, 25.1, 23.2, 25.6, 25.3, 29.8 ], "color": "#222b3a", "complete": [ true, true, true, true, true, true, true, true, true, true, true ], "training": false } ] }, { "title": "Evaluation tool calls", "desc": "Calls per graded rollout \u00b7 lower is better", "series": [ { "label": "OpenCode", "x": [ 0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000 ], "y": [ 11.764, 10.8, 10.104, 10.964, 10.504, 9.896, 9.152, 8.172, 8.084, 7.112, 9.148 ], "color": "#7552dd", "complete": [ true, true, true, true, true, true, true, true, true, true, true ], "training": false }, { "label": "Claude Code", "x": [ 0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000 ], "y": [ 13.444, 12.48, 12.156, 12.104, 11.9, 11.512, 10.624, 10.44, 10.528, 10.08, 10.528 ], "color": "#169b89", "complete": [ true, true, true, true, true, true, true, true, true, true, true ], "training": false }, { "label": "Codex", "x": [ 0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000 ], "y": [ 15.444, 15.22, 13.576, 15.164, 14.968, 14.58, 14.364, 15.244, 14.496, 15.088, 14.46 ], "color": "#267bc1", "complete": [ true, true, true, true, true, true, true, true, true, true, true ], "training": false }, { "label": "Mini-SWE-Agent", "x": [ 0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000 ], "y": [ 15.62, 15.172, 14.708, 15.244, 14.892, 14.256, 13.14, 13.608, 13.776, 13.84, 13.736 ], "color": "#df8940", "complete": [ true, true, true, true, true, true, true, true, true, true, true ], "training": false } ] } ] }