% Generated by docs/results/make_tables.py from runs/ (read-only). Needs only \usepackage{booktabs}. \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Runs used and their recorded configuration (run\_config.json).} \label{tab:runs} \begin{tabular}{llcccclccc} \toprule Run & Config & Profile & Seed & Split & Commit & Tree & Degraded & Run status & vs first run \\ \midrule 20261103T174646Z\_C5\_experiment & C5 & experiment & 2026 & test & ad70ec2 & clean & False & running & same \\ 20261103T212359Z\_C5\_experiment\_resume & C5 & experiment & 2026 & test & ad70ec2 & dirty (plan file only) & False & budget\_stop & same \\ 20261103T225140Z\_C5\_experiment\_resume2 & C5 & experiment & 2026 & test & 93bc701 & dirty (plan file only) & False & finished & same \\ 20261103T235556Z\_C4\_experiment & C4 & experiment & 2026 & test & ea4db4b & dirty (plan file only) & False & finished & same \\ 20261104T194056Z\_C3\_experiment & C3 & experiment & 2026 & test & 6c61d69 & clean & False & finished & same \\ 20261104T232056Z\_C2\_experiment\_check & C2 & experiment & 2026 & test & 6c61d69 & clean & False & finished & same \\ 20261105T131443Z\_C1\_experiment & C1 & experiment & 2026 & test & 5ed7507 & clean & False & running & same \\ 20261107T200233Z\_C1\_experiment\_resume & C1 & experiment & 2026 & test & 2c8eae5 & dirty (plan file only) & False & finished & same \\ \bottomrule \end{tabular} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Integrity checks of the analysed rows.} \label{tab:checks} \begin{tabular}{lclccc} \toprule Config & Rows & Status counts & lean\_down & split/config mismatch & area/difficulty mismatch \\ \midrule C1 & 30 & budget\_exhausted: 7, failed: 3, solved: 20 & 0 & 0 & 0 \\ C2 & 30 & claimed\_solved: 30 & 0 & 0 & 0 \\ C3 & 30 & budget\_exhausted: 3, failed: 7, solved: 20 & 0 & 0 & 0 \\ C4 & 30 & failed: 16, solved: 14 & 0 & 0 & 0 \\ C5 & 30 & failed: 6, solved: 24 & 0 & 0 & 0 \\ \bottomrule \end{tabular} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Verified success rate SR with Wilson 95 \% intervals (C1, C3--C5; pass@1 = SR, one run per problem), and the claimed rate and FPR$_{\mathrm{B}}$ (Eq. 16) of Baseline B (C2).} \label{tab:sr} \begin{tabular}{lccccc} \toprule Config & Solved & SR (\%) & Wilson 95 \% CI (\%) & C2 claimed rate (\%) {[}Wilson{]} & FPR$_{\mathrm{B}}$ (\%) {[}Wilson{]} \\ \midrule C1 & 20/30 & 66.7 & {[}48.8, 80.8{]} & --- & --- \\ C3 & 20/30 & 66.7 & {[}48.8, 80.8{]} & --- & --- \\ C4 & 14/30 & 46.7 & {[}30.2, 63.9{]} & --- & --- \\ C5 & 24/30 & 80.0 & {[}62.7, 90.5{]} & --- & --- \\ C2 (Baseline B) & --- & --- & --- & 30/30 = 100.0 {[}88.6, 100.0{]} & 11/30 = 36.7 {[}21.9, 54.5{]} \\ \bottomrule \end{tabular} \vspace{2pt} \parbox{\linewidth}{\scriptsize C2 is not verified, so it has no SR; it enters only through its claimed rate $B^{+}$/n and FPR$_{\mathrm{B}}$ (plan §7.1). Of the 30 claimed proofs the formal check verified 19 and did not verify 11; 0 left out of FPR$_{\mathrm{B}}$.} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Baseline B: claimed proofs and their formal check. FPR$_{\mathrm{B}}$ is an upper bound on the true false-positive rate.} \label{tab:fprb} \begin{tabular}{ll} \toprule Quantity & Value \\ \midrule Problems & 30 \\ Claimed ($B^{+}$) & 30 \\ Not claimed & 0 \\ Check status among $B^{+}$ & not\_verified: 11, verified: 19 \\ Verified (Acc on the assembled theorem) & 19 \\ Not verified (budget\_exhausted included) & 11 \\ Left out of FPR$_{\mathrm{B}}$ (provider\_error, budget\_stop, error, lean\_down) & 0 \\ FPR$_{\mathrm{B}}$ = not verified / (verified + not verified) & 11/30 = 36.7 \% {[}21.9, 54.5{]} \\ $B^{+}$ problems for the manual review (§4.2) & 10 drawn with random.Random(2026).sample(sorted ids, 10) (listed below) \\ \bottomrule \end{tabular} \vspace{2pt} \parbox{\linewidth}{\scriptsize Manual review set ($B^{+}$ not verified): amc12b\_2002\_p3, imo\_1964\_p1\_2, imo\_1965\_p2, imo\_1978\_p5, imo\_1979\_p1, imo\_1984\_p6, imo\_1987\_p6, imo\_1993\_p5, putnam\_1965\_b2, putnam\_1995\_b1.} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Planned comparisons: two-sided exact McNemar test on outcomes paired by problem, Bonferroni-corrected for 3 tests (reject at p $<$ 0.0167; adjusted p = min(1, 3p)). b: first configuration solved and second not; c: the reverse. $\Delta$SR = SR(first) $-$ SR(second), percentage points, with a paired bootstrap 95 \% CI (10 000 resamples of problems, seed 2026, percentile method).} \label{tab:mcnemar} \begin{tabular}{lcccccccc} \toprule ID & Comparison & b & c & $\Delta$SR (pp) & Paired bootstrap 95 \% CI (pp) & Exact p & Bonferroni p & Reject at 0.05/3 \\ \midrule H1 & C5 vs C1 & 4 & 0 & +13.3 & {[}+3.3, +26.7{]} & 0.1250 & 0.3750 & no \\ H2 & C5 vs C3 & 5 & 1 & +13.3 & {[}+0.0, +30.0{]} & 0.2188 & 0.6562 & no \\ H3 & C5 vs C4 & 10 & 0 & +33.3 & {[}+16.7, +50.0{]} & 0.0020 & 0.0059 & yes \\ \bottomrule \end{tabular} \vspace{2pt} \parbox{\linewidth}{\scriptsize n = 30, so power is low: a non-significant result is no evidence of a difference, not evidence of no difference. H2 is $\Delta_{\mathrm{RAG}}$ = SR$_{\mathrm{C5}}$ $-$ SR$_{\mathrm{C3}}$; H3 is the Critic contribution SR$_{\mathrm{C5}}$ $-$ SR$_{\mathrm{C4}}$.} \end{table*} \begin{table}[t] \centering \footnotesize \setlength{\tabcolsep}{3pt} \caption{Exploratory comparisons listed in plan §3: exact McNemar, no correction, no claims.} \label{tab:mcnemar-expl} \begin{tabular}{lccccc} \toprule ID & Comparison & b & c & $\Delta$SR (pp) & Exact p (uncorrected) \\ \midrule E1 & C3 vs C4 & 9 & 3 & +20.0 & 0.1460 \\ E2 & C1 vs C3 & 3 & 3 & +0.0 & 1.0000 \\ E3 & C1 vs C4 & 8 & 2 & +20.0 & 0.1094 \\ \bottomrule \end{tabular} \end{table} \begin{table}[t] \centering \footnotesize \setlength{\tabcolsep}{3pt} \caption{Per-cell solved/n (descriptive: no tests, no intervals per cell). C2 counts claimed (self-assessed) solutions, not verified ones; --- marks an empty cell.} \label{tab:cells} \begin{tabular}{llcccccc} \toprule Difficulty & Area & n & C1 & C3 & C4 & C5 & C2 (claimed) \\ \midrule easy & Algebra & 5 & 5/5 & 5/5 & 4/5 & 5/5 & 5/5 \\ easy & Number theory & 5 & 5/5 & 4/5 & 5/5 & 5/5 & 5/5 \\ easy & Combinatorics & 0 & --- & --- & --- & --- & --- \\ medium & Algebra & 5 & 4/5 & 5/5 & 1/5 & 5/5 & 5/5 \\ medium & Number theory & 4 & 2/4 & 2/4 & 2/4 & 4/4 & 4/4 \\ medium & Combinatorics & 1 & 0/1 & 0/1 & 1/1 & 1/1 & 1/1 \\ hard & Algebra & 3 & 1/3 & 2/3 & 0/3 & 1/3 & 3/3 \\ hard & Number theory & 4 & 1/4 & 1/4 & 0/4 & 1/4 & 4/4 \\ hard & Combinatorics & 3 & 2/3 & 1/3 & 1/3 & 2/3 & 3/3 \\ \bottomrule \end{tabular} \end{table} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Margins of the grid by difficulty and by area: solved/n, then SR \% {[}Wilson 95 \% CI{]} (descriptive; no tests). C2: claimed solutions.} \label{tab:margins} \begin{tabular}{lcccccc} \toprule Config & easy (n = 10) & medium (n = 10) & hard (n = 10) & Algebra (n = 13) & Number theory (n = 13) & Combinatorics (n = 4) \\ \midrule C1 & 10/10 & 6/10 & 4/10 & 10/13 & 8/13 & 2/4 \\ & 100.0 {[}72.2, 100.0{]} & 60.0 {[}31.3, 83.2{]} & 40.0 {[}16.8, 68.7{]} & 76.9 {[}49.7, 91.8{]} & 61.5 {[}35.5, 82.3{]} & 50.0 {[}15.0, 85.0{]} \\ C3 & 9/10 & 7/10 & 4/10 & 12/13 & 7/13 & 1/4 \\ & 90.0 {[}59.6, 98.2{]} & 70.0 {[}39.7, 89.2{]} & 40.0 {[}16.8, 68.7{]} & 92.3 {[}66.7, 98.6{]} & 53.8 {[}29.1, 76.8{]} & 25.0 {[}4.6, 69.9{]} \\ C4 & 9/10 & 4/10 & 1/10 & 5/13 & 7/13 & 2/4 \\ & 90.0 {[}59.6, 98.2{]} & 40.0 {[}16.8, 68.7{]} & 10.0 {[}1.8, 40.4{]} & 38.5 {[}17.7, 64.5{]} & 53.8 {[}29.1, 76.8{]} & 50.0 {[}15.0, 85.0{]} \\ C5 & 10/10 & 10/10 & 4/10 & 11/13 & 10/13 & 3/4 \\ & 100.0 {[}72.2, 100.0{]} & 100.0 {[}72.2, 100.0{]} & 40.0 {[}16.8, 68.7{]} & 84.6 {[}57.8, 95.7{]} & 76.9 {[}49.7, 91.8{]} & 75.0 {[}30.1, 95.4{]} \\ C2 (claimed) & 10/10 & 10/10 & 10/10 & 13/13 & 13/13 & 4/4 \\ & 100.0 {[}72.2, 100.0{]} & 100.0 {[}72.2, 100.0{]} & 100.0 {[}72.2, 100.0{]} & 100.0 {[}77.2, 100.0{]} & 100.0 {[}77.2, 100.0{]} & 100.0 {[}51.0, 100.0{]} \\ \bottomrule \end{tabular} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Efficiency per solved problem: sum over all 30 problems (failed ones included) divided by the number solved, with 95 \% bootstrap CIs (10 000 resamples of problems, seed 2026, percentile). Nominal cost (cache hits priced as fresh calls) is the compared figure (plan §1); real cost is from the kept rows only.} \label{tab:eff-solved} \begin{tabular}{lcccccc} \toprule Config & Solved & Calls / solved & k tokens / solved & Nominal USD / solved & Real USD / solved & Wall min / solved \\ \midrule C1 & 20 & 12.85 {[}7.19, 22.60{]} & 219.7 {[}114.6, 394.7{]} & 0.1285 {[}0.0668, 0.2313{]} & 0.1235 {[}0.0643, 0.2218{]} & 17.7 {[}9.2, 32.0{]} \\ C2 (per claimed) & 30 & 2.17 {[}2.00, 2.40{]} & 5.3 {[}2.8, 8.6{]} & 0.0027 {[}0.0013, 0.0047{]} & 0.0027 {[}0.0013, 0.0047{]} & 0.4 {[}0.2, 0.7{]} \\ C3 & 20 & 26.50 {[}16.15, 43.67{]} & 150.4 {[}79.3, 274.6{]} & 0.0675 {[}0.0335, 0.1279{]} & 0.0675 {[}0.0335, 0.1278{]} & 9.2 {[}4.6, 17.3{]} \\ C4 & 14 & 9.43 {[}5.72, 16.67{]} & 70.7 {[}38.2, 137.8{]} & 0.0278 {[}0.0135, 0.0571{]} & 0.0000 {[}0.0000, 0.0000{]} & 0.2 {[}0.1, 0.3{]} \\ C5 & 24 & 16.92 {[}10.63, 26.21{]} & 148.0 {[}74.4, 266.4{]} & 0.0566 {[}0.0245, 0.1097{]} & 0.0490 {[}0.0236, 0.0897{]} & 7.6 {[}3.3, 14.6{]} \\ \bottomrule \end{tabular} \vspace{2pt} \parbox{\linewidth}{\scriptsize C2 has no verified solutions: its ratio uses claimed solutions and excludes the claim check (calls, tokens, cost and wall time up to claim\_check\_start).} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Mean per problem with 95 \% bootstrap CIs (C2 excludes the claim check).} \label{tab:eff-problem} \begin{tabular}{lccccc} \toprule Config & Calls / problem & k tokens / problem & Nominal USD / problem & Real USD / problem & Wall min / problem \\ \midrule C1 & 8.57 {[}5.83, 11.33{]} & 146.4 {[}91.1, 203.5{]} & 0.0857 {[}0.0532, 0.1192{]} & 0.0823 {[}0.0505, 0.1152{]} & 11.8 {[}7.2, 16.5{]} \\ C2 & 2.17 {[}2.00, 2.40{]} & 5.3 {[}2.8, 8.6{]} & 0.0027 {[}0.0013, 0.0047{]} & 0.0027 {[}0.0013, 0.0047{]} & 0.4 {[}0.2, 0.7{]} \\ C3 & 17.67 {[}12.97, 22.37{]} & 100.3 {[}63.8, 140.8{]} & 0.0450 {[}0.0269, 0.0658{]} & 0.0450 {[}0.0269, 0.0658{]} & 6.1 {[}3.7, 9.0{]} \\ C4 & 4.40 {[}3.37, 5.43{]} & 33.0 {[}22.8, 44.4{]} & 0.0130 {[}0.0081, 0.0186{]} & 0.0000 {[}0.0000, 0.0000{]} & 0.1 {[}0.1, 0.1{]} \\ C5 & 13.53 {[}9.57, 17.60{]} & 118.4 {[}67.9, 178.7{]} & 0.0453 {[}0.0225, 0.0737{]} & 0.0392 {[}0.0218, 0.0588{]} & 6.1 {[}3.0, 9.7{]} \\ \bottomrule \end{tabular} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Totals over the 30 kept rows. The ledger column sums every paid call of the configuration runs, including attempts discarded by the merge (Deviations) and, for C2, the claim check.} \label{tab:totals} \begin{tabular}{lccccccc} \toprule Config & LLM calls & Cached calls & Cached (\%) & Tokens (M) & Nominal USD (rows) & Real USD (rows) & Real USD (ledger, all runs) \\ \midrule C1 & 257 & 47 & 18.3 & 4.393 & 2.5700 & 2.4695 & 2.5679 \\ C2 & 65 & 0 & 0.0 & 0.158 & 0.0814 & 0.0814 & 0.7083 \\ C3 & 530 & 18 & 3.4 & 3.009 & 1.3508 & 1.3500 & 1.3500 \\ C4 & 132 & 132 & 100.0 & 0.990 & 0.3887 & 0.0000 & 0.0000 \\ C5 & 406 & 34 & 8.4 & 3.551 & 1.3590 & 1.1770 & 1.3585 \\ \bottomrule \end{tabular} \vspace{2pt} \parbox{\linewidth}{\scriptsize C2 claim check (not in the C2 figures of the other columns): 77 calls (11 cached), 0.6274 nominal, 0.6269 real.} \end{table*} % RNF-3: C5 calls per solved = 406/24 = 16.92 (met) \begin{table}[t] \centering \footnotesize \setlength{\tabcolsep}{3pt} \caption{Median and interquartile range of LLM calls and wall time per problem (supplementary; not in the plan).} \label{tab:medians} \begin{tabular}{lcc} \toprule Config & Calls, median {[}Q1, Q3{]} & Wall min, median {[}Q1, Q3{]} \\ \midrule C1 & 4.5 {[}1.0, 18.0{]} & 2.7 {[}0.6, 24.2{]} \\ C2 & 2.0 {[}2.0, 2.0{]} & 0.1 {[}0.1, 0.3{]} \\ C3 & 16.5 {[}3.2, 28.0{]} & 3.7 {[}0.6, 6.7{]} \\ C4 & 5.0 {[}1.0, 5.8{]} & 0.1 {[}0.0, 0.1{]} \\ C5 & 11.5 {[}1.0, 23.8{]} & 2.4 {[}0.2, 5.2{]} \\ \bottomrule \end{tabular} \end{table} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Wall-clock time per problem by difficulty in minutes, mean / median {[}Q1, Q3{]} (descriptive; not in the plan). C2 excludes the claim check (time to claim\_check\_start in trace.jsonl).} \label{tab:wall-diff} \begin{tabular}{lccc} \toprule Config & easy (n = 10): mean / median {[}Q1, Q3{]} & medium (n = 10): mean / median {[}Q1, Q3{]} & hard (n = 10): mean / median {[}Q1, Q3{]} \\ \midrule C1 & 0.7 / 0.4 {[}0.1, 1.1{]} & 14.3 / 16.2 {[}1.8, 23.2{]} & 20.3 / 25.3 {[}7.0, 32.4{]} \\ C2 & 0.1 / 0.1 {[}0.1, 0.2{]} & 0.1 / 0.1 {[}0.1, 0.2{]} & 1.0 / 0.5 {[}0.2, 1.1{]} \\ C3 & 0.8 / 0.1 {[}0.0, 1.0{]} & 4.0 / 2.6 {[}1.2, 6.0{]} & 13.6 / 12.3 {[}6.6, 19.8{]} \\ C4 & 0.1 / 0.0 {[}0.0, 0.1{]} & 0.1 / 0.1 {[}0.0, 0.1{]} & 0.1 / 0.1 {[}0.1, 0.1{]} \\ C5 & 0.4 / 0.0 {[}0.0, 0.5{]} & 2.7 / 2.6 {[}0.7, 4.5{]} & 15.1 / 14.1 {[}4.0, 21.0{]} \\ \bottomrule \end{tabular} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Refinement robustness. A C5 problem counts as solved at k if every subgoal of its accepted plan used at most k attempts (subgoal\_solved events in trace.jsonl). $\hat p$ is the first-attempt success rate of the LLM-attempted subgoals of C5 (all plans, all problems).} \label{tab:srk} \begin{tabular}{lccccc} \toprule k & SR$_{\mathrm{C5}}$(k) {[}Wilson{]} (\%) & $\hat\Delta$(k) = SR(k) $-$ SR(k$-$1) (pp) & Indep. $\Delta$(k) = $\hat p$(1$-\hat p$)$^{k-1}$ (\%) & Subgoals solved at attempt k & Solved at k / reached k \\ \midrule 1 & 15/30 = 50.0 {[}33.2, 66.8{]} & +20.0 & 40.9 & 18/44 = 40.9 & 18/44 = 40.9 \\ 2 & 19/30 = 63.3 {[}45.5, 78.1{]} & +13.3 & 24.2 & 8/44 = 18.2 & 8/26 = 30.8 \\ 3 & 24/30 = 80.0 {[}62.7, 90.5{]} & +16.7 & 14.3 & 5/44 = 11.4 & 5/18 = 27.8 \\ \bottomrule \end{tabular} \vspace{2pt} \parbox{\linewidth}{\scriptsize SR$_{\mathrm{C5}}$(0) = 9/30 (accepted plan with no LLM subgoal attempt). $\hat p$ = 18/44 = 40.9 \%. k = 1 from C4 (no retries, no Critic advice): 14/30 = 46.7 \% {[}30.2, 63.9{]}. The last two columns are subgoal-level diagnostics (supplementary).} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Error taxonomy: count (relative frequency, \%) of each $\kappa$ class of Eq. 11 over all failed attempts in the kept rows.} \label{tab:kappa} \begin{tabular}{lccccccc} \toprule Config & Failed attempts & syntax & missing lemma & tactic failure & misformalization & timeout & planning \\ \midrule C1 & 186 & 60 (32.3) & 50 (26.9) & 68 (36.6) & 0 (0.0) & 8 (4.3) & 0 (0.0) \\ C2 (claim check) & 45 & 14 (31.1) & 8 (17.8) & 22 (48.9) & 0 (0.0) & 1 (2.2) & 0 (0.0) \\ C3 & 82 & 20 (24.4) & 17 (20.7) & 42 (51.2) & 0 (0.0) & 2 (2.4) & 1 (1.2) \\ C4 & 16 & 5 (31.2) & 4 (25.0) & 7 (43.8) & 0 (0.0) & 0 (0.0) & 0 (0.0) \\ C5 & 57 & 19 (33.3) & 18 (31.6) & 18 (31.6) & 0 (0.0) & 2 (3.5) & 0 (0.0) \\ \bottomrule \end{tabular} \vspace{2pt} \parbox{\linewidth}{\scriptsize C3--C5: classes recorded by the Critic. C1: attempts classified offline with policy.classify (first 3 Lean messages, 300 characters each, as stored; no k\_max escalation). C2 has no verified attempts of its own: its row is the claim-check loop.} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Error taxonomy per configuration and area: count (relative frequency within the row, \%).} \label{tab:kappa-area} \begin{tabular}{lcccccccc} \toprule Config & Area & Failed attempts & syntax & missing lemma & tactic failure & misformalization & timeout & planning \\ \midrule C1 & Algebra & 77 & 14 (18.2) & 40 (51.9) & 22 (28.6) & 0 (0.0) & 1 (1.3) & 0 (0.0) \\ C1 & Number theory & 83 & 27 (32.5) & 3 (3.6) & 46 (55.4) & 0 (0.0) & 7 (8.4) & 0 (0.0) \\ C1 & Combinatorics & 26 & 19 (73.1) & 7 (26.9) & 0 (0.0) & 0 (0.0) & 0 (0.0) & 0 (0.0) \\ \midrule C2 (claim check) & Algebra & 14 & 5 (35.7) & 4 (28.6) & 5 (35.7) & 0 (0.0) & 0 (0.0) & 0 (0.0) \\ C2 (claim check) & Number theory & 23 & 5 (21.7) & 4 (17.4) & 13 (56.5) & 0 (0.0) & 1 (4.3) & 0 (0.0) \\ C2 (claim check) & Combinatorics & 8 & 4 (50.0) & 0 (0.0) & 4 (50.0) & 0 (0.0) & 0 (0.0) & 0 (0.0) \\ \midrule C3 & Algebra & 18 & 3 (16.7) & 9 (50.0) & 5 (27.8) & 0 (0.0) & 0 (0.0) & 1 (5.6) \\ C3 & Number theory & 43 & 13 (30.2) & 3 (7.0) & 26 (60.5) & 0 (0.0) & 1 (2.3) & 0 (0.0) \\ C3 & Combinatorics & 21 & 4 (19.0) & 5 (23.8) & 11 (52.4) & 0 (0.0) & 1 (4.8) & 0 (0.0) \\ \midrule C4 & Algebra & 8 & 1 (12.5) & 3 (37.5) & 4 (50.0) & 0 (0.0) & 0 (0.0) & 0 (0.0) \\ C4 & Number theory & 6 & 2 (33.3) & 1 (16.7) & 3 (50.0) & 0 (0.0) & 0 (0.0) & 0 (0.0) \\ C4 & Combinatorics & 2 & 2 (100.0) & 0 (0.0) & 0 (0.0) & 0 (0.0) & 0 (0.0) & 0 (0.0) \\ \midrule C5 & Algebra & 25 & 7 (28.0) & 8 (32.0) & 10 (40.0) & 0 (0.0) & 0 (0.0) & 0 (0.0) \\ C5 & Number theory & 24 & 8 (33.3) & 9 (37.5) & 6 (25.0) & 0 (0.0) & 1 (4.2) & 0 (0.0) \\ C5 & Combinatorics & 8 & 4 (50.0) & 1 (12.5) & 2 (25.0) & 0 (0.0) & 1 (12.5) & 0 (0.0) \\ \bottomrule \end{tabular} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Final status and stage of the unsolved problems (C2: claim and check outcome of all problems). Stages verify:sgN, formalize:sgN, ... are grouped by prefix.} \label{tab:final-fail} \begin{tabular}{lcp{0.7\linewidth}} \toprule Config & Problems & status @ stage: count \\ \midrule C1 & 10 & budget\_exhausted: 7; failed: 3 \\ C2 & 30 & claimed, check verified: 19; claimed, check not\_verified: 11 \\ C3 & 10 & failed @ verify: 5; budget\_exhausted @ formalize: 3; failed @ formalize: 2 \\ C4 & 16 & failed @ verify: 11; failed @ formalize: 5 \\ C5 & 6 & failed @ verify: 4; failed @ formalize: 2 \\ \bottomrule \end{tabular} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Share of subgoals closed by the deterministic tactic pre-pass vs the LLM (solved\_by in trace.jsonl): a possible confound shared by C3--C5 (supplementary; not in the plan).} \label{tab:prepass} \begin{tabular}{lccccc} \toprule Config & Pre-pass (accepted plans) & LLM (accepted plans) & Pre-pass (all subgoal\_solved) & Solved with no LLM subgoal & Solved \\ \midrule C3 & 7/32 = 21.9 & 25/32 = 78.1 & 8/39 = 20.5 & 8 & 20 \\ C4 & 9/18 = 50.0 & 9/18 = 50.0 & 11/21 = 52.4 & 9 & 14 \\ C5 & 11/39 = 28.2 & 28/39 = 71.8 & 11/42 = 26.2 & 9 & 24 \\ \bottomrule \end{tabular} \end{table*} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Counts of provider\_error, budget\_stop, error-type statuses and lean\_down rows in the analysed rows (degraded runs: see the runs table).} \label{tab:infra} \begin{tabular}{lcccccc} \toprule Config & provider\_error & budget\_stop & error & node\_error & invalid\_statement & lean\_down \\ \midrule C1 & 0 & 0 & 0 & 0 & 0 & 0 \\ C2 & 0 & 0 & 0 & 0 & 0 & 0 \\ C3 & 0 & 0 & 0 & 0 & 0 & 0 \\ C4 & 0 & 0 & 0 & 0 & 0 & 0 \\ C5 & 0 & 0 & 0 & 0 & 0 & 0 \\ \bottomrule \end{tabular} \end{table*} \begin{table}[t] \centering \footnotesize \setlength{\tabcolsep}{3pt} \caption{Problems solved on test by some configuration that carry a note in data/curation/fidelity\_review.yaml, and the team's manual review (post\_test\_review in that file). The verified SR is unchanged by the review.} \label{tab:fidelity} \begin{tabular}{llp{0.5\linewidth}} \toprule Problem & Solved by & Manual review \\ \midrule amc12b\_2003\_p6 & C1, C3, C4, C5 & 2026-11-07: counts as solved; report note: The formalization proves that u 0 = +-2/sqrt 3 (a necessary condition), not that -2*sqrt(3)/3 is attained; this is enough to rule out the other options. \\ mathd\_algebra\_234 & C1, C3, C4, C5 & 2026-11-07: counts as solved; no note \\ mathd\_algebra\_455 & C1, C3, C4, C5 & 2026-11-07: counts as solved; no note \\ putnam\_1965\_b2 & C1, C3, C5 & 2026-11-07: counts as solved; no note \\ \bottomrule \end{tabular} \end{table} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Per-problem outcomes on test (1 = verified solved).} \label{tab:per-problem} \begin{tabular}{lllccccl} \toprule Problem & Difficulty & Area & C1 & C3 & C4 & C5 & C2 (claim, check) \\ \midrule algebra\_absapbon1pabsapbleqsumabsaon1pabsa & medium & Algebra & 0 & 1 & 0 & 1 & claimed, verified \\ amc12\_2000\_p20 & medium & Algebra & 1 & 1 & 0 & 1 & claimed, verified \\ amc12\_2001\_p9 & medium & Algebra & 1 & 1 & 0 & 1 & claimed, verified \\ amc12a\_2008\_p15 & medium & Number theory & 0 & 0 & 0 & 1 & claimed, not\_verified \\ amc12a\_2008\_p25 & medium & Algebra & 1 & 1 & 0 & 1 & claimed, verified \\ amc12b\_2002\_p3 & medium & Number theory & 0 & 1 & 1 & 1 & claimed, not\_verified \\ amc12b\_2003\_p6 & medium & Algebra & 1 & 1 & 1 & 1 & claimed, verified \\ imo\_1964\_p1\_2 & hard & Number theory & 1 & 1 & 0 & 1 & claimed, not\_verified \\ imo\_1965\_p2 & hard & Algebra & 1 & 1 & 0 & 1 & claimed, not\_verified \\ imo\_1978\_p5 & hard & Algebra & 0 & 0 & 0 & 0 & claimed, not\_verified \\ imo\_1979\_p1 & hard & Number theory & 0 & 0 & 0 & 0 & claimed, not\_verified \\ imo\_1984\_p6 & hard & Number theory & 0 & 0 & 0 & 0 & claimed, not\_verified \\ imo\_1987\_p6 & hard & Number theory & 0 & 0 & 0 & 0 & claimed, not\_verified \\ imo\_1993\_p5 & hard & Algebra & 0 & 1 & 0 & 0 & claimed, not\_verified \\ mathd\_algebra\_160 & easy & Algebra & 1 & 1 & 1 & 1 & claimed, verified \\ mathd\_algebra\_206 & easy & Algebra & 1 & 1 & 0 & 1 & claimed, verified \\ mathd\_algebra\_234 & easy & Algebra & 1 & 1 & 1 & 1 & claimed, verified \\ mathd\_algebra\_288 & easy & Algebra & 1 & 1 & 1 & 1 & claimed, verified \\ mathd\_algebra\_455 & easy & Algebra & 1 & 1 & 1 & 1 & claimed, verified \\ mathd\_numbertheory\_321 & easy & Number theory & 1 & 0 & 1 & 1 & claimed, verified \\ mathd\_numbertheory\_328 & easy & Number theory & 1 & 1 & 1 & 1 & claimed, verified \\ mathd\_numbertheory\_345 & easy & Number theory & 1 & 1 & 1 & 1 & claimed, verified \\ mathd\_numbertheory\_430 & easy & Number theory & 1 & 1 & 1 & 1 & claimed, verified \\ mathd\_numbertheory\_64 & easy & Number theory & 1 & 1 & 1 & 1 & claimed, verified \\ numbertheory\_3pow2pownm1mod2pownp3eq2pownp2 & medium & Number theory & 1 & 0 & 0 & 1 & claimed, verified \\ numbertheory\_notequiv2i2jasqbsqdiv8 & medium & Number theory & 1 & 1 & 1 & 1 & claimed, verified \\ numbertheory\_sumkmulnckeqnmul2pownm1 & medium & Combinatorics & 0 & 0 & 1 & 1 & claimed, verified \\ putnam\_1964\_b2 & hard & Combinatorics & 1 & 0 & 1 & 1 & claimed, verified \\ putnam\_1965\_b2 & hard & Combinatorics & 1 & 1 & 0 & 1 & claimed, not\_verified \\ putnam\_1995\_b1 & hard & Combinatorics & 0 & 0 & 0 & 0 & claimed, not\_verified \\ \bottomrule \end{tabular} \end{table*} \begin{table}[t] \centering \footnotesize \setlength{\tabcolsep}{3pt} \caption{Hard test problems per verified configuration (supplementary; post hoc, descriptive, not in the analysis plan).} \label{tab:hard} \begin{tabular}{llccccc} \toprule Problem & Area & C1 & C3 & C4 & C5 & Solved by \\ \midrule imo\_1964\_p1\_2 & Number theory & 1 & 1 & 0 & 1 & 3 \\ imo\_1965\_p2 & Algebra & 1 & 1 & 0 & 1 & 3 \\ imo\_1978\_p5 & Algebra & 0 & 0 & 0 & 0 & 0 \\ imo\_1979\_p1 & Number theory & 0 & 0 & 0 & 0 & 0 \\ imo\_1984\_p6 & Number theory & 0 & 0 & 0 & 0 & 0 \\ imo\_1987\_p6 & Number theory & 0 & 0 & 0 & 0 & 0 \\ imo\_1993\_p5 & Algebra & 0 & 1 & 0 & 0 & 1 \\ putnam\_1964\_b2 & Combinatorics & 1 & 0 & 1 & 1 & 3 \\ putnam\_1965\_b2 & Combinatorics & 1 & 1 & 0 & 1 & 3 \\ putnam\_1995\_b1 & Combinatorics & 0 & 0 & 0 & 0 & 0 \\ \midrule Solved & & 4/10 & 4/10 & 1/10 & 4/10 & \\ \bottomrule \end{tabular} \vspace{2pt} \parbox{\linewidth}{\scriptsize Post hoc, descriptive (not in the analysis plan): no tests, no intervals. 1 = verified solved. C1 and C5 solve the same hard problems (4 and 4); 5 hard problems are solved by no configuration (imo\_1978\_p5, imo\_1979\_p1, imo\_1984\_p6, imo\_1987\_p6, putnam\_1995\_b1); C3 and C5 solve 4 and 4 hard problems, but only C3 solves imo\_1993\_p5 and only C5 solves putnam\_1964\_b2.} \end{table} \begin{table*}[t] \centering \scriptsize \setlength{\tabcolsep}{3pt} \caption{Exploratory, post hoc: C5 with DeepSeek-Flash vs DeepSeek-V4-Pro on the hard test problems (descriptive).} \label{tab:pro-hard} \begin{tabular}{lcccclcccc} \toprule Problem & Flash solved & Flash calls & Flash USD & Flash min & Pro outcome & Pro solved & Pro calls & Pro USD & Pro min \\ \midrule imo\_1964\_p1\_2 & 1 & 15 & 0.0261 & 3.4 & not attempted & --- & --- & --- & --- \\ imo\_1965\_p2 & 1 & 14 & 0.0312 & 3.7 & failed & 0 & 30 & 0.3608 & 33.6 \\ imo\_1978\_p5 & 0 & 31 & 0.1347 & 39.6 & provider\_error (connection drop, not a system failure) & --- & 30 & 0.2938 & 40.7 \\ imo\_1979\_p1 & 0 & 28 & 0.1463 & 19.7 & failed & 0 & 24 & 0.2722 & 29.3 \\ imo\_1984\_p6 & 0 & 32 & 0.3366 & 27.6 & interrupted & --- & --- & --- & --- \\ imo\_1987\_p6 & 0 & 36 & 0.1793 & 21.4 & budget\_exhausted & 0 & 40 & 0.4607 & 57.4 \\ imo\_1993\_p5 & 0 & 27 & 0.0790 & 10.6 & failed & 0 & 24 & 0.3807 & 53.6 \\ putnam\_1964\_b2 & 1 & 9 & 0.0160 & 2.3 & not attempted & --- & --- & --- & --- \\ putnam\_1965\_b2 & 1 & 17 & 0.0409 & 4.9 & not attempted & --- & --- & --- & --- \\ putnam\_1995\_b1 & 0 & 25 & 0.1268 & 17.5 & not attempted & --- & --- & --- & --- \\ \midrule Both finished (4) & 1/4 & 105 & 0.4357 & 55.5 & & 0/4 & 118 & 1.4744 & 173.9 \\ All hard (10) & 4/10 & 234 & 1.1169 & 150.9 & & --- & --- & --- & --- \\ \bottomrule \end{tabular} \vspace{2pt} \parbox{\linewidth}{\scriptsize Post hoc, exploratory (decided 2026-11-08, after all confirmatory runs): descriptive only, no tests, no intervals; it changes no main result. C5-Flash = the kept C5 rows of the main test result; C5-Pro = run 20261108T130803Z\_C5\_experiment\_pro\_pro\_hard (profile experiment\_pro: deepseek-v4-pro in every role, everything else as frozen), stopped by the team and not resumed. Cost is nominal USD (cache hits priced as fresh calls). provider\_error, interrupted and not attempted do not count as failures of Pro and are left out of its solved count; only problems both runs finished are compared. imo\_1984\_p6 was interrupted after 4 paid calls (0.1725 USD real, 18.1 min) and left no row. Real cost of the Pro run: 1.9407 USD (151 paid calls, all in the ledger); project ledger total: 9.3003 USD.} \end{table*}