formal-mas / docs /results /check_explained.py
NicolasT08's picture
Formal-MAS source code
dfe1826 verified
Raw History Blame Contribute Delete
4.8 kB
"""Check that every number in docs/results_explained.md matches docs/results/results.md.
Usage (from the repository root, after `make_tables.py`):
.venv/Scripts/python.exe docs/results/check_explained.py
results.md is generated by make_tables.py from runs/; results_explained.md is hand-written around
copies of its tables. The checks:
1. Generated tables. A Markdown table in results_explained.md whose header row appears in
results.md must appear there verbatim, every row in the same order (a single block).
2. Notes. Every `*Note.* ...` paragraph must appear verbatim in results.md.
3. Every other number (prose and hand-written tables) must equal a number in results.md
rounded to the same decimals (so "1.47" matches 1.4744, "174" matches 173.9). Skipped:
headings, inline code, section references (§5.3, §7.1–7.3), dates (2026-11-08), link targets
and digits that are part of a name (C5, H1, p6).
Limitation: rule 3 checks that a number exists in results.md, not that it is the right one in
context. Decimals rarely match by chance; small integers (counts) almost always do, so for them
the check only catches numbers that appear nowhere in results.md.
Exit code 0 if everything matches, 1 otherwise (each mismatch is printed with its line number).
Reads only; writes nothing.
"""
from __future__ import annotations
import re
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
RESULTS = HERE / "results.md"
EXPLAINED = HERE.parent / "results_explained.md"
NUMBER = re.compile(r"(?<![\w.\-−])\d+(?:\.\d+)?(?![\w])")
SKIP = [
re.compile(r"`[^`]*`"), # inline code (ids, paths, labels)
re.compile(r"\]\([^)]*\)"), # link targets
re.compile(r"§\s?\d+(?:\.\d+)?(?:\s?[–-]\s?\d+(?:\.\d+)?)?"), # section references and ranges
re.compile(r"\d{4}-\d{2}-\d{2}(?:T[\d:.]+Z?)?"), # dates and timestamps
re.compile(r"\d{2}:\d{2}(?::\d{2})?"), # clock times
]
def table_blocks(lines: list[str]) -> list[tuple[int, list[str]]]:
"""(1-based start line, lines) of each Markdown table."""
blocks, cur, start = [], [], 0
for i, line in enumerate(lines, 1):
if line.startswith("|"):
if not cur:
start = i
cur.append(line)
elif cur:
blocks.append((start, cur))
cur = []
if cur:
blocks.append((start, cur))
return blocks
def decimals(tok: str) -> int:
return len(tok.split(".")[1]) if "." in tok else 0
def main() -> int:
res = RESULTS.read_text(encoding="utf-8")
exp_lines = EXPLAINED.read_text(encoding="utf-8").splitlines()
res_lines = res.splitlines()
res_values = sorted({float(t) for t in re.findall(r"\d+(?:\.\d+)?", res)})
res_headers = {line for line in res_lines if line.startswith("|")}
errors: list[str] = []
counts = {"tables": 0, "notes": 0, "numbers": 0}
generated_lines: set[int] = set()
for start, block in table_blocks(exp_lines):
if block[0] not in res_headers:
continue # hand-written table: its numbers go through rule 3
counts["tables"] += 1
generated_lines.update(range(start, start + len(block)))
if "\n".join(block) not in res:
bad = [f"line {start + k}: {row[:100]}" for k, row in enumerate(block) if row not in res_headers]
errors.append(f"table at line {start} is not a verbatim block of results.md"
+ ("".join(f"\n {b}" for b in bad) if bad else " (rows reordered or missing)"))
for i, line in enumerate(exp_lines, 1):
if i in generated_lines or line.startswith("#"):
continue
if line.startswith("*Note.*"):
counts["notes"] += 1
if line not in res_lines:
errors.append(f"line {i}: note not found verbatim in results.md: {line[:100]}")
continue
text = line
for pat in SKIP:
text = pat.sub(" ", text)
for m in NUMBER.finditer(text):
tok = m.group()
counts["numbers"] += 1
d, v = decimals(tok), float(tok)
if not any(abs(round(x, d) - v) < 1e-9 for x in res_values):
errors.append(f"line {i}: {tok} not in results.md (at {d} decimals): {line.strip()[:100]}")
print(f"checked {counts['tables']} generated tables, {counts['notes']} notes and {counts['numbers']} other numbers "
f"in {EXPLAINED.name} against {RESULTS.name}")
if errors:
print(f"FAIL: {len(errors)} mismatch(es)")
for e in errors:
print(f" - {e}")
return 1
print("OK: every number matches")
return 0
if __name__ == "__main__":
sys.exit(main())