Add an executor benchmark that keeps its results

Choosing a local executor model, and tuning its knobs, needs measured success rates across repeated runs, comparable with every model measured before. - Four canned milestones under nisse_dev/bench/tasks: write a function, fix a bug a failing test exposes, thread an option through several files, and find one failure in output far past max_output_chars. Each is an approved plan over a seed project, judged by a hidden check the model never sees; a reference solution proves each is solvable and that its check tells the difference - Runs use the agent's own client wiring, now shared as make_llm_clients, with delegation and ask_expert off: an escalation is a failure of the model under test - Each run is stored as one JSON line in bench/results/, committed, with the model's backing name and declared facts, a note on how it was served, the task's content hash, the agent's commit and tuning, the outcome, turns, misbehaviour counts, context trims, tokens, time and LiteLLM's cost; a fallback answer can never count as a success - The report rebuilds the comparison from every stored run, with Wilson 95% intervals, marking runs against older task versions - scripts/benchmark.py runs and reports; scripts/evaluate_model.sh takes a new model through the context probe, the capability matrix and the benchmark in one command - Clients keep LiteLLM's per-call cost, and BudgetedClient counts the requests it had to trim - The tuning guide gains "Benchmarking models" Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-25 00:35:24 +00:00
1 parent b2bbf70b7b
commit 50b46821d7
@@ -0,0 +1,17 @@
+"""The executor benchmark: canned milestones, run repeatedly per model.
+
+Choosing an executor model, and later tuning its knobs, should rest on
+measured success rates, not on one run or on reputation. Each task is a fixed
+approved plan over a seed project, with a hidden check the model never sees;
+success means the check passes, not that the model said it finished. Runs use
+the agent's own client wiring (context budget, declared limits, tool_choice
+only where declared), so what is measured is what the agent would do.
+
+Results are stored, one JSON line per run, with everything needed to compare
+them later: the model's identity and declared facts, a note on how it was
+served, the task's content hash, the agent's commit, and its tuning. A new
+model then needs only its own runs to be compared with everything measured
+before it.
+
+See scripts/benchmark.py, and docs/Tuning-Guide.md for how to read results.
+"""
@@ -0,0 +1,117 @@
+"""Stored benchmark results, and the report that compares them.
+
+Every run is one JSON line. The report is rebuilt from all of them each
+time, so a model benchmarked today is read alongside every model measured
+before it, as long as the task is still the same version.
+"""
+
+import json
+import math
+import statistics
+from collections import defaultdict
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from nisse_dev.bench.runner import RunRecord
+
+
+def results_file(directory: Path, model: str, when: datetime | None = None) -> Path:
+    day = (when or datetime.now(UTC)).strftime("%Y-%m-%d")
+    return directory / f"{day}-{model}.jsonl"
+
+
+def append(path: Path, record: RunRecord) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a") as out:
+        out.write(json.dumps(record.to_json(), sort_keys=True) + "\n")
+
+
+def load(directory: Path) -> list[dict[str, Any]]:
+    records = []
+    for path in sorted(directory.glob("*.jsonl")):
+        for line in path.read_text().splitlines():
+            if line.strip():
+                records.append(json.loads(line))
+    return records
+
+
+def wilson(successes: int, runs: int, z: float = 1.96) -> tuple[float, float]:
+    """A 95% confidence interval for a success rate, sound at small counts.
+
+    Three successes in five runs and five in five overlap here, which is the
+    point: small samples of a noisy model do not separate much."""
+    if runs == 0:
+        return 0.0, 1.0
+    p = successes / runs
+    denominator = 1 + z * z / runs
+    centre = (p + z * z / (2 * runs)) / denominator
+    half = z * math.sqrt(p * (1 - p) / runs + z * z / (4 * runs * runs)) / denominator
+    return max(0.0, centre - half), min(1.0, centre + half)
+
+
+@dataclass
+class Row:
+    model: str
+    note: str
+    task: str
+    runs: int
+    successes: int
+    median_turns: float
+    mean_wall_s: float
+    cost_per_run: float
+    stale: bool
+
+    @property
+    def interval(self) -> tuple[float, float]:
+        return wilson(self.successes, self.runs)
+
+
+def summarise(
+    records: Iterable[dict[str, Any]], current_hashes: dict[str, str]
+) -> list[Row]:
+    """One row per (model, how it was served, task, task version)."""
+    groups: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
+    for r in records:
+        groups[(r["backing"], r.get("note", ""), r["task"], r["task_hash"])].append(r)
+    rows = []
+    for (backing, note, task, task_hash), runs in sorted(groups.items()):
+        rows.append(
+            Row(
+                model=backing,
+                note=note,
+                task=task,
+                runs=len(runs),
+                successes=sum(1 for r in runs if r["success"]),
+                median_turns=statistics.median(r["turns"] for r in runs),
+                mean_wall_s=statistics.fmean(r["wall_s"] for r in runs),
+                cost_per_run=statistics.fmean(r["cost"] for r in runs),
+                stale=current_hashes.get(task) != task_hash,
+            )
+        )
+    return rows
+
+
+def render(rows: list[Row]) -> str:
+    header = (
+        f"{'model':<34} {'note':<22} {'task':<15} {'success':>8} {'95% CI':>11} "
+        f"{'turns':>6} {'time':>7} {'$/run':>8}"
+    )
+    lines = [header, "-" * len(header)]
+    for row in rows:
+        low, high = row.interval
+        task = row.task + (" (old)" if row.stale else "")
+        lines.append(
+            f"{row.model[:34]:<34} {row.note[:22]:<22} {task[:15]:<15} "
+            f"{row.successes:>3}/{row.runs:<4} {low:>4.0%}-{high:<5.0%} "
+            f"{row.median_turns:>6.0f} {row.mean_wall_s:>6.0f}s {row.cost_per_run:>8.4f}"
+        )
+    if any(row.stale for row in rows):
+        lines.append("")
+        lines.append(
+            "(old) = measured against an earlier version of the task; "
+            "not comparable with current runs."
+        )
+    return "\n".join(lines)
@@ -0,0 +1,218 @@
+"""Running one task once with one executor model, and recording it."""
+
+import io
+import subprocess
+import tempfile
+import time
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from nisse_dev.bench.tasks import Task
+from nisse_dev.canonical import (
+    CanonicalEvent,
+    PlanSubmitted,
+    StoredEvent,
+    ToolCall,
+    ToolResult,
+    UserMessage,
+)
+from nisse_dev.catalog import ModelFacts
+from nisse_dev.config import Tuning
+from nisse_dev.executor import LLMExecutor
+from nisse_dev.llm import RoleMap
+from nisse_dev.oplog import OpLog
+from nisse_dev.state import StateStore
+from nisse_dev.tools import Workspace
+
+
+class MeteringClient:
+    """Counts what a run spends, and notices a fallback: an answer from a
+    model other than the one asked for is not a measurement of it."""
+
+    def __init__(self, inner: Any, facts: ModelFacts) -> None:
+        self._inner = inner
+        self._facts = facts
+        self.calls = 0
+        self.prose_turns = 0
+        self.tokens_in = 0
+        self.tokens_out = 0
+        self.cost = 0.0
+        self.fallbacks: list[str] = []
+
+    def complete(
+        self,
+        events: list[StoredEvent],
+        tools: list[dict[str, Any]] | None,
+        model: str,
+        system: str | None = None,
+    ) -> list[CanonicalEvent]:
+        self.calls += 1
+        produced = self._inner.complete(events, tools, model, system=system)
+        usage = getattr(self._inner, "last_usage", {}) or {}
+        self.tokens_in += int(usage.get("input", 0))
+        self.tokens_out += int(usage.get("output", 0))
+        self.cost += float(getattr(self._inner, "last_cost", 0.0) or 0.0)
+        answered = getattr(self._inner, "last_model", None)
+        if not self._facts.answered(answered):
+            self.fallbacks.append(str(answered))
+        if not any(isinstance(e, ToolCall) for e in produced):
+            self.prose_turns += 1
+        return produced
+
+
+@dataclass
+class RunRecord:
+    """One run, with everything needed to compare it with runs made later
+    against other models, other servers, or other tuning."""
+
+    ts: str
+    model: str
+    backing: str
+    provider: str
+    model_info: dict[str, Any]
+    note: str
+    task: str
+    task_hash: str
+    commit: str
+    tuning: dict[str, Any]
+    success: bool
+    outcome: str
+    outcome_note: str
+    turns: int
+    prose_turns: int
+    failed_commands: int
+    invented_tools: int
+    trimmed_requests: int
+    tokens_in: int
+    tokens_out: int
+    cost: float
+    wall_s: float
+    fallbacks: list[str] = field(default_factory=list)
+    check_output: str = ""
+
+    def to_json(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+def agent_commit(repo: Path | None = None) -> str:
+    """The agent's commit, marked when the tree had uncommitted changes."""
+    where = repo or Path(__file__).resolve().parent
+    try:
+        sha = subprocess.run(
+            ["git", "rev-parse", "--short", "HEAD"],
+            cwd=where,
+            capture_output=True,
+            text=True,
+            check=True,
+        ).stdout.strip()
+        dirty = subprocess.run(
+            ["git", "status", "--porcelain"],
+            cwd=where,
+            capture_output=True,
+            text=True,
+            check=True,
+        ).stdout.strip()
+    except (OSError, subprocess.CalledProcessError):
+        return "unknown"
+    return f"{sha}+dirty" if dirty else sha
+
+
+def _count(events: list[StoredEvent]) -> tuple[int, int]:
+    tools = {
+        e.event.call_id: e.event.tool for e in events if isinstance(e.event, ToolCall)
+    }
+    failed = invented = 0
+    for stored in events:
+        event = stored.event
+        if isinstance(event, ToolResult) and not event.success:
+            if event.result.startswith("no such tool"):
+                invented += 1
+            elif tools.get(event.call_id) == "run_command":
+                failed += 1
+    return failed, invented
+
+
+def run_task(
+    task: Task,
+    facts: ModelFacts,
+    client: Any,
+    tuning: Tuning,
+    *,
+    note: str = "",
+    commit: str = "",
+    workdir: Path | None = None,
+) -> RunRecord:
+    """Run one task once: seed a workspace, hand the executor the approved
+    plan, then judge the result with the hidden check.
+
+    Delegation and ask_expert are off: an escalation is a failure of the
+    model under test, not a frontier model's success."""
+    with tempfile.TemporaryDirectory(dir=workdir) as tmp:
+        root = Path(tmp)
+        workspace = Workspace(
+            root / "work",
+            timeout_s=tuning.tools.command_timeout_s,
+            max_output_chars=tuning.tools.max_output_chars,
+        )
+        task.seed_into(workspace.root)
+        store = StateStore(root / "state.db", oplog=OpLog("nisse-bench", io.StringIO()))
+        store.append_event(PlanSubmitted(plan=task.plan, model="benchmark"))
+        store.append_event(UserMessage(text="start milestone 1", source="benchmark"))
+
+        meter = MeteringClient(client, facts)
+        trims_before = getattr(client, "trimmed_requests", 0)
+        model = facts.alias
+        roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
+        executor = LLMExecutor(
+            meter,
+            store,
+            roles,
+            workspace,
+            max_turns=tuning.executor.max_turns,
+            failure_limit=tuning.executor.failure_limit,
+        )
+        started = time.monotonic()
+        try:
+            outcome = executor.run(1)
+            status, detail = outcome.status, outcome.note
+        except Exception as exc:  # noqa: BLE001 -- recorded, the benchmark goes on
+            status, detail = "error", repr(exc)
+        wall = time.monotonic() - started
+
+        success, check_output = task.run_check(workspace.root)
+        failed, invented = _count(store.read_events())
+        store.close()
+
+    return RunRecord(
+        ts=datetime.now(UTC).isoformat(timespec="seconds"),
+        model=model,
+        backing=facts.backing,
+        provider=facts.provider,
+        model_info={
+            "max_input_tokens": facts.max_input_tokens,
+            "max_output_tokens": facts.max_output_tokens,
+            "supports_tool_choice": facts.supports_tool_choice,
+            "supports_function_calling": facts.supports_tools,
+        },
+        note=note,
+        task=task.name,
+        task_hash=task.content_hash,
+        commit=commit,
+        tuning=asdict(tuning),
+        success=success and not meter.fallbacks,
+        outcome=status,
+        outcome_note=detail[:500],
+        turns=meter.calls,
+        prose_turns=meter.prose_turns,
+        failed_commands=failed,
+        invented_tools=invented,
+        trimmed_requests=getattr(client, "trimmed_requests", 0) - trims_before,
+        tokens_in=meter.tokens_in,
+        tokens_out=meter.tokens_out,
+        cost=round(meter.cost, 6),
+        wall_s=round(wall, 1),
+        fallbacks=meter.fallbacks,
+        check_output=check_output,
+    )
@@ -0,0 +1,102 @@
+"""Benchmark tasks: a seed project, an approved plan, and a hidden check.
+
+Each task is a directory under ``tasks/``:
+
+- ``task.yaml``: ``title``, ``plan`` (the approved milestone plan, written the
+  way the planner is asked to write them), and ``check`` (a shell command run
+  in the workspace afterwards).
+- ``seed/``: the project the executor starts from.
+- ``hidden/``: files the check needs, copied in only after the run, into
+  ``.bench_check/``. The executor never sees them.
+- ``solution/``: a reference solution, overlaid on the seed by the tests to
+  prove each task can be solved and that its check tells the difference.
+"""
+
+import hashlib
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+TASKS_DIR = Path(__file__).parent / "tasks"
+CHECK_DIR = ".bench_check"
+
+
+@dataclass(frozen=True)
+class Task:
+    name: str
+    title: str
+    plan: str
+    check: str
+    path: Path
+
+    @property
+    def content_hash(self) -> str:
+        """What the task is, so a changed task is never compared with runs of
+        its earlier version. Covers everything the model or the check sees;
+        not the reference solution."""
+        digest = hashlib.sha256()
+        for part in ("task.yaml", "seed", "hidden"):
+            root = self.path / part
+            files = [root] if root.is_file() else sorted(root.rglob("*"))
+            for file in files:
+                if file.is_file() and "__pycache__" not in file.parts:
+                    digest.update(str(file.relative_to(self.path)).encode())
+                    digest.update(file.read_bytes())
+        return digest.hexdigest()[:16]
+
+    def seed_into(self, workspace: Path) -> None:
+        shutil.copytree(self.path / "seed", workspace, dirs_exist_ok=True)
+
+    def apply_solution(self, workspace: Path) -> None:
+        shutil.copytree(self.path / "solution", workspace, dirs_exist_ok=True)
+
+    def run_check(self, workspace: Path, timeout_s: int = 300) -> tuple[bool, str]:
+        """Copy the hidden files in and run the check. Returns whether it
+        passed, and its output for the record."""
+        target = workspace / CHECK_DIR
+        if target.exists():
+            shutil.rmtree(target)  # never trust a directory the model made
+        shutil.copytree(self.path / "hidden", target)
+        try:
+            result = subprocess.run(
+                self.check,
+                shell=True,
+                cwd=workspace,
+                capture_output=True,
+                text=True,
+                timeout=timeout_s,
+                check=False,
+            )
+        except subprocess.TimeoutExpired:
+            return False, f"check timed out after {timeout_s}s"
+        output = (result.stdout + result.stderr)[-2000:]
+        return result.returncode == 0, output
+
+
+def load_task(path: Path) -> Task:
+    spec = yaml.safe_load((path / "task.yaml").read_text())
+    return Task(
+        name=path.name,
+        title=str(spec["title"]),
+        plan=str(spec["plan"]).strip(),
+        check=str(spec["check"]),
+        path=path,
+    )
+
+
+def load_tasks(names: list[str] | None = None, root: Path = TASKS_DIR) -> list[Task]:
+    available = {
+        p.name: p for p in sorted(root.iterdir()) if (p / "task.yaml").exists()
+    }
+    if names:
+        unknown = [n for n in names if n not in available]
+        if unknown:
+            raise ValueError(
+                f"unknown task(s): {', '.join(unknown)}; "
+                f"available: {', '.join(available)}"
+            )
+        return [load_task(available[n]) for n in names]
+    return [load_task(p) for p in available.values()]
@@ -0,0 +1,11 @@
+from inventory import Item, low_stock, total_value
+
+
+def test_total_value_counts_every_unit() -> None:
+    items = [Item("bolt", 0.25, 100), Item("nut", 0.10, 40)]
+    assert total_value(items) == 29.0
+
+
+def test_low_stock_is_alphabetical() -> None:
+    items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100), Item("anchor", 3.0, 1)]
+    assert low_stock(items) == ["anchor", "washer"]
@@ -0,0 +1,27 @@
+from pathlib import Path
+
+import pytest
+from inventory import Item, low_stock, total_value
+
+
+def test_the_visible_tests_were_not_changed() -> None:
+    original = (Path(__file__).parent / "original_test_inventory.py.txt").read_text()
+    assert Path("test_inventory.py").read_text() == original
+
+
+@pytest.mark.parametrize(
+    ("items", "expected"),
+    [
+        ([], 0.0),
+        ([Item("bolt", 0.25, 100), Item("nut", 0.10, 40)], 29.0),
+        ([Item("gear", 12.5, 3)], 37.5),
+        ([Item("empty", 9.99, 0)], 0.0),
+    ],
+)
+def test_total_value(items: list[Item], expected: float) -> None:
+    assert total_value(items) == pytest.approx(expected)
+
+
+def test_low_stock_still_works() -> None:
+    items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100)]
+    assert low_stock(items) == ["washer"]
@@ -0,0 +1,23 @@
+"""A tiny inventory."""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class Item:
+    name: str
+    unit_price: float
+    quantity: int
+
+
+def total_value(items: list[Item]) -> float:
+    """The value of all stock: each item's unit price times its quantity."""
+    total = 0.0
+    for item in items:
+        total += item.unit_price
+    return total
+
+
+def low_stock(items: list[Item], threshold: int = 5) -> list[str]:
+    """Names of items with fewer than `threshold` units, alphabetically."""
+    return sorted(item.name for item in items if item.quantity < threshold)
@@ -0,0 +1,11 @@
+from inventory import Item, low_stock, total_value
+
+
+def test_total_value_counts_every_unit() -> None:
+    items = [Item("bolt", 0.25, 100), Item("nut", 0.10, 40)]
+    assert total_value(items) == 29.0
+
+
+def test_low_stock_is_alphabetical() -> None:
+    items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100), Item("anchor", 3.0, 1)]
+    assert low_stock(items) == ["anchor", "washer"]
@@ -0,0 +1,23 @@
+"""A tiny inventory."""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class Item:
+    name: str
+    unit_price: float
+    quantity: int
+
+
+def total_value(items: list[Item]) -> float:
+    """The value of all stock: each item's unit price times its quantity."""
+    total = 0.0
+    for item in items:
+        total += item.unit_price * item.quantity
+    return total
+
+
+def low_stock(items: list[Item], threshold: int = 5) -> list[str]:
+    """Names of items with fewer than `threshold` units, alphabetically."""
+    return sorted(item.name for item in items if item.quantity < threshold)
@@ -0,0 +1,12 @@
+title: Fix a bug a failing test exposes
+check: python -m pytest -q .bench_check
+plan: |
+  Milestone 1: fix the failing inventory test.
+
+  1. Run `python -m pytest -q` and read the failure.
+  2. The failing test is correct; the bug is in `inventory.py`. Do not change
+     any test file.
+  3. Fix `total_value` so that each item contributes its unit price times its
+     quantity.
+  4. Run `python -m pytest -q` until every test passes.
+  5. Call finish.
@@ -0,0 +1,11 @@
+import pytest
+from rates import RATES
+
+
+def test_every_region_is_present_once() -> None:
+    assert sorted(RATES) == [f"R{n:03d}" for n in range(300)]
+
+
+@pytest.mark.parametrize("n", range(300))
+def test_rate(n: int) -> None:
+    assert RATES[f"R{n:03d}"] == pytest.approx(n % 25 / 100)
@@ -0,0 +1,304 @@
+"""Tax rate per region code."""
+
+RATES = {
+    "R000": 0.00,
+    "R001": 0.01,
+    "R002": 0.02,
+    "R003": 0.03,
+    "R004": 0.04,
+    "R005": 0.05,
+    "R006": 0.06,
+    "R007": 0.07,
+    "R008": 0.08,
+    "R009": 0.09,
+    "R010": 0.10,
+    "R011": 0.11,
+    "R012": 0.12,
+    "R013": 0.13,
+    "R014": 0.14,
+    "R015": 0.15,
+    "R016": 0.16,
+    "R017": 0.17,
+    "R018": 0.18,
+    "R019": 0.19,
+    "R020": 0.20,
+    "R021": 0.21,
+    "R022": 0.22,
+    "R023": 0.23,
+    "R024": 0.24,
+    "R025": 0.00,
+    "R026": 0.01,
+    "R027": 0.02,
+    "R028": 0.03,
+    "R029": 0.04,
+    "R030": 0.05,
+    "R031": 0.06,
+    "R032": 0.07,
+    "R033": 0.08,
+    "R034": 0.09,
+    "R035": 0.10,
+    "R036": 0.11,
+    "R037": 0.12,
+    "R038": 0.13,
+    "R039": 0.14,
+    "R040": 0.15,
+    "R041": 0.16,
+    "R042": 0.17,
+    "R043": 0.18,
+    "R044": 0.19,
+    "R045": 0.20,
+    "R046": 0.21,
+    "R047": 0.22,
+    "R048": 0.23,
+    "R049": 0.24,
+    "R050": 0.00,
+    "R051": 0.01,
+    "R052": 0.02,
+    "R053": 0.03,
+    "R054": 0.04,
+    "R055": 0.05,
+    "R056": 0.06,
+    "R057": 0.07,
+    "R058": 0.08,
+    "R059": 0.09,
+    "R060": 0.10,
+    "R061": 0.11,
+    "R062": 0.12,
+    "R063": 0.13,
+    "R064": 0.14,
+    "R065": 0.15,
+    "R066": 0.16,
+    "R067": 0.17,
+    "R068": 0.18,
+    "R069": 0.19,
+    "R070": 0.20,
+    "R071": 0.21,
+    "R072": 0.22,
+    "R073": 0.23,
+    "R074": 0.24,
+    "R075": 0.00,
+    "R076": 0.01,
+    "R077": 0.02,
+    "R078": 0.03,
+    "R079": 0.04,
+    "R080": 0.05,
+    "R081": 0.06,
+    "R082": 0.07,
+    "R083": 0.08,
+    "R084": 0.09,
+    "R085": 0.10,
+    "R086": 0.11,
+    "R087": 0.12,
+    "R088": 0.13,
+    "R089": 0.14,
+    "R090": 0.15,
+    "R091": 0.16,
+    "R092": 0.17,
+    "R093": 0.18,
+    "R094": 0.19,
+    "R095": 0.20,
+    "R096": 0.21,
+    "R097": 0.22,
+    "R098": 0.23,
+    "R099": 0.24,
+    "R100": 0.00,
+    "R101": 0.01,
+    "R102": 0.02,
+    "R103": 0.03,
+    "R104": 0.04,
+    "R105": 0.05,
+    "R106": 0.06,
+    "R107": 0.07,
+    "R108": 0.08,
+    "R109": 0.09,
+    "R110": 0.10,
+    "R111": 0.11,
+    "R112": 0.12,
+    "R113": 0.13,
+    "R114": 0.14,
+    "R115": 0.15,
+    "R116": 0.16,
+    "R117": 0.17,
+    "R118": 0.18,
+    "R119": 0.19,
+    "R120": 0.20,
+    "R121": 0.21,
+    "R122": 0.22,
+    "R123": 0.23,
+    "R124": 0.24,
+    "R125": 0.00,
+    "R126": 0.01,
+    "R127": 0.02,
+    "R128": 0.03,
+    "R129": 0.04,
+    "R130": 0.05,
+    "R131": 0.06,
+    "R132": 0.07,
+    "R133": 0.08,
+    "R134": 0.09,
+    "R135": 0.10,
+    "R136": 0.11,
+    "R137": 0.12,
+    "R138": 0.13,
+    "R139": 0.14,
+    "R140": 0.15,
+    "R141": 0.16,
+    "R142": 0.17,
+    "R143": 0.18,
+    "R144": 0.19,
+    "R145": 0.20,
+    "R146": 0.21,
+    "R147": 0.22,
+    "R148": 0.23,
+    "R149": 0.24,
+    "R150": 0.00,
+    "R151": 0.01,
+    "R152": 0.02,
+    "R153": 0.03,
+    "R154": 0.04,
+    "R155": 0.05,
+    "R156": 0.06,
+    "R157": 0.07,
+    "R158": 0.08,
+    "R159": 0.09,
+    "R160": 0.10,
+    "R161": 0.11,
+    "R162": 0.12,
+    "R163": 0.13,
+    "R164": 0.14,
+    "R165": 0.15,
+    "R166": 0.16,
+    "R167": 0.17,
+    "R168": 0.18,
+    "R169": 0.19,
+    "R170": 0.20,
+    "R171": 0.21,
+    "R172": 0.22,
+    "R173": 0.23,
+    "R174": 0.24,
+    "R175": 0.00,
+    "R176": 0.01,
+    "R177": 0.02,
+    "R178": 0.03,
+    "R179": 0.04,
+    "R180": 0.05,
+    "R181": 0.06,
+    "R182": 0.07,
+    "R183": 0.08,
+    "R184": 0.09,
+    "R185": 0.10,
+    "R186": 0.11,
+    "R187": 0.12,
+    "R188": 0.13,
+    "R189": 0.14,
+    "R190": 0.15,
+    "R191": 0.16,
+    "R192": 0.17,
+    "R193": 0.18,
+    "R194": 0.19,
+    "R195": 0.20,
+    "R196": 0.21,
+    "R197": 0.22,
+    "R198": 0.23,
+    "R199": 0.24,
+    "R200": 0.00,
+    "R201": 0.01,
+    "R202": 0.02,
+    "R203": 0.03,
+    "R204": 0.04,
+    "R205": 0.05,
+    "R206": 0.06,
+    "R207": 0.07,
+    "R208": 0.08,
+    "R209": 0.09,
+    "R210": 0.10,
+    "R211": 0.11,
+    "R212": 0.12,
+    "R213": 0.13,
+    "R214": 0.14,
+    "R215": 0.15,
+    "R216": 0.16,
+    "R217": 0.17,
+    "R218": 0.18,
+    "R219": 0.19,
+    "R220": 0.20,
+    "R221": 0.21,
+    "R222": 0.22,
+    "R223": 0.23,
+    "R224": 0.24,
+    "R225": 0.00,
+    "R226": 0.01,
+    "R227": 0.02,
+    "R228": 0.03,
+    "R229": 0.04,
+    "R230": 0.05,
+    "R231": 0.06,
+    "R232": 0.07,
+    "R233": 0.08,
+    "R234": 0.09,
+    "R235": 0.10,
+    "R236": 0.11,
+    "R237": 0.21,
+    "R238": 0.13,
+    "R239": 0.14,
+    "R240": 0.15,
+    "R241": 0.16,
+    "R242": 0.17,
+    "R243": 0.18,
+    "R244": 0.19,
+    "R245": 0.20,
+    "R246": 0.21,
+    "R247": 0.22,
+    "R248": 0.23,
+    "R249": 0.24,
+    "R250": 0.00,
+    "R251": 0.01,
+    "R252": 0.02,
+    "R253": 0.03,
+    "R254": 0.04,
+    "R255": 0.05,
+    "R256": 0.06,
+    "R257": 0.07,
+    "R258": 0.08,
+    "R259": 0.09,
+    "R260": 0.10,
+    "R261": 0.11,
+    "R262": 0.12,
+    "R263": 0.13,
+    "R264": 0.14,
+    "R265": 0.15,
+    "R266": 0.16,
+    "R267": 0.17,
+    "R268": 0.18,
+    "R269": 0.19,
+    "R270": 0.20,
+    "R271": 0.21,
+    "R272": 0.22,
+    "R273": 0.23,
+    "R274": 0.24,
+    "R275": 0.00,
+    "R276": 0.01,
+    "R277": 0.02,
+    "R278": 0.03,
+    "R279": 0.04,
+    "R280": 0.05,
+    "R281": 0.06,
+    "R282": 0.07,
+    "R283": 0.08,
+    "R284": 0.09,
+    "R285": 0.10,
+    "R286": 0.11,
+    "R287": 0.12,
+    "R288": 0.13,
+    "R289": 0.14,
+    "R290": 0.15,
+    "R291": 0.16,
+    "R292": 0.17,
+    "R293": 0.18,
+    "R294": 0.19,
+    "R295": 0.20,
+    "R296": 0.21,
+    "R297": 0.22,
+    "R298": 0.23,
+    "R299": 0.24,
+}
@@ -0,0 +1,7 @@
+import pytest
+from rates import RATES
+
+
+@pytest.mark.parametrize("code", [f"R{n:03d}" for n in range(300)])
+def test_rate(code: str) -> None:
+    assert RATES[code] == pytest.approx(int(code[1:]) % 25 / 100)
@@ -0,0 +1,304 @@
+"""Tax rate per region code."""
+
+RATES = {
+    "R000": 0.00,
+    "R001": 0.01,
+    "R002": 0.02,
+    "R003": 0.03,
+    "R004": 0.04,
+    "R005": 0.05,
+    "R006": 0.06,
+    "R007": 0.07,
+    "R008": 0.08,
+    "R009": 0.09,
+    "R010": 0.10,
+    "R011": 0.11,
+    "R012": 0.12,
+    "R013": 0.13,
+    "R014": 0.14,
+    "R015": 0.15,
+    "R016": 0.16,
+    "R017": 0.17,
+    "R018": 0.18,
+    "R019": 0.19,
+    "R020": 0.20,
+    "R021": 0.21,
+    "R022": 0.22,
+    "R023": 0.23,
+    "R024": 0.24,
+    "R025": 0.00,
+    "R026": 0.01,
+    "R027": 0.02,
+    "R028": 0.03,
+    "R029": 0.04,
+    "R030": 0.05,
+    "R031": 0.06,
+    "R032": 0.07,
+    "R033": 0.08,
+    "R034": 0.09,
+    "R035": 0.10,
+    "R036": 0.11,
+    "R037": 0.12,
+    "R038": 0.13,
+    "R039": 0.14,
+    "R040": 0.15,
+    "R041": 0.16,
+    "R042": 0.17,
+    "R043": 0.18,
+    "R044": 0.19,
+    "R045": 0.20,
+    "R046": 0.21,
+    "R047": 0.22,
+    "R048": 0.23,
+    "R049": 0.24,
+    "R050": 0.00,
+    "R051": 0.01,
+    "R052": 0.02,
+    "R053": 0.03,
+    "R054": 0.04,
+    "R055": 0.05,
+    "R056": 0.06,
+    "R057": 0.07,
+    "R058": 0.08,
+    "R059": 0.09,
+    "R060": 0.10,
+    "R061": 0.11,
+    "R062": 0.12,
+    "R063": 0.13,
+    "R064": 0.14,
+    "R065": 0.15,
+    "R066": 0.16,
+    "R067": 0.17,
+    "R068": 0.18,
+    "R069": 0.19,
+    "R070": 0.20,
+    "R071": 0.21,
+    "R072": 0.22,
+    "R073": 0.23,
+    "R074": 0.24,
+    "R075": 0.00,
+    "R076": 0.01,
+    "R077": 0.02,
+    "R078": 0.03,
+    "R079": 0.04,
+    "R080": 0.05,
+    "R081": 0.06,
+    "R082": 0.07,
+    "R083": 0.08,
+    "R084": 0.09,
+    "R085": 0.10,
+    "R086": 0.11,
+    "R087": 0.12,
+    "R088": 0.13,
+    "R089": 0.14,
+    "R090": 0.15,
+    "R091": 0.16,
+    "R092": 0.17,
+    "R093": 0.18,
+    "R094": 0.19,
+    "R095": 0.20,
+    "R096": 0.21,
+    "R097": 0.22,
+    "R098": 0.23,
+    "R099": 0.24,
+    "R100": 0.00,
+    "R101": 0.01,
+    "R102": 0.02,
+    "R103": 0.03,
+    "R104": 0.04,
+    "R105": 0.05,
+    "R106": 0.06,
+    "R107": 0.07,
+    "R108": 0.08,
+    "R109": 0.09,
+    "R110": 0.10,
+    "R111": 0.11,
+    "R112": 0.12,
+    "R113": 0.13,
+    "R114": 0.14,
+    "R115": 0.15,
+    "R116": 0.16,
+    "R117": 0.17,
+    "R118": 0.18,
+    "R119": 0.19,
+    "R120": 0.20,
+    "R121": 0.21,
+    "R122": 0.22,
+    "R123": 0.23,
+    "R124": 0.24,
+    "R125": 0.00,
+    "R126": 0.01,
+    "R127": 0.02,
+    "R128": 0.03,
+    "R129": 0.04,
+    "R130": 0.05,
+    "R131": 0.06,
+    "R132": 0.07,
+    "R133": 0.08,
+    "R134": 0.09,
+    "R135": 0.10,
+    "R136": 0.11,
+    "R137": 0.12,
+    "R138": 0.13,
+    "R139": 0.14,
+    "R140": 0.15,
+    "R141": 0.16,
+    "R142": 0.17,
+    "R143": 0.18,
+    "R144": 0.19,
+    "R145": 0.20,
+    "R146": 0.21,
+    "R147": 0.22,
+    "R148": 0.23,
+    "R149": 0.24,
+    "R150": 0.00,
+    "R151": 0.01,
+    "R152": 0.02,
+    "R153": 0.03,
+    "R154": 0.04,
+    "R155": 0.05,
+    "R156": 0.06,
+    "R157": 0.07,
+    "R158": 0.08,
+    "R159": 0.09,
+    "R160": 0.10,
+    "R161": 0.11,
+    "R162": 0.12,
+    "R163": 0.13,
+    "R164": 0.14,
+    "R165": 0.15,
+    "R166": 0.16,
+    "R167": 0.17,
+    "R168": 0.18,
+    "R169": 0.19,
+    "R170": 0.20,
+    "R171": 0.21,
+    "R172": 0.22,
+    "R173": 0.23,
+    "R174": 0.24,
+    "R175": 0.00,
+    "R176": 0.01,
+    "R177": 0.02,
+    "R178": 0.03,
+    "R179": 0.04,
+    "R180": 0.05,
+    "R181": 0.06,
+    "R182": 0.07,
+    "R183": 0.08,
+    "R184": 0.09,
+    "R185": 0.10,
+    "R186": 0.11,
+    "R187": 0.12,
+    "R188": 0.13,
+    "R189": 0.14,
+    "R190": 0.15,
+    "R191": 0.16,
+    "R192": 0.17,
+    "R193": 0.18,
+    "R194": 0.19,
+    "R195": 0.20,
+    "R196": 0.21,
+    "R197": 0.22,
+    "R198": 0.23,
+    "R199": 0.24,
+    "R200": 0.00,
+    "R201": 0.01,
+    "R202": 0.02,
+    "R203": 0.03,
+    "R204": 0.04,
+    "R205": 0.05,
+    "R206": 0.06,
+    "R207": 0.07,
+    "R208": 0.08,
+    "R209": 0.09,
+    "R210": 0.10,
+    "R211": 0.11,
+    "R212": 0.12,
+    "R213": 0.13,
+    "R214": 0.14,
+    "R215": 0.15,
+    "R216": 0.16,
+    "R217": 0.17,
+    "R218": 0.18,
+    "R219": 0.19,
+    "R220": 0.20,
+    "R221": 0.21,
+    "R222": 0.22,
+    "R223": 0.23,
+    "R224": 0.24,
+    "R225": 0.00,
+    "R226": 0.01,
+    "R227": 0.02,
+    "R228": 0.03,
+    "R229": 0.04,
+    "R230": 0.05,
+    "R231": 0.06,
+    "R232": 0.07,
+    "R233": 0.08,
+    "R234": 0.09,
+    "R235": 0.10,
+    "R236": 0.11,
+    "R237": 0.12,
+    "R238": 0.13,
+    "R239": 0.14,
+    "R240": 0.15,
+    "R241": 0.16,
+    "R242": 0.17,
+    "R243": 0.18,
+    "R244": 0.19,
+    "R245": 0.20,
+    "R246": 0.21,
+    "R247": 0.22,
+    "R248": 0.23,
+    "R249": 0.24,
+    "R250": 0.00,
+    "R251": 0.01,
+    "R252": 0.02,
+    "R253": 0.03,
+    "R254": 0.04,
+    "R255": 0.05,
+    "R256": 0.06,
+    "R257": 0.07,
+    "R258": 0.08,
+    "R259": 0.09,
+    "R260": 0.10,
+    "R261": 0.11,
+    "R262": 0.12,
+    "R263": 0.13,
+    "R264": 0.14,
+    "R265": 0.15,
+    "R266": 0.16,
+    "R267": 0.17,
+    "R268": 0.18,
+    "R269": 0.19,
+    "R270": 0.20,
+    "R271": 0.21,
+    "R272": 0.22,
+    "R273": 0.23,
+    "R274": 0.24,
+    "R275": 0.00,
+    "R276": 0.01,
+    "R277": 0.02,
+    "R278": 0.03,
+    "R279": 0.04,
+    "R280": 0.05,
+    "R281": 0.06,
+    "R282": 0.07,
+    "R283": 0.08,
+    "R284": 0.09,
+    "R285": 0.10,
+    "R286": 0.11,
+    "R287": 0.12,
+    "R288": 0.13,
+    "R289": 0.14,
+    "R290": 0.15,
+    "R291": 0.16,
+    "R292": 0.17,
+    "R293": 0.18,
+    "R294": 0.19,
+    "R295": 0.20,
+    "R296": 0.21,
+    "R297": 0.22,
+    "R298": 0.23,
+    "R299": 0.24,
+}
@@ -0,0 +1,12 @@
+title: Find one failure in long test output
+check: python -m pytest -q .bench_check
+plan: |
+  Milestone 1: fix the one wrong entry in the rate table.
+
+  1. Run `python -m pytest -v`. Exactly one test fails, and its output names a
+     region code.
+  2. The tests are correct. In `rates.py`, the rate for region code RNNN must be
+     (NNN mod 25) / 100. Correct only the failing region's entry; leave every
+     other line as it is.
+  3. Run `python -m pytest -q` until every test passes.
+  4. Call finish.
@@ -0,0 +1,33 @@
+import subprocess
+import sys
+
+from greeter.cli import main
+from greeter.core import greet
+
+
+def test_default_is_unchanged() -> None:
+    assert greet("Bob") == "Hello, Bob!"
+
+
+def test_shout() -> None:
+    assert greet("Bob", shout=True) == "HELLO, BOB!"
+
+
+def test_cli_default(capsys) -> None:
+    assert main(["Ada"]) == 0
+    assert capsys.readouterr().out == "Hello, Ada!\n"
+
+
+def test_cli_shout(capsys) -> None:
+    assert main(["--shout", "Ada"]) == 0
+    assert capsys.readouterr().out == "HELLO, ADA!\n"
+
+
+def test_module_entry_point() -> None:
+    result = subprocess.run(
+        [sys.executable, "-m", "greeter", "--shout", "Bob"],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    assert result.stdout == "HELLO, BOB!\n"
@@ -0,0 +1,3 @@
+from greeter.cli import main
+
+raise SystemExit(main())
@@ -0,0 +1,11 @@
+import argparse
+
+from greeter.core import greet
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="greeter")
+    parser.add_argument("name")
+    args = parser.parse_args(argv)
+    print(greet(args.name))
+    return 0
@@ -0,0 +1,2 @@
+def greet(name: str) -> str:
+    return f"Hello, {name}!"
@@ -0,0 +1,11 @@
+from greeter.cli import main
+from greeter.core import greet
+
+
+def test_greet() -> None:
+    assert greet("Bob") == "Hello, Bob!"
+
+
+def test_cli_prints_the_greeting(capsys) -> None:
+    assert main(["Bob"]) == 0
+    assert capsys.readouterr().out == "Hello, Bob!\n"
@@ -0,0 +1,12 @@
+import argparse
+
+from greeter.core import greet
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="greeter")
+    parser.add_argument("name")
+    parser.add_argument("--shout", action="store_true")
+    args = parser.parse_args(argv)
+    print(greet(args.name, shout=args.shout))
+    return 0
@@ -0,0 +1,3 @@
+def greet(name: str, shout: bool = False) -> str:
+    greeting = f"Hello, {name}!"
+    return greeting.upper() if shout else greeting
@@ -0,0 +1,16 @@
+title: Thread a new option through several files
+check: python -m pytest -q .bench_check
+plan: |
+  Milestone 1: add a --shout option to the greeter.
+
+  1. In `greeter/core.py`, give `greet` a keyword argument `shout: bool = False`.
+     When it is true, return the whole greeting in upper case:
+     greet("Bob", shout=True) == "HELLO, BOB!". The default behaviour is unchanged.
+  2. In `greeter/cli.py`, add a `--shout` flag (action="store_true") and pass it
+     to `greet`.
+  3. Add two tests to `tests/test_greeter.py`: one for greet with shout=True, and
+     one that calls `main(["--shout", "Bob"])` and checks the printed output
+     using pytest's capsys fixture.
+  4. Run `python -m pytest -q` until every test passes, and check that
+     `python -m greeter --shout Bob` prints `HELLO, BOB!`.
+  5. Call finish.
@@ -0,0 +1,19 @@
+import pytest
+from slugify import slugify
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("Hello, World!", "hello-world"),
+        ("  Nisse  Agent  ", "nisse-agent"),
+        ("a--b", "a-b"),
+        ("", ""),
+        ("Python 3.12 Release", "python-3-12-release"),
+        ("--leading and trailing--", "leading-and-trailing"),
+        ("MiXeD CaSe", "mixed-case"),
+        ("!!!", ""),
+    ],
+)
+def test_slugify(text: str, expected: str) -> None:
+    assert slugify(text) == expected
@@ -0,0 +1 @@
+A small utilities project.
@@ -0,0 +1,5 @@
+import re
+
+
+def slugify(text: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
@@ -0,0 +1,13 @@
+title: Write a small function and its tests
+check: python -m pytest -q .bench_check
+plan: |
+  Milestone 1: add a slugify function.
+
+  1. Create `slugify.py` in the workspace root, defining `slugify(text: str) -> str`.
+  2. It lowercases the text, replaces every run of characters that are not a-z or
+     0-9 with a single hyphen, and strips hyphens from both ends. Examples:
+     "Hello, World!" -> "hello-world"; "  Nisse  Agent  " -> "nisse-agent";
+     "a--b" -> "a-b"; "" -> "".
+  3. Create `test_slugify.py` with pytest tests covering the four examples.
+  4. Run `python -m pytest -q test_slugify.py`, and fix anything until it passes.
+  5. Call finish.