Add an executor benchmark that keeps its results

Choosing a local executor model, and tuning its knobs, needs measured success rates across repeated runs, comparable with every model measured before. - Four canned milestones under nisse_dev/bench/tasks: write a function, fix a bug a failing test exposes, thread an option through several files, and find one failure in output far past max_output_chars. Each is an approved plan over a seed project, judged by a hidden check the model never sees; a reference solution proves each is solvable and that its check tells the difference - Runs use the agent's own client wiring, now shared as make_llm_clients, with delegation and ask_expert off: an escalation is a failure of the model under test - Each run is stored as one JSON line in bench/results/, committed, with the model's backing name and declared facts, a note on how it was served, the task's content hash, the agent's commit and tuning, the outcome, turns, misbehaviour counts, context trims, tokens, time and LiteLLM's cost; a fallback answer can never count as a success - The report rebuilds the comparison from every stored run, with Wilson 95% intervals, marking runs against older task versions - scripts/benchmark.py runs and reports; scripts/evaluate_model.sh takes a new model through the context probe, the capability matrix and the benchmark in one command - Clients keep LiteLLM's per-call cost, and BudgetedClient counts the requests it had to trim - The tuning guide gains "Benchmarking models" Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-25 00:35:24 +00:00
1 parent b2bbf70b7b
commit 50b46821d7
@@ -0,0 +1,11 @@
+from inventory import Item, low_stock, total_value
+
+
+def test_total_value_counts_every_unit() -> None:
+    items = [Item("bolt", 0.25, 100), Item("nut", 0.10, 40)]
+    assert total_value(items) == 29.0
+
+
+def test_low_stock_is_alphabetical() -> None:
+    items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100), Item("anchor", 3.0, 1)]
+    assert low_stock(items) == ["anchor", "washer"]
@@ -0,0 +1,27 @@
+from pathlib import Path
+
+import pytest
+from inventory import Item, low_stock, total_value
+
+
+def test_the_visible_tests_were_not_changed() -> None:
+    original = (Path(__file__).parent / "original_test_inventory.py.txt").read_text()
+    assert Path("test_inventory.py").read_text() == original
+
+
+@pytest.mark.parametrize(
+    ("items", "expected"),
+    [
+        ([], 0.0),
+        ([Item("bolt", 0.25, 100), Item("nut", 0.10, 40)], 29.0),
+        ([Item("gear", 12.5, 3)], 37.5),
+        ([Item("empty", 9.99, 0)], 0.0),
+    ],
+)
+def test_total_value(items: list[Item], expected: float) -> None:
+    assert total_value(items) == pytest.approx(expected)
+
+
+def test_low_stock_still_works() -> None:
+    items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100)]
+    assert low_stock(items) == ["washer"]
@@ -0,0 +1,23 @@
+"""A tiny inventory."""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class Item:
+    name: str
+    unit_price: float
+    quantity: int
+
+
+def total_value(items: list[Item]) -> float:
+    """The value of all stock: each item's unit price times its quantity."""
+    total = 0.0
+    for item in items:
+        total += item.unit_price
+    return total
+
+
+def low_stock(items: list[Item], threshold: int = 5) -> list[str]:
+    """Names of items with fewer than `threshold` units, alphabetically."""
+    return sorted(item.name for item in items if item.quantity < threshold)
@@ -0,0 +1,11 @@
+from inventory import Item, low_stock, total_value
+
+
+def test_total_value_counts_every_unit() -> None:
+    items = [Item("bolt", 0.25, 100), Item("nut", 0.10, 40)]
+    assert total_value(items) == 29.0
+
+
+def test_low_stock_is_alphabetical() -> None:
+    items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100), Item("anchor", 3.0, 1)]
+    assert low_stock(items) == ["anchor", "washer"]
@@ -0,0 +1,23 @@
+"""A tiny inventory."""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class Item:
+    name: str
+    unit_price: float
+    quantity: int
+
+
+def total_value(items: list[Item]) -> float:
+    """The value of all stock: each item's unit price times its quantity."""
+    total = 0.0
+    for item in items:
+        total += item.unit_price * item.quantity
+    return total
+
+
+def low_stock(items: list[Item], threshold: int = 5) -> list[str]:
+    """Names of items with fewer than `threshold` units, alphabetically."""
+    return sorted(item.name for item in items if item.quantity < threshold)
@@ -0,0 +1,12 @@
+title: Fix a bug a failing test exposes
+check: python -m pytest -q .bench_check
+plan: |
+  Milestone 1: fix the failing inventory test.
+
+  1. Run `python -m pytest -q` and read the failure.
+  2. The failing test is correct; the bug is in `inventory.py`. Do not change
+     any test file.
+  3. Fix `total_value` so that each item contributes its unit price times its
+     quantity.
+  4. Run `python -m pytest -q` until every test passes.
+  5. Call finish.
@@ -0,0 +1,11 @@
+import pytest
+from rates import RATES
+
+
+def test_every_region_is_present_once() -> None:
+    assert sorted(RATES) == [f"R{n:03d}" for n in range(300)]
+
+
+@pytest.mark.parametrize("n", range(300))
+def test_rate(n: int) -> None:
+    assert RATES[f"R{n:03d}"] == pytest.approx(n % 25 / 100)
@@ -0,0 +1,304 @@
+"""Tax rate per region code."""
+
+RATES = {
+    "R000": 0.00,
+    "R001": 0.01,
+    "R002": 0.02,
+    "R003": 0.03,
+    "R004": 0.04,
+    "R005": 0.05,
+    "R006": 0.06,
+    "R007": 0.07,
+    "R008": 0.08,
+    "R009": 0.09,
+    "R010": 0.10,
+    "R011": 0.11,
+    "R012": 0.12,
+    "R013": 0.13,
+    "R014": 0.14,
+    "R015": 0.15,
+    "R016": 0.16,
+    "R017": 0.17,
+    "R018": 0.18,
+    "R019": 0.19,
+    "R020": 0.20,
+    "R021": 0.21,
+    "R022": 0.22,
+    "R023": 0.23,
+    "R024": 0.24,
+    "R025": 0.00,
+    "R026": 0.01,
+    "R027": 0.02,
+    "R028": 0.03,
+    "R029": 0.04,
+    "R030": 0.05,
+    "R031": 0.06,
+    "R032": 0.07,
+    "R033": 0.08,
+    "R034": 0.09,
+    "R035": 0.10,
+    "R036": 0.11,
+    "R037": 0.12,
+    "R038": 0.13,
+    "R039": 0.14,
+    "R040": 0.15,
+    "R041": 0.16,
+    "R042": 0.17,
+    "R043": 0.18,
+    "R044": 0.19,
+    "R045": 0.20,
+    "R046": 0.21,
+    "R047": 0.22,
+    "R048": 0.23,
+    "R049": 0.24,
+    "R050": 0.00,
+    "R051": 0.01,
+    "R052": 0.02,
+    "R053": 0.03,
+    "R054": 0.04,
+    "R055": 0.05,
+    "R056": 0.06,
+    "R057": 0.07,
+    "R058": 0.08,
+    "R059": 0.09,
+    "R060": 0.10,
+    "R061": 0.11,
+    "R062": 0.12,
+    "R063": 0.13,
+    "R064": 0.14,
+    "R065": 0.15,
+    "R066": 0.16,
+    "R067": 0.17,
+    "R068": 0.18,
+    "R069": 0.19,
+    "R070": 0.20,
+    "R071": 0.21,
+    "R072": 0.22,
+    "R073": 0.23,
+    "R074": 0.24,
+    "R075": 0.00,
+    "R076": 0.01,
+    "R077": 0.02,
+    "R078": 0.03,
+    "R079": 0.04,
+    "R080": 0.05,
+    "R081": 0.06,
+    "R082": 0.07,
+    "R083": 0.08,
+    "R084": 0.09,
+    "R085": 0.10,
+    "R086": 0.11,
+    "R087": 0.12,
+    "R088": 0.13,
+    "R089": 0.14,
+    "R090": 0.15,
+    "R091": 0.16,
+    "R092": 0.17,
+    "R093": 0.18,
+    "R094": 0.19,
+    "R095": 0.20,
+    "R096": 0.21,
+    "R097": 0.22,
+    "R098": 0.23,
+    "R099": 0.24,
+    "R100": 0.00,
+    "R101": 0.01,
+    "R102": 0.02,
+    "R103": 0.03,
+    "R104": 0.04,
+    "R105": 0.05,
+    "R106": 0.06,
+    "R107": 0.07,
+    "R108": 0.08,
+    "R109": 0.09,
+    "R110": 0.10,
+    "R111": 0.11,
+    "R112": 0.12,
+    "R113": 0.13,
+    "R114": 0.14,
+    "R115": 0.15,
+    "R116": 0.16,
+    "R117": 0.17,
+    "R118": 0.18,
+    "R119": 0.19,
+    "R120": 0.20,
+    "R121": 0.21,
+    "R122": 0.22,
+    "R123": 0.23,
+    "R124": 0.24,
+    "R125": 0.00,
+    "R126": 0.01,
+    "R127": 0.02,
+    "R128": 0.03,
+    "R129": 0.04,
+    "R130": 0.05,
+    "R131": 0.06,
+    "R132": 0.07,
+    "R133": 0.08,
+    "R134": 0.09,
+    "R135": 0.10,
+    "R136": 0.11,
+    "R137": 0.12,
+    "R138": 0.13,
+    "R139": 0.14,
+    "R140": 0.15,
+    "R141": 0.16,
+    "R142": 0.17,
+    "R143": 0.18,
+    "R144": 0.19,
+    "R145": 0.20,
+    "R146": 0.21,
+    "R147": 0.22,
+    "R148": 0.23,
+    "R149": 0.24,
+    "R150": 0.00,
+    "R151": 0.01,
+    "R152": 0.02,
+    "R153": 0.03,
+    "R154": 0.04,
+    "R155": 0.05,
+    "R156": 0.06,
+    "R157": 0.07,
+    "R158": 0.08,
+    "R159": 0.09,
+    "R160": 0.10,
+    "R161": 0.11,
+    "R162": 0.12,
+    "R163": 0.13,
+    "R164": 0.14,
+    "R165": 0.15,
+    "R166": 0.16,
+    "R167": 0.17,
+    "R168": 0.18,
+    "R169": 0.19,
+    "R170": 0.20,
+    "R171": 0.21,
+    "R172": 0.22,
+    "R173": 0.23,
+    "R174": 0.24,
+    "R175": 0.00,
+    "R176": 0.01,
+    "R177": 0.02,
+    "R178": 0.03,
+    "R179": 0.04,
+    "R180": 0.05,
+    "R181": 0.06,
+    "R182": 0.07,
+    "R183": 0.08,
+    "R184": 0.09,
+    "R185": 0.10,
+    "R186": 0.11,
+    "R187": 0.12,
+    "R188": 0.13,
+    "R189": 0.14,
+    "R190": 0.15,
+    "R191": 0.16,
+    "R192": 0.17,
+    "R193": 0.18,
+    "R194": 0.19,
+    "R195": 0.20,
+    "R196": 0.21,
+    "R197": 0.22,
+    "R198": 0.23,
+    "R199": 0.24,
+    "R200": 0.00,
+    "R201": 0.01,
+    "R202": 0.02,
+    "R203": 0.03,
+    "R204": 0.04,
+    "R205": 0.05,
+    "R206": 0.06,
+    "R207": 0.07,
+    "R208": 0.08,
+    "R209": 0.09,
+    "R210": 0.10,
+    "R211": 0.11,
+    "R212": 0.12,
+    "R213": 0.13,
+    "R214": 0.14,
+    "R215": 0.15,
+    "R216": 0.16,
+    "R217": 0.17,
+    "R218": 0.18,
+    "R219": 0.19,
+    "R220": 0.20,
+    "R221": 0.21,
+    "R222": 0.22,
+    "R223": 0.23,
+    "R224": 0.24,
+    "R225": 0.00,
+    "R226": 0.01,
+    "R227": 0.02,
+    "R228": 0.03,
+    "R229": 0.04,
+    "R230": 0.05,
+    "R231": 0.06,
+    "R232": 0.07,
+    "R233": 0.08,
+    "R234": 0.09,
+    "R235": 0.10,
+    "R236": 0.11,
+    "R237": 0.21,
+    "R238": 0.13,
+    "R239": 0.14,
+    "R240": 0.15,
+    "R241": 0.16,
+    "R242": 0.17,
+    "R243": 0.18,
+    "R244": 0.19,
+    "R245": 0.20,
+    "R246": 0.21,
+    "R247": 0.22,
+    "R248": 0.23,
+    "R249": 0.24,
+    "R250": 0.00,
+    "R251": 0.01,
+    "R252": 0.02,
+    "R253": 0.03,
+    "R254": 0.04,
+    "R255": 0.05,
+    "R256": 0.06,
+    "R257": 0.07,
+    "R258": 0.08,
+    "R259": 0.09,
+    "R260": 0.10,
+    "R261": 0.11,
+    "R262": 0.12,
+    "R263": 0.13,
+    "R264": 0.14,
+    "R265": 0.15,
+    "R266": 0.16,
+    "R267": 0.17,
+    "R268": 0.18,
+    "R269": 0.19,
+    "R270": 0.20,
+    "R271": 0.21,
+    "R272": 0.22,
+    "R273": 0.23,
+    "R274": 0.24,
+    "R275": 0.00,
+    "R276": 0.01,
+    "R277": 0.02,
+    "R278": 0.03,
+    "R279": 0.04,
+    "R280": 0.05,
+    "R281": 0.06,
+    "R282": 0.07,
+    "R283": 0.08,
+    "R284": 0.09,
+    "R285": 0.10,
+    "R286": 0.11,
+    "R287": 0.12,
+    "R288": 0.13,
+    "R289": 0.14,
+    "R290": 0.15,
+    "R291": 0.16,
+    "R292": 0.17,
+    "R293": 0.18,
+    "R294": 0.19,
+    "R295": 0.20,
+    "R296": 0.21,
+    "R297": 0.22,
+    "R298": 0.23,
+    "R299": 0.24,
+}
@@ -0,0 +1,7 @@
+import pytest
+from rates import RATES
+
+
+@pytest.mark.parametrize("code", [f"R{n:03d}" for n in range(300)])
+def test_rate(code: str) -> None:
+    assert RATES[code] == pytest.approx(int(code[1:]) % 25 / 100)
@@ -0,0 +1,304 @@
+"""Tax rate per region code."""
+
+RATES = {
+    "R000": 0.00,
+    "R001": 0.01,
+    "R002": 0.02,
+    "R003": 0.03,
+    "R004": 0.04,
+    "R005": 0.05,
+    "R006": 0.06,
+    "R007": 0.07,
+    "R008": 0.08,
+    "R009": 0.09,
+    "R010": 0.10,
+    "R011": 0.11,
+    "R012": 0.12,
+    "R013": 0.13,
+    "R014": 0.14,
+    "R015": 0.15,
+    "R016": 0.16,
+    "R017": 0.17,
+    "R018": 0.18,
+    "R019": 0.19,
+    "R020": 0.20,
+    "R021": 0.21,
+    "R022": 0.22,
+    "R023": 0.23,
+    "R024": 0.24,
+    "R025": 0.00,
+    "R026": 0.01,
+    "R027": 0.02,
+    "R028": 0.03,
+    "R029": 0.04,
+    "R030": 0.05,
+    "R031": 0.06,
+    "R032": 0.07,
+    "R033": 0.08,
+    "R034": 0.09,
+    "R035": 0.10,
+    "R036": 0.11,
+    "R037": 0.12,
+    "R038": 0.13,
+    "R039": 0.14,
+    "R040": 0.15,
+    "R041": 0.16,
+    "R042": 0.17,
+    "R043": 0.18,
+    "R044": 0.19,
+    "R045": 0.20,
+    "R046": 0.21,
+    "R047": 0.22,
+    "R048": 0.23,
+    "R049": 0.24,
+    "R050": 0.00,
+    "R051": 0.01,
+    "R052": 0.02,
+    "R053": 0.03,
+    "R054": 0.04,
+    "R055": 0.05,
+    "R056": 0.06,
+    "R057": 0.07,
+    "R058": 0.08,
+    "R059": 0.09,
+    "R060": 0.10,
+    "R061": 0.11,
+    "R062": 0.12,
+    "R063": 0.13,
+    "R064": 0.14,
+    "R065": 0.15,
+    "R066": 0.16,
+    "R067": 0.17,
+    "R068": 0.18,
+    "R069": 0.19,
+    "R070": 0.20,
+    "R071": 0.21,
+    "R072": 0.22,
+    "R073": 0.23,
+    "R074": 0.24,
+    "R075": 0.00,
+    "R076": 0.01,
+    "R077": 0.02,
+    "R078": 0.03,
+    "R079": 0.04,
+    "R080": 0.05,
+    "R081": 0.06,
+    "R082": 0.07,
+    "R083": 0.08,
+    "R084": 0.09,
+    "R085": 0.10,
+    "R086": 0.11,
+    "R087": 0.12,
+    "R088": 0.13,
+    "R089": 0.14,
+    "R090": 0.15,
+    "R091": 0.16,
+    "R092": 0.17,
+    "R093": 0.18,
+    "R094": 0.19,
+    "R095": 0.20,
+    "R096": 0.21,
+    "R097": 0.22,
+    "R098": 0.23,
+    "R099": 0.24,
+    "R100": 0.00,
+    "R101": 0.01,
+    "R102": 0.02,
+    "R103": 0.03,
+    "R104": 0.04,
+    "R105": 0.05,
+    "R106": 0.06,
+    "R107": 0.07,
+    "R108": 0.08,
+    "R109": 0.09,
+    "R110": 0.10,
+    "R111": 0.11,
+    "R112": 0.12,
+    "R113": 0.13,
+    "R114": 0.14,
+    "R115": 0.15,
+    "R116": 0.16,
+    "R117": 0.17,
+    "R118": 0.18,
+    "R119": 0.19,
+    "R120": 0.20,
+    "R121": 0.21,
+    "R122": 0.22,
+    "R123": 0.23,
+    "R124": 0.24,
+    "R125": 0.00,
+    "R126": 0.01,
+    "R127": 0.02,
+    "R128": 0.03,
+    "R129": 0.04,
+    "R130": 0.05,
+    "R131": 0.06,
+    "R132": 0.07,
+    "R133": 0.08,
+    "R134": 0.09,
+    "R135": 0.10,
+    "R136": 0.11,
+    "R137": 0.12,
+    "R138": 0.13,
+    "R139": 0.14,
+    "R140": 0.15,
+    "R141": 0.16,
+    "R142": 0.17,
+    "R143": 0.18,
+    "R144": 0.19,
+    "R145": 0.20,
+    "R146": 0.21,
+    "R147": 0.22,
+    "R148": 0.23,
+    "R149": 0.24,
+    "R150": 0.00,
+    "R151": 0.01,
+    "R152": 0.02,
+    "R153": 0.03,
+    "R154": 0.04,
+    "R155": 0.05,
+    "R156": 0.06,
+    "R157": 0.07,
+    "R158": 0.08,
+    "R159": 0.09,
+    "R160": 0.10,
+    "R161": 0.11,
+    "R162": 0.12,
+    "R163": 0.13,
+    "R164": 0.14,
+    "R165": 0.15,
+    "R166": 0.16,
+    "R167": 0.17,
+    "R168": 0.18,
+    "R169": 0.19,
+    "R170": 0.20,
+    "R171": 0.21,
+    "R172": 0.22,
+    "R173": 0.23,
+    "R174": 0.24,
+    "R175": 0.00,
+    "R176": 0.01,
+    "R177": 0.02,
+    "R178": 0.03,
+    "R179": 0.04,
+    "R180": 0.05,
+    "R181": 0.06,
+    "R182": 0.07,
+    "R183": 0.08,
+    "R184": 0.09,
+    "R185": 0.10,
+    "R186": 0.11,
+    "R187": 0.12,
+    "R188": 0.13,
+    "R189": 0.14,
+    "R190": 0.15,
+    "R191": 0.16,
+    "R192": 0.17,
+    "R193": 0.18,
+    "R194": 0.19,
+    "R195": 0.20,
+    "R196": 0.21,
+    "R197": 0.22,
+    "R198": 0.23,
+    "R199": 0.24,
+    "R200": 0.00,
+    "R201": 0.01,
+    "R202": 0.02,
+    "R203": 0.03,
+    "R204": 0.04,
+    "R205": 0.05,
+    "R206": 0.06,
+    "R207": 0.07,
+    "R208": 0.08,
+    "R209": 0.09,
+    "R210": 0.10,
+    "R211": 0.11,
+    "R212": 0.12,
+    "R213": 0.13,
+    "R214": 0.14,
+    "R215": 0.15,
+    "R216": 0.16,
+    "R217": 0.17,
+    "R218": 0.18,
+    "R219": 0.19,
+    "R220": 0.20,
+    "R221": 0.21,
+    "R222": 0.22,
+    "R223": 0.23,
+    "R224": 0.24,
+    "R225": 0.00,
+    "R226": 0.01,
+    "R227": 0.02,
+    "R228": 0.03,
+    "R229": 0.04,
+    "R230": 0.05,
+    "R231": 0.06,
+    "R232": 0.07,
+    "R233": 0.08,
+    "R234": 0.09,
+    "R235": 0.10,
+    "R236": 0.11,
+    "R237": 0.12,
+    "R238": 0.13,
+    "R239": 0.14,
+    "R240": 0.15,
+    "R241": 0.16,
+    "R242": 0.17,
+    "R243": 0.18,
+    "R244": 0.19,
+    "R245": 0.20,
+    "R246": 0.21,
+    "R247": 0.22,
+    "R248": 0.23,
+    "R249": 0.24,
+    "R250": 0.00,
+    "R251": 0.01,
+    "R252": 0.02,
+    "R253": 0.03,
+    "R254": 0.04,
+    "R255": 0.05,
+    "R256": 0.06,
+    "R257": 0.07,
+    "R258": 0.08,
+    "R259": 0.09,
+    "R260": 0.10,
+    "R261": 0.11,
+    "R262": 0.12,
+    "R263": 0.13,
+    "R264": 0.14,
+    "R265": 0.15,
+    "R266": 0.16,
+    "R267": 0.17,
+    "R268": 0.18,
+    "R269": 0.19,
+    "R270": 0.20,
+    "R271": 0.21,
+    "R272": 0.22,
+    "R273": 0.23,
+    "R274": 0.24,
+    "R275": 0.00,
+    "R276": 0.01,
+    "R277": 0.02,
+    "R278": 0.03,
+    "R279": 0.04,
+    "R280": 0.05,
+    "R281": 0.06,
+    "R282": 0.07,
+    "R283": 0.08,
+    "R284": 0.09,
+    "R285": 0.10,
+    "R286": 0.11,
+    "R287": 0.12,
+    "R288": 0.13,
+    "R289": 0.14,
+    "R290": 0.15,
+    "R291": 0.16,
+    "R292": 0.17,
+    "R293": 0.18,
+    "R294": 0.19,
+    "R295": 0.20,
+    "R296": 0.21,
+    "R297": 0.22,
+    "R298": 0.23,
+    "R299": 0.24,
+}
@@ -0,0 +1,12 @@
+title: Find one failure in long test output
+check: python -m pytest -q .bench_check
+plan: |
+  Milestone 1: fix the one wrong entry in the rate table.
+
+  1. Run `python -m pytest -v`. Exactly one test fails, and its output names a
+     region code.
+  2. The tests are correct. In `rates.py`, the rate for region code RNNN must be
+     (NNN mod 25) / 100. Correct only the failing region's entry; leave every
+     other line as it is.
+  3. Run `python -m pytest -q` until every test passes.
+  4. Call finish.
@@ -0,0 +1,33 @@
+import subprocess
+import sys
+
+from greeter.cli import main
+from greeter.core import greet
+
+
+def test_default_is_unchanged() -> None:
+    assert greet("Bob") == "Hello, Bob!"
+
+
+def test_shout() -> None:
+    assert greet("Bob", shout=True) == "HELLO, BOB!"
+
+
+def test_cli_default(capsys) -> None:
+    assert main(["Ada"]) == 0
+    assert capsys.readouterr().out == "Hello, Ada!\n"
+
+
+def test_cli_shout(capsys) -> None:
+    assert main(["--shout", "Ada"]) == 0
+    assert capsys.readouterr().out == "HELLO, ADA!\n"
+
+
+def test_module_entry_point() -> None:
+    result = subprocess.run(
+        [sys.executable, "-m", "greeter", "--shout", "Bob"],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    assert result.stdout == "HELLO, BOB!\n"
@@ -0,0 +1,3 @@
+from greeter.cli import main
+
+raise SystemExit(main())
@@ -0,0 +1,11 @@
+import argparse
+
+from greeter.core import greet
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="greeter")
+    parser.add_argument("name")
+    args = parser.parse_args(argv)
+    print(greet(args.name))
+    return 0
@@ -0,0 +1,2 @@
+def greet(name: str) -> str:
+    return f"Hello, {name}!"
@@ -0,0 +1,11 @@
+from greeter.cli import main
+from greeter.core import greet
+
+
+def test_greet() -> None:
+    assert greet("Bob") == "Hello, Bob!"
+
+
+def test_cli_prints_the_greeting(capsys) -> None:
+    assert main(["Bob"]) == 0
+    assert capsys.readouterr().out == "Hello, Bob!\n"
@@ -0,0 +1,12 @@
+import argparse
+
+from greeter.core import greet
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="greeter")
+    parser.add_argument("name")
+    parser.add_argument("--shout", action="store_true")
+    args = parser.parse_args(argv)
+    print(greet(args.name, shout=args.shout))
+    return 0
@@ -0,0 +1,3 @@
+def greet(name: str, shout: bool = False) -> str:
+    greeting = f"Hello, {name}!"
+    return greeting.upper() if shout else greeting
@@ -0,0 +1,16 @@
+title: Thread a new option through several files
+check: python -m pytest -q .bench_check
+plan: |
+  Milestone 1: add a --shout option to the greeter.
+
+  1. In `greeter/core.py`, give `greet` a keyword argument `shout: bool = False`.
+     When it is true, return the whole greeting in upper case:
+     greet("Bob", shout=True) == "HELLO, BOB!". The default behaviour is unchanged.
+  2. In `greeter/cli.py`, add a `--shout` flag (action="store_true") and pass it
+     to `greet`.
+  3. Add two tests to `tests/test_greeter.py`: one for greet with shout=True, and
+     one that calls `main(["--shout", "Bob"])` and checks the printed output
+     using pytest's capsys fixture.
+  4. Run `python -m pytest -q` until every test passes, and check that
+     `python -m greeter --shout Bob` prints `HELLO, BOB!`.
+  5. Call finish.
@@ -0,0 +1,19 @@
+import pytest
+from slugify import slugify
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("Hello, World!", "hello-world"),
+        ("  Nisse  Agent  ", "nisse-agent"),
+        ("a--b", "a-b"),
+        ("", ""),
+        ("Python 3.12 Release", "python-3-12-release"),
+        ("--leading and trailing--", "leading-and-trailing"),
+        ("MiXeD CaSe", "mixed-case"),
+        ("!!!", ""),
+    ],
+)
+def test_slugify(text: str, expected: str) -> None:
+    assert slugify(text) == expected
@@ -0,0 +1 @@
+A small utilities project.
@@ -0,0 +1,5 @@
+import re
+
+
+def slugify(text: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
@@ -0,0 +1,13 @@
+title: Write a small function and its tests
+check: python -m pytest -q .bench_check
+plan: |
+  Milestone 1: add a slugify function.
+
+  1. Create `slugify.py` in the workspace root, defining `slugify(text: str) -> str`.
+  2. It lowercases the text, replaces every run of characters that are not a-z or
+     0-9 with a single hyphen, and strips hyphens from both ends. Examples:
+     "Hello, World!" -> "hello-world"; "  Nisse  Agent  " -> "nisse-agent";
+     "a--b" -> "a-b"; "" -> "".
+  3. Create `test_slugify.py` with pytest tests covering the four examples.
+  4. Run `python -m pytest -q test_slugify.py`, and fix anything until it passes.
+  5. Call finish.