Add four harder benchmark tasks

All seven newer local models passed the original four tasks 20 of 20, so the benchmark no longer told them apart. The new tasks keep plans explicit, as the planner is told to write them, and make the work itself harder: - trace-bug: a wrong multi-currency total whose cause is a shared rate helper; discounts and tax go through it too, so a fix at the call site fails the check. - refactor-rename: rename load_config across a ten-module package, leaving load_config_defaults and reload_config alone. - spec-parser: implement parse_duration from an eight-rule SPEC.md given three examples, and write tests for every rule. - long-milestone: eight dependent steps adding due dates to a task tracker, from the model through storage versions to the CLI. Self-tests prove that a call-site fix, a blind text replace and a partial rename all fail their checks. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-27 22:43:39 +00:00
1 parent f2c48d8b16
commit 25fd57351d
@@ -0,0 +1,186 @@
+import json
+import os
+import subprocess
+import sys
+from datetime import date
+from pathlib import Path
+
+import pytest
+from tracker import storage
+from tracker.core import add_task, complete, overdue, sort_tasks
+from tracker.formatting import format_task
+from tracker.models import Task
+
+TODAY = date(2026, 10, 5)
+
+
+# step 1
+def test_task_has_an_optional_due_date() -> None:
+    assert Task(1, "a").due is None
+    assert Task(1, "a", False, date(2026, 1, 2)).due == date(2026, 1, 2)
+
+
+# step 2
+def test_save_writes_version_2_with_iso_dates(tmp_path: Path) -> None:
+    path = tmp_path / "t.json"
+    storage.save(path, [Task(1, "a", due=date(2026, 10, 1)), Task(2, "b")])
+    data = json.loads(path.read_text())
+    assert data["version"] == 2
+    assert [t["due"] for t in data["tasks"]] == ["2026-10-01", None]
+
+
+def test_round_trip_keeps_dates(tmp_path: Path) -> None:
+    path = tmp_path / "t.json"
+    tasks = [Task(1, "a", due=date(2026, 10, 1)), Task(2, "b", done=True)]
+    storage.save(path, tasks)
+    assert storage.load(path) == tasks
+
+
+def test_version_1_files_still_load(tmp_path: Path) -> None:
+    path = tmp_path / "t.json"
+    path.write_text(
+        json.dumps({"version": 1, "tasks": [{"id": 4, "title": "old", "done": True}]})
+    )
+    assert storage.load(path) == [Task(4, "old", done=True, due=None)]
+
+
+@pytest.mark.parametrize("version", [0, 3, 99])
+def test_other_versions_are_refused(tmp_path: Path, version: int) -> None:
+    path = tmp_path / "t.json"
+    path.write_text(json.dumps({"version": version, "tasks": []}))
+    with pytest.raises(ValueError, match=f"unsupported tasks file version: {version}"):
+        storage.load(path)
+
+
+# step 3
+def test_add_task_takes_a_due_date() -> None:
+    tasks: list[Task] = []
+    assert add_task(tasks, "a", due=date(2026, 1, 1)).due == date(2026, 1, 1)
+    assert add_task(tasks, "b").due is None
+
+
+# step 4
+def test_sort_order() -> None:
+    tasks = [
+        Task(1, "undated"),
+        Task(2, "late", due=date(2026, 12, 1)),
+        Task(3, "early", due=date(2026, 9, 1)),
+        Task(4, "done dated", done=True, due=date(2026, 1, 1)),
+        Task(5, "done undated", done=True),
+        Task(6, "early twin", due=date(2026, 9, 1)),
+        Task(7, "undated twin"),
+    ]
+    assert [t.id for t in sort_tasks(tasks)] == [3, 6, 2, 1, 7, 4, 5]
+
+
+# step 5
+def test_overdue() -> None:
+    tasks = [
+        Task(1, "due today", due=TODAY),
+        Task(2, "yesterday", due=date(2026, 10, 4)),
+        Task(3, "done late", done=True, due=date(2026, 1, 1)),
+        Task(4, "undated"),
+        Task(5, "long ago", due=date(2026, 1, 1)),
+    ]
+    assert [t.id for t in overdue(tasks, TODAY)] == [5, 2]
+
+
+# step 6
+@pytest.mark.parametrize(
+    ("task", "line"),
+    [
+        (Task(3, "Buy milk"), "[ ] 3 Buy milk"),
+        (
+            Task(4, "Call Bob", done=True, due=date(2026, 10, 1)),
+            "[x] 4 Call Bob (due 2026-10-01)",
+        ),
+        (Task(5, "Pay rent", due=date(2026, 9, 1)), "[!] 5 Pay rent (due 2026-09-01)"),
+        (Task(6, "Soon", due=date(2026, 10, 9)), "[ ] 6 Soon (due 2026-10-09)"),
+        (Task(7, "Today", due=TODAY), "[ ] 7 Today (due 2026-10-05)"),
+    ],
+)
+def test_format(task: Task, line: str) -> None:
+    assert format_task(task, TODAY) == line
+
+
+# step 7, end to end through the real command line
+def run(tmp_path: Path, *args: str) -> subprocess.CompletedProcess[str]:
+    env = {
+        **os.environ,
+        "TRACKER_FILE": str(tmp_path / "t.json"),
+        "TRACKER_TODAY": "2026-10-05",
+    }
+    return subprocess.run(
+        [sys.executable, "-m", "tracker", *args],
+        capture_output=True,
+        text=True,
+        env=env,
+        cwd=Path.cwd(),
+        check=False,
+    )
+
+
+def test_command_line_flow(tmp_path: Path) -> None:
+    assert run(tmp_path, "add", "Buy milk").returncode == 0
+    assert run(tmp_path, "add", "Pay rent", "--due", "2026-10-01").returncode == 0
+    assert run(tmp_path, "add", "Dentist", "--due", "2026-11-20").returncode == 0
+    assert run(tmp_path, "done", "1").returncode == 0
+    listing = run(tmp_path, "list")
+    assert listing.stdout.splitlines() == [
+        "[!] 2 Pay rent (due 2026-10-01)",
+        "[ ] 3 Dentist (due 2026-11-20)",
+        "[x] 1 Buy milk",
+    ], listing.stderr
+    assert run(tmp_path, "list", "--overdue").stdout.splitlines() == [
+        "[!] 2 Pay rent (due 2026-10-01)"
+    ]
+
+
+def test_a_bad_due_date_is_a_usage_error(tmp_path: Path) -> None:
+    result = run(tmp_path, "add", "x", "--due", "2026-13-40")
+    assert result.returncode == 2
+    assert "invalid --due date: 2026-13-40" in result.stderr
+
+
+def test_today_defaults_to_the_real_date(tmp_path: Path) -> None:
+    env = {**os.environ, "TRACKER_FILE": str(tmp_path / "t.json")}
+    env.pop("TRACKER_TODAY", None)
+    subprocess.run(
+        [sys.executable, "-m", "tracker", "add", "Ancient", "--due", "2000-01-01"],
+        env=env,
+        cwd=Path.cwd(),
+        check=True,
+        capture_output=True,
+    )
+    listing = subprocess.run(
+        [sys.executable, "-m", "tracker", "list", "--overdue"],
+        env=env,
+        cwd=Path.cwd(),
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert listing.stdout.splitlines() == ["[!] 1 Ancient (due 2000-01-01)"]
+
+
+def test_existing_behaviour_is_kept() -> None:
+    tasks: list[Task] = []
+    add_task(tasks, "a")
+    add_task(tasks, "b")
+    complete(tasks, 1)
+    assert [t.id for t in sort_tasks(tasks)] == [2, 1]
+
+
+# step 8
+def test_the_projects_own_tests_pass_and_cover_due_dates() -> None:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "-q", "tests"],
+        capture_output=True,
+        text=True,
+        check=False,
+        cwd=Path.cwd(),
+    )
+    assert result.returncode == 0, result.stdout[-1500:]
+    written = "\n".join(p.read_text() for p in Path("tests").glob("test_*.py"))
+    for name in ("due", "overdue", "version"):
+        assert name in written, f"no test mentions {name!r}"
@@ -0,0 +1,26 @@
+import pytest
+from tracker.core import add_task, complete, sort_tasks
+from tracker.models import Task
+
+
+def test_ids_count_up() -> None:
+    tasks: list[Task] = []
+    assert [add_task(tasks, t).id for t in ("a", "b", "c")] == [1, 2, 3]
+
+
+def test_a_title_is_required() -> None:
+    with pytest.raises(ValueError):
+        add_task([], "   ")
+
+
+def test_complete_and_sort() -> None:
+    tasks: list[Task] = []
+    for title in ("a", "b", "c"):
+        add_task(tasks, title)
+    complete(tasks, 1)
+    assert [t.id for t in sort_tasks(tasks)] == [2, 3, 1]
+
+
+def test_completing_an_unknown_task() -> None:
+    with pytest.raises(KeyError):
+        complete([], 7)
@@ -0,0 +1,25 @@
+from pathlib import Path
+
+from tracker import storage
+from tracker.cli import main
+from tracker.models import Task
+
+
+def test_round_trip(tmp_path: Path) -> None:
+    path = tmp_path / "t.json"
+    storage.save(path, [Task(1, "a"), Task(2, "b", done=True)])
+    assert storage.load(path) == [Task(1, "a"), Task(2, "b", done=True)]
+
+
+def test_no_file_is_no_tasks(tmp_path: Path) -> None:
+    assert storage.load(tmp_path / "missing.json") == []
+
+
+def test_cli_add_done_list(tmp_path: Path, capsys) -> None:
+    path = str(tmp_path / "t.json")
+    main(["--file", path, "add", "Buy milk"])
+    main(["--file", path, "add", "Call Bob"])
+    main(["--file", path, "done", "1"])
+    capsys.readouterr()
+    main(["--file", path, "list"])
+    assert capsys.readouterr().out.splitlines() == ["[ ] 2 Call Bob", "[x] 1 Buy milk"]
@@ -0,0 +1 @@
+"""A small command-line task tracker."""
@@ -0,0 +1,3 @@
+from tracker.cli import main
+
+raise SystemExit(main())
@@ -0,0 +1,41 @@
+"""python -m tracker [--file PATH] {add,list,done} ...
+
+The file defaults to $TRACKER_FILE, or tasks.json in the current directory.
+"""
+
+import argparse
+import os
+from pathlib import Path
+
+from tracker import storage
+from tracker.core import add_task, complete, sort_tasks
+from tracker.formatting import format_task
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="tracker")
+    parser.add_argument("--file", default=os.environ.get("TRACKER_FILE", "tasks.json"))
+    commands = parser.add_subparsers(dest="command", required=True)
+    add = commands.add_parser("add", help="add a task")
+    add.add_argument("title")
+    commands.add_parser("list", help="list tasks")
+    done = commands.add_parser("done", help="mark a task done")
+    done.add_argument("id", type=int)
+    args = parser.parse_args(argv)
+
+    path = Path(args.file)
+    tasks = storage.load(path)
+    if args.command == "add":
+        task = add_task(tasks, args.title)
+        storage.save(path, tasks)
+        print(f"added {task.id}")
+    elif args.command == "done":
+        try:
+            complete(tasks, args.id)
+        except KeyError as exc:
+            parser.error(str(exc.args[0]))
+        storage.save(path, tasks)
+    else:
+        for task in sort_tasks(tasks):
+            print(format_task(task))
+    return 0
@@ -0,0 +1,25 @@
+"""Operations on a list of tasks."""
+
+from tracker.models import Task
+
+
+def add_task(tasks: list[Task], title: str) -> Task:
+    title = title.strip()
+    if not title:
+        raise ValueError("a task needs a title")
+    task = Task(id=max((t.id for t in tasks), default=0) + 1, title=title)
+    tasks.append(task)
+    return task
+
+
+def complete(tasks: list[Task], task_id: int) -> Task:
+    for task in tasks:
+        if task.id == task_id:
+            task.done = True
+            return task
+    raise KeyError(f"no task {task_id}")
+
+
+def sort_tasks(tasks: list[Task]) -> list[Task]:
+    """Tasks not done first, then by id."""
+    return sorted(tasks, key=lambda t: (t.done, t.id))
@@ -0,0 +1,8 @@
+"""One line per task."""
+
+from tracker.models import Task
+
+
+def format_task(task: Task) -> str:
+    mark = "x" if task.done else " "
+    return f"[{mark}] {task.id} {task.title}"
@@ -0,0 +1,10 @@
+"""The task record."""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class Task:
+    id: int
+    title: str
+    done: bool = False
@@ -0,0 +1,24 @@
+"""Tasks are kept in one JSON file."""
+
+import json
+from pathlib import Path
+
+from tracker.models import Task
+
+VERSION = 1
+
+
+def load(path: Path) -> list[Task]:
+    """Every task in the file; an empty list when there is no file yet."""
+    if not path.exists():
+        return []
+    data = json.loads(path.read_text())
+    return [Task(id=t["id"], title=t["title"], done=t["done"]) for t in data["tasks"]]
+
+
+def save(path: Path, tasks: list[Task]) -> None:
+    data = {
+        "version": VERSION,
+        "tasks": [{"id": t.id, "title": t.title, "done": t.done} for t in tasks],
+    }
+    path.write_text(json.dumps(data, indent=2))
@@ -0,0 +1,76 @@
+import json
+from datetime import date
+from pathlib import Path
+
+import pytest
+from tracker import storage
+from tracker.cli import main
+from tracker.core import add_task, overdue, sort_tasks
+from tracker.formatting import format_task
+from tracker.models import Task
+
+
+def test_round_trip(tmp_path: Path) -> None:
+    path = tmp_path / "t.json"
+    tasks = [Task(1, "a"), Task(2, "b", done=True, due=date(2026, 10, 1))]
+    storage.save(path, tasks)
+    assert storage.load(path) == tasks
+    assert json.loads(path.read_text())["version"] == 2
+
+
+def test_no_file_is_no_tasks(tmp_path: Path) -> None:
+    assert storage.load(tmp_path / "missing.json") == []
+
+
+def test_version_1_files_load(tmp_path: Path) -> None:
+    path = tmp_path / "t.json"
+    path.write_text(
+        json.dumps({"version": 1, "tasks": [{"id": 1, "title": "a", "done": False}]})
+    )
+    assert storage.load(path) == [Task(1, "a")]
+
+
+def test_unknown_versions_are_refused(tmp_path: Path) -> None:
+    path = tmp_path / "t.json"
+    path.write_text(json.dumps({"version": 9, "tasks": []}))
+    with pytest.raises(ValueError, match="unsupported tasks file version: 9"):
+        storage.load(path)
+
+
+def test_due_dates_sort_and_overdue() -> None:
+    tasks: list[Task] = []
+    add_task(tasks, "undated")
+    add_task(tasks, "later", due=date(2026, 12, 1))
+    add_task(tasks, "sooner", due=date(2026, 9, 1))
+    assert [t.title for t in sort_tasks(tasks)] == ["sooner", "later", "undated"]
+    assert [t.title for t in overdue(tasks, date(2026, 10, 1))] == ["sooner"]
+
+
+def test_format_marks_overdue() -> None:
+    today = date(2026, 10, 5)
+    assert format_task(Task(5, "Pay rent", due=date(2026, 9, 1)), today) == (
+        "[!] 5 Pay rent (due 2026-09-01)"
+    )
+    assert format_task(Task(3, "Buy milk"), today) == "[ ] 3 Buy milk"
+
+
+def test_cli_add_done_list(tmp_path: Path, capsys, monkeypatch) -> None:
+    monkeypatch.setenv("TRACKER_TODAY", "2026-10-05")
+    path = str(tmp_path / "t.json")
+    main(["--file", path, "add", "Buy milk"])
+    main(["--file", path, "add", "Pay rent", "--due", "2026-10-01"])
+    main(["--file", path, "done", "1"])
+    capsys.readouterr()
+    main(["--file", path, "list"])
+    assert capsys.readouterr().out.splitlines() == [
+        "[!] 2 Pay rent (due 2026-10-01)",
+        "[x] 1 Buy milk",
+    ]
+    main(["--file", path, "list", "--overdue"])
+    assert capsys.readouterr().out.splitlines() == ["[!] 2 Pay rent (due 2026-10-01)"]
+
+
+def test_cli_rejects_a_bad_date(tmp_path: Path) -> None:
+    with pytest.raises(SystemExit) as caught:
+        main(["--file", str(tmp_path / "t.json"), "add", "x", "--due", "soon"])
+    assert caught.value.code == 2
@@ -0,0 +1,57 @@
+"""python -m tracker [--file PATH] {add,list,done} ...
+
+The file defaults to $TRACKER_FILE, or tasks.json in the current directory.
+"""
+
+import argparse
+import os
+from datetime import date
+from pathlib import Path
+
+from tracker import storage
+from tracker.core import add_task, complete, overdue, sort_tasks
+from tracker.formatting import format_task
+
+
+def today() -> date:
+    value = os.environ.get("TRACKER_TODAY")
+    return date.fromisoformat(value) if value else date.today()  # noqa: DTZ011 -- the plan asks for the local date
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="tracker")
+    parser.add_argument("--file", default=os.environ.get("TRACKER_FILE", "tasks.json"))
+    commands = parser.add_subparsers(dest="command", required=True)
+    add = commands.add_parser("add", help="add a task")
+    add.add_argument("title")
+    add.add_argument("--due", help="due date, YYYY-MM-DD")
+    listing = commands.add_parser("list", help="list tasks")
+    listing.add_argument("--overdue", action="store_true", help="only overdue tasks")
+    done = commands.add_parser("done", help="mark a task done")
+    done.add_argument("id", type=int)
+    args = parser.parse_args(argv)
+
+    path = Path(args.file)
+    tasks = storage.load(path)
+    now = today()
+    if args.command == "add":
+        due = None
+        if args.due is not None:
+            try:
+                due = date.fromisoformat(args.due)
+            except ValueError:
+                parser.error(f"invalid --due date: {args.due}")
+        task = add_task(tasks, args.title, due=due)
+        storage.save(path, tasks)
+        print(f"added {task.id}")
+    elif args.command == "done":
+        try:
+            complete(tasks, args.id)
+        except KeyError as exc:
+            parser.error(str(exc.args[0]))
+        storage.save(path, tasks)
+    else:
+        shown = overdue(tasks, now) if args.overdue else sort_tasks(tasks)
+        for task in shown:
+            print(format_task(task, now))
+    return 0
@@ -0,0 +1,34 @@
+"""Operations on a list of tasks."""
+
+from datetime import date
+
+from tracker.models import Task
+
+
+def add_task(tasks: list[Task], title: str, *, due: date | None = None) -> Task:
+    title = title.strip()
+    if not title:
+        raise ValueError("a task needs a title")
+    task = Task(id=max((t.id for t in tasks), default=0) + 1, title=title, due=due)
+    tasks.append(task)
+    return task
+
+
+def complete(tasks: list[Task], task_id: int) -> Task:
+    for task in tasks:
+        if task.id == task_id:
+            task.done = True
+            return task
+    raise KeyError(f"no task {task_id}")
+
+
+def sort_tasks(tasks: list[Task]) -> list[Task]:
+    """Not done first; then dated tasks, earliest first; then undated; then id."""
+    return sorted(
+        tasks,
+        key=lambda t: (t.done, t.due is None, t.due or date.max, t.id),
+    )
+
+
+def overdue(tasks: list[Task], today: date) -> list[Task]:
+    return [t for t in sort_tasks(tasks) if not t.done and t.due and t.due < today]
@@ -0,0 +1,16 @@
+"""One line per task."""
+
+from datetime import date
+
+from tracker.models import Task
+
+
+def format_task(task: Task, today: date) -> str:
+    if task.done:
+        mark = "x"
+    elif task.due and task.due < today:
+        mark = "!"
+    else:
+        mark = " "
+    due = f" (due {task.due.isoformat()})" if task.due else ""
+    return f"[{mark}] {task.id} {task.title}{due}"
@@ -0,0 +1,12 @@
+"""The task record."""
+
+from dataclasses import dataclass
+from datetime import date
+
+
+@dataclass
+class Task:
+    id: int
+    title: str
+    done: bool = False
+    due: date | None = None
@@ -0,0 +1,44 @@
+"""Tasks are kept in one JSON file."""
+
+import json
+from datetime import date
+from pathlib import Path
+
+from tracker.models import Task
+
+VERSION = 2
+
+
+def load(path: Path) -> list[Task]:
+    """Every task in the file; an empty list when there is no file yet."""
+    if not path.exists():
+        return []
+    data = json.loads(path.read_text())
+    version = data.get("version")
+    if version not in (1, 2):
+        raise ValueError(f"unsupported tasks file version: {version}")
+    return [
+        Task(
+            id=t["id"],
+            title=t["title"],
+            done=t["done"],
+            due=date.fromisoformat(t["due"]) if t.get("due") else None,
+        )
+        for t in data["tasks"]
+    ]
+
+
+def save(path: Path, tasks: list[Task]) -> None:
+    data = {
+        "version": VERSION,
+        "tasks": [
+            {
+                "id": t.id,
+                "title": t.title,
+                "done": t.done,
+                "due": t.due.isoformat() if t.due else None,
+            }
+            for t in tasks
+        ],
+    }
+    path.write_text(json.dumps(data, indent=2))
@@ -0,0 +1,40 @@
+title: Carry out a long milestone whose steps build on each other
+check: python -m pytest -q .bench_check
+plan: |
+  Milestone 1: add due dates to the task tracker.
+
+  Run `python -m pytest -q` after each step; every test must still pass
+  before you start the next one. Use `datetime.date` for dates throughout.
+
+  1. `tracker/models.py`: add a field `due: date | None = None` to `Task`,
+     after `done`.
+  2. `tracker/storage.py`: `save` writes `"version": 2`, and each task's
+     `"due"` as an ISO string ("2026-10-01") or null. `load` reads version 1
+     files (their tasks have no "due"; use None) and version 2 files. Any
+     other version raises `ValueError("unsupported tasks file version: N")`,
+     with N the version found.
+  3. `tracker/core.py`: `add_task` takes a keyword argument
+     `due: date | None = None` and stores it on the new task.
+  4. `tracker/core.py`: change `sort_tasks` so that tasks not done come
+     before tasks done; within each of those two groups, tasks with a due
+     date come first, earliest due date first, then tasks without one; ties
+     are broken by id.
+  5. `tracker/core.py`: add `overdue(tasks: list[Task], today: date) -> list[Task]`:
+     tasks not done whose due date is before `today`, in `sort_tasks` order.
+  6. `tracker/formatting.py`: `format_task` takes a second argument
+     `today: date`. A task with a due date gets " (due 2026-10-01)" after its
+     title. An overdue task (as in step 5) shows `[!]` instead of `[ ]`.
+     Examples: "[ ] 3 Buy milk", "[x] 4 Call Bob (due 2026-10-01)",
+     "[!] 5 Pay rent (due 2026-09-01)".
+  7. `tracker/cli.py`: pass today's date to `format_task`. Today is the
+     `TRACKER_TODAY` environment variable (YYYY-MM-DD) when it is set, else
+     `date.today()`. `add` gains an option `--due YYYY-MM-DD`; an invalid date
+     must call `parser.error("invalid --due date: VALUE")`, which exits with
+     status 2. `list` gains a flag `--overdue` that lists only overdue tasks.
+  8. Add tests under `tests/` for each of steps 1-7, run `python -m pytest -q`
+     until everything passes, then check the whole flow by running:
+     `TRACKER_FILE=/tmp/tracker-check.json TRACKER_TODAY=2026-10-05 python -m tracker add "Pay rent" --due 2026-10-01`
+     then the same with `list --overdue`, which must print
+     `[!] 1 Pay rent (due 2026-10-01)`. Delete /tmp/tracker-check.json
+     before and after.
+  9. Call finish.