Add four harder benchmark tasks
All seven newer local models passed the original four tasks 20 of 20, so the benchmark no longer told them apart. The new tasks keep plans explicit, as the planner is told to write them, and make the work itself harder: - trace-bug: a wrong multi-currency total whose cause is a shared rate helper; discounts and tax go through it too, so a fix at the call site fails the check. - refactor-rename: rename load_config across a ten-module package, leaving load_config_defaults and reload_config alone. - spec-parser: implement parse_duration from an eight-rule SPEC.md given three examples, and write tests for every rule. - long-milestone: eight dependent steps adding due dates to a task tracker, from the model through storage versions to the CLI. Self-tests prove that a call-site fix, a blind text replace and a partial rename all fail their checks. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f2c48d8b16
commit
25fd57351d
67 files changed
+2115
-2
No files matched your search
@@ -36,11 +36,15 @@ FACTS = ModelFacts(
|
||||
# --- the tasks themselves ---------------------------------------------------
|
||||
|
||||
|
||||
def test_the_bundled_tasks_are_the_four_planned() -> None:
|
||||
def test_the_bundled_tasks_are_the_eight_planned() -> None:
|
||||
assert [t.name for t in TASKS] == [
|
||||
"fix-bug",
|
||||
"long-milestone",
|
||||
"long-output",
|
||||
"multi-file",
|
||||
"refactor-rename",
|
||||
"spec-parser",
|
||||
"trace-bug",
|
||||
"write-function",
|
||||
]
|
||||
|
||||
@@ -85,6 +89,48 @@ def test_long_output_really_is_long(tmp_path: Path) -> None:
|
||||
assert "R237" in run.stdout[-3000:] # the failure is named at the end
|
||||
|
||||
|
||||
def test_trace_bug_refuses_a_fix_at_the_call_site(tmp_path: Path) -> None:
|
||||
"""Fixing only the currency path passes the visible test, but the same
|
||||
shared helper is wrong for discounts and tax: the check must notice."""
|
||||
[task] = load_tasks(["trace-bug"])
|
||||
task.seed_into(tmp_path)
|
||||
currency = tmp_path / "invoicing" / "currency.py"
|
||||
currency.write_text(
|
||||
currency.read_text()
|
||||
.replace("apply_rate, money, to_cents", "money, to_cents")
|
||||
.replace(
|
||||
"return apply_rate(amount, rate(from_currency, to_currency))",
|
||||
"return to_cents(amount * rate(from_currency, to_currency))",
|
||||
)
|
||||
)
|
||||
passed, output = task.run_check(tmp_path)
|
||||
assert not passed
|
||||
assert "test_fractional_discount" in output
|
||||
|
||||
|
||||
def test_refactor_rename_refuses_a_blind_replace(tmp_path: Path) -> None:
|
||||
"""A plain text replace also renames load_config_defaults and
|
||||
reload_config, which must keep their names."""
|
||||
[task] = load_tasks(["refactor-rename"])
|
||||
task.seed_into(tmp_path)
|
||||
for path in [*tmp_path.rglob("*.py"), *tmp_path.rglob("*.md")]:
|
||||
path.write_text(path.read_text().replace("load_config", "read_settings"))
|
||||
passed, output = task.run_check(tmp_path)
|
||||
assert not passed
|
||||
assert "test_the_similar_names_were_left_alone" in output
|
||||
|
||||
|
||||
def test_refactor_rename_refuses_a_partial_rename(tmp_path: Path) -> None:
|
||||
[task] = load_tasks(["refactor-rename"])
|
||||
task.seed_into(tmp_path)
|
||||
task.apply_solution(tmp_path)
|
||||
readme = tmp_path / "README.md"
|
||||
readme.write_text(readme.read_text().replace("read_settings", "load_config", 1))
|
||||
passed, output = task.run_check(tmp_path)
|
||||
assert not passed
|
||||
assert "README.md" in output
|
||||
|
||||
|
||||
def test_a_changed_task_gets_a_new_hash(tmp_path: Path) -> None:
|
||||
[task] = load_tasks(["write-function"])
|
||||
shutil.copytree(task.path, tmp_path / "write-function")
|
||||
|
||||
Reference in new issue
Block a user