diff --git a/nisse-dev/src/nisse_dev/bench/tasks.py b/nisse-dev/src/nisse_dev/bench/tasks.py index 71afa12..aac6916 100644 --- a/nisse-dev/src/nisse_dev/bench/tasks.py +++ b/nisse-dev/src/nisse_dev/bench/tasks.py @@ -5,7 +5,8 @@ Each task is a directory under ``tasks/``: - ``task.yaml``: ``title``, ``plan`` (the approved milestone plan, written the way the driver restates them), and ``check`` (a shell command run in the workspace afterwards). -- ``seed/``: the project the driver starts from. +- ``seed/``: the project the driver starts from, committed as a git + repository's first commit (``GIT_SEED``). - ``hidden/``: files the check needs, copied in only after the run, into ``.bench_check/``. The driver never sees them. - ``solution/``: a reference solution, overlaid on the seed by the tests to @@ -28,6 +29,18 @@ if TYPE_CHECKING: TASKS_DIR = Path(__file__).parent / "tasks" CHECK_DIR = ".bench_check" +# The seed is committed as a git repository's first commit, as the agent's +# own workspace is a clone: a model that starts with `git log` or `git +# status` shouldn't fail where it would succeed in production. +GIT_SEED = ( + "git init -q -b main && git add -A && " + "git -c user.name=nisse-bench -c user.email=bench@nisse.invalid " + "commit -q -m seed" +) +# How a workspace is set up, part of every task's hash: runs from before a +# change here are not comparable with runs after it. +SEED_SETUP = f"seed committed: {GIT_SEED}" + @dataclass(frozen=True) class Task: @@ -42,7 +55,7 @@ class Task: """What the task is, so a changed task is never compared with runs of its earlier version. Covers everything the model or the check sees; not the reference solution.""" - digest = hashlib.sha256() + digest = hashlib.sha256(SEED_SETUP.encode()) for part in ("task.yaml", "seed", "hidden"): root = self.path / part files = [root] if root.is_file() else sorted(root.rglob("*")) @@ -54,6 +67,7 @@ class Task: def seed_into(self, workspace: Path) -> None: shutil.copytree(self.path / "seed", workspace, dirs_exist_ok=True) + subprocess.run(GIT_SEED, shell=True, cwd=workspace, check=True) def apply_solution(self, workspace: Path) -> None: shutil.copytree(self.path / "solution", workspace, dirs_exist_ok=True) @@ -84,6 +98,9 @@ class Task: def seed_workspace(self, workspace: "Workspace") -> None: workspace.upload_tree(self.path / "seed") + result = workspace.host.execute(GIT_SEED, cwd=str(workspace.root), timeout_s=60) + if result.exit_code != 0: + raise RuntimeError(f"committing the seed failed: {result.stderr.strip()}") def check_workspace( self, workspace: "Workspace", timeout_s: int = 300 diff --git a/nisse-dev/tests/test_bench.py b/nisse-dev/tests/test_bench.py index c4a518e..fd21d23 100644 --- a/nisse-dev/tests/test_bench.py +++ b/nisse-dev/tests/test_bench.py @@ -151,6 +151,33 @@ def test_a_changed_task_gets_a_new_hash(tmp_path: Path) -> None: assert copy.content_hash != task.content_hash +def test_the_seed_is_a_git_repository_with_one_clean_commit(tmp_path: Path) -> None: + # The agent works in a clone; a model that starts with `git log` must not + # fail here where it would succeed in production. + import subprocess + + [task] = load_tasks(["write-function"]) + task.seed_into(tmp_path) + + def git(*args: str) -> str: + return subprocess.run( + ["git", *args], cwd=tmp_path, capture_output=True, text=True, check=True + ).stdout + + assert git("log", "--oneline").count("\n") == 1 + assert git("status", "--short") == "" + + +def test_how_the_seed_is_set_up_is_part_of_the_hash(monkeypatch) -> None: + # Runs from before the seed was committed are not comparable. + import nisse_dev.bench.tasks as tasks_module + + [task] = load_tasks(["write-function"]) + before = task.content_hash + monkeypatch.setattr(tasks_module, "SEED_SETUP", "seed copied") + assert task.content_hash != before + + def test_an_unknown_task_is_refused() -> None: with pytest.raises(ValueError, match="unknown task"): load_tasks(["nope"]) @@ -211,6 +238,17 @@ def test_a_solved_task_is_a_success(tmp_path: Path) -> None: assert record.tuning["driver"]["max_turns"] == 40 +def test_the_model_can_read_the_history_on_the_build_vm(tmp_path: Path) -> None: + [task] = load_tasks(["write-function"]) + turns = [ + [call("run_command", "g", command="git log --oneline && git status --short")], + *SOLVE_SLUGIFY, + ] + record = run_task(task, FACTS, Scripted(turns), Tuning(), host=HOST, note="n") + assert record.success + assert record.failed_commands == 0 + + def test_saying_finished_is_not_success(tmp_path: Path) -> None: # Success is the hidden check passing, never the model's own claim. [task] = load_tasks(["write-function"])