Merge pull request 'Benchmark: workspaces are git repositories' (#177) from bench-git-workspace into main

nisse-dev/src/nisse_dev/bench/tasks.py

@@ -5,7 +5,8 @@ Each task is a directory under ``tasks/``:
 - ``task.yaml``: ``title``, ``plan`` (the approved milestone plan, written the
   way the driver restates them), and ``check`` (a shell command run
   in the workspace afterwards).
-- ``seed/``: the project the driver starts from.
+- ``seed/``: the project the driver starts from, committed as a git
+  repository's first commit (``GIT_SEED``).
 - ``hidden/``: files the check needs, copied in only after the run, into
   ``.bench_check/``. The driver never sees them.
 - ``solution/``: a reference solution, overlaid on the seed by the tests to
@@ -28,6 +29,18 @@ if TYPE_CHECKING:
 TASKS_DIR = Path(__file__).parent / "tasks"
 CHECK_DIR = ".bench_check"
 
+# The seed is committed as a git repository's first commit, as the agent's
+# own workspace is a clone: a model that starts with `git log` or `git
+# status` shouldn't fail where it would succeed in production.
+GIT_SEED = (
+    "git init -q -b main && git add -A && "
+    "git -c user.name=nisse-bench -c user.email=bench@nisse.invalid "
+    "commit -q -m seed"
+)
+# How a workspace is set up, part of every task's hash: runs from before a
+# change here are not comparable with runs after it.
+SEED_SETUP = f"seed committed: {GIT_SEED}"
+
 
 @dataclass(frozen=True)
 class Task:
@@ -42,7 +55,7 @@ class Task:
         """What the task is, so a changed task is never compared with runs of
         its earlier version. Covers everything the model or the check sees;
         not the reference solution."""
-        digest = hashlib.sha256()
+        digest = hashlib.sha256(SEED_SETUP.encode())
         for part in ("task.yaml", "seed", "hidden"):
             root = self.path / part
             files = [root] if root.is_file() else sorted(root.rglob("*"))
@@ -54,6 +67,7 @@ class Task:
 
     def seed_into(self, workspace: Path) -> None:
         shutil.copytree(self.path / "seed", workspace, dirs_exist_ok=True)
+        subprocess.run(GIT_SEED, shell=True, cwd=workspace, check=True)
 
     def apply_solution(self, workspace: Path) -> None:
         shutil.copytree(self.path / "solution", workspace, dirs_exist_ok=True)
@@ -84,6 +98,9 @@ class Task:
 
     def seed_workspace(self, workspace: "Workspace") -> None:
         workspace.upload_tree(self.path / "seed")
+        result = workspace.host.execute(GIT_SEED, cwd=str(workspace.root), timeout_s=60)
+        if result.exit_code != 0:
+            raise RuntimeError(f"committing the seed failed: {result.stderr.strip()}")
 
     def check_workspace(
         self, workspace: "Workspace", timeout_s: int = 300

nisse-dev/tests/test_bench.py

@@ -151,6 +151,33 @@ def test_a_changed_task_gets_a_new_hash(tmp_path: Path) -> None:
     assert copy.content_hash != task.content_hash
 
 
+def test_the_seed_is_a_git_repository_with_one_clean_commit(tmp_path: Path) -> None:
+    # The agent works in a clone; a model that starts with `git log` must not
+    # fail here where it would succeed in production.
+    import subprocess
+
+    [task] = load_tasks(["write-function"])
+    task.seed_into(tmp_path)
+
+    def git(*args: str) -> str:
+        return subprocess.run(
+            ["git", *args], cwd=tmp_path, capture_output=True, text=True, check=True
+        ).stdout
+
+    assert git("log", "--oneline").count("\n") == 1
+    assert git("status", "--short") == ""
+
+
+def test_how_the_seed_is_set_up_is_part_of_the_hash(monkeypatch) -> None:
+    # Runs from before the seed was committed are not comparable.
+    import nisse_dev.bench.tasks as tasks_module
+
+    [task] = load_tasks(["write-function"])
+    before = task.content_hash
+    monkeypatch.setattr(tasks_module, "SEED_SETUP", "seed copied")
+    assert task.content_hash != before
+
+
 def test_an_unknown_task_is_refused() -> None:
     with pytest.raises(ValueError, match="unknown task"):
         load_tasks(["nope"])
@@ -211,6 +238,17 @@ def test_a_solved_task_is_a_success(tmp_path: Path) -> None:
     assert record.tuning["driver"]["max_turns"] == 40
 
 
+def test_the_model_can_read_the_history_on_the_build_vm(tmp_path: Path) -> None:
+    [task] = load_tasks(["write-function"])
+    turns = [
+        [call("run_command", "g", command="git log --oneline && git status --short")],
+        *SOLVE_SLUGIFY,
+    ]
+    record = run_task(task, FACTS, Scripted(turns), Tuning(), host=HOST, note="n")
+    assert record.success
+    assert record.failed_commands == 0
+
+
 def test_saying_finished_is_not_success(tmp_path: Path) -> None:
     # Success is the hidden check passing, never the model's own claim.
     [task] = load_tasks(["write-function"])