Run every command on the build VM over SSH (M1.3)

execution.SSHHost.execute is now the only way the agent runs anything
(L13). Commands travel as a script on SSH's stdin, so the secrets handed
to a call are never on a command line, and every value handed out is
redacted from the output. Timeouts kill the remote process group, and
an unreachable host is reported as such, not as a failed command.

Workspace and GitRepo run on the build VM. The clone uses the
repository's SSH URL, so no credential is stored in it (#89); fetches
and pushes get the push key over stdin, in a short-lived ssh-agent,
and PRs are still opened from the agent's host with its API token.
The bench runs on the build host too.

Tests reach "the build VM" through a throwaway local sshd (nisse_dev
.testing); CI installs openssh-server. The live push test now pushes
with the sandbox's deploy key (NISSE_PUSH_KEY).

Fixes #89.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE

nisse-dev/src/nisse_dev/bench/runner.py

@@ -1,9 +1,11 @@
 """Running one task once with one executor model, and recording it."""
 
 import io
+import shlex
 import subprocess
 import tempfile
 import time
+import uuid
 from dataclasses import asdict, dataclass, field
 from datetime import UTC, datetime
 from pathlib import Path
@@ -20,6 +22,7 @@ from nisse_dev.canonical import (
 )
 from nisse_dev.catalog import ModelFacts
 from nisse_dev.config import Tuning
+from nisse_dev.execution import SSHHost
 from nisse_dev.executor import LLMExecutor
 from nisse_dev.llm import RoleMap
 from nisse_dev.oplog import OpLog
@@ -150,6 +153,7 @@ def run_task(
     client: Any,
     tuning: Tuning,
     *,
+    host: SSHHost,
     note: str = "",
     commit: str = "",
     workdir: Path | None = None,
@@ -161,12 +165,14 @@ def run_task(
     model under test, not a frontier model's success."""
     with tempfile.TemporaryDirectory(dir=workdir) as tmp:
         root = Path(tmp)
+        # The workspace is on the build VM, like the agent's own (L13).
         workspace = Workspace(
-            root / "work",
+            host,
+            f"/tmp/nisse-bench/{uuid.uuid4().hex}",
             timeout_s=tuning.tools.command_timeout_s,
             max_output_chars=tuning.tools.max_output_chars,
         )
-        task.seed_into(workspace.root)
+        task.seed_workspace(workspace)
         store = StateStore(root / "state.db", oplog=OpLog("nisse-bench", io.StringIO()))
         # One work item per run, as the agent's own tasks are (L1).
         store.open_work_item(
@@ -198,7 +204,8 @@ def run_task(
             status, detail = "error", repr(exc)
         wall = time.monotonic() - started
 
-        success, check_output = task.run_check(workspace.root)
+        success, check_output = task.check_workspace(workspace)
+        host.execute(f"rm -rf -- {shlex.quote(str(workspace.root))}", timeout_s=60)
         failed, invented = _count(store.transcript("bench", "executor").read())
         store.close()
 

nisse-dev/src/nisse_dev/bench/tasks.py

@@ -13,13 +13,18 @@ Each task is a directory under ``tasks/``:
 """
 
 import hashlib
+import shlex
 import shutil
 import subprocess
 from dataclasses import dataclass
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import yaml
 
+if TYPE_CHECKING:
+    from nisse_dev.tools import Workspace
+
 TASKS_DIR = Path(__file__).parent / "tasks"
 CHECK_DIR = ".bench_check"
 
@@ -75,6 +80,26 @@ class Task:
         output = (result.stdout + result.stderr)[-2000:]
         return result.returncode == 0, output
 
+    # -- on the build VM, where the agent's runs happen (L13) ---------------
+
+    def seed_workspace(self, workspace: "Workspace") -> None:
+        workspace.upload_tree(self.path / "seed")
+
+    def check_workspace(
+        self, workspace: "Workspace", timeout_s: int = 300
+    ) -> tuple[bool, str]:
+        """run_check, on the build VM: copy the hidden files in fresh (never
+        trusting a directory the model made) and run the check there."""
+        root = shlex.quote(str(workspace.root / CHECK_DIR))
+        workspace.host.execute(f"rm -rf -- {root}", timeout_s=60)
+        workspace.upload_tree(self.path / "hidden", CHECK_DIR)
+        result = workspace.host.execute(
+            self.check, cwd=str(workspace.root), timeout_s=timeout_s
+        )
+        if result.timed_out:
+            return False, f"check timed out after {timeout_s}s"
+        return result.exit_code == 0, (result.stdout + result.stderr)[-2000:]
+
 
 def load_task(path: Path) -> Task:
     spec = yaml.safe_load((path / "task.yaml").read_text())