Run every command on the build VM over SSH (M1.3)

execution.SSHHost.execute is now the only way the agent runs anything
(L13). Commands travel as a script on SSH's stdin, so the secrets handed
to a call are never on a command line, and every value handed out is
redacted from the output. Timeouts kill the remote process group, and
an unreachable host is reported as such, not as a failed command.

Workspace and GitRepo run on the build VM. The clone uses the
repository's SSH URL, so no credential is stored in it (#89); fetches
and pushes get the push key over stdin, in a short-lived ssh-agent,
and PRs are still opened from the agent's host with its API token.
The bench runs on the build host too.

Tests reach "the build VM" through a throwaway local sshd (nisse_dev
.testing); CI installs openssh-server. The live push test now pushes
with the sandbox's deploy key (NISSE_PUSH_KEY).

Fixes #89.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE

scripts/benchmark.py

@@ -14,7 +14,7 @@ import os
 import sys
 from pathlib import Path
 
-from nisse_dev.agent import make_llm_clients
+from nisse_dev.agent import build_host, make_llm_clients
 from nisse_dev.bench.results import append, load, render, results_file, summarise
 from nisse_dev.bench.runner import agent_commit, run_task
 from nisse_dev.bench.tasks import load_tasks
@@ -46,6 +46,7 @@ def run(args: argparse.Namespace) -> int:
     catalog = ModelCatalog.discover(config.llm_base_url, config.llm_token)
     catalog.require_limits({m: m for m in models})
     normalized, _passthrough = make_llm_clients(config, catalog)
+    host = build_host(config)  # runs happen on the build VM, as the agent's do
     commit = agent_commit(REPO)
 
     for model in models:
@@ -60,6 +61,7 @@ def run(args: argparse.Namespace) -> int:
                     facts,
                     normalized,
                     config.tuning,
+                    host=host,
                     note=args.note,
                     commit=commit,
                 )

scripts/bringup.sh

@@ -17,7 +17,8 @@
 # Layout, under $E2E (default /root/nisse-e2e):
 #   nisse-dev.yaml, .env    the run config and its secrets (made by hand once)
 #   code/                   the exported code under test, with its own .venv
-#   state.db, project/      the agent's state and workspace
+#   state.db                the agent's state (its workspace is on the build VM)
+#   ssh/                    the agent's key for the build VM, and push keys
 #   run-info, collected/    what was run, and the output of `collect`
 #   runs/<time>/            earlier runs, set aside by `start`
 set -euo pipefail
@@ -42,12 +43,15 @@ cmd_start() {
     for item in "$E2E"/state.db* "$E2E/project" "$E2E/collected" "$E2E/run-info"; do
         [ -e "$item" ] && items+=("$item")
     done
+    local stamp
+    stamp=$(date +%Y%m%d-%H%M%S)
     if [ ${#items[@]} -gt 0 ]; then
-        aside=$E2E/runs/$(date +%Y%m%d-%H%M%S)
+        aside=$E2E/runs/$stamp
         mkdir -p "$aside"
         mv "${items[@]}" "$aside/"
         echo "previous run set aside in $aside"
     fi
+    remote_aside "$stamp"
 
     # Export the code under test, so edits to the working tree can't change a
     # running agent. Uncommitted changes are not part of a run.
@@ -67,6 +71,24 @@ cmd_start() {
     echo "started $UNIT at ${sha:0:8} ($ref)"
 }
 
+# The workspace lives on the build VM (L13): set the last run's aside there
+# too, so each run starts from a fresh clone.
+remote_aside() {
+    local stamp=$1 where
+    where=$(python3 - "$E2E/nisse-dev.yaml" <<'PY'
+import sys, yaml
+c = yaml.safe_load(open(sys.argv[1]))
+b = c["build_host"]
+print(b.get("user", "root"), b["address"], b.get("port", 22), b["key"], c["workspace"])
+PY
+) || die "cannot read build_host and workspace from the config"
+    read -r user address port key workspace <<< "$where"
+    ssh -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \
+        -o UserKnownHostsFile="$E2E/ssh/known_hosts" "$user@$address" \
+        "if [ -e '$workspace' ]; then mv '$workspace' '$workspace.$stamp' && echo 'build VM: set aside $workspace.$stamp'; fi" \
+        || die "cannot reach the build VM $address"
+}
+
 cmd_stop() {
     running || { echo "not running"; return 0; }
     systemctl stop "$UNIT"   # SIGTERM: the agent shuts down cleanly (main.py)