Run every command on the build VM over SSH (M1.3)
CI / checks (push) Skipped
CI / checks (pull_request) Failing after 55s

execution.SSHHost.execute is now the only way the agent runs anything
(L13). Commands travel as a script on SSH's stdin, so the secrets handed
to a call are never on a command line, and every value handed out is
redacted from the output. Timeouts kill the remote process group, and
an unreachable host is reported as such, not as a failed command.

Workspace and GitRepo run on the build VM. The clone uses the
repository's SSH URL, so no credential is stored in it (#89); fetches
and pushes get the push key over stdin, in a short-lived ssh-agent,
and PRs are still opened from the agent's host with its API token.
The bench runs on the build host too.

Tests reach "the build VM" through a throwaway local sshd (nisse_dev
.testing); CI installs openssh-server. The live push test now pushes
with the sandbox's deploy key (NISSE_PUSH_KEY).

Fixes #89.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
This commit is contained in:
dongarandClaude Opus 5.5 committed 2026-10-02 21:39:46 +00:00
1 parent b7f1f3371c
commit 761ebdb857
32 files changed
+1098 -280

No files matched your search

+21 -6
View File
@@ -20,6 +20,17 @@ from nisse_dev.bench.tasks import CHECK_DIR, load_tasks
from nisse_dev.canonical import AssistantText, ToolCall
from nisse_dev.catalog import ModelFacts
from nisse_dev.config import ExecutorTuning, Tuning
from nisse_dev.execution import SSHHost
# The build VM for every test here: the session's throwaway sshd.
HOST: SSHHost
@pytest.fixture(autouse=True)
def _build_host(ssh_host: SSHHost) -> None:
global HOST
HOST = ssh_host
TASKS = load_tasks()
FACTS = ModelFacts(
@@ -187,7 +198,9 @@ SOLVE_SLUGIFY = [
def test_a_solved_task_is_a_success(tmp_path: Path) -> None:
[task] = load_tasks(["write-function"])
record = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), note="n")
record = run_task(
task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), host=HOST, note="n"
)
assert record.success
assert record.outcome == "completed"
@@ -212,7 +225,7 @@ def test_saying_finished_is_not_success(tmp_path: Path) -> None:
],
[call("finish", "f", summary="done")],
]
record = run_task(task, FACTS, Scripted(wrong), Tuning())
record = run_task(task, FACTS, Scripted(wrong), Tuning(), host=HOST)
assert record.outcome == "completed"
assert not record.success
@@ -227,7 +240,7 @@ def test_misbehaviour_is_counted(tmp_path: Path) -> None:
# A higher failure limit, so the invented tool and the failed command
# don't escalate before the prose turn is reached.
tuning = Tuning(executor=ExecutorTuning(failure_limit=5))
record = run_task(task, FACTS, Scripted(turns), tuning)
record = run_task(task, FACTS, Scripted(turns), tuning, host=HOST)
assert record.invented_tools == 1
assert record.failed_commands == 1
assert record.prose_turns >= 1
@@ -237,7 +250,7 @@ def test_misbehaviour_is_counted(tmp_path: Path) -> None:
def test_a_fallback_answer_can_never_be_a_success(tmp_path: Path) -> None:
[task] = load_tasks(["write-function"])
client = Scripted(list(SOLVE_SLUGIFY), answered="claude-sonnet-5")
record = run_task(task, FACTS, client, Tuning())
record = run_task(task, FACTS, client, Tuning(), host=HOST)
assert record.fallbacks == ["claude-sonnet-5"] * 3
assert not record.success # the check passed, but not for this model
@@ -255,7 +268,9 @@ def test_wilson_intervals_behave() -> None:
def record_json(**overrides: Any) -> dict[str, Any]:
[task] = load_tasks(["write-function"])
base = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning()).to_json()
base = run_task(
task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), host=HOST
).to_json()
base.update(overrides)
return base
@@ -338,7 +353,7 @@ def test_a_run_records_its_rate_limit_waits(tmp_path: Path) -> None:
[task] = load_tasks(["write-function"])
client = Scripted(list(SOLVE_SLUGIFY))
client.last_rate_limit_wait_s = 20.0 # every call waited once
record = run_task(task, FACTS, client, Tuning())
record = run_task(task, FACTS, client, Tuning(), host=HOST)
assert record.rate_limit_wait_s == 60.0