Run every command on the build VM over SSH (M1.3)
CI / checks (push) Skipped
CI / checks (pull_request) Failing after 55s
CI / checks (push) Skipped
CI / checks (pull_request) Failing after 55s
execution.SSHHost.execute is now the only way the agent runs anything (L13). Commands travel as a script on SSH's stdin, so the secrets handed to a call are never on a command line, and every value handed out is redacted from the output. Timeouts kill the remote process group, and an unreachable host is reported as such, not as a failed command. Workspace and GitRepo run on the build VM. The clone uses the repository's SSH URL, so no credential is stored in it (#89); fetches and pushes get the push key over stdin, in a short-lived ssh-agent, and PRs are still opened from the agent's host with its API token. The bench runs on the build host too. Tests reach "the build VM" through a throwaway local sshd (nisse_dev .testing); CI installs openssh-server. The live push test now pushes with the sandbox's deploy key (NISSE_PUSH_KEY). Fixes #89. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
This commit is contained in:
1 parent
b7f1f3371c
commit
761ebdb857
32 files changed
+1098
-280
No files matched your search
@@ -20,6 +20,17 @@ from nisse_dev.bench.tasks import CHECK_DIR, load_tasks
|
||||
from nisse_dev.canonical import AssistantText, ToolCall
|
||||
from nisse_dev.catalog import ModelFacts
|
||||
from nisse_dev.config import ExecutorTuning, Tuning
|
||||
from nisse_dev.execution import SSHHost
|
||||
|
||||
# The build VM for every test here: the session's throwaway sshd.
|
||||
HOST: SSHHost
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _build_host(ssh_host: SSHHost) -> None:
|
||||
global HOST
|
||||
HOST = ssh_host
|
||||
|
||||
|
||||
TASKS = load_tasks()
|
||||
FACTS = ModelFacts(
|
||||
@@ -187,7 +198,9 @@ SOLVE_SLUGIFY = [
|
||||
|
||||
def test_a_solved_task_is_a_success(tmp_path: Path) -> None:
|
||||
[task] = load_tasks(["write-function"])
|
||||
record = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), note="n")
|
||||
record = run_task(
|
||||
task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), host=HOST, note="n"
|
||||
)
|
||||
|
||||
assert record.success
|
||||
assert record.outcome == "completed"
|
||||
@@ -212,7 +225,7 @@ def test_saying_finished_is_not_success(tmp_path: Path) -> None:
|
||||
],
|
||||
[call("finish", "f", summary="done")],
|
||||
]
|
||||
record = run_task(task, FACTS, Scripted(wrong), Tuning())
|
||||
record = run_task(task, FACTS, Scripted(wrong), Tuning(), host=HOST)
|
||||
assert record.outcome == "completed"
|
||||
assert not record.success
|
||||
|
||||
@@ -227,7 +240,7 @@ def test_misbehaviour_is_counted(tmp_path: Path) -> None:
|
||||
# A higher failure limit, so the invented tool and the failed command
|
||||
# don't escalate before the prose turn is reached.
|
||||
tuning = Tuning(executor=ExecutorTuning(failure_limit=5))
|
||||
record = run_task(task, FACTS, Scripted(turns), tuning)
|
||||
record = run_task(task, FACTS, Scripted(turns), tuning, host=HOST)
|
||||
assert record.invented_tools == 1
|
||||
assert record.failed_commands == 1
|
||||
assert record.prose_turns >= 1
|
||||
@@ -237,7 +250,7 @@ def test_misbehaviour_is_counted(tmp_path: Path) -> None:
|
||||
def test_a_fallback_answer_can_never_be_a_success(tmp_path: Path) -> None:
|
||||
[task] = load_tasks(["write-function"])
|
||||
client = Scripted(list(SOLVE_SLUGIFY), answered="claude-sonnet-5")
|
||||
record = run_task(task, FACTS, client, Tuning())
|
||||
record = run_task(task, FACTS, client, Tuning(), host=HOST)
|
||||
assert record.fallbacks == ["claude-sonnet-5"] * 3
|
||||
assert not record.success # the check passed, but not for this model
|
||||
|
||||
@@ -255,7 +268,9 @@ def test_wilson_intervals_behave() -> None:
|
||||
|
||||
def record_json(**overrides: Any) -> dict[str, Any]:
|
||||
[task] = load_tasks(["write-function"])
|
||||
base = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning()).to_json()
|
||||
base = run_task(
|
||||
task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), host=HOST
|
||||
).to_json()
|
||||
base.update(overrides)
|
||||
return base
|
||||
|
||||
@@ -338,7 +353,7 @@ def test_a_run_records_its_rate_limit_waits(tmp_path: Path) -> None:
|
||||
[task] = load_tasks(["write-function"])
|
||||
client = Scripted(list(SOLVE_SLUGIFY))
|
||||
client.last_rate_limit_wait_s = 20.0 # every call waited once
|
||||
record = run_task(task, FACTS, client, Tuning())
|
||||
record = run_task(task, FACTS, client, Tuning(), host=HOST)
|
||||
assert record.rate_limit_wait_s == 60.0
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user