Run every command on the build VM over SSH (M1.3) #93

Merged
dongar merged 5 commits from ssh-execution into main 2026-10-02 22:44:22 +00:00
36 changed files with 1492 additions and 316 deletions

No files matched your search

+2
View File
@@ -11,6 +11,8 @@ NISSE_LLM_TOKEN=replace-me
NISSE_MATRIX_TOKEN=replace-me NISSE_MATRIX_TOKEN=replace-me
# gitea.token: the Gitea service account's scoped token (B2) # gitea.token: the Gitea service account's scoped token (B2)
NISSE_GITEA_TOKEN=replace-me NISSE_GITEA_TOKEN=replace-me
# The live push test reads the sandbox's deploy key from the path in
# tests/integration/nisse-test.yaml (gitea.push_key), not from here.
# Where the config file is, if not ./nisse-dev.yaml. Where this file is, if # Where the config file is, if not ./nisse-dev.yaml. Where this file is, if
# not ./.env, is NISSE_ENV, which has to be set in the real environment. # not ./.env, is NISSE_ENV, which has to be set in the real environment.
+18
View File
@@ -24,6 +24,15 @@ jobs:
- name: Sync workspace - name: Sync workspace
run: uv sync --all-packages run: uv sync --all-packages
- name: Install sshd
# The tests reach "the build VM" over real SSH, through a throwaway
# sshd the test fixture starts (nisse_dev/testing.py).
run: |
SUDO=$(command -v sudo || true)
$SUDO apt-get update -qq
$SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null
$SUDO mkdir -p /run/sshd
- name: Lint - name: Lint
run: uv run ruff check . run: uv run ruff check .
@@ -33,6 +42,15 @@ jobs:
- name: Unit tests - name: Unit tests
run: uv run pytest -m "not integration" run: uv run pytest -m "not integration"
- name: Push key
# The sandbox's deploy key, where nisse-test.yaml expects it.
env:
NISSE_PUSH_KEY: ${{ secrets.NISSE_PUSH_KEY }}
run: |
mkdir -p /tmp/nisse-test
printf '%s\n' "$NISSE_PUSH_KEY" > /tmp/nisse-test/push_key
chmod 600 /tmp/nisse-test/push_key
- name: Integration tests - name: Integration tests
# Everything but the secrets is in tests/integration/nisse-test.yaml, # Everything but the secrets is in tests/integration/nisse-test.yaml,
# which refers to these by name. A missing secret fails the live # which refers to these by name. A missing secret fails the live
+18
View File
@@ -0,0 +1,18 @@
"""Fixtures shared by every test tree."""
from collections.abc import Iterator
from pathlib import Path
import pytest
from nisse_dev.execution import SSHHost
from nisse_dev.testing import local_sshd
@pytest.fixture(scope="session")
def ssh_host(tmp_path_factory: pytest.TempPathFactory) -> Iterator[SSHHost]:
"""A throwaway sshd on this machine, standing in for the build VM (L13).
It runs as the current user, so remote paths are this machine's paths and
tests can inspect what a command did."""
with local_sshd(Path(tmp_path_factory.mktemp("sshd"))) as host:
yield host
+7 -6
View File
@@ -543,8 +543,8 @@ event, work item, cause); every verdict, escalation and external effect is logge
- **MCP tools are the one deliberate exception:** fixed code against an API, with - **MCP tools are the one deliberate exception:** fixed code against an API, with
the model choosing only arguments. They still pass through P3, P4 and L4, and their the model choosing only arguments. They still pass through P3, P4 and L4, and their
output is redacted (L12). output is redacted (L12).
- **Tests use SSH too,** against a throwaway `sshd` container, so the tested path is - **Tests use SSH too,** against a throwaway `sshd`, so the tested path is the
the production path. production path.
--- ---
@@ -674,9 +674,10 @@ Installing on any other host is dangerous.
fresh and this one resumes from its branch. fresh and this one resumes from its branch.
- **Publish only the work:** what the agent changed, counted from where the - **Publish only the work:** what the agent changed, counted from where the
milestone started, never build artifacts; anything left out is reported. milestone started, never build artifacts; anything left out is reported.
- **Pushes run on the build VM** with a key the model's commands never receive - **Pushes run on the build VM** with a push credential (a deploy key, or a
(L12); PRs are opened from the agent's container with its API token, which never push-only account's token) the model's commands never receive (L12); PRs are
reaches a build VM. Gitea is the only copy that has to survive. opened from the agent's container with its API token, which never reaches a
build VM. Gitea is the only copy that has to survive.
- **Publishing is idempotent** (L8). A milestone with no changes is done if it wasn't - **Publishing is idempotent** (L8). A milestone with no changes is done if it wasn't
expected to change code; otherwise it's raised with the operator. expected to change code; otherwise it's raised with the operator.
- **The agent never owns or administers a repository:** whoever does can turn - **The agent never owns or administers a repository:** whoever does can turn
@@ -727,7 +728,7 @@ execution:
**Secrets are namespaced by project.** A build VM's commands receive only their **Secrets are namespaced by project.** A build VM's commands receive only their
project's, and each secret's configuration says whether the model's commands may project's, and each secret's configuration says whether the model's commands may
see it or only harness operations use it (the push key). Anything the model's see it or only harness operations use it (the push credential). Anything the model's
commands can see must be scoped for testing, never production. commands can see must be scoped for testing, never production.
**Provisioning:** build VMs come from a template via Terraform; eventually at **Provisioning:** build VMs come from a template via Terraform; eventually at
+4 -2
View File
@@ -354,8 +354,10 @@ if the implementation differs.
- **D3 preflight:** a project's needs are read from its files (`Cargo.toml`, - **D3 preflight:** a project's needs are read from its files (`Cargo.toml`,
`pyproject.toml`, `package.json`) or a declaration in the repository. `pyproject.toml`, `package.json`) or a declaration in the repository.
- **D4 publishing:** standard excludes are configured in each clone, and only what - **D4 publishing:** standard excludes are configured in each clone, and only what
the work produced is committed. A push hands the key over stdin to a the work produced is committed. The push credential is handed to each push
short-lived `ssh-agent` on the build VM, for that one push. only: a deploy key over stdin to a short-lived `ssh-agent` on the build VM, or
a push-only account's HTTPS token as that command's secrets, read by a
credential helper that names environment variables, never values.
- **D7:** test services use ports 18000–18099, so opening access later is a - **D7:** test services use ports 18000–18099, so opening access later is a
firewall change. The inference host is behind LiteLLM, so it is not a second firewall change. The inference host is behind LiteLLM, so it is not a second
egress exception. egress exception.
+8 -1
View File
@@ -9,9 +9,15 @@
# silently doing nothing. # silently doing nothing.
state_db: ~/nisse-demo-workspace/state.db state_db: ~/nisse-demo-workspace/state.db
workspace: ~/nisse-demo-workspace/project # the clone workspace: /root/work/demo # the clone, on the build host
project: demo project: demo
build_host: # L13: every command runs here, over SSH
address: build-demo.example
key: ~/.ssh/nisse-dev # the agent's key for the build host
# user: root
# port: 22
llm: # L9: LiteLLM is the only LLM destination llm: # L9: LiteLLM is the only LLM destination
base_url: http://litellm.example:4000 base_url: http://litellm.example:4000
token: ${NISSE_LLM_TOKEN} token: ${NISSE_LLM_TOKEN}
@@ -41,6 +47,7 @@ gitea: # D4, D5
repo: owner/demo-repo repo: owner/demo-repo
operators: [your-gitea-login] operators: [your-gitea-login]
base_branch: main base_branch: main
push_key: ~/.ssh/push-demo-repo # a deploy key with write access (D4)
# What each knob does, and how to choose a value: docs/Tuning-Guide.md. # What each knob does, and how to choose a value: docs/Tuning-Guide.md.
# These are the defaults; the section can be left out entirely. The file is # These are the defaults; the section can be left out entirely. The file is
+60 -17
View File
@@ -18,9 +18,10 @@ from nisse_dev.config import AgentConfig
from nisse_dev.context import BudgetedClient from nisse_dev.context import BudgetedClient
from nisse_dev.control import Orchestrator from nisse_dev.control import Orchestrator
from nisse_dev.delegation import Caps, Delegation from nisse_dev.delegation import Caps, Delegation
from nisse_dev.execution import SSHHost
from nisse_dev.executor import LLMExecutor from nisse_dev.executor import LLMExecutor
from nisse_dev.expert import ExpertDesk from nisse_dev.expert import ExpertDesk
from nisse_dev.git import GitRepo, authenticated_url from nisse_dev.git import GitRepo, HTTPSToken, PushCredential, SSHKey
from nisse_dev.gitea import Comment, GiteaClient from nisse_dev.gitea import Comment, GiteaClient
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient
from nisse_dev.matrix import MatrixTransport from nisse_dev.matrix import MatrixTransport
@@ -164,20 +165,59 @@ def make_llm_clients(
return normalized, passthrough return normalized, passthrough
def prepare_workspace(config: AgentConfig, account: str) -> GitRepo: def build_host(config: AgentConfig) -> SSHHost:
"""Ensure the workspace is a clone of the project repo. """The tool VM the workspace lives on (L13). Its known-hosts file and
connection socket live beside the state store."""
Cloning here rather than on `use <project>` is the single-project return SSHHost(
shortcut: resolving a project name to a repo is issue #27's work. config.build_host.address,
""" key=config.build_host.key,
repo = GitRepo(config.workspace) user=config.build_host.user,
remote = authenticated_url( port=config.build_host.port,
config.gitea_url, config.gitea_repo, account, config.gitea_token state_dir=config.state_db.parent / "ssh",
) )
if not (config.workspace / ".git").exists():
config.workspace.parent.mkdir(parents=True, exist_ok=True)
GitRepo(config.workspace.parent).clone(remote, config.workspace) def push_credential(config: AgentConfig) -> PushCredential | None:
"""What pushes go by: a push-only account's token over HTTPS, or a
deploy key over SSH (D4)."""
if config.push_user and config.push_token:
return HTTPSToken(config.push_user, config.push_token)
if config.push_key is None:
return None
try:
return SSHKey(config.push_key.read_text())
except OSError as exc:
raise RuntimeError(
f"cannot read the push key {config.push_key}: {exc}"
) from exc
def remote_url(gitea: GiteaClient, repo: str, credential: PushCredential | None) -> str:
"""The credential-free URL the workspace clones from and pushes to."""
if isinstance(credential, SSHKey):
return gitea.ssh_url(repo)
return gitea.clone_url(repo)
def prepare_workspace(
host: SSHHost,
config: AgentConfig,
account: str,
remote: str,
credential: PushCredential | None,
) -> GitRepo:
"""Ensure the workspace on the build VM is a clone of the project repo.
The clone uses the repository's SSH URL, so no credential is ever stored
in it (#89); the push credential is handed to the clone and nothing else.
Cloning here rather than when a work item starts is the single-project
shortcut that M1.4 and D3's fresh workspace replace.
"""
repo = GitRepo(host, config.workspace)
if not repo.exists():
repo.clone(remote, credential=credential)
repo.configure_identity(account, f"{account}@users.noreply.local") repo.configure_identity(account, f"{account}@users.noreply.local")
repo.install_excludes() # build artifacts are never the work (#80)
return repo return repo
@@ -204,9 +244,13 @@ def build_agent(
# anything depends on it. # anything depends on it.
account = gitea.whoami() account = gitea.whoami()
git_repo = prepare_workspace(config, account) host = build_host(config)
credential = push_credential(config)
remote = remote_url(gitea, config.gitea_repo, credential)
git_repo = prepare_workspace(host, config, account, remote, credential)
tuning = config.tuning tuning = config.tuning
workspace = Workspace( workspace = Workspace(
host,
config.workspace, config.workspace,
timeout_s=tuning.tools.command_timeout_s, timeout_s=tuning.tools.command_timeout_s,
max_output_chars=tuning.tools.max_output_chars, max_output_chars=tuning.tools.max_output_chars,
@@ -249,9 +293,8 @@ def build_agent(
gitea, gitea,
config.gitea_repo, config.gitea_repo,
git_repo, git_repo,
base_url=config.gitea_url, remote_url=remote,
account=account, push_credential=credential,
token=config.gitea_token,
base_branch=config.base_branch, base_branch=config.base_branch,
) )
+10 -3
View File
@@ -1,9 +1,11 @@
"""Running one task once with one executor model, and recording it.""" """Running one task once with one executor model, and recording it."""
import io import io
import shlex
import subprocess import subprocess
import tempfile import tempfile
import time import time
import uuid
from dataclasses import asdict, dataclass, field from dataclasses import asdict, dataclass, field
from datetime import UTC, datetime from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
@@ -20,6 +22,7 @@ from nisse_dev.canonical import (
) )
from nisse_dev.catalog import ModelFacts from nisse_dev.catalog import ModelFacts
from nisse_dev.config import Tuning from nisse_dev.config import Tuning
from nisse_dev.execution import SSHHost
from nisse_dev.executor import LLMExecutor from nisse_dev.executor import LLMExecutor
from nisse_dev.llm import RoleMap from nisse_dev.llm import RoleMap
from nisse_dev.oplog import OpLog from nisse_dev.oplog import OpLog
@@ -150,6 +153,7 @@ def run_task(
client: Any, client: Any,
tuning: Tuning, tuning: Tuning,
*, *,
host: SSHHost,
note: str = "", note: str = "",
commit: str = "", commit: str = "",
workdir: Path | None = None, workdir: Path | None = None,
@@ -161,12 +165,14 @@ def run_task(
model under test, not a frontier model's success.""" model under test, not a frontier model's success."""
with tempfile.TemporaryDirectory(dir=workdir) as tmp: with tempfile.TemporaryDirectory(dir=workdir) as tmp:
root = Path(tmp) root = Path(tmp)
# The workspace is on the build VM, like the agent's own (L13).
workspace = Workspace( workspace = Workspace(
root / "work", host,
f"/tmp/nisse-bench/{uuid.uuid4().hex}",
timeout_s=tuning.tools.command_timeout_s, timeout_s=tuning.tools.command_timeout_s,
max_output_chars=tuning.tools.max_output_chars, max_output_chars=tuning.tools.max_output_chars,
) )
task.seed_into(workspace.root) task.seed_workspace(workspace)
store = StateStore(root / "state.db", oplog=OpLog("nisse-bench", io.StringIO())) store = StateStore(root / "state.db", oplog=OpLog("nisse-bench", io.StringIO()))
# One work item per run, as the agent's own tasks are (L1). # One work item per run, as the agent's own tasks are (L1).
store.open_work_item( store.open_work_item(
@@ -198,7 +204,8 @@ def run_task(
status, detail = "error", repr(exc) status, detail = "error", repr(exc)
wall = time.monotonic() - started wall = time.monotonic() - started
success, check_output = task.run_check(workspace.root) success, check_output = task.check_workspace(workspace)
host.execute(f"rm -rf -- {shlex.quote(str(workspace.root))}", timeout_s=60)
failed, invented = _count(store.transcript("bench", "executor").read()) failed, invented = _count(store.transcript("bench", "executor").read())
store.close() store.close()
+25
View File
@@ -13,13 +13,18 @@ Each task is a directory under ``tasks/``:
""" """
import hashlib import hashlib
import shlex
import shutil import shutil
import subprocess import subprocess
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING
import yaml import yaml
if TYPE_CHECKING:
from nisse_dev.tools import Workspace
TASKS_DIR = Path(__file__).parent / "tasks" TASKS_DIR = Path(__file__).parent / "tasks"
CHECK_DIR = ".bench_check" CHECK_DIR = ".bench_check"
@@ -75,6 +80,26 @@ class Task:
output = (result.stdout + result.stderr)[-2000:] output = (result.stdout + result.stderr)[-2000:]
return result.returncode == 0, output return result.returncode == 0, output
# -- on the build VM, where the agent's runs happen (L13) ---------------
def seed_workspace(self, workspace: "Workspace") -> None:
workspace.upload_tree(self.path / "seed")
def check_workspace(
self, workspace: "Workspace", timeout_s: int = 300
) -> tuple[bool, str]:
"""run_check, on the build VM: copy the hidden files in fresh (never
trusting a directory the model made) and run the check there."""
root = shlex.quote(str(workspace.root / CHECK_DIR))
workspace.host.execute(f"rm -rf -- {root}", timeout_s=60)
workspace.upload_tree(self.path / "hidden", CHECK_DIR)
result = workspace.host.execute(
self.check, cwd=str(workspace.root), timeout_s=timeout_s
)
if result.timed_out:
return False, f"check timed out after {timeout_s}s"
return result.exit_code == 0, (result.stdout + result.stderr)[-2000:]
def load_task(path: Path) -> Task: def load_task(path: Path) -> Task:
spec = yaml.safe_load((path / "task.yaml").read_text()) spec = yaml.safe_load((path / "task.yaml").read_text())
+61 -5
View File
@@ -47,8 +47,19 @@ REQUIRED_KEYS = (
"gitea.token", "gitea.token",
"gitea.repo", "gitea.repo",
"gitea.operators", "gitea.operators",
"build_host.address",
"build_host.key",
)
OPTIONAL_KEYS = (
"llm.max_output_tokens",
"llm.request_timeout_s",
"gitea.base_branch",
"gitea.push_key",
"gitea.push_user",
"gitea.push_token",
"build_host.user",
"build_host.port",
) )
OPTIONAL_KEYS = ("llm.max_output_tokens", "llm.request_timeout_s", "gitea.base_branch")
_REFERENCE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") _REFERENCE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
@@ -205,21 +216,41 @@ def _read_file(path: Path, errors: list[str]) -> dict[str, Any]:
return loaded return loaded
@dataclass(frozen=True)
class BuildHost:
"""The tool VM the workspace lives on (L13, D3). ``key`` is the path to
the agent's SSH private key for it."""
address: str
key: Path
user: str = "root"
port: int = 22
@dataclass(frozen=True) @dataclass(frozen=True)
class AgentConfig: class AgentConfig:
state_db: Path state_db: Path
workspace: Path workspace: Path
project: str project: str
llm_base_url: str llm_base_url: str
llm_token: str # Credentials never appear in a repr: a failing test or a log line
# would print them.
llm_token: str = field(repr=False)
roles: RoleMap roles: RoleMap
matrix: MatrixConfig matrix: MatrixConfig
operators: frozenset[str] operators: frozenset[str]
gitea_url: str gitea_url: str
gitea_token: str gitea_token: str = field(repr=False)
gitea_repo: str gitea_repo: str
gitea_operators: frozenset[str] gitea_operators: frozenset[str]
build_host: BuildHost
base_branch: str = "main" base_branch: str = "main"
# What may push to the project repository, handed to each push on the
# build VM and to nothing else (D4): a push-only account's HTTPS token
# (push_user, push_token), or the path to a deploy key (push_key).
push_key: Path | None = None
push_user: str | None = None
push_token: str | None = field(default=None, repr=False)
max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS
request_timeout_s: float = DEFAULT_REQUEST_TIMEOUT_S request_timeout_s: float = DEFAULT_REQUEST_TIMEOUT_S
tuning: Tuning = field(default_factory=Tuning) tuning: Tuning = field(default_factory=Tuning)
@@ -300,9 +331,23 @@ class AgentConfig:
raise ValueError raise ValueError
except (TypeError, ValueError): except (TypeError, ValueError):
errors.append("llm.request_timeout_s: must be a positive number of seconds") errors.append("llm.request_timeout_s: must be a positive number of seconds")
port = values.get("build_host.port", 22)
try:
port = int(port)
if not 0 < port < 65536:
raise ValueError
except (TypeError, ValueError):
errors.append("build_host.port: must be a TCP port number")
push_user = values.get("gitea.push_user")
push_token = values.get("gitea.push_token")
if bool(push_user) != bool(push_token):
errors.append("gitea.push_user and gitea.push_token: set both or neither")
if push_token and values.get("gitea.push_key"):
errors.append("gitea: set a push token or a push key, not both")
if errors: if errors:
raise ConfigError("cannot start: " + "; ".join(errors)) raise ConfigError("cannot start: " + "; ".join(errors))
push_key = values.get("gitea.push_key")
config = cls( config = cls(
state_db=Path(values["state_db"]).expanduser(), state_db=Path(values["state_db"]).expanduser(),
workspace=Path(values["workspace"]).expanduser(), workspace=Path(values["workspace"]).expanduser(),
@@ -326,6 +371,15 @@ class AgentConfig:
gitea_token=str(values["gitea.token"]), gitea_token=str(values["gitea.token"]),
gitea_repo=str(values["gitea.repo"]), gitea_repo=str(values["gitea.repo"]),
gitea_operators=_people(values["gitea.operators"]), gitea_operators=_people(values["gitea.operators"]),
build_host=BuildHost(
address=str(values["build_host.address"]),
key=Path(values["build_host.key"]).expanduser(),
user=str(values.get("build_host.user") or "root"),
port=port,
),
push_key=Path(push_key).expanduser() if push_key else None,
push_user=str(push_user) if push_user else None,
push_token=str(push_token) if push_token else None,
base_branch=str(values.get("gitea.base_branch") or "main"), base_branch=str(values.get("gitea.base_branch") or "main"),
max_output_tokens=max_output, max_output_tokens=max_output,
request_timeout_s=request_timeout, request_timeout_s=request_timeout,
@@ -337,7 +391,9 @@ class AgentConfig:
def _refuse_secrets_inside_the_clone(self, env: Mapping[str, str]) -> None: def _refuse_secrets_inside_the_clone(self, env: Mapping[str, str]) -> None:
"""The workspace is a git clone the agent commits with ``git add -A`` """The workspace is a git clone the agent commits with ``git add -A``
and pushes. Anything inside it reaches the project repository, so a and pushes. It lives on the build VM, so this only bites when the
build host is this machine (tests, a local bring-up), and there it
matters exactly as before. Anything inside it reaches the project repository, so a
state store or an env file placed there would publish credentials as a state store or an env file placed there would publish credentials as a
side effect of the agent working correctly — silently, and once. The side effect of the agent working correctly — silently, and once. The
config file holds no secrets itself, but it names where they come config file holds no secrets itself, but it names where they come
@@ -367,7 +423,7 @@ class AgentConfig:
"""What the agent may safely announce about how it is configured.""" """What the agent may safely announce about how it is configured."""
return { return {
"project": self.project, "project": self.project,
"workspace": str(self.workspace), "workspace": f"{self.build_host.address}:{self.workspace}",
"repo": self.gitea_repo, "repo": self.gitea_repo,
"planner": self.roles.planner, "planner": self.roles.planner,
"executor": self.roles.executor, "executor": self.roles.executor,
+207
View File
@@ -0,0 +1,207 @@
"""Running commands on a tool VM over SSH (architecture L13).
``SSHHost.execute`` is the only way the agent runs anything: there is no
local execution path, so the host the agent lives on never runs a command a
model chose. The harness binds the host; a model never names one.
How a command travels. The remote side runs ``bash -s`` and reads a small
script from SSH's stdin: the secrets handed to this one call as ``export``
lines, a ``cd``, then the command under ``timeout``. Nothing secret is ever on
a command line (``ps`` and ``/proc/*/cmdline`` would show it); the values exist
only in that shell and the command's environment. The command's own stdin is
``/dev/null``, or a here-document carrying ``stdin``, so it can never read the
rest of the script.
Every value handed out is redacted from what comes back (L12): a test that
prints its environment must not put a key into the transcript.
The connection is reused across calls (OpenSSH's ControlMaster), so a tool
call costs one round trip, not a handshake.
"""
import os
import secrets as _random
import shlex
import subprocess
import tempfile
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from pathlib import Path
# The remote command's exit code is passed back as is, except that 255 is
# what ssh itself uses for "could not connect"; a command exiting 255 is
# reported as 254 so the two can't be confused.
SSH_FAILED = 255
_TIMED_OUT = 124 # coreutils `timeout`
_KILL_GRACE_S = 5
# The local backstop, beyond the remote timeout, for a connection that hangs.
_LOCAL_SLACK_S = 30
REDACTED = "[redacted]"
class HostUnavailable(RuntimeError):
"""The host could not be reached: not something the command did."""
@dataclass(frozen=True)
class Result:
exit_code: int
stdout: str
stderr: str
timed_out: bool = False
@property
def ok(self) -> bool:
return self.exit_code == 0 and not self.timed_out
def redact(text: str, values: Iterable[str]) -> str:
"""Mask every non-trivial secret value in ``text``. Longest first, so a
value containing another is masked whole."""
for value in sorted((v for v in values if len(v) >= 4), key=len, reverse=True):
text = text.replace(value, REDACTED)
return text
def _heredoc(body: str) -> tuple[str, str]:
"""A here-document redirection whose delimiter can't occur in ``body``."""
while True:
tag = f"NISSE_EOF_{_random.token_hex(8)}"
if tag not in body:
return f"<<'{tag}'", f"\n{body}\n{tag}"
def _script(
command: str,
cwd: str | None,
secrets: Mapping[str, str],
stdin: str | None,
timeout_s: int,
) -> str:
lines = [f"export {name}={shlex.quote(value)}" for name, value in secrets.items()]
if cwd is not None:
lines.append(f"cd -- {shlex.quote(cwd)} || exit 125")
redirect, body = ("< /dev/null", "") if stdin is None else _heredoc(stdin)
# `timeout` runs the command in its own process group and signals the
# whole group, so a backgrounded grandchild doesn't outlive it.
lines.append(
f"timeout --kill-after={_KILL_GRACE_S} {timeout_s} "
f"bash -c {shlex.quote(command)} {redirect}{body}"
)
lines.append("rc=$?; [ $rc -eq 255 ] && rc=254; exit $rc")
return "\n".join(lines) + "\n"
class SSHHost:
"""One tool VM, reached as ``user@address`` with a private key."""
def __init__(
self,
address: str,
*,
key: Path,
state_dir: Path,
user: str = "root",
port: int = 22,
) -> None:
self.address = address
self.user = user
self.port = port
self._key = Path(key)
state_dir = Path(state_dir)
state_dir.mkdir(parents=True, exist_ok=True)
self._known_hosts = state_dir / "known_hosts"
# A Unix socket's path is limited to about 100 bytes, so the shared
# connection's socket lives in a short private directory rather than
# beside the state, named by %C, ssh's hash of the connection.
sockets = Path(tempfile.gettempdir()) / f"nisse-ssh-{os.getuid()}"
sockets.mkdir(mode=0o700, exist_ok=True)
self._control = sockets / "%C"
def describe(self) -> str:
return f"{self.user}@{self.address}"
def _ssh(self) -> list[str]:
return [
"ssh",
"-i",
str(self._key),
"-p",
str(self.port),
"-o",
"BatchMode=yes",
"-o",
"IdentitiesOnly=yes",
"-o",
f"UserKnownHostsFile={self._known_hosts}",
"-o",
"StrictHostKeyChecking=accept-new",
"-o",
"ControlMaster=auto",
"-o",
f"ControlPath={self._control}",
"-o",
"ControlPersist=120",
"-o",
"ConnectTimeout=15",
"-o",
"ServerAliveInterval=15",
"-o",
"LogLevel=ERROR",
f"{self.user}@{self.address}",
"bash -s",
]
def execute(
self,
command: str,
*,
cwd: str | None = None,
secrets: Mapping[str, str] | None = None,
stdin: str | None = None,
timeout_s: int = 120,
) -> Result:
"""Run ``command`` with bash in ``cwd``. ``secrets`` become environment
variables of this one command and are redacted from its output.
Raises HostUnavailable when the host can't be reached."""
handed_out = dict(secrets or {})
script = _script(command, cwd, handed_out, stdin, timeout_s)
try:
completed = subprocess.run(
self._ssh(),
input=script.encode(),
capture_output=True,
timeout=timeout_s + _KILL_GRACE_S + _LOCAL_SLACK_S,
check=False,
)
except subprocess.TimeoutExpired as exc:
raise HostUnavailable(
f"{self.describe()} stopped responding during a command"
) from exc
except OSError as exc:
raise HostUnavailable(f"could not run ssh: {exc}") from exc
stdout = completed.stdout.decode(errors="replace")
stderr = completed.stderr.decode(errors="replace")
if completed.returncode == SSH_FAILED:
raise HostUnavailable(
f"could not reach {self.describe()}: "
+ redact(stderr.strip() or "ssh failed", handed_out.values())
)
values = handed_out.values()
return Result(
exit_code=completed.returncode,
stdout=redact(stdout, values),
stderr=redact(stderr, values),
timed_out=completed.returncode in (_TIMED_OUT, 137),
)
def close(self) -> None:
"""End the shared connection, if one is open."""
args = self._ssh()[:-2] # without the destination and the command
subprocess.run(
[*args, "-O", "exit", f"{self.user}@{self.address}"],
capture_output=True,
timeout=10,
check=False,
)
+1 -1
View File
@@ -134,7 +134,7 @@ class LLMExecutor:
result = self._delegate.run( result = self._delegate.run(
transcript.as_actor("helper"), transcript.as_actor("helper"),
brief=brief, brief=brief,
branch=f"delegation/milestone-{milestone}-{failures}", branch=f"delegation-milestone-{milestone}-{failures}",
) )
if result.usable: if result.usable:
return None return None
+212 -59
View File
@@ -1,111 +1,272 @@
"""Git operations for the agent's workspace (architecture D4). """Git operations in the work item's workspace, on its build VM (D4, L13).
Push authentication uses the Gitea service-account token embedded in the Every command runs on the build VM over SSH. Remotes are plain URLs with no
remote URL. The token is never written to disk in the repo config and is credential in them, so nothing secret is ever written to the clone's
scrubbed from anything this module raises or returns, so a failed push cannot configuration (#89). Operations that reach the remote (clone, fetch, push,
leak the credential into a transcript, a log line, or a PR comment. ls-remote) are handed the push credential for that one operation, and the
model's commands never receive it:
- an HTTPS token (the default: HTTPS reaches the forge from everywhere) goes
to git through a credential helper that names environment variables, never
values; the values are that one command's secrets, so they are never on a
command line and are redacted from its output;
- an SSH key is fed over stdin to a short-lived ``ssh-agent`` on the VM, gone
when the command ends.
Pull requests are opened from the agent's own host with its Gitea API token
(publish.py), which never reaches the VM.
""" """
import subprocess import shlex
from dataclasses import dataclass from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from nisse_dev.execution import HostUnavailable, SSHHost
TIMEOUT_S = 120 TIMEOUT_S = 120
# ssh for git on the VM: no prompts, and trust Gitea's host key on first use.
_GIT_SSH = "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new"
class GitError(RuntimeError): class GitError(RuntimeError):
pass pass
# Build artifacts and caches that are never the work (D4, #80). Kept out of
# every commit the harness makes, tracked or not, and excluded in the clone
# so they don't even show as changes. Conservative on purpose: a pattern here
# must never match a project's real source.
ARTIFACT_PATTERNS = (
"__pycache__/",
"*.py[cod]",
".pytest_cache/",
".mypy_cache/",
".ruff_cache/",
".tox/",
".nox/",
"*.egg-info/",
".coverage",
"htmlcov/",
".venv/",
"node_modules/",
".DS_Store",
)
_EXCLUDES_MARK = "# nisse-dev: build artifacts are never the work (D4)"
def _pathspec(pattern: str) -> str:
"""A pathspec leaving ``pattern`` out wherever it occurs in the tree."""
glob = pattern.rstrip("/")
tail = "/**" if pattern.endswith("/") else ""
return f":(exclude,glob)**/{glob}{tail}"
@dataclass(frozen=True)
class HTTPSToken:
"""A push-only account's token, for HTTPS remotes (D4)."""
username: str
# Never in a repr: a failing test or a log line would print it.
token: str = field(repr=False)
@dataclass(frozen=True)
class SSHKey:
"""A deploy key's private half, for SSH remotes (D4)."""
private_key: str = field(repr=False)
PushCredential = HTTPSToken | SSHKey
# git asks the helper for credentials; it answers from the environment, so
# the command line names variables, never values.
_HELPER = (
'!f() { test "$1" = get || return 0; '
'echo "username=$NISSE_PUSH_USER"; echo "password=$NISSE_PUSH_TOKEN"; }; f'
)
def _with_key(command: str, known_hosts: str | None) -> str:
"""Run ``command`` with the key on stdin loaded into a throwaway agent."""
git_ssh = _GIT_SSH
if known_hosts is not None:
git_ssh += f" -o UserKnownHostsFile={shlex.quote(known_hosts)}"
return (
'eval "$(ssh-agent -s)" > /dev/null && '
"ssh-add -q - 2> /dev/null && "
f"GIT_SSH_COMMAND={shlex.quote(git_ssh)} {command}; "
"rc=$?; ssh-agent -k > /dev/null 2>&1; exit $rc"
)
@dataclass(frozen=True) @dataclass(frozen=True)
class GitRepo: class GitRepo:
host: SSHHost
root: Path root: Path
# Where git's ssh on the VM records the forge's host key; the default is
# the VM user's own known_hosts.
known_hosts: str | None = None
def _run(self, *args: str, secret: str | None = None) -> str: def _git(
try: self,
completed = subprocess.run( *args: str,
["git", *args], credential: PushCredential | None = None,
cwd=self.root, cwd: str | None = None,
capture_output=True, raw: bool = False,
text=True, ) -> str:
timeout=TIMEOUT_S, quoted = " ".join(shlex.quote(a) for a in args)
check=False, command, stdin, secrets = f"git {quoted}", None, {}
if isinstance(credential, SSHKey):
command = _with_key(command, self.known_hosts)
stdin = credential.private_key
elif isinstance(credential, HTTPSToken):
# The empty helper first clears any configured on the VM.
command = (
"GIT_TERMINAL_PROMPT=0 git -c credential.helper= "
f"-c credential.helper={shlex.quote(_HELPER)} {quoted}"
) )
except (OSError, subprocess.TimeoutExpired) as exc: secrets = {
raise GitError(f"git {args[0]} failed: {_scrub(str(exc), secret)}") from exc "NISSE_PUSH_USER": credential.username,
if completed.returncode != 0: "NISSE_PUSH_TOKEN": credential.token,
}
try:
result = self.host.execute(
command,
cwd=cwd or str(self.root),
stdin=stdin,
secrets=secrets,
timeout_s=TIMEOUT_S,
)
except HostUnavailable as exc:
raise GitError(f"git {args[0]} failed: {exc}") from exc
if result.timed_out:
raise GitError(f"git {args[0]} timed out after {TIMEOUT_S}s")
if result.exit_code != 0:
raise GitError( raise GitError(
f"git {args[0]} failed: " f"git {args[0]} failed: "
f"{_scrub(completed.stderr.strip() or completed.stdout.strip(), secret)}" f"{result.stderr.strip() or result.stdout.strip()}"
) )
return completed.stdout.strip() # Raw for output whose leading whitespace means something (porcelain).
return result.stdout if raw else result.stdout.strip()
# -- inspection ------------------------------------------------------- # -- inspection -------------------------------------------------------
def current_branch(self) -> str: def current_branch(self) -> str:
# --show-current rather than rev-parse: it answers correctly on a # --show-current rather than rev-parse: it answers correctly on a
# freshly initialised repo, where HEAD points at an unborn branch. # freshly initialised repo, where HEAD points at an unborn branch.
return self._run("branch", "--show-current") return self._git("branch", "--show-current")
def head_sha(self) -> str: def head_sha(self) -> str:
return self._run("rev-parse", "HEAD") return self._git("rev-parse", "HEAD")
def has_changes(self) -> bool: def has_changes(self) -> bool:
return bool(self._run("status", "--porcelain")) return bool(self._git("status", "--porcelain"))
def commits_since(self, ref: str) -> int: def commits_since(self, ref: str) -> int:
"""How many commits HEAD has that ref lacks. The executor may commit """How many commits HEAD has that ref lacks. The executor may commit
its own work, so a clean tree does not mean nothing to publish.""" its own work, so a clean tree does not mean nothing to publish."""
return int(self._run("rev-list", "--count", f"{ref}..HEAD")) return int(self._git("rev-list", "--count", f"{ref}..HEAD"))
def remote_sha( def remote_sha(
self, remote_url: str, branch: str, *, token: str | None = None self, remote_url: str, branch: str, *, credential: PushCredential | None = None
) -> str | None: ) -> str | None:
"""What the remote's branch points at, or None if it has no such """What the remote's branch points at, or None if it has no such
branch. Pushes go by URL, so there is no tracking ref to read.""" branch. Pushes go by URL, so there is no tracking ref to read."""
out = self._run("ls-remote", remote_url, f"refs/heads/{branch}", secret=token) out = self._git(
"ls-remote", remote_url, f"refs/heads/{branch}", credential=credential
)
return out.split()[0] if out else None return out.split()[0] if out else None
def exists(self) -> bool:
"""Whether the workspace is already a git repository."""
try:
result = self.host.execute(
f"test -d {shlex.quote(str(self.root / '.git'))}", timeout_s=TIMEOUT_S
)
except HostUnavailable as exc:
raise GitError(f"could not check the workspace: {exc}") from exc
return result.exit_code == 0
# -- mutation --------------------------------------------------------- # -- mutation ---------------------------------------------------------
def init(self, *, default_branch: str = "main") -> None: def init(self, *, default_branch: str = "main") -> None:
self.root.mkdir(parents=True, exist_ok=True) self.host.execute(f"mkdir -p -- {shlex.quote(str(self.root))}")
self._run("init", "-q", "-b", default_branch) self._git("init", "-q", "-b", default_branch)
def clone(self, remote_url: str, dest: Path, *, token: str | None = None) -> None: def clone(
self.root.mkdir(parents=True, exist_ok=True) self, remote_url: str, *, credential: PushCredential | None = None
self._run("clone", "-q", remote_url, str(dest), secret=token) ) -> None:
"""Clone into the workspace root. The remote URL carries no
credential, so none is stored in the clone's configuration."""
parent = str(self.root.parent)
self.host.execute(f"mkdir -p -- {shlex.quote(parent)}")
self._git(
"clone", "-q", remote_url, str(self.root), credential=credential, cwd=parent
)
def configure_identity(self, name: str, email: str) -> None: def configure_identity(self, name: str, email: str) -> None:
self._run("config", "user.name", name) self._git("config", "user.name", name)
self._run("config", "user.email", email) self._git("config", "user.email", email)
def checkout_new_branch(self, name: str) -> None: def checkout_new_branch(self, name: str) -> None:
self._run("checkout", "-q", "-b", name) self._git("checkout", "-q", "-b", name)
def checkout_existing_branch(self, name: str) -> None: def checkout_existing_branch(self, name: str) -> None:
self._run("checkout", "-q", name) self._git("checkout", "-q", name)
def branch_exists(self, name: str) -> bool: def branch_exists(self, name: str) -> bool:
try: try:
self._run("rev-parse", "--verify", "--quiet", f"refs/heads/{name}") self._git("rev-parse", "--verify", "--quiet", f"refs/heads/{name}")
except GitError: except GitError:
return False return False
return True return True
def install_excludes(self) -> None:
"""List the build artifacts in the clone's own exclude file, so they
never appear as changes. Local to the clone: nothing in the project
changes. Idempotent."""
path = shlex.quote(str(self.root / ".git" / "info" / "exclude"))
block = "\n".join([_EXCLUDES_MARK, *ARTIFACT_PATTERNS])
result = self.host.execute(
f'mkdir -p -- "$(dirname -- {path})" && touch -- {path} && '
f"grep -qxF {shlex.quote(_EXCLUDES_MARK)} {path} || "
f"cat >> {path}",
stdin=block,
timeout_s=TIMEOUT_S,
)
if not result.ok:
raise GitError(f"could not set excludes: {result.stderr.strip()}")
def commit_all(self, message: str) -> str | None: def commit_all(self, message: str) -> str | None:
"""Returns the new commit sha, or None when there was nothing to """Commit the work: every change except build artifacts, which are
commit — an empty milestone is a fact to report, not an error.""" left out even when the project tracks them (#80). Returns the new
commit sha, or None when there was nothing to commit — an empty
milestone is a fact to report, not an error."""
if not self.has_changes(): if not self.has_changes():
return None return None
self._run("add", "-A") self._git("add", "-A", "--", ".", *(_pathspec(p) for p in ARTIFACT_PATTERNS))
self._run("commit", "-q", "-m", message) staged = self._git("diff", "--cached", "--name-only")
if not staged:
return None
self._git("commit", "-q", "-m", message)
return self.head_sha() return self.head_sha()
def push(self, remote_url: str, branch: str, *, token: str | None = None) -> None: def left_out(self) -> list[str]:
# The URL is passed per-invocation rather than stored as a remote, so """Changes not in the last commit: build artifacts the harness left
# the token never lands in .git/config. out, to be reported rather than silently dropped (D4)."""
self._run("push", "-q", remote_url, f"HEAD:refs/heads/{branch}", secret=token) out = self._git("status", "--porcelain", "--untracked-files=all", raw=True)
return sorted(line[3:] for line in out.splitlines() if line)
def push(
self, remote_url: str, branch: str, *, credential: PushCredential | None = None
) -> None:
# By URL, per invocation, rather than a configured remote with a
# credential in it.
self._git(
"push", "-q", remote_url, f"HEAD:refs/heads/{branch}", credential=credential
)
def discard_branch(self, branch: str, *, back_to: str) -> None: def discard_branch(self, branch: str, *, back_to: str) -> None:
"""Throw away a branch and everything on it. """Throw away a branch and everything on it.
@@ -114,20 +275,12 @@ class GitRepo:
atomicity that reconciliation gives an interrupted transition, and atomicity that reconciliation gives an interrupted transition, and
half-finished work must not survive to be built upon. half-finished work must not survive to be built upon.
""" """
self._run("checkout", "-q", "--force", back_to) self._git("checkout", "-q", "--force", back_to)
self._run("branch", "-q", "-D", branch) self._git("branch", "-q", "-D", branch)
def delete_remote_branch( def delete_remote_branch(
self, remote_url: str, branch: str, *, token: str | None = None self, remote_url: str, branch: str, *, credential: PushCredential | None = None
) -> None: ) -> None:
self._run("push", "-q", remote_url, f":refs/heads/{branch}", secret=token) self._git(
"push", "-q", remote_url, f":refs/heads/{branch}", credential=credential
)
def authenticated_url(base_url: str, repo: str, user: str, token: str) -> str:
host = base_url.split("://", 1)[-1].rstrip("/")
scheme = base_url.split("://", 1)[0] if "://" in base_url else "https"
return f"{scheme}://{user}:{token}@{host}/{repo}.git"
def _scrub(text: str, secret: str | None) -> str:
return text.replace(secret, "***") if secret else text
+8
View File
@@ -70,6 +70,14 @@ class GiteaClient:
) )
return response.json() if response.content else None return response.json() if response.content else None
def ssh_url(self, repo: str) -> str:
"""The repository's SSH clone URL, which carries no credential."""
return str(self._request("GET", f"/repos/{repo}")["ssh_url"])
def clone_url(self, repo: str) -> str:
"""The repository's HTTPS clone URL, which carries no credential."""
return str(self._request("GET", f"/repos/{repo}")["clone_url"])
def whoami(self) -> str: def whoami(self) -> str:
return str(self._request("GET", "/user")["login"]) return str(self._request("GET", "/user")["login"])
+2 -2
View File
@@ -16,7 +16,7 @@ import re
import threading import threading
import uuid import uuid
from collections.abc import Callable from collections.abc import Callable
from dataclasses import dataclass from dataclasses import dataclass, field
from urllib.parse import unquote from urllib.parse import unquote
import httpx import httpx
@@ -40,7 +40,7 @@ class TransportError(RuntimeError):
class MatrixConfig: class MatrixConfig:
homeserver: str homeserver: str
user_id: str user_id: str
token: str token: str = field(repr=False) # never in a repr or a log line
room_id: str room_id: str
+48 -15
View File
@@ -7,9 +7,10 @@ PR exists rather than repeating the push-and-open — the spec's own example of
an idempotent transition reconciled against external truth. an idempotent transition reconciled against external truth.
""" """
from collections.abc import Sequence
from dataclasses import dataclass from dataclasses import dataclass
from nisse_dev.git import GitRepo, authenticated_url from nisse_dev.git import GitRepo, PushCredential
from nisse_dev.gitea import ( from nisse_dev.gitea import (
GiteaClient, GiteaClient,
GiteaError, GiteaError,
@@ -36,9 +37,8 @@ class MilestonePublisher:
repo: str, repo: str,
git_repo: GitRepo, git_repo: GitRepo,
*, *,
base_url: str, remote_url: str,
account: str, push_credential: PushCredential | None = None,
token: str,
base_branch: str = "main", base_branch: str = "main",
) -> None: ) -> None:
self._store = store self._store = store
@@ -46,16 +46,20 @@ class MilestonePublisher:
self._repo = repo self._repo = repo
self._git = git_repo self._git = git_repo
self._base_branch = base_branch self._base_branch = base_branch
self._remote = authenticated_url(base_url, repo, account, token) # A URL with no credential in it; the push credential is handed to
self._token = token # each push on the build VM and nothing else (D4, L12).
self._remote = remote_url
self._credential = push_credential
def reconcilers(self) -> dict[str, Reconciler]: def reconcilers(self) -> dict[str, Reconciler]:
return {OPEN_PR_INTENT: pull_request_reconciler(self._client, self._repo)} return {OPEN_PR_INTENT: pull_request_reconciler(self._client, self._repo)}
def branch_for(self, task_id: str, milestone: int) -> str: def branch_for(self, task_id: str, milestone: int) -> str:
# Namespaced by task so re-running a project against the same repo # Namespaced by task so re-running a project against the same repo
# cannot collide with an earlier run's branch. # cannot collide with an earlier run's branch. Flat, with no "/":
return f"{task_id}/milestone-{milestone}" # git stores branch names as paths, so a branch named after the task
# (which a model did make, live) would block "<task>/milestone-1".
return f"{task_id}-milestone-{milestone}"
def publish( def publish(
self, self,
@@ -77,6 +81,8 @@ class MilestonePublisher:
return PublishResult(None, "no changes to publish") return PublishResult(None, "no changes to publish")
# The executor committed its work itself; publish that. # The executor committed its work itself; publish that.
sha = self._git.head_sha() sha = self._git.head_sha()
# D4: what was left out (build artifacts) is reported, not dropped.
left_out = self._git.left_out()
intent_id = self._store.begin_intent( intent_id = self._store.begin_intent(
OPEN_PR_INTENT, OPEN_PR_INTENT,
@@ -84,7 +90,7 @@ class MilestonePublisher:
cause=f"publishing milestone {milestone}", cause=f"publishing milestone {milestone}",
work_item=task_id, # owned by the work item it publishes (L1) work_item=task_id, # owned by the work item it publishes (L1)
) )
self._git.push(self._remote, branch, token=self._token) self._git.push(self._remote, branch, credential=self._credential)
# Idempotent: if a PR for this branch is already open (a crash after # Idempotent: if a PR for this branch is already open (a crash after
# opening it, or a response lost on the way back), the push above # opening it, or a response lost on the way back), the push above
# has updated it; adopt it rather than fail on a second. # has updated it; adopt it rather than fail on a second.
@@ -96,17 +102,19 @@ class MilestonePublisher:
return PublishResult( return PublishResult(
existing, existing,
f"PR #{existing.number} was already open for this milestone; " f"PR #{existing.number} was already open for this milestone; "
f"pushed to it: {existing.url}", f"pushed to it: {existing.url}" + _left_out_note(left_out),
) )
pr = self._client.open_pull_request( pr = self._client.open_pull_request(
self._repo, self._repo,
head=branch, head=branch,
base=self._base_branch, base=self._base_branch,
title=f"Milestone {milestone}: {project}", title=f"Milestone {milestone}: {project}",
body=_pr_body(milestone, plan, service_url), body=_pr_body(milestone, plan, service_url, left_out),
) )
self._store.commit_intent(intent_id, cause=f"opened PR #{pr.number}") self._store.commit_intent(intent_id, cause=f"opened PR #{pr.number}")
return PublishResult(pr, f"opened PR #{pr.number}: {pr.url}") return PublishResult(
pr, f"opened PR #{pr.number}: {pr.url}" + _left_out_note(left_out)
)
def published_pull_request( def published_pull_request(
self, task_id: str, milestone: int self, task_id: str, milestone: int
@@ -149,14 +157,30 @@ class MilestonePublisher:
if sha is None: if sha is None:
# The executor may have committed the fix itself. # The executor may have committed the fix itself.
sha = self._git.head_sha() sha = self._git.head_sha()
pushed = self._git.remote_sha(self._remote, branch, token=self._token) pushed = self._git.remote_sha(
self._remote, branch, credential=self._credential
)
if sha == pushed: if sha == pushed:
return None return None
self._git.push(self._remote, branch, token=self._token) self._git.push(self._remote, branch, credential=self._credential)
return sha return sha
def _pr_body(milestone: int, plan: str, service_url: str | None) -> str: def _left_out_note(paths: list[str]) -> str:
if not paths:
return ""
shown = ", ".join(paths[:5]) + (
f" and {len(paths) - 5} more" if len(paths) > 5 else ""
)
return f" (left out {len(paths)} build artifact(s): {shown})"
def _pr_body(
milestone: int,
plan: str,
service_url: str | None,
left_out: Sequence[str] = (),
) -> str:
parts = [ parts = [
f"Milestone {milestone}, implemented by nisse-dev against the approved plan.", f"Milestone {milestone}, implemented by nisse-dev against the approved plan.",
"", "",
@@ -169,4 +193,13 @@ def _pr_body(milestone: int, plan: str, service_url: str | None) -> str:
# so a reviewer can exercise it. Nothing reports one yet; the browser # so a reviewer can exercise it. Nothing reports one yet; the browser
# tool (D7) is what will. # tool (D7) is what will.
parts += ["", "## Running test service", "", service_url] parts += ["", "## Running test service", "", service_url]
if left_out:
parts += [
"",
"## Left out",
"",
"Build artifacts in the workspace, never part of the work:",
"",
*(f"- `{path}`" for path in left_out),
]
return "\n".join(parts) return "\n".join(parts)
+101
View File
@@ -0,0 +1,101 @@
"""Test support: a throwaway sshd standing in for a tool VM (L13).
Runs the system's sshd as the current user, on a free local port, with a
host key and client key made for the occasion. Tests then reach "the build
VM" over real SSH, so the code path under test is the production path.
"""
import getpass
import os
import socket
import subprocess
import time
from collections.abc import Iterator
from contextlib import contextmanager
from pathlib import Path
from nisse_dev.execution import SSHHost
SSHD = "/usr/sbin/sshd"
def _free_port() -> int:
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
return int(sock.getsockname()[1])
def _keygen(path: Path) -> None:
subprocess.run(
["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(path)], check=True
)
@contextmanager
def local_sshd(root: Path) -> Iterator[SSHHost]:
"""An SSHHost for a private sshd on 127.0.0.1, torn down afterwards."""
root.mkdir(parents=True, exist_ok=True)
host_key, client_key = root / "host_key", root / "client_key"
_keygen(host_key)
_keygen(client_key)
authorized = root / "authorized_keys"
authorized.write_text(client_key.with_suffix(".pub").read_text())
authorized.chmod(0o600)
port = _free_port()
config = root / "sshd_config"
config.write_text(
"\n".join(
[
f"Port {port}",
"ListenAddress 127.0.0.1",
f"HostKey {host_key}",
f"AuthorizedKeysFile {authorized}",
f"PidFile {root / 'sshd.pid'}",
"PasswordAuthentication no",
"KbdInteractiveAuthentication no",
"PubkeyAuthentication yes",
"PermitRootLogin prohibit-password",
"StrictModes no",
"UsePAM no",
"MaxSessions 50",
"MaxStartups 50",
# Stands in for a provisioned build VM: sessions see the test
# runner's PATH, so its Python and tools are "installed".
f"SetEnv PATH={os.environ.get('PATH', '/usr/bin:/bin')}",
"",
]
)
)
process = subprocess.Popen(
[SSHD, "-D", "-e", "-f", str(config)],
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
)
user = getpass.getuser()
host = SSHHost(
"127.0.0.1", key=client_key, state_dir=root / "client", user=user, port=port
)
try:
deadline = time.monotonic() + 10
while True:
if process.poll() is not None:
raise RuntimeError(
"sshd exited: " + (process.stderr.read() or b"").decode()
if process.stderr
else "sshd exited"
)
try:
with socket.create_connection(("127.0.0.1", port), timeout=0.5):
break
except OSError:
if time.monotonic() > deadline:
raise RuntimeError("sshd did not start") from None
time.sleep(0.05)
yield host
finally:
host.close()
process.terminate()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
+102 -77
View File
@@ -1,26 +1,27 @@
"""Executor tools (architecture L9: reduced schema, flat arguments). """Executor tools (architecture L9: reduced schema, flat arguments; L13).
Deliberately unsandboxed. The VM is the trust boundary (D7) and tool use is Every tool runs on the work item's build VM over SSH (execution.py); the
auto-approved inside an approved unit of work (L3), so per-command agent's own host never runs a command a model chose. Within the VM nothing is
confinement would add failure modes without adding safety. The workspace is sandboxed: the VM is the trust boundary (D7), so per-command confinement would
where work happens — the default for relative paths and the command cwd — not add failure modes without adding safety. The workspace is where work happens,
a wall. Protecting agent state from the agent is a mount-layout concern the default for relative paths and the command's directory, not a wall.
(L8/D7), not this module's job.
What *is* enforced here is operational: commands cannot hang the loop, cannot What *is* enforced here is operational: commands cannot hang the loop and
consume the context window with megabytes of build log, and do not inherit cannot flood the context window with megabytes of build log. The agent's own
the agent's own credentials. credentials are never on the build VM to inherit.
""" """
import os import base64
import shutil import io
import signal import shlex
import subprocess import tarfile
from collections.abc import Mapping
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from nisse_dev.context import head_and_tail from nisse_dev.context import head_and_tail
from nisse_dev.execution import HostUnavailable, Result, SSHHost
# Tools that change or verify something. Reading a file or listing a # Tools that change or verify something. Reading a file or listing a
# directory is a model looking around, not doing the work, so neither counts # directory is a model looking around, not doing the work, so neither counts
@@ -88,12 +89,6 @@ EXECUTOR_TOOLS: list[dict[str, Any]] = [
] ]
def _child_env() -> dict[str, str]:
"""The project's build and test code is arbitrary; it has no business
reading the agent's LiteLLM, Matrix, or Gitea credentials."""
return {k: v for k, v in os.environ.items() if not k.startswith("NISSE_")}
@dataclass(frozen=True) @dataclass(frozen=True)
class ToolOutcome: class ToolOutcome:
result: str result: str
@@ -101,30 +96,53 @@ class ToolOutcome:
class Workspace: class Workspace:
"""The work item's workspace: a directory on its tool VM (L13, D3).
Every operation runs there over SSH; nothing touches the agent's own
host. ``root`` is the path on that host.
"""
def __init__( def __init__(
self, self,
root: Path, host: SSHHost,
root: Path | str,
*, *,
timeout_s: int = DEFAULT_TIMEOUT_S, timeout_s: int = DEFAULT_TIMEOUT_S,
max_output_chars: int = MAX_OUTPUT_CHARS, max_output_chars: int = MAX_OUTPUT_CHARS,
secrets: Mapping[str, str] | None = None,
) -> None: ) -> None:
self.host = host
self.root = Path(root) self.root = Path(root)
self.root.mkdir(parents=True, exist_ok=True)
self._timeout_s = timeout_s self._timeout_s = timeout_s
self._max_output_chars = max_output_chars self._max_output_chars = max_output_chars
# The project secrets the model's commands may see (D7); redacted from
# whatever comes back.
self._secrets = dict(secrets or {})
self._run(f"mkdir -p -- {shlex.quote(str(self.root))}")
def _run(
self, command: str, *, stdin: str | None = None, cwd: str | None = None
) -> Result:
return self.host.execute(
command, cwd=cwd, stdin=stdin, timeout_s=self._timeout_s
)
def _truncate(self, text: str) -> str: def _truncate(self, text: str) -> str:
return head_and_tail(text, self._max_output_chars) return head_and_tail(text, self._max_output_chars)
def _resolve(self, path: str) -> Path: def _resolve(self, path: str) -> str:
candidate = Path(path) candidate = Path(path)
return candidate if candidate.is_absolute() else self.root / candidate return str(candidate if candidate.is_absolute() else self.root / candidate)
def _inside(self, target: Path) -> bool: def _inside(self, target: str) -> bool:
try: result = self._run(
return target.resolve().is_relative_to(self.root.resolve()) f"realpath -m -- {shlex.quote(target)} {shlex.quote(str(self.root))}"
except OSError: )
lines = result.stdout.splitlines()
if not result.ok or len(lines) != 2:
return False return False
resolved, root = (Path(line) for line in lines)
return resolved == root or root in resolved.parents
def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome: def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome:
try: try:
@@ -143,13 +161,24 @@ class Workspace:
return ToolOutcome(f"no such tool: {tool}", success=False) return ToolOutcome(f"no such tool: {tool}", success=False)
except KeyError as exc: except KeyError as exc:
return ToolOutcome(f"missing argument {exc} for {tool}", success=False) return ToolOutcome(f"missing argument {exc} for {tool}", success=False)
except HostUnavailable as exc:
# Not the work failing: the build VM is out of reach.
return ToolOutcome(f"the build VM is unreachable: {exc}", success=False)
def read_file(self, path: str) -> ToolOutcome: def read_file(self, path: str) -> ToolOutcome:
target = self._resolve(path) target = self._resolve(path)
# base64 on the wire, so the bytes arrive intact and decoding is
# strict here, as it was for a local file.
result = self._run(f"base64 -w0 -- {shlex.quote(target)}")
if not result.ok:
return ToolOutcome(
f"could not read {path}: {result.stderr.strip()}", success=False
)
try: try:
return ToolOutcome(self._truncate(target.read_text()), success=True) text = base64.b64decode(result.stdout).decode()
except (OSError, UnicodeDecodeError) as exc: except (ValueError, UnicodeDecodeError) as exc:
return ToolOutcome(f"could not read {path}: {exc}", success=False) return ToolOutcome(f"could not read {path}: {exc}", success=False)
return ToolOutcome(self._truncate(text), success=True)
def write_file(self, path: str, content: str) -> ToolOutcome: def write_file(self, path: str, content: str) -> ToolOutcome:
target = self._resolve(path) target = self._resolve(path)
@@ -165,21 +194,25 @@ class Workspace:
"relative paths.", "relative paths.",
success=False, success=False,
) )
try: quoted = shlex.quote(target)
target.parent.mkdir(parents=True, exist_ok=True) result = self._run(
target.write_text(content) f'mkdir -p -- "$(dirname -- {quoted})" && base64 -d > {quoted}',
except OSError as exc: stdin=base64.b64encode(content.encode()).decode(),
return ToolOutcome(f"could not write {path}: {exc}", success=False) )
if not result.ok:
return ToolOutcome(
f"could not write {path}: {result.stderr.strip()}", success=False
)
return ToolOutcome(f"wrote {len(content)} characters to {path}", success=True) return ToolOutcome(f"wrote {len(content)} characters to {path}", success=True)
def list_dir(self, path: str) -> ToolOutcome: def list_dir(self, path: str) -> ToolOutcome:
target = self._resolve(path) target = self._resolve(path)
try: result = self._run(f"ls -1Ap -- {shlex.quote(target)}")
entries = sorted( if not result.ok:
f"{p.name}/" if p.is_dir() else p.name for p in target.iterdir() return ToolOutcome(
f"could not list {path}: {result.stderr.strip()}", success=False
) )
except OSError as exc: entries = sorted(line for line in result.stdout.splitlines() if line)
return ToolOutcome(f"could not list {path}: {exc}", success=False)
return ToolOutcome( return ToolOutcome(
self._truncate("\n".join(entries) or "(empty)"), success=True self._truncate("\n".join(entries) or "(empty)"), success=True
) )
@@ -187,56 +220,48 @@ class Workspace:
def describe(self) -> str: def describe(self) -> str:
"""Told to the model, so it does not have to guess where it is.""" """Told to the model, so it does not have to guess where it is."""
return ( return (
f"Your workspace is {self.root}. It is a git repository and the " f"Your workspace is {self.root} on the build VM. It is a git "
"only place your work is recorded; commands run there by default. " "repository and the only place your work is recorded; commands run "
"Use relative paths." "there by default. Use relative paths."
) )
def run_command(self, command: str) -> ToolOutcome: def run_command(self, command: str) -> ToolOutcome:
try: try:
# Popen rather than run(): on timeout, run() kills only the direct result = self.host.execute(
# child, and TimeoutExpired carries no pid, so a backgrounded
# grandchild (a dev server, say) would survive. Holding the pid
# lets us kill the whole session.
process = subprocess.Popen(
command, command,
shell=True, cwd=str(self.root),
cwd=self.root, secrets=self._secrets,
env=_child_env(), timeout_s=self._timeout_s,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.DEVNULL,
text=True,
start_new_session=True,
) )
except OSError as exc: except HostUnavailable as exc:
return ToolOutcome(f"could not run command: {exc}", success=False) return ToolOutcome(f"the build VM is unreachable: {exc}", success=False)
try: if result.timed_out:
stdout, stderr = process.communicate(timeout=self._timeout_s)
except subprocess.TimeoutExpired:
self._kill_group(process)
return ToolOutcome( return ToolOutcome(
f"command timed out after {self._timeout_s}s: {command}", f"command timed out after {self._timeout_s}s: {command}",
success=False, success=False,
) )
body = self._truncate( body = self._truncate(
f"exit code: {process.returncode}\nstdout:\n{stdout}\nstderr:\n{stderr}" f"exit code: {result.exit_code}\nstdout:\n{result.stdout}"
f"\nstderr:\n{result.stderr}"
) )
return ToolOutcome(body, success=process.returncode == 0) return ToolOutcome(body, success=result.exit_code == 0)
@staticmethod def upload_tree(self, local: Path, dest: str = ".") -> None:
def _kill_group(process: "subprocess.Popen[str]") -> None: """Copy a local directory's contents into the workspace (the bench
try: seeding a task); raises RuntimeError if it fails."""
os.killpg(os.getpgid(process.pid), signal.SIGKILL) buffer = io.BytesIO()
except (ProcessLookupError, PermissionError): with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
process.kill() for entry in sorted(Path(local).iterdir()):
# Drain the pipes of the now-dead group so no zombie is left behind. archive.add(entry, arcname=entry.name)
try: target = shlex.quote(self._resolve(dest))
process.communicate(timeout=5) result = self._run(
except subprocess.TimeoutExpired: f"mkdir -p -- {target} && base64 -d | tar -xzf - -C {target}",
pass stdin=base64.b64encode(buffer.getvalue()).decode(),
)
if not result.ok:
raise RuntimeError(f"upload to {dest} failed: {result.stderr.strip()}")
def reset(self) -> None: def reset(self) -> None:
"""Discard workspace contents; used by tests, not by the agent.""" """Discard workspace contents; used by tests, not by the agent."""
shutil.rmtree(self.root, ignore_errors=True) root = shlex.quote(str(self.root))
self.root.mkdir(parents=True, exist_ok=True) self._run(f"rm -rf -- {root} && mkdir -p -- {root}")
+48 -21
View File
@@ -3,6 +3,7 @@ delegation harness and the comment poller — were fully built, fully tested,
and reachable from nothing but their own tests. Unit tests that construct a and reachable from nothing but their own tests. Unit tests that construct a
component directly cannot catch that; only assembly can.""" component directly cannot catch that; only assembly can."""
import subprocess
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@@ -13,7 +14,7 @@ from test_config import SECRETS, tree
from nisse_dev.agent import build_agent, discover_models, prepare_workspace from nisse_dev.agent import build_agent, discover_models, prepare_workspace
from nisse_dev.catalog import ModelCatalog, UnknownModel from nisse_dev.catalog import ModelCatalog, UnknownModel
from nisse_dev.config import AgentConfig from nisse_dev.config import AgentConfig
from nisse_dev.git import GitRepo from nisse_dev.execution import SSHHost
from nisse_dev.oplog import OpLog from nisse_dev.oplog import OpLog
# What the test roles resolve to, as LiteLLM would report them. # What the test roles resolve to, as LiteLLM would report them.
@@ -36,47 +37,77 @@ TEST_CATALOG = ModelCatalog.from_payload(
) )
# The build VM for every test here: the session's throwaway sshd.
HOST: SSHHost
@pytest.fixture(autouse=True)
def _build_host(ssh_host: SSHHost) -> None:
global HOST
HOST = ssh_host
def config_for(tmp_path: Path, **changes: str) -> AgentConfig: def config_for(tmp_path: Path, **changes: str) -> AgentConfig:
return AgentConfig.from_tree( return AgentConfig.from_tree(
tree( tree(
state_db=str(tmp_path / "state.db"), state_db=str(tmp_path / "state.db"),
workspace=str(tmp_path / "work"), workspace=str(tmp_path / "work"),
build_host__address=HOST.address,
build_host__user=HOST.user,
build_host__port=str(HOST.port),
build_host__key=str(HOST._key),
**changes, **changes,
), ),
SECRETS, SECRETS,
) )
def git(*args: str) -> None:
subprocess.run(["git", *args], check=True, capture_output=True)
@pytest.fixture @pytest.fixture
def origin(tmp_path: Path) -> Path: def origin(tmp_path: Path) -> Path:
"""A bare repo standing in for the project on the forge.""" """A bare repo standing in for the project on the forge."""
remote = tmp_path / "origin.git" remote = tmp_path / "origin.git"
GitRepo(tmp_path)._run("init", "-q", "--bare", "-b", "main", str(remote)) git("init", "-q", "--bare", "-b", "main", str(remote))
seed = GitRepo(tmp_path / "seed") seed = tmp_path / "seed"
seed.init() git("init", "-q", "-b", "main", str(seed))
seed.configure_identity("seed", "seed@test") (seed / "README.md").write_text("project\n")
(seed.root / "README.md").write_text("project\n") git("-C", str(seed), "add", "-A")
seed.commit_all("seed") git(
seed.push(str(remote), "main") "-C",
str(seed),
"-c",
"user.name=s",
"-c",
"user.email=s@t",
"commit",
"-qm",
"s",
)
git("-C", str(seed), "push", "-q", str(remote), "HEAD:refs/heads/main")
return remote return remote
def test_workspace_is_cloned_when_absent(tmp_path: Path, origin: Path) -> None: def test_workspace_is_cloned_on_the_build_host_when_absent(
config = config_for(tmp_path, gitea__url=f"file://{origin}") tmp_path: Path, origin: Path
# authenticated_url would mangle a file:// remote, so clone directly. ) -> None:
GitRepo(tmp_path).clone(str(origin), config.workspace) config = config_for(tmp_path)
repo = prepare_workspace(config, "nisse-dev-testing") repo = prepare_workspace(HOST, config, "nisse-dev-testing", str(origin), None)
assert (config.workspace / "README.md").exists() assert (config.workspace / "README.md").exists()
assert repo.current_branch() == "main" assert repo.current_branch() == "main"
# The clone's configuration holds the plain URL it was given (#89).
assert str(origin) in (config.workspace / ".git" / "config").read_text()
def test_workspace_is_reused_when_already_cloned(tmp_path: Path, origin: Path) -> None: def test_workspace_is_reused_when_already_cloned(tmp_path: Path, origin: Path) -> None:
config = config_for(tmp_path) config = config_for(tmp_path)
GitRepo(tmp_path).clone(str(origin), config.workspace) prepare_workspace(HOST, config, "nisse-dev-testing", str(origin), None)
(config.workspace / "local-work.txt").write_text("in progress") (config.workspace / "local-work.txt").write_text("in progress")
prepare_workspace(config, "nisse-dev-testing") prepare_workspace(HOST, config, "nisse-dev-testing", str(origin), None)
# Re-cloning would silently destroy uncommitted work. # Re-cloning would silently destroy uncommitted work.
assert (config.workspace / "local-work.txt").exists() assert (config.workspace / "local-work.txt").exists()
@@ -110,7 +141,6 @@ class FakeTransport:
def build_offline(tmp_path: Path, origin: Path, monkeypatch: Any) -> Any: def build_offline(tmp_path: Path, origin: Path, monkeypatch: Any) -> Any:
"""Assemble the real graph with only the network boundaries faked.""" """Assemble the real graph with only the network boundaries faked."""
config = config_for(tmp_path) config = config_for(tmp_path)
GitRepo(tmp_path).clone(str(origin), config.workspace)
monkeypatch.setattr( monkeypatch.setattr(
"nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing" "nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing"
@@ -121,8 +151,7 @@ def build_offline(tmp_path: Path, origin: Path, monkeypatch: Any) -> Any:
) )
monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport()) monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport())
monkeypatch.setattr( monkeypatch.setattr(
"nisse_dev.agent.authenticated_url", "nisse_dev.agent.GiteaClient.clone_url", lambda self, repo: str(origin)
lambda base, repo, user, token: str(origin),
) )
return build_agent(config) return build_agent(config)
@@ -317,7 +346,6 @@ def test_tuning_reaches_every_component(
tools=ToolsTuning(command_timeout_s=300, max_output_chars=2000), tools=ToolsTuning(command_timeout_s=300, max_output_chars=2000),
), ),
) )
GitRepo(tmp_path).clone(str(origin), config.workspace)
monkeypatch.setattr( monkeypatch.setattr(
"nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing" "nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing"
) )
@@ -327,8 +355,7 @@ def test_tuning_reaches_every_component(
) )
monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport()) monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport())
monkeypatch.setattr( monkeypatch.setattr(
"nisse_dev.agent.authenticated_url", "nisse_dev.agent.GiteaClient.clone_url", lambda self, repo: str(origin)
lambda base, repo, user, token: str(origin),
) )
agent = build_agent(config) agent = build_agent(config)
executor = agent.orchestrator._executor executor = agent.orchestrator._executor
+21 -6
View File
@@ -20,6 +20,17 @@ from nisse_dev.bench.tasks import CHECK_DIR, load_tasks
from nisse_dev.canonical import AssistantText, ToolCall from nisse_dev.canonical import AssistantText, ToolCall
from nisse_dev.catalog import ModelFacts from nisse_dev.catalog import ModelFacts
from nisse_dev.config import ExecutorTuning, Tuning from nisse_dev.config import ExecutorTuning, Tuning
from nisse_dev.execution import SSHHost
# The build VM for every test here: the session's throwaway sshd.
HOST: SSHHost
@pytest.fixture(autouse=True)
def _build_host(ssh_host: SSHHost) -> None:
global HOST
HOST = ssh_host
TASKS = load_tasks() TASKS = load_tasks()
FACTS = ModelFacts( FACTS = ModelFacts(
@@ -187,7 +198,9 @@ SOLVE_SLUGIFY = [
def test_a_solved_task_is_a_success(tmp_path: Path) -> None: def test_a_solved_task_is_a_success(tmp_path: Path) -> None:
[task] = load_tasks(["write-function"]) [task] = load_tasks(["write-function"])
record = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), note="n") record = run_task(
task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), host=HOST, note="n"
)
assert record.success assert record.success
assert record.outcome == "completed" assert record.outcome == "completed"
@@ -212,7 +225,7 @@ def test_saying_finished_is_not_success(tmp_path: Path) -> None:
], ],
[call("finish", "f", summary="done")], [call("finish", "f", summary="done")],
] ]
record = run_task(task, FACTS, Scripted(wrong), Tuning()) record = run_task(task, FACTS, Scripted(wrong), Tuning(), host=HOST)
assert record.outcome == "completed" assert record.outcome == "completed"
assert not record.success assert not record.success
@@ -227,7 +240,7 @@ def test_misbehaviour_is_counted(tmp_path: Path) -> None:
# A higher failure limit, so the invented tool and the failed command # A higher failure limit, so the invented tool and the failed command
# don't escalate before the prose turn is reached. # don't escalate before the prose turn is reached.
tuning = Tuning(executor=ExecutorTuning(failure_limit=5)) tuning = Tuning(executor=ExecutorTuning(failure_limit=5))
record = run_task(task, FACTS, Scripted(turns), tuning) record = run_task(task, FACTS, Scripted(turns), tuning, host=HOST)
assert record.invented_tools == 1 assert record.invented_tools == 1
assert record.failed_commands == 1 assert record.failed_commands == 1
assert record.prose_turns >= 1 assert record.prose_turns >= 1
@@ -237,7 +250,7 @@ def test_misbehaviour_is_counted(tmp_path: Path) -> None:
def test_a_fallback_answer_can_never_be_a_success(tmp_path: Path) -> None: def test_a_fallback_answer_can_never_be_a_success(tmp_path: Path) -> None:
[task] = load_tasks(["write-function"]) [task] = load_tasks(["write-function"])
client = Scripted(list(SOLVE_SLUGIFY), answered="claude-sonnet-5") client = Scripted(list(SOLVE_SLUGIFY), answered="claude-sonnet-5")
record = run_task(task, FACTS, client, Tuning()) record = run_task(task, FACTS, client, Tuning(), host=HOST)
assert record.fallbacks == ["claude-sonnet-5"] * 3 assert record.fallbacks == ["claude-sonnet-5"] * 3
assert not record.success # the check passed, but not for this model assert not record.success # the check passed, but not for this model
@@ -255,7 +268,9 @@ def test_wilson_intervals_behave() -> None:
def record_json(**overrides: Any) -> dict[str, Any]: def record_json(**overrides: Any) -> dict[str, Any]:
[task] = load_tasks(["write-function"]) [task] = load_tasks(["write-function"])
base = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning()).to_json() base = run_task(
task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), host=HOST
).to_json()
base.update(overrides) base.update(overrides)
return base return base
@@ -338,7 +353,7 @@ def test_a_run_records_its_rate_limit_waits(tmp_path: Path) -> None:
[task] = load_tasks(["write-function"]) [task] = load_tasks(["write-function"])
client = Scripted(list(SOLVE_SLUGIFY)) client = Scripted(list(SOLVE_SLUGIFY))
client.last_rate_limit_wait_s = 20.0 # every call waited once client.last_rate_limit_wait_s = 20.0 # every call waited once
record = run_task(task, FACTS, client, Tuning()) record = run_task(task, FACTS, client, Tuning(), host=HOST)
assert record.rate_limit_wait_s == 60.0 assert record.rate_limit_wait_s == 60.0
+39
View File
@@ -32,6 +32,7 @@ COMPLETE: dict[str, Any] = {
"repo": "don/demo", "repo": "don/demo",
"operators": ["don"], "operators": ["don"],
}, },
"build_host": {"address": "build.test", "key": "~/.ssh/nisse-dev"},
} }
SECRETS = { SECRETS = {
"TEST_LLM_TOKEN": "sk-x", "TEST_LLM_TOKEN": "sk-x",
@@ -223,6 +224,9 @@ gitea:
token: ${TEST_GITEA_TOKEN} token: ${TEST_GITEA_TOKEN}
repo: don/demo repo: don/demo
operators: [don] operators: [don]
build_host:
address: build.test
key: ~/.ssh/nisse-dev
""" """
@@ -328,3 +332,38 @@ def test_context_tuning_is_read_and_checked() -> None:
assert build().tuning.context.keep_recent_turns == 6 assert build().tuning.context.keep_recent_turns == 6
with pytest.raises(ConfigError, match="safety_margin: must be less than 1"): with pytest.raises(ConfigError, match="safety_margin: must be less than 1"):
build(tuning={"context": {"safety_margin": 1}}) build(tuning={"context": {"safety_margin": 1}})
def test_the_build_host_is_required_and_defaults_to_root_on_22() -> None:
config = build()
assert (config.build_host.address, config.build_host.user) == ("build.test", "root")
assert config.build_host.port == 22
assert config.build_host.key.is_absolute()
with pytest.raises(ConfigError, match="build_host.address"):
build(build_host__address=None)
def test_a_bad_build_host_port_is_reported() -> None:
with pytest.raises(ConfigError, match="build_host.port"):
build(build_host__port="ssh")
def test_the_push_key_is_optional_and_a_path() -> None:
assert build().push_key is None
assert build(gitea__push_key="~/k").push_key == Path("~/k").expanduser()
def test_a_push_token_needs_its_user_and_excludes_a_key() -> None:
config = build(gitea__push_user="pusher", gitea__push_token="${TEST_GITEA_TOKEN}")
assert (config.push_user, config.push_token) == ("pusher", "gt")
with pytest.raises(ConfigError, match="set both or neither"):
build(gitea__push_user="pusher")
with pytest.raises(ConfigError, match="not both"):
build(gitea__push_user="p", gitea__push_token="t", gitea__push_key="~/k")
def test_no_credential_appears_in_the_config_repr() -> None:
config = build(gitea__push_user="pusher", gitea__push_token="${TEST_GITEA_TOKEN}")
text = repr(config)
for secret in SECRETS.values():
assert secret not in text
+4 -3
View File
@@ -22,6 +22,7 @@ from nisse_dev.delegation import (
DelegationResult, DelegationResult,
delegation_reconciler, delegation_reconciler,
) )
from nisse_dev.execution import SSHHost
from nisse_dev.git import GitRepo from nisse_dev.git import GitRepo
from nisse_dev.llm import BudgetExhausted, LLMError, UpstreamUnavailable from nisse_dev.llm import BudgetExhausted, LLMError, UpstreamUnavailable
from nisse_dev.oplog import OpLog from nisse_dev.oplog import OpLog
@@ -73,8 +74,8 @@ def store(tmp_path: Path) -> StateStore:
@pytest.fixture @pytest.fixture
def repo(tmp_path: Path) -> GitRepo: def repo(ssh_host: SSHHost, tmp_path: Path) -> GitRepo:
r = GitRepo(tmp_path / "work") r = GitRepo(ssh_host, tmp_path / "work")
r.init() r.init()
r.configure_identity("bot", "bot@example.test") r.configure_identity("bot", "bot@example.test")
(r.root / "seed.txt").write_text("seed") (r.root / "seed.txt").write_text("seed")
@@ -84,7 +85,7 @@ def repo(tmp_path: Path) -> GitRepo:
@pytest.fixture @pytest.fixture
def ws(repo: GitRepo) -> Workspace: def ws(repo: GitRepo) -> Workspace:
return Workspace(repo.root, timeout_s=10) return Workspace(repo.host, repo.root, timeout_s=10)
ITEM = "t" # the work item these delegations run in ITEM = "t" # the work item these delegations run in
+77
View File
@@ -0,0 +1,77 @@
"""Running commands on a tool VM over SSH (L13), against a throwaway sshd."""
import os
from pathlib import Path
import pytest
from nisse_dev.execution import REDACTED, HostUnavailable, SSHHost, redact
def test_exit_codes_and_streams_come_back(ssh_host: SSHHost) -> None:
result = ssh_host.execute("echo out; echo err >&2; exit 3")
assert (result.exit_code, result.stdout, result.stderr) == (3, "out\n", "err\n")
assert not result.ok
def test_cwd_and_stdin(ssh_host: SSHHost, tmp_path: Path) -> None:
result = ssh_host.execute("pwd; cat", cwd=str(tmp_path), stdin="line one\nline two")
assert result.stdout == f"{tmp_path}\nline one\nline two\n"
def test_stdin_that_looks_like_a_delimiter_survives(ssh_host: SSHHost) -> None:
payload = "NISSE_EOF\nEOF\n'quoted' $notexpanded `nor this`"
assert ssh_host.execute("cat", stdin=payload).stdout == payload + "\n"
def test_without_stdin_a_command_cannot_read_the_script(ssh_host: SSHHost) -> None:
# The script arrives on the same channel; the command must see /dev/null.
result = ssh_host.execute("cat; echo done")
assert result.stdout == "done\n"
def test_a_missing_cwd_fails_before_running(ssh_host: SSHHost, tmp_path: Path) -> None:
marker = tmp_path / "ran"
result = ssh_host.execute(f"touch {marker}", cwd=str(tmp_path / "absent"))
assert result.exit_code == 125
assert not marker.exists()
def test_a_command_exiting_255_is_not_mistaken_for_ssh(ssh_host: SSHHost) -> None:
assert ssh_host.execute("exit 255").exit_code == 254
def test_an_unreachable_host_raises(tmp_path: Path) -> None:
host = SSHHost("127.0.0.1", key=tmp_path / "k", state_dir=tmp_path / "s", port=1)
with pytest.raises(HostUnavailable):
host.execute("true")
def test_a_timeout_is_reported(ssh_host: SSHHost) -> None:
result = ssh_host.execute("sleep 30", timeout_s=1)
assert result.timed_out
def test_secrets_reach_the_command_and_never_its_output(ssh_host: SSHHost) -> None:
value = "s-" + os.urandom(12).hex()
result = ssh_host.execute(
'echo "$API_KEY"; echo "$API_KEY" >&2', secrets={"API_KEY": value}
)
assert result.ok
assert value not in result.stdout + result.stderr
assert result.stdout == f"{REDACTED}\n"
def test_secrets_are_never_on_a_command_line(ssh_host: SSHHost, tmp_path: Path) -> None:
# Made up here, so no command line in this test run can contain it except
# by leaking it.
value = "s-" + os.urandom(12).hex()
seen = tmp_path / "ps.txt"
ssh_host.execute(f"ps -eo args > {seen}", secrets={"API_KEY": value})
assert "bash -s" in seen.read_text() # the listing did include our shell
assert value not in seen.read_text()
def test_redact_masks_longest_values_first() -> None:
assert redact("abc-abcdef", ["abcdef", "abc-abcdef"]) == REDACTED
assert redact("short ab stays", ["ab"]) == "short ab stays" # too short to mask
+3 -2
View File
@@ -11,6 +11,7 @@ from nisse_dev.canonical import (
ToolCall, ToolCall,
ToolResult, ToolResult,
) )
from nisse_dev.execution import SSHHost
from nisse_dev.executor import LLMExecutor from nisse_dev.executor import LLMExecutor
from nisse_dev.llm import LLMError, RoleMap from nisse_dev.llm import LLMError, RoleMap
from nisse_dev.oplog import OpLog from nisse_dev.oplog import OpLog
@@ -62,8 +63,8 @@ def store(tmp_path: Path) -> StateStore:
@pytest.fixture @pytest.fixture
def ws(tmp_path: Path) -> Workspace: def ws(ssh_host: SSHHost, tmp_path: Path) -> Workspace:
return Workspace(tmp_path / "work", timeout_s=10) return Workspace(ssh_host, tmp_path / "work", timeout_s=10)
ITEM = "t" # the work item these tests run in ITEM = "t" # the work item these tests run in
+209 -35
View File
@@ -1,18 +1,28 @@
"""Git on the build VM (D4, L13): here, a throwaway sshd on this machine."""
import shlex
import subprocess
from pathlib import Path from pathlib import Path
import pytest import pytest
from nisse_dev.git import GitError, GitRepo, _scrub, authenticated_url from nisse_dev.execution import SSHHost
from nisse_dev.git import GitError, GitRepo, HTTPSToken, SSHKey
@pytest.fixture @pytest.fixture
def repo(tmp_path: Path) -> GitRepo: def repo(ssh_host: SSHHost, tmp_path: Path) -> GitRepo:
r = GitRepo(tmp_path / "work") r = GitRepo(ssh_host, tmp_path / "work")
r.init() r.init()
r.configure_identity("nisse-dev-testing", "nisse@example.test") r.configure_identity("nisse-dev-testing", "nisse@example.test")
return r return r
def bare_remote(path: Path) -> str:
subprocess.run(["git", "init", "-q", "--bare", "-b", "main", str(path)], check=True)
return str(path)
def test_init_creates_repo_on_default_branch(repo: GitRepo) -> None: def test_init_creates_repo_on_default_branch(repo: GitRepo) -> None:
assert repo.current_branch() == "main" assert repo.current_branch() == "main"
@@ -40,31 +50,8 @@ def test_has_changes_tracks_the_worktree(repo: GitRepo) -> None:
assert repo.has_changes() assert repo.has_changes()
def test_authenticated_url_shape() -> None: def test_git_error_on_bad_command(ssh_host: SSHHost, tmp_path: Path) -> None:
url = authenticated_url("https://gitea.test/", "o/r", "bot", "secret") repo = GitRepo(ssh_host, tmp_path / "not-a-repo")
assert url == "https://bot:secret@gitea.test/o/r.git"
def test_push_failure_does_not_leak_the_token(repo: GitRepo) -> None:
(repo.root / "a.txt").write_text("a")
repo.commit_all("first")
url = authenticated_url("https://127.0.0.1:1", "o/r", "bot", "sup3rsecret")
with pytest.raises(GitError) as excinfo:
repo.push(url, "main", token="sup3rsecret")
# git redacts credentials from its own messages, but that is git's
# behaviour, not ours; the scrub below is the guarantee we control.
assert "sup3rsecret" not in str(excinfo.value)
def test_scrub_removes_the_token_when_git_does_echo_it() -> None:
assert _scrub("failed to push to https://bot:s3cret@h/o/r.git", "s3cret") == (
"failed to push to https://bot:***@h/o/r.git"
)
assert _scrub("no secret here", None) == "no secret here"
def test_git_error_on_bad_command(tmp_path: Path) -> None:
repo = GitRepo(tmp_path / "not-a-repo")
repo.root.mkdir() repo.root.mkdir()
with pytest.raises(GitError): with pytest.raises(GitError):
repo.head_sha() repo.head_sha()
@@ -73,15 +60,202 @@ def test_git_error_on_bad_command(tmp_path: Path) -> None:
def test_commits_since_and_remote_sha_against_a_local_remote( def test_commits_since_and_remote_sha_against_a_local_remote(
repo: GitRepo, tmp_path: Path repo: GitRepo, tmp_path: Path
) -> None: ) -> None:
remote = GitRepo(tmp_path / "remote.git") remote = bare_remote(tmp_path / "remote.git")
remote.root.mkdir()
remote._run("init", "-q", "--bare")
(repo.root / "a.txt").write_text("a") (repo.root / "a.txt").write_text("a")
base = repo.commit_all("base") base = repo.commit_all("base")
repo.push(str(remote.root), "main") repo.push(remote, "main")
repo._run("update-ref", "refs/remotes/origin/main", "HEAD") repo._git("update-ref", "refs/remotes/origin/main", "HEAD")
assert repo.remote_sha(str(remote.root), "main") == base assert repo.remote_sha(remote, "main") == base
assert repo.remote_sha(str(remote.root), "absent") is None assert repo.remote_sha(remote, "absent") is None
(repo.root / "b.txt").write_text("b") (repo.root / "b.txt").write_text("b")
repo.commit_all("ahead") repo.commit_all("ahead")
assert repo.commits_since("origin/main") == 1 assert repo.commits_since("origin/main") == 1
# --- the push key (#89) ------------------------------------------------------
@pytest.fixture
def ssh_remote(ssh_host: SSHHost, tmp_path: Path) -> tuple[str, SSHKey]:
"""A bare repository reached by git over SSH, and the key that may push
to it: the forge, as the build VM sees it."""
remote = bare_remote(tmp_path / "forge.git")
url = f"ssh://{ssh_host.user}@127.0.0.1:{ssh_host.port}{remote}"
return url, SSHKey(ssh_host._key.read_text())
def test_clone_and_push_with_the_key_store_no_credential(
ssh_host: SSHHost, tmp_path: Path, ssh_remote: tuple[str, SSHKey]
) -> None:
url, key = ssh_remote
seed = GitRepo(ssh_host, tmp_path / "seed", known_hosts=str(tmp_path / "kh"))
seed.init()
seed.configure_identity("t", "t@example.test")
(seed.root / "a.txt").write_text("a")
seed.commit_all("first")
seed.push(url, "main", credential=key)
clone = GitRepo(ssh_host, tmp_path / "clone", known_hosts=str(tmp_path / "kh"))
clone.clone(url, credential=key)
assert (clone.root / "a.txt").read_text() == "a"
config = (clone.root / ".git" / "config").read_text()
assert "PRIVATE KEY" not in config
# The URL recorded is the credential-free one we gave.
assert url in config
def test_the_key_is_gone_when_the_command_ends(
ssh_host: SSHHost, tmp_path: Path, ssh_remote: tuple[str, SSHKey]
) -> None:
url, key = ssh_remote
repo = GitRepo(ssh_host, tmp_path / "w", known_hosts=str(tmp_path / "kh"))
repo.init()
repo.configure_identity("t", "t@example.test")
(repo.root / "a.txt").write_text("a")
repo.commit_all("first")
before = _agents()
repo.push(url, "main", credential=key)
assert _agents() == before # the throwaway ssh-agent was killed
def test_a_failed_push_still_kills_its_agent(
ssh_host: SSHHost, tmp_path: Path, ssh_remote: tuple[str, SSHKey]
) -> None:
_, key = ssh_remote
repo = GitRepo(ssh_host, tmp_path / "w", known_hosts=str(tmp_path / "kh"))
repo.init()
before = _agents()
with pytest.raises(GitError):
repo.push(
f"ssh://nobody@127.0.0.1:{ssh_host.port}/nowhere.git",
"main",
credential=key,
)
assert _agents() == before
def _agents() -> set[str]:
"""Live ssh-agents. A killed one lingers as a zombie in a CI container,
where nothing reaps orphans, so zombies don't count."""
out = subprocess.run(
["pgrep", "-x", "ssh-agent"], capture_output=True, text=True, check=False
).stdout
live = set()
for pid in out.split():
try:
stat = Path(f"/proc/{pid}/stat").read_text()
except FileNotFoundError:
continue
if stat.rsplit(")", 1)[1].split()[0] != "Z":
live.add(pid)
return live
def test_credentials_never_appear_in_a_repr() -> None:
assert "s3cr3t" not in repr(HTTPSToken("pusher", "s3cr3t"))
assert "PRIVATE" not in repr(SSHKey("-----BEGIN OPENSSH PRIVATE KEY-----"))
def test_a_remote_url_is_quoted_not_interpreted(repo: GitRepo, tmp_path: Path) -> None:
marker = tmp_path / "pwned"
with pytest.raises(GitError):
repo.push(f"x; touch {shlex.quote(str(marker))}", "main")
assert not marker.exists()
class RecordingHost:
"""Captures what would be sent to the build VM."""
def __init__(self) -> None:
self.calls: list[dict] = []
def execute(self, command: str, **kwargs):
from nisse_dev.execution import Result
self.calls.append({"command": command, **kwargs})
return Result(0, "", "")
def test_an_https_token_travels_only_as_a_secret(tmp_path: Path) -> None:
host = RecordingHost()
repo = GitRepo(host, tmp_path) # type: ignore[arg-type]
token = "tok-" + "9f3a" * 6
repo.push(
"https://gitea.test/o/r.git", "m1", credential=HTTPSToken("pusher", token)
)
[call] = host.calls
assert token not in call["command"] # never on a command line
assert "pusher" not in call["command"]
assert call["secrets"] == {"NISSE_PUSH_USER": "pusher", "NISSE_PUSH_TOKEN": token}
# Git asks a helper that reads the environment, and any helper configured
# on the VM is cleared first.
assert "credential.helper= " in call["command"]
assert "$NISSE_PUSH_TOKEN" in call["command"]
assert "GIT_TERMINAL_PROMPT=0" in call["command"]
def test_an_https_token_answers_git_through_the_helper(
ssh_host: SSHHost, tmp_path: Path
) -> None:
# The helper as git runs it, on the build host: asked to "get", it
# answers from the environment; asked anything else, nothing.
from nisse_dev.git import _HELPER
body = _HELPER.removeprefix("!")
asked = ssh_host.execute(
f"{body} get",
secrets={"NISSE_PUSH_USER": "pusher", "NISSE_PUSH_TOKEN": "tok-1234567"},
)
# Both travel as secrets, so both are masked in what comes back.
assert asked.stdout == "username=[redacted]\npassword=[redacted]\n"
stored = ssh_host.execute(f"{body} store", secrets={"NISSE_PUSH_TOKEN": "x"})
assert stored.stdout == ""
# --- build artifacts are never the work (#80, D4) ----------------------------
def test_artifacts_are_left_out_even_when_the_project_tracks_them(
repo: GitRepo,
) -> None:
# The sandbox's main tracks .pyc files (an earlier run published them).
cache = repo.root / "pkg" / "__pycache__"
cache.mkdir(parents=True)
(repo.root / "pkg" / "m.py").write_text("x = 1\n")
(cache / "m.cpython-312.pyc").write_bytes(b"old")
repo._git("add", "-A")
repo._git("commit", "-qm", "a project that tracks its caches")
# A test run: the tracked .pyc changes, new caches appear, and the work.
(cache / "m.cpython-312.pyc").write_bytes(b"new")
(cache / "m.cpython-310.pyc").write_bytes(b"x")
(repo.root / ".pytest_cache").mkdir()
(repo.root / ".pytest_cache" / "v").write_text("x")
(repo.root / "pkg" / "m.py").write_text("x = 2\n")
assert repo.commit_all("the work") is not None
committed = repo._git("show", "--name-only", "--format=", "HEAD").split()
assert committed == ["pkg/m.py"]
assert repo.left_out() == [
".pytest_cache/v",
"pkg/__pycache__/m.cpython-310.pyc",
"pkg/__pycache__/m.cpython-312.pyc",
]
def test_only_artifacts_changed_means_nothing_to_commit(repo: GitRepo) -> None:
(repo.root / "__pycache__").mkdir()
(repo.root / "__pycache__" / "a.pyc").write_bytes(b"x")
assert repo.commit_all("nothing real") is None
def test_excludes_hide_new_artifacts_and_install_once(repo: GitRepo) -> None:
repo.install_excludes()
repo.install_excludes()
exclude = (repo.root / ".git" / "info" / "exclude").read_text()
assert exclude.count("__pycache__/") == 1
(repo.root / "node_modules").mkdir()
(repo.root / "node_modules" / "x.js").write_text("x")
(repo.root / "real.py").write_text("x")
# Even the model's own `git add -A` now skips them.
assert repo._git("status", "--porcelain", "--untracked-files=all") == "?? real.py"
+2 -6
View File
@@ -1,5 +1,4 @@
import json import json
from pathlib import Path
import httpx import httpx
import pytest import pytest
@@ -200,7 +199,6 @@ def test_intent_without_a_branch_is_unverifiable() -> None:
def test_closing_comments_first_then_closes_issue_and_pr() -> None: def test_closing_comments_first_then_closes_issue_and_pr() -> None:
from nisse_dev.approvals import PlanBoard from nisse_dev.approvals import PlanBoard
from nisse_dev.git import GitRepo
from nisse_dev.publish import MilestonePublisher from nisse_dev.publish import MilestonePublisher
seen: list[tuple[str, str, dict]] = [] seen: list[tuple[str, str, dict]] = []
@@ -220,10 +218,8 @@ def test_closing_comments_first_then_closes_issue_and_pr() -> None:
None, # type: ignore[arg-type] -- closing touches neither store nor git None, # type: ignore[arg-type] -- closing touches neither store nor git
gitea, gitea,
"o/r", "o/r",
GitRepo(Path(".")), None, # type: ignore[arg-type]
base_url="https://gitea.test", remote_url="ssh://git@gitea.test/o/r.git",
account="bot",
token="tok",
) )
publisher.close_pull_request(20, "abandoned by don") publisher.close_pull_request(20, "abandoned by don")
+63 -19
View File
@@ -5,12 +5,22 @@ from typing import Any
import pytest import pytest
from nisse_dev.git import GitRepo from nisse_dev.execution import SSHHost
from nisse_dev.git import GitRepo, HTTPSToken
from nisse_dev.gitea import GiteaError, PullRequest from nisse_dev.gitea import GiteaError, PullRequest
from nisse_dev.oplog import OpLog from nisse_dev.oplog import OpLog
from nisse_dev.publish import OPEN_PR_INTENT, MilestonePublisher from nisse_dev.publish import OPEN_PR_INTENT, MilestonePublisher
from nisse_dev.state import StateStore from nisse_dev.state import StateStore
# The build VM for every test here: the session's throwaway sshd.
HOST: SSHHost
@pytest.fixture(autouse=True)
def _build_host(ssh_host: SSHHost) -> None:
global HOST
HOST = ssh_host
class FakeGitea: class FakeGitea:
def __init__(self, *, fail: bool = False) -> None: def __init__(self, *, fail: bool = False) -> None:
@@ -46,7 +56,7 @@ class FakeGit:
def __init__( def __init__(
self, root: Path, *, changes: bool = True, committed: bool = False self, root: Path, *, changes: bool = True, committed: bool = False
) -> None: ) -> None:
self.repo = GitRepo(root) self.repo = GitRepo(HOST, root)
self.repo.init() self.repo.init()
self.repo.configure_identity("bot", "bot@example.test") self.repo.configure_identity("bot", "bot@example.test")
(root / "README.md").write_text("sandbox") (root / "README.md").write_text("sandbox")
@@ -67,7 +77,7 @@ class FakeGit:
return self.repo.commits_since(ref) return self.repo.commits_since(ref)
def remote_sha( def remote_sha(
self, remote_url: str, branch: str, *, token: str | None = None self, remote_url: str, branch: str, *, credential: object = None
) -> str | None: ) -> str | None:
return self.remote.get(branch) return self.remote.get(branch)
@@ -80,7 +90,10 @@ class FakeGit:
def commit_all(self, message: str) -> str | None: def commit_all(self, message: str) -> str | None:
return self.repo.commit_all(message) return self.repo.commit_all(message)
def push(self, remote_url: str, branch: str, *, token: str | None = None) -> None: def left_out(self) -> list[str]:
return self.repo.left_out()
def push(self, remote_url: str, branch: str, *, credential: object = None) -> None:
self.pushed.append(branch) self.pushed.append(branch)
self.remote[branch] = self.repo.head_sha() self.remote[branch] = self.repo.head_sha()
@@ -100,9 +113,8 @@ def build(store: StateStore, git: Any, client: Any) -> MilestonePublisher:
client, client,
"o/sandbox", "o/sandbox",
git, git,
base_url="https://gitea.test", remote_url="ssh://git@gitea.test/o/sandbox.git",
account="bot", push_credential=HTTPSToken("pusher", "tok"),
token="tok",
) )
@@ -115,14 +127,46 @@ def test_publish_commits_pushes_and_opens_a_pr(
task_id="demo-abc", milestone=2, project="demo", plan="the plan" task_id="demo-abc", milestone=2, project="demo", plan="the plan"
) )
assert git.pushed == ["demo-abc/milestone-2"] assert git.pushed == ["demo-abc-milestone-2"]
assert client.opened[0]["head"] == "demo-abc/milestone-2" assert client.opened[0]["head"] == "demo-abc-milestone-2"
assert client.opened[0]["base"] == "main" assert client.opened[0]["base"] == "main"
assert "the plan" in client.opened[0]["body"] assert "the plan" in client.opened[0]["body"]
assert result.pull_request is not None assert result.pull_request is not None
assert "opened PR #1" in result.note assert "opened PR #1" in result.note
def test_a_branch_named_after_the_task_does_not_block_publishing(
store: StateStore, tmp_path: Path
) -> None:
# Live run 3: the executor ran `git checkout -b <task id>` itself. With
# "<task>/milestone-1" as the harness's branch, git refused it, since a
# branch name can't also be a directory of other branch names.
git = FakeGit(tmp_path / "work")
git.repo.checkout_new_branch("demo-abc")
result = build(store, git, FakeGitea()).publish(
task_id="demo-abc", milestone=1, project="demo", plan="p"
)
assert result.pull_request is not None
assert git.pushed == ["demo-abc-milestone-1"]
def test_left_out_artifacts_are_reported_not_published(
store: StateStore, tmp_path: Path
) -> None:
git = FakeGit(tmp_path / "work")
(tmp_path / "work" / "__pycache__").mkdir()
(tmp_path / "work" / "__pycache__" / "work.cpython-310.pyc").write_bytes(b"x")
client = FakeGitea()
result = build(store, git, client).publish(
task_id="t", milestone=1, project="demo", plan="p"
)
assert "left out 1 build artifact(s): __pycache__/work.cpython-310.pyc" in (
result.note
)
assert "## Left out" in client.opened[0]["body"]
assert "`__pycache__/work.cpython-310.pyc`" in client.opened[0]["body"]
def test_intent_is_recorded_then_committed(store: StateStore, tmp_path: Path) -> None: def test_intent_is_recorded_then_committed(store: StateStore, tmp_path: Path) -> None:
build(store, FakeGit(tmp_path / "work"), FakeGitea()).publish( build(store, FakeGit(tmp_path / "work"), FakeGitea()).publish(
task_id="t", milestone=1, project="demo", plan="p" task_id="t", milestone=1, project="demo", plan="p"
@@ -141,7 +185,7 @@ def test_failure_after_intent_leaves_it_pending_for_reconciliation(
[pending] = store.pending_intents() [pending] = store.pending_intents()
assert pending.kind == OPEN_PR_INTENT assert pending.kind == OPEN_PR_INTENT
assert pending.payload["branch"] == "t/milestone-1" assert pending.payload["branch"] == "t-milestone-1"
assert pending.payload["sha"] assert pending.payload["sha"]
@@ -162,7 +206,7 @@ def test_nothing_to_commit_is_reported_not_raised(
def test_branch_is_namespaced_by_task(store: StateStore, tmp_path: Path) -> None: def test_branch_is_namespaced_by_task(store: StateStore, tmp_path: Path) -> None:
publisher = build(store, FakeGit(tmp_path / "work"), FakeGitea()) publisher = build(store, FakeGit(tmp_path / "work"), FakeGitea())
assert publisher.branch_for("greet-9f2", 3) == "greet-9f2/milestone-3" assert publisher.branch_for("greet-9f2", 3) == "greet-9f2-milestone-3"
def test_reconciler_settles_an_intent_left_by_a_crash( def test_reconciler_settles_an_intent_left_by_a_crash(
@@ -174,9 +218,9 @@ def test_reconciler_settles_an_intent_left_by_a_crash(
client = FakeGitea() client = FakeGitea()
publisher = build(store, FakeGit(tmp_path / "work"), client) publisher = build(store, FakeGit(tmp_path / "work"), client)
intent_id = store.begin_intent( intent_id = store.begin_intent(
OPEN_PR_INTENT, {"branch": "t/milestone-1"}, cause="crash simulation" OPEN_PR_INTENT, {"branch": "t-milestone-1"}, cause="crash simulation"
) )
client.existing.append("t/milestone-1") client.existing.append("t-milestone-1")
reconcilers = publisher.reconcilers() reconcilers = publisher.reconcilers()
assert set(reconcilers) == {OPEN_PR_INTENT} assert set(reconcilers) == {OPEN_PR_INTENT}
@@ -196,8 +240,8 @@ def test_work_the_executor_committed_itself_is_published(
task_id="t", milestone=1, project="demo", plan="p" task_id="t", milestone=1, project="demo", plan="p"
) )
assert result.pull_request is not None assert result.pull_request is not None
assert git.pushed == ["t/milestone-1"] assert git.pushed == ["t-milestone-1"]
assert git.remote["t/milestone-1"] == git.head_sha() assert git.remote["t-milestone-1"] == git.head_sha()
def test_a_followup_the_executor_committed_itself_is_pushed( def test_a_followup_the_executor_committed_itself_is_pushed(
@@ -210,7 +254,7 @@ def test_a_followup_the_executor_committed_itself_is_pushed(
git.repo.commit_all("Address review") git.repo.commit_all("Address review")
sha = publisher.push_followup(task_id="t", milestone=1, message="m") sha = publisher.push_followup(task_id="t", milestone=1, message="m")
assert sha == git.head_sha() assert sha == git.head_sha()
assert git.pushed == ["t/milestone-1", "t/milestone-1"] assert git.pushed == ["t-milestone-1", "t-milestone-1"]
# Nothing new since that push: no second push, and it says so. # Nothing new since that push: no second push, and it says so.
assert publisher.push_followup(task_id="t", milestone=1, message="m") is None assert publisher.push_followup(task_id="t", milestone=1, message="m") is None
assert len(git.pushed) == 2 assert len(git.pushed) == 2
@@ -230,15 +274,15 @@ def test_publishing_again_adopts_the_open_pr_instead_of_opening_another(
assert len(client.opened) == 1 assert len(client.opened) == 1
assert result.pull_request is not None and result.pull_request.number == 1 assert result.pull_request is not None and result.pull_request.number == 1
assert "already open" in result.note assert "already open" in result.note
assert git.pushed == ["t/milestone-1", "t/milestone-1"] # updated it assert git.pushed == ["t-milestone-1", "t-milestone-1"] # updated it
assert store.pending_intents() == [] assert store.pending_intents() == []
def test_a_closed_pr_is_not_adopted(store: StateStore, tmp_path: Path) -> None: def test_a_closed_pr_is_not_adopted(store: StateStore, tmp_path: Path) -> None:
git = FakeGit(tmp_path / "work") git = FakeGit(tmp_path / "work")
client = FakeGitea() client = FakeGitea()
client.existing.append("t/milestone-1") client.existing.append("t-milestone-1")
client.closed.add("t/milestone-1") client.closed.add("t-milestone-1")
publisher = build(store, git, client) publisher = build(store, git, client)
assert publisher.published_pull_request("t", 1) is None assert publisher.published_pull_request("t", 1) is None
publisher.publish(task_id="t", milestone=1, project="demo", plan="p") publisher.publish(task_id="t", milestone=1, project="demo", plan="p")
@@ -121,7 +121,7 @@ class Publisher:
return {"open_pr": lambda intent: intent.payload["branch"] in self._gitea.prs} return {"open_pr": lambda intent: intent.payload["branch"] in self._gitea.prs}
def publish(self, **kwargs: Any) -> PublishResult: def publish(self, **kwargs: Any) -> PublishResult:
branch = f"{kwargs['task_id']}/milestone-{kwargs['milestone']}" branch = f"{kwargs['task_id']}-milestone-{kwargs['milestone']}"
self.attempts += 1 self.attempts += 1
intent = self._store.begin_intent("open_pr", {"branch": branch}, cause="t") intent = self._store.begin_intent("open_pr", {"branch": branch}, cause="t")
if branch in self._gitea.prs: if branch in self._gitea.prs:
@@ -135,7 +135,7 @@ class Publisher:
) )
def published_pull_request(self, task_id: str, milestone: int) -> Any: def published_pull_request(self, task_id: str, milestone: int) -> Any:
branch = f"{task_id}/milestone-{milestone}" branch = f"{task_id}-milestone-{milestone}"
if branch in self._gitea.prs: if branch in self._gitea.prs:
return PullRequest(number=20, state="open", head=branch, url="u") return PullRequest(number=20, state="open", head=branch, url="u")
return None return None
+51 -6
View File
@@ -4,12 +4,13 @@ from pathlib import Path
import pytest import pytest
from nisse_dev.execution import REDACTED, SSHHost
from nisse_dev.tools import EXECUTOR_TOOLS, MAX_OUTPUT_CHARS, Workspace from nisse_dev.tools import EXECUTOR_TOOLS, MAX_OUTPUT_CHARS, Workspace
@pytest.fixture @pytest.fixture
def ws(tmp_path: Path) -> Workspace: def ws(ssh_host: SSHHost, tmp_path: Path) -> Workspace:
return Workspace(tmp_path / "work", timeout_s=10) return Workspace(ssh_host, tmp_path / "work", timeout_s=10)
def test_manifest_is_small_and_flat() -> None: def test_manifest_is_small_and_flat() -> None:
@@ -71,8 +72,8 @@ def test_command_runs_in_the_workspace(ws: Workspace) -> None:
assert "marker.txt" in ws.run_command("ls").result assert "marker.txt" in ws.run_command("ls").result
def test_command_timeout_is_reported(tmp_path: Path) -> None: def test_command_timeout_is_reported(ssh_host: SSHHost, tmp_path: Path) -> None:
ws = Workspace(tmp_path / "work", timeout_s=1) ws = Workspace(ssh_host, tmp_path / "work", timeout_s=1)
outcome = ws.run_command("sleep 30") outcome = ws.run_command("sleep 30")
assert not outcome.success assert not outcome.success
assert "timed out" in outcome.result assert "timed out" in outcome.result
@@ -90,8 +91,10 @@ def _is_running(pid: int) -> bool:
return stat.rsplit(")", 1)[1].split()[0] != "Z" return stat.rsplit(")", 1)[1].split()[0] != "Z"
def test_timeout_kills_the_whole_process_group(tmp_path: Path) -> None: def test_timeout_kills_the_whole_process_group(
ws = Workspace(tmp_path / "work", timeout_s=1) ssh_host: SSHHost, tmp_path: Path
) -> None:
ws = Workspace(ssh_host, tmp_path / "work", timeout_s=1)
pidfile = ws.root / "child.pid" pidfile = ws.root / "child.pid"
# A backgrounded grandchild must not outlive the timeout. # A backgrounded grandchild must not outlive the timeout.
ws.run_command(f"(sleep 30 & echo $! > {pidfile}); sleep 30") ws.run_command(f"(sleep 30 & echo $! > {pidfile}); sleep 30")
@@ -156,3 +159,45 @@ def test_escaping_via_parent_traversal_is_refused(ws: Workspace) -> None:
def test_describe_names_the_root(ws: Workspace) -> None: def test_describe_names_the_root(ws: Workspace) -> None:
assert str(ws.root) in ws.describe() assert str(ws.root) in ws.describe()
# --- on the build VM (L13) -----------------------------------------------------
def test_project_secrets_reach_commands_and_are_redacted(
ssh_host: SSHHost, tmp_path: Path
) -> None:
value = "tok-" + os.urandom(8).hex()
ws = Workspace(ssh_host, tmp_path / "work", secrets={"TEST_TOKEN": value})
outcome = ws.run_command('test -n "$TEST_TOKEN" && echo "token=$TEST_TOKEN"')
assert outcome.success # the command had it
assert value not in outcome.result
assert f"token={REDACTED}" in outcome.result
def test_an_unreachable_build_vm_is_a_failed_tool_call(tmp_path: Path) -> None:
ws = Workspace.__new__(Workspace) # no host to create the root on
ws.host = SSHHost(
"127.0.0.1", key=tmp_path / "no-key", state_dir=tmp_path / "s", port=1
)
ws.root = tmp_path / "work"
ws._timeout_s, ws._max_output_chars, ws._secrets = 5, MAX_OUTPUT_CHARS, {}
outcome = ws.dispatch("run_command", {"command": "true"})
assert not outcome.success
assert "unreachable" in outcome.result
def test_upload_tree_copies_a_directory_in(ws: Workspace, tmp_path: Path) -> None:
src = tmp_path / "seed"
(src / "pkg").mkdir(parents=True)
(src / "pkg" / "m.py").write_text("x = 1\n")
ws.upload_tree(src)
assert (ws.root / "pkg" / "m.py").read_text() == "x = 1\n"
ws.upload_tree(src, "copy")
assert (ws.root / "copy" / "pkg" / "m.py").exists()
def test_non_text_files_read_as_a_failure(ws: Workspace) -> None:
(ws.root / "blob.bin").write_bytes(b"\xff\xfe\x00")
outcome = ws.read_file("blob.bin")
assert not outcome.success
+3 -1
View File
@@ -14,7 +14,7 @@ import os
import sys import sys
from pathlib import Path from pathlib import Path
from nisse_dev.agent import make_llm_clients from nisse_dev.agent import build_host, make_llm_clients
from nisse_dev.bench.results import append, load, render, results_file, summarise from nisse_dev.bench.results import append, load, render, results_file, summarise
from nisse_dev.bench.runner import agent_commit, run_task from nisse_dev.bench.runner import agent_commit, run_task
from nisse_dev.bench.tasks import load_tasks from nisse_dev.bench.tasks import load_tasks
@@ -46,6 +46,7 @@ def run(args: argparse.Namespace) -> int:
catalog = ModelCatalog.discover(config.llm_base_url, config.llm_token) catalog = ModelCatalog.discover(config.llm_base_url, config.llm_token)
catalog.require_limits({m: m for m in models}) catalog.require_limits({m: m for m in models})
normalized, _passthrough = make_llm_clients(config, catalog) normalized, _passthrough = make_llm_clients(config, catalog)
host = build_host(config) # runs happen on the build VM, as the agent's do
commit = agent_commit(REPO) commit = agent_commit(REPO)
for model in models: for model in models:
@@ -60,6 +61,7 @@ def run(args: argparse.Namespace) -> int:
facts, facts,
normalized, normalized,
config.tuning, config.tuning,
host=host,
note=args.note, note=args.note,
commit=commit, commit=commit,
) )
+24 -2
View File
@@ -17,7 +17,8 @@
# Layout, under $E2E (default /root/nisse-e2e): # Layout, under $E2E (default /root/nisse-e2e):
# nisse-dev.yaml, .env the run config and its secrets (made by hand once) # nisse-dev.yaml, .env the run config and its secrets (made by hand once)
# code/ the exported code under test, with its own .venv # code/ the exported code under test, with its own .venv
# state.db, project/ the agent's state and workspace # state.db the agent's state (its workspace is on the build VM)
# ssh/ the agent's key for the build VM, and push keys
# run-info, collected/ what was run, and the output of `collect` # run-info, collected/ what was run, and the output of `collect`
# runs/<time>/ earlier runs, set aside by `start` # runs/<time>/ earlier runs, set aside by `start`
set -euo pipefail set -euo pipefail
@@ -42,12 +43,15 @@ cmd_start() {
for item in "$E2E"/state.db* "$E2E/project" "$E2E/collected" "$E2E/run-info"; do for item in "$E2E"/state.db* "$E2E/project" "$E2E/collected" "$E2E/run-info"; do
[ -e "$item" ] && items+=("$item") [ -e "$item" ] && items+=("$item")
done done
local stamp
stamp=$(date +%Y%m%d-%H%M%S)
if [ ${#items[@]} -gt 0 ]; then if [ ${#items[@]} -gt 0 ]; then
aside=$E2E/runs/$(date +%Y%m%d-%H%M%S) aside=$E2E/runs/$stamp
mkdir -p "$aside" mkdir -p "$aside"
mv "${items[@]}" "$aside/" mv "${items[@]}" "$aside/"
echo "previous run set aside in $aside" echo "previous run set aside in $aside"
fi fi
remote_aside "$stamp"
# Export the code under test, so edits to the working tree can't change a # Export the code under test, so edits to the working tree can't change a
# running agent. Uncommitted changes are not part of a run. # running agent. Uncommitted changes are not part of a run.
@@ -67,6 +71,24 @@ cmd_start() {
echo "started $UNIT at ${sha:0:8} ($ref)" echo "started $UNIT at ${sha:0:8} ($ref)"
} }
# The workspace lives on the build VM (L13): set the last run's aside there
# too, so each run starts from a fresh clone.
remote_aside() {
local stamp=$1 where
where=$(python3 - "$E2E/nisse-dev.yaml" <<'PY'
import sys, yaml
c = yaml.safe_load(open(sys.argv[1]))
b = c["build_host"]
print(b.get("user", "root"), b["address"], b.get("port", 22), b["key"], c["workspace"])
PY
) || die "cannot read build_host and workspace from the config"
read -r user address port key workspace <<< "$where"
ssh -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \
-o UserKnownHostsFile="$E2E/ssh/known_hosts" "$user@$address" \
"if [ -e '$workspace' ]; then mv '$workspace' '$workspace.$stamp' && echo 'build VM: set aside $workspace.$stamp'; fi" \
|| die "cannot reach the build VM $address"
}
cmd_stop() { cmd_stop() {
running || { echo "not running"; return 0; } running || { echo "not running"; return 0; }
systemctl stop "$UNIT" # SIGTERM: the agent shuts down cleanly (main.py) systemctl stop "$UNIT" # SIGTERM: the agent shuts down cleanly (main.py)
+10 -2
View File
@@ -7,11 +7,15 @@
# the live tests that need it, rather than skipping them, so a broken # the live tests that need it, rather than skipping them, so a broken
# credential cannot hide as a skip. # credential cannot hide as a skip.
# Unused by the live tests, which work in pytest's temporary directories; # Unused by the live tests, which work in pytest's temporary directories on a
# present because every agent config has them. # throwaway local sshd (the `ssh_host` fixture); present because every agent
# config has them.
state_db: /tmp/nisse-test/state.db state_db: /tmp/nisse-test/state.db
workspace: /tmp/nisse-test/work workspace: /tmp/nisse-test/work
project: nisse-dev-testing project: nisse-dev-testing
build_host:
address: 127.0.0.1
key: /tmp/nisse-test/unused-key
llm: llm:
base_url: https://litellm.dongar.net base_url: https://litellm.dongar.net
@@ -38,3 +42,7 @@ gitea:
token: ${NISSE_GITEA_TOKEN} token: ${NISSE_GITEA_TOKEN}
repo: dongar/nisse-dev-testing-sandbox # the sandbox, never a real project repo: dongar/nisse-dev-testing-sandbox # the sandbox, never a real project
operators: [dongar] operators: [dongar]
# What pushes go by (D4): the sandbox's deploy key, whose private half CI
# writes to this path from the NISSE_PUSH_KEY secret (locally: copy it
# there). An HTTPS push token works too: push_user plus push_token.
push_key: /tmp/nisse-test/push_key
+5 -2
View File
@@ -10,19 +10,22 @@ from nisse_dev.bench.runner import run_task
from nisse_dev.bench.tasks import load_tasks from nisse_dev.bench.tasks import load_tasks
from nisse_dev.catalog import ModelCatalog from nisse_dev.catalog import ModelCatalog
from nisse_dev.config import AgentConfig from nisse_dev.config import AgentConfig
from nisse_dev.execution import SSHHost
pytestmark = pytest.mark.integration pytestmark = pytest.mark.integration
def test_the_simplest_task_passes_live( def test_the_simplest_task_passes_live(
live_config: AgentConfig, tmp_path: Path live_config: AgentConfig, tmp_path: Path, ssh_host: SSHHost
) -> None: ) -> None:
catalog = ModelCatalog.discover(live_config.llm_base_url, live_config.llm_token) catalog = ModelCatalog.discover(live_config.llm_base_url, live_config.llm_token)
normalized, _ = make_llm_clients(live_config, catalog) normalized, _ = make_llm_clients(live_config, catalog)
facts = catalog.facts(live_config.roles.executor) facts = catalog.facts(live_config.roles.executor)
[task] = load_tasks(["write-function"]) [task] = load_tasks(["write-function"])
record = run_task(task, facts, normalized, live_config.tuning, workdir=tmp_path) record = run_task(
task, facts, normalized, live_config.tuning, host=ssh_host, workdir=tmp_path
)
assert record.fallbacks == [] assert record.fallbacks == []
assert record.success, ( assert record.success, (
+19 -8
View File
@@ -7,6 +7,7 @@ from pathlib import Path
import pytest import pytest
from nisse_dev.delegation import Caps, Delegation from nisse_dev.delegation import Caps, Delegation
from nisse_dev.execution import SSHHost
from nisse_dev.git import GitRepo from nisse_dev.git import GitRepo
from nisse_dev.llm import AnthropicPassthroughClient from nisse_dev.llm import AnthropicPassthroughClient
from nisse_dev.oplog import OpLog from nisse_dev.oplog import OpLog
@@ -23,9 +24,13 @@ it, and run `python -m pytest test_slugify.py -q` until it passes."""
def build( def build(
tmp_path: Path, caps: Caps, client: AnthropicPassthroughClient, model: str ssh_host: SSHHost,
tmp_path: Path,
caps: Caps,
client: AnthropicPassthroughClient,
model: str,
) -> tuple[Delegation, GitRepo, StateStore]: ) -> tuple[Delegation, GitRepo, StateStore]:
repo = GitRepo(tmp_path / "work") repo = GitRepo(ssh_host, tmp_path / "work")
repo.init() repo.init()
repo.configure_identity("nisse-dev-testing", "n@t.test") repo.configure_identity("nisse-dev-testing", "n@t.test")
(repo.root / "README.md").write_text("sandbox\n") (repo.root / "README.md").write_text("sandbox\n")
@@ -34,7 +39,7 @@ def build(
delegation = Delegation( delegation = Delegation(
client, client,
store, store,
Workspace(repo.root, timeout_s=60), Workspace(ssh_host, repo.root, timeout_s=60),
repo, repo,
model=model, model=model,
caps=caps, caps=caps,
@@ -43,10 +48,13 @@ def build(
def test_delegate_completes_a_scoped_subtask( def test_delegate_completes_a_scoped_subtask(
tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str tmp_path: Path,
passthrough: AnthropicPassthroughClient,
live_model: str,
ssh_host: SSHHost,
) -> None: ) -> None:
delegation, repo, store = build( delegation, repo, store = build(
tmp_path, Caps(turns=12, wall_clock_s=300.0), passthrough, live_model ssh_host, tmp_path, Caps(turns=12, wall_clock_s=300.0), passthrough, live_model
) )
result = delegation.run( result = delegation.run(
@@ -66,19 +74,22 @@ def test_delegate_completes_a_scoped_subtask(
# The delegate's own work is on its branch, and it really runs. # The delegate's own work is on its branch, and it really runs.
assert repo.current_branch() == "live/delegation-1" assert repo.current_branch() == "live/delegation-1"
check = Workspace(repo.root, timeout_s=60).run_command( check = Workspace(ssh_host, repo.root, timeout_s=60).run_command(
"python -m pytest test_slugify.py -q" "python -m pytest test_slugify.py -q"
) )
assert check.success, f"delegate's own tests do not pass: {check.result[:400]}" assert check.success, f"delegate's own tests do not pass: {check.result[:400]}"
def test_a_cap_breach_discards_the_work( def test_a_cap_breach_discards_the_work(
tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str tmp_path: Path,
passthrough: AnthropicPassthroughClient,
live_model: str,
ssh_host: SSHHost,
) -> None: ) -> None:
# One turn cannot finish the brief, so this exercises the abort path # One turn cannot finish the brief, so this exercises the abort path
# against a real model rather than a scripted one. # against a real model rather than a scripted one.
delegation, repo, store = build( delegation, repo, store = build(
tmp_path, Caps(turns=1, wall_clock_s=120.0), passthrough, live_model ssh_host, tmp_path, Caps(turns=1, wall_clock_s=120.0), passthrough, live_model
) )
result = delegation.run( result = delegation.run(
+15 -11
View File
@@ -10,8 +10,10 @@ from pathlib import Path
import pytest import pytest
from nisse_dev.agent import push_credential, remote_url
from nisse_dev.config import AgentConfig from nisse_dev.config import AgentConfig
from nisse_dev.git import GitError, GitRepo, authenticated_url from nisse_dev.execution import SSHHost
from nisse_dev.git import GitError, GitRepo
from nisse_dev.gitea import GiteaClient, pull_request_reconciler from nisse_dev.gitea import GiteaClient, pull_request_reconciler
from nisse_dev.state import Intent from nisse_dev.state import Intent
@@ -30,10 +32,8 @@ def intent_for(branch: str) -> Intent:
def test_push_open_pr_and_reconcile( def test_push_open_pr_and_reconcile(
tmp_path: Path, live_config: AgentConfig, gitea: GiteaClient tmp_path: Path, live_config: AgentConfig, gitea: GiteaClient, ssh_host: SSHHost
) -> None: ) -> None:
base_url = live_config.gitea_url
token = live_config.gitea_token
repo_path = live_config.gitea_repo # the sandbox repo_path = live_config.gitea_repo # the sandbox
client = gitea client = gitea
@@ -48,16 +48,20 @@ def test_push_open_pr_and_reconcile(
# Before the PR exists, external truth says the effect has not happened. # Before the PR exists, external truth says the effect has not happened.
assert reconcile(intent_for(branch)) is False assert reconcile(intent_for(branch)) is False
repo = GitRepo(tmp_path / "work") # Pushed the way the agent does: from the build VM (here the throwaway
# sshd), with the push credential the config names, never the API token.
key = push_credential(live_config)
assert key is not None, "nisse-test.yaml names no push credential (D4)"
url = remote_url(client, repo_path, key)
repo = GitRepo(ssh_host, tmp_path / "work", known_hosts=str(tmp_path / "kh"))
repo.init() repo.init()
repo.configure_identity(account, f"{account}@example.test") repo.configure_identity(account, f"{account}@example.test")
(repo.root / f"{branch}.txt").write_text("live test artifact\n") (repo.root / f"{branch}.txt").write_text("live test artifact\n")
assert repo.commit_all("Live test commit") is not None assert repo.commit_all("Live test commit") is not None
url = authenticated_url(base_url, repo_path, account, token)
pr = None pr = None
try: try:
repo.push(url, branch, token=token) repo.push(url, branch, credential=key)
pr = client.open_pull_request( pr = client.open_pull_request(
repo_path, repo_path,
head=branch, head=branch,
@@ -75,8 +79,8 @@ def test_push_open_pr_and_reconcile(
assert reconcile(intent_for(branch)) is True assert reconcile(intent_for(branch)) is True
# D4 is only real if the server enforces it. The agent's own token # D4 is only real if the server enforces it. The agent's own token
# must not be able to merge the PR it just opened, and must not be # must not be able to merge the PR it just opened, and its push credential
# able to push to the protected default branch at all. # must not be able to push to the protected default branch at all.
merged = client._http.post( merged = client._http.post(
f"/repos/{repo_path}/pulls/{pr.number}/merge", json={"Do": "merge"} f"/repos/{repo_path}/pulls/{pr.number}/merge", json={"Do": "merge"}
) )
@@ -85,9 +89,9 @@ def test_push_open_pr_and_reconcile(
"the human gate" "the human gate"
) )
with pytest.raises(GitError): with pytest.raises(GitError):
repo.push(url, "main", token=token) repo.push(url, "main", credential=key)
finally: finally:
if pr is not None: if pr is not None:
client.close_pull_request(repo_path, pr.number) client.close_pull_request(repo_path, pr.number)
repo.delete_remote_branch(url, branch, token=token) repo.delete_remote_branch(url, branch, credential=key)
client.close() client.close()
+3 -2
View File
@@ -19,6 +19,7 @@ from nisse_dev.control import (
PR_OPEN, PR_OPEN,
Orchestrator, Orchestrator,
) )
from nisse_dev.execution import SSHHost
from nisse_dev.executor import LLMExecutor from nisse_dev.executor import LLMExecutor
from nisse_dev.llm import LiteLLMClient, RoleMap from nisse_dev.llm import LiteLLMClient, RoleMap
from nisse_dev.oplog import OpLog from nisse_dev.oplog import OpLog
@@ -59,7 +60,7 @@ def _drive_to_execution(
def test_plan_approve_execute_produces_a_real_file( def test_plan_approve_execute_produces_a_real_file(
tmp_path: Path, litellm: LiteLLMClient, live_model: str tmp_path: Path, litellm: LiteLLMClient, live_model: str, ssh_host: SSHHost
) -> None: ) -> None:
# Both roles use the capable model here: the point is the whole gated # Both roles use the capable model here: the point is the whole gated
# loop over a real proxy, not local-model competence (measured separately). # loop over a real proxy, not local-model competence (measured separately).
@@ -67,7 +68,7 @@ def test_plan_approve_execute_produces_a_real_file(
roles = RoleMap(executor=model, planner=model, helper=model, expert=model) roles = RoleMap(executor=model, planner=model, helper=model, expert=model)
client = litellm client = litellm
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
workspace = Workspace(tmp_path / "work", timeout_s=60) workspace = Workspace(ssh_host, tmp_path / "work", timeout_s=60)
transport = InMemoryTransport() transport = InMemoryTransport()
orchestrator = Orchestrator( orchestrator = Orchestrator(
store, store,