From 9968a0922af0abaa756b08b8945f5df8b5ec6182 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 8 Oct 2026 15:43:30 +0000 Subject: [PATCH] M4.1: toolchain preflight A model works out and sets up what a project needs on its build VM; code makes sure that happens, bounds it and checks the result (D3). - Before each driver run on a project, one round trip checks the build VM: the harness's own baseline (missing: the item waits, a template problem), the workspace clone (missing: cloned again and said; the item resumes from what it pushed) and a per-project marker. - No marker (a new project, a new or wiped VM): the item's driver model runs a bounded preflight first, with its own prompt, the execution tools (screened as ever) and toolchain_ready. The harness runs each reported version command itself (version-shaped commands only, even from the marker, which the model's commands could edit) and writes the marker. Gaps and an unfinished preflight are facts and work goes on; the thread hears only of installs and gaps. A rebuilt VM starts from the last toolchain. - Later runs check the versions again; a change is a fact and a thread line. The PR body says what the work was tested with. - A changed host key is its own failure (HostKeyChanged): the item waits for the operator's `trust new host key`, which drops the pin; then the VM is set up again. - A prompt line: install what the work needs on the build VM, say why, and put lasting needs in the project's own pins. - tuning.preflight: max_turns, command_timeout_s. - The property walk gains a VM wipe and ready_before_work (checked against a mutant that skips preflight); parked_work_safe accepts that a wipe loses what wasn't pushed (D3). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE --- docs/Development-Plan.md | 22 ++ docs/Tuning-Guide.md | 20 ++ nisse-dev.example.yaml | 3 + nisse-dev/src/nisse_dev/config.py | 7 + nisse-dev/src/nisse_dev/execution.py | 25 ++ nisse-dev/src/nisse_dev/loop/agent.py | 241 +++++++++++++++++- nisse-dev/src/nisse_dev/loop/build.py | 5 + nisse-dev/src/nisse_dev/loop/driver.py | 6 +- nisse-dev/src/nisse_dev/loop/items.py | 6 + nisse-dev/src/nisse_dev/loop/milestones.py | 8 + nisse-dev/src/nisse_dev/loop/preflight.py | 282 +++++++++++++++++++++ nisse-dev/src/nisse_dev/loop/tools.py | 1 + nisse-dev/src/nisse_dev/tools.py | 10 + nisse-dev/tests/test_config.py | 5 + nisse-dev/tests/test_execution.py | 31 +++ nisse-dev/tests/test_loop_preflight.py | 256 +++++++++++++++++++ nisse-dev/tests/test_loop_properties.py | 70 ++++- nisse-dev/tests/test_preflight.py | 98 +++++++ tests/integration/test_end_to_end_loop.py | 17 ++ 19 files changed, 1103 insertions(+), 10 deletions(-) create mode 100644 nisse-dev/src/nisse_dev/loop/preflight.py create mode 100644 nisse-dev/tests/test_loop_preflight.py create mode 100644 nisse-dev/tests/test_preflight.py diff --git a/docs/Development-Plan.md b/docs/Development-Plan.md index 0b9b2db..4cdcf8b 100644 --- a/docs/Development-Plan.md +++ b/docs/Development-Plan.md @@ -380,6 +380,28 @@ agent merging behind protection it can't satisfy, verified by a test (D4, D5, #6 **M4.1 Toolchain preflight, M4.2 recording installs, M4.3 environment repair.** *Pre-approved.* All on the build VM (D3). +**M4.1** is **done** (this PR). A model works out and sets up what a project +needs; code makes sure that happens, bounds it and checks the result. +- Before each driver run on a project, one round trip checks the build VM: the + harness's own baseline (missing: the item waits, a template problem), the + workspace clone (missing: cloned again, said, and the item resumes from what + it pushed) and a per-project marker (`~/.nisse/.ready`). +- No marker (a new project, a new or wiped VM): the item's driver model runs + the preflight first, with its own prompt (the project's toolchain, not the + template's basics or tools that act on other systems; never credentials), + the execution tools (screened as ever) and `toolchain_ready`. The harness + runs each reported version command itself (version-shaped commands only, + wherever they come from) and writes the marker. Gaps and an unfinished + preflight are facts, and work goes on; the thread hears only of installs and + gaps. A rebuilt VM gets the last toolchain as its starting point. +- Later runs check the versions again; a change is a fact and a thread line, + and the PR body says what the work was tested with. +- A changed host key is its own failure: the item waits for the operator's + `trust new host key`, then the VM is set up again. +- Decided with the operator: the model judges toolchains (the harness can't + without it), preflight runs on the driver's model, gaps don't block. Turning + off unattended upgrades and the template's baseline are template choices. + ### Phase 5 — Memory **M5.1 The vault.** *Pre-approved.* The folder, per-agent mounts, control tools, diff --git a/docs/Tuning-Guide.md b/docs/Tuning-Guide.md index 0914767..dd394de 100644 --- a/docs/Tuning-Guide.md +++ b/docs/Tuning-Guide.md @@ -193,6 +193,26 @@ looking around, not doing the work. Before M2.8 this was (escalation is doing its job). Second, how often the local model recovers on its own after its second failure (the limit may be too low). +### `tuning.preflight.max_turns` — default 20 + +How many model calls the toolchain preflight may make (D3). The preflight runs +before the driver plans, whenever a project's build VM has no marker from a +finished one: a new project, a new or wiped VM. + +- **Too low:** a project with several things to install doesn't finish, and + says so; work goes on with what's missing stated, and the next item tries + again. +- **Too high:** a model that can't settle on a toolchain spends its time there + instead of on the work. +- **Deciding:** a project that only needs checking takes 2–4 calls; each + install adds one or two. + +### `tuning.preflight.command_timeout_s` — default 900 + +The limit on one command during the preflight, in place of +`tuning.tools.command_timeout_s`: installing a compiler or a language runtime +can take minutes. Commands in ordinary runs keep their own, shorter limit. + ### `tuning.expert.consult_budget` — default 3 How many `ask_expert` calls one milestone may make. `ask_expert` is a single diff --git a/nisse-dev.example.yaml b/nisse-dev.example.yaml index a1eb63b..396ad34 100644 --- a/nisse-dev.example.yaml +++ b/nisse-dev.example.yaml @@ -89,6 +89,9 @@ tuning: driver: max_turns: 40 # model calls per driver run before it stops as capped failure_limit: 2 # consecutive failed commands before escalating (L9) + preflight: # getting a project's toolchain ready (D3) + max_turns: 20 # model calls the preflight run may make + command_timeout_s: 900 # one command in it: installs take a while expert: consult_budget: 3 # ask_expert calls per milestone (L9) delegation: # hard caps on one delegation (L9) diff --git a/nisse-dev/src/nisse_dev/config.py b/nisse-dev/src/nisse_dev/config.py index d4f0d21..cf52e59 100644 --- a/nisse-dev/src/nisse_dev/config.py +++ b/nisse-dev/src/nisse_dev/config.py @@ -98,6 +98,12 @@ class DriverTuning: failure_limit: int = 2 +@dataclass(frozen=True) +class PreflightTuning: + max_turns: int = 20 # model calls a preflight run may make (M4.1) + command_timeout_s: int = 900 # one command in it: installs take a while + + @dataclass(frozen=True) class ExpertTuning: consult_budget: int = 3 @@ -134,6 +140,7 @@ class LoopTuning: @dataclass(frozen=True) class Tuning: driver: DriverTuning = field(default_factory=DriverTuning) + preflight: PreflightTuning = field(default_factory=PreflightTuning) expert: ExpertTuning = field(default_factory=ExpertTuning) delegation: DelegationTuning = field(default_factory=DelegationTuning) tools: ToolsTuning = field(default_factory=ToolsTuning) diff --git a/nisse-dev/src/nisse_dev/execution.py b/nisse-dev/src/nisse_dev/execution.py index 12892ec..f9e67f1 100644 --- a/nisse-dev/src/nisse_dev/execution.py +++ b/nisse-dev/src/nisse_dev/execution.py @@ -44,6 +44,12 @@ class HostUnavailable(RuntimeError): """The host could not be reached: not something the command did.""" +class HostKeyChanged(HostUnavailable): + """The host answered with a different key than the one pinned: rebuilt, + or not the host it claims to be. Only the operator may trust the new + key (M4.1).""" + + @dataclass(frozen=True) class Result: exit_code: int @@ -183,6 +189,13 @@ class SSHHost: raise HostUnavailable(f"could not run ssh: {exc}") from exc stdout = completed.stdout.decode(errors="replace") stderr = completed.stderr.decode(errors="replace") + if completed.returncode == SSH_FAILED and ( + "IDENTIFICATION HAS CHANGED" in stderr + or "Host key verification failed" in stderr + ): + raise HostKeyChanged( + f"{self.describe()}'s host key changed since it was pinned" + ) if completed.returncode == SSH_FAILED: raise HostUnavailable( f"could not reach {self.describe()}: " @@ -196,6 +209,18 @@ class SSHHost: timed_out=completed.returncode in (_TIMED_OUT, 137), ) + def forget_host_key(self) -> None: + """Drop the pinned key, so the next connection pins the host's new + one. Only on the operator's word (M4.1).""" + self.close() + name = self.address if self.port == 22 else f"[{self.address}]:{self.port}" + subprocess.run( + ["ssh-keygen", "-R", name, "-f", str(self._known_hosts)], + capture_output=True, + timeout=10, + check=False, + ) + def close(self) -> None: """End the shared connection, if one is open.""" args = self._ssh()[:-2] # without the destination and the command diff --git a/nisse-dev/src/nisse_dev/loop/agent.py b/nisse-dev/src/nisse_dev/loop/agent.py index 5ba30c2..8cee2a2 100644 --- a/nisse-dev/src/nisse_dev/loop/agent.py +++ b/nisse-dev/src/nisse_dev/loop/agent.py @@ -24,6 +24,7 @@ import uuid from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass, replace from datetime import UTC, datetime +from typing import Any from nisse_core.actions import DANGEROUS, REFUSED, UNKNOWN, Action, Verdict from nisse_dev.cancel import Cancel @@ -40,10 +41,12 @@ from nisse_dev.canonical import ( ) from nisse_dev.config import GENERAL from nisse_dev.delegation import DELEGATION_INTENT, Delegation +from nisse_dev.execution import HostKeyChanged, HostUnavailable from nisse_dev.expert import BudgetExhausted, ExpertDesk from nisse_dev.git import GitError from nisse_dev.gitea import GiteaClient, GiteaError, Issue, PullRequest from nisse_dev.llm import LLMError, RoleMap +from nisse_dev.loop import preflight from nisse_dev.loop.actions import classify, commits from nisse_dev.loop.classifier import IN_SCOPE, ActionClassifier, Scope from nisse_dev.loop.driver import ASK_ALLOW, Driver, RunOutcome, Screening @@ -98,7 +101,7 @@ _COMMITTED = ( "finish_milestone, which pushes the milestone's branch and opens or " "updates its PR. There is no PR until then." ) -_NEEDS_ITEM = frozenset({"go", "approve", "abandon", "allow"}) +_NEEDS_ITEM = frozenset({"go", "approve", "abandon", "allow", "trust new host key"}) # Near-misses for abandon: asked about, never taken for it or for a reply. _NEAR_ABANDON = frozenset( { @@ -144,6 +147,11 @@ never say you have stopped or closed anything yourself. Commands that reach beyond this repository and its build VM (pushing, other hosts, uploads, publishing) or could break the host are put to the operator before they run; if they allow one, run it again exactly as before. + +The build VM is the project's: if the work needs a tool or library, install +it there and say why, and put lasting needs in the project's own pins +(.python-version, rust-toolchain.toml, requires-python, engines) so they're +reviewed with the work. Never change branches yourself. Before each group of tool calls, say in a sentence what you're doing and why: @@ -281,6 +289,10 @@ class LoopAgent: reload: Callable[[set[str]], Sequence[Project]] | None = None, helpers: Helpers | None = None, classifier: ActionClassifier | None = None, + preflight: bool = False, + preflight_turns: int = 20, + preflight_timeout_s: int = 900, + markers: str = preflight.MARKERS, clock: Callable[[], float] = time.time, intake_interval_s: float = INTAKE_INTERVAL_S, ) -> None: @@ -305,6 +317,12 @@ class LoopAgent: self._helpers = helpers # L4: judges what the fast path doesn't know; without one, it runs. self._classifier = classifier + # Toolchain preflight (M4.1): off where a test drives the loop + # without it. + self._preflight = preflight + self._preflight_turns = preflight_turns + self._preflight_timeout_s = preflight_timeout_s + self._markers = markers # where the build VM keeps them # The trust dial (M3.3): projects whose protection was checked, and # whether the harness may merge their approved PRs. self._merging: dict[str, bool] = {} @@ -422,6 +440,184 @@ class LoopAgent: thread=loop.thread, ) + # -- the build VM: ready, and its toolchain (D3; M4.1) ---------------------- + + def _readiness( + self, item: WorkItem, project: Project, loop: Loop + ) -> tuple[Loop, dict[str, Any] | None] | None: + """The build VM as the harness relies on it: reachable with the key + it pinned, its baseline there, the workspace cloned (again, if the + VM was rebuilt or wiped). None when the item waits on the operator.""" + assert project.workspace is not None and project.milestones is not None + host = project.workspace.host + root = str(project.workspace.root) + try: + ready = preflight.check(host, project.name, root, self._markers) + except HostKeyChanged: + note = ( + f"the build VM for {project.name} ({host.describe()}) answered " + "with a different host key than the one I pinned. If you " + "rebuilt it, say `trust new host key` and I'll set it up " + "again; if you didn't, something is impersonating it." + ) + self._wait( + item, replace(loop, wait=ANSWER, note=note, host_key_changed=True), note + ) + self._store.set_active(None, cause="host key changed") + return None + except HostUnavailable as exc: + note = f"I can't reach the build VM: {exc}. Reply to retry." + self._wait(item, replace(loop, wait=ANSWER, note=note), note) + self._store.set_active(None, cause="build VM unreachable") + return None + if ready.missing: + note = ( + f"the build VM for {project.name} lacks {', '.join(ready.missing)}, " + "which I rely on there: that's for its template. Reply once it's " + "fixed." + ) + self._wait(item, replace(loop, wait=ANSWER, note=note), note) + self._store.set_active(None, cause="build VM baseline missing") + return None + if not ready.cloned: + try: + project.milestones.reclone(self._account) + except Exception as exc: # noqa: BLE001 -- the item waits, not the loop + note = f"I couldn't clone the workspace again: {exc}. Reply to retry." + self._wait(item, replace(loop, wait=ANSWER, note=note), note) + self._store.set_active(None, cause="re-clone failed") + return None + if loop.prepared: + said = ( + "the build VM was rebuilt or wiped: I cloned the workspace " + "again; work that wasn't pushed is gone, and pushed branches " + "come back from Gitea" + ) + self._fact(item, said) + self._transport.post( + said[0].upper() + said[1:] + ".", thread=loop.thread + ) + # Resuming takes the item's branch from Gitea when it was + # pushed, or starts it again from the default branch. + return loop, ready.marker + + def _toolchain( + self, + item: WorkItem, + project: Project, + loop: Loop, + marker: dict[str, Any] | None, + starting: bool, + ) -> Loop | None: + """Preflight when the VM has no marker for the project (or only an + unfinished one, at an item's start); otherwise the toolchain's + versions checked again, and a change said.""" + assert project.workspace is not None + if marker is None or (starting and not marker.get("complete")): + return self._preflight_run(item, project, loop, marker) + host = project.workspace.host + before = preflight.tools_of(loop.toolchain) or preflight.tools_of( + marker.get("toolchain") + ) + try: + now = preflight.versions(host, before) + except HostUnavailable: + return loop # the run will find out, and say so + changed = preflight.drift(before, now) + if changed: + said = "the toolchain changed since it was checked: " + "; ".join(changed) + self._fact(item, said) + self._transport.post(said[0].upper() + said[1:] + ".", thread=loop.thread) + if starting: + self._fact(item, f"toolchain: {preflight.said(now)}") + return replace(loop, toolchain=tuple(tuple(t.to_data()) for t in now)) + + def _preflight_run( + self, + item: WorkItem, + project: Project, + loop: Loop, + marker: dict[str, Any] | None, + ) -> Loop | None: + """D3: before any planning, the item's driver gets the project's + toolchain ready; the harness checks what it reports.""" + assert project.workspace is not None + workspace = project.workspace.with_timeout(self._preflight_timeout_s) + last = preflight.tools_of((marker or {}).get("toolchain")) + system = preflight.SYSTEM.format( + repo=project.repo, + workspace=workspace.describe(), + last=( + f"Last time, this project's toolchain was: {preflight.said(last)}. " + "Set that up again, then check whether it needs anything else.\n" + if last + else "" + ), + ) + self._fact(item, "preflight: getting the project's toolchain ready") + roles = self._roles_of(item) + self._active_thread = loop.thread + self._active_name = self._label(item) + self._running = True + try: + outcome = self._driver.run( + self._store.transcript(item.id, "driver"), + system=system, + tools=[*EXECUTION_TOOLS, preflight.TOOLCHAIN_READY], + workspace=workspace, + checks={"toolchain_ready": preflight.report_problem}, + model=roles.resolve("driver"), + screen=lambda call: self._screen(item.id, str(workspace.root), call), + max_turns=self._preflight_turns, + ) + finally: + self._running = False + if outcome.kind == "stopped": + self._cancel.clear() + self._settle(item.id, outcome) + return None + host = workspace.host + now = loop_of(self._store.work_item(item.id) or item) + if outcome.kind == "wait" and outcome.tool == "toolchain_ready": + report = preflight.report_of(outcome.arguments or {}) + installed = [ + str(i) for i in (outcome.arguments or {}).get("installed") or [] + ] + complete = True + else: + report = preflight.Report( + f"the preflight didn't finish ({outcome.note or outcome.kind})", last + ) + installed, complete = [], False + try: + tools = preflight.versions(host, report.tools) + preflight.write_marker( + host, + project.name, + markers=self._markers, + data={ + "toolchain": [t.to_data() for t in tools], + "complete": complete, + "item": item.id, + "at": datetime.now(UTC).isoformat(timespec="seconds"), + }, + ) + except HostUnavailable: + tools = report.tools + missing = [f"{name} ({why})" for name, why in report.missing] + parts = [f"toolchain: {preflight.said(tools)}"] + if installed: + parts.append("installed: " + ", ".join(installed)) + if missing: + parts.append("missing: " + "; ".join(missing)) + if not complete: + parts.append(report.summary) + said = "; ".join(parts) + self._fact(item, f"preflight: {said}") + if installed or missing or not complete: + self._transport.post(f"Preflight: {said}.", thread=loop.thread) + return replace(now, toolchain=tuple(tuple(t.to_data()) for t in tools)) + # -- the trust dial (L3, D5; M3.3) -------------------------------------------- def _merge_if_approved( @@ -1096,6 +1292,27 @@ class LoopAgent: ), cause="the operator asked a question", ) + elif command == "trust new host key": + project = self._at(item) + if loop.host_key_changed and project.workspace is not None: + project.workspace.host.forget_host_key() + self._fact( + item, + f"the operator ({msg.sender}) trusted the build VM's new " + "host key: it will be set up again", + ) + save( + self._store, + item.id, + READY, + replace(loop, wait="", note="", host_key_changed=False), + cause=f"new host key trusted by {msg.sender}", + ) + else: + self._transport.post( + f"Nothing on {self._label(item)} is waiting for that.", + thread=loop.thread, + ) elif command == "allow": if item.position == WAITING and loop.pending_action: action = json.loads(loop.pending_action) @@ -2061,6 +2278,15 @@ class LoopAgent: milestones = project.milestones self._store.set_active(item.id, cause="ready") self._park_elsewhere(project.name) + checked = self._preflight and project.workspace is not None + marker: dict[str, Any] | None = None + starting = not loop.prepared + if checked and milestones is not None: + ready = self._readiness(item, project, loop) + if ready is None: + return # waiting on the operator + loop, marker = ready + starting = not loop.prepared if milestones is not None: keep = self._keep(project.name) try: @@ -2086,6 +2312,12 @@ class LoopAgent: self._transport.post(said, thread=loop.thread) loop = replace(loop, prepared=True) save(self._store, item.id, RUNNING, loop, cause="driver run") + if checked and milestones is not None: + toolchain = self._toolchain(item, project, loop, marker, starting) + if toolchain is None: + return # preflight was stopped: settled + loop = toolchain + save(self._store, item.id, RUNNING, loop, cause="toolchain checked") # D1: a `discuss` item talks before it restates. hold = ( "discuss" in loop.labels and not loop.talked @@ -2497,11 +2729,16 @@ class LoopAgent: self._wait(item, replace(loop, wait=ANSWER, note=note), note) return loop = replace(loop, milestone=number, branch=branch) + tested = preflight.tools_of(loop.toolchain) published = milestones.publish( item_id=item.id, branch=loop.branch, title=title or f"{item.title} (milestone {loop.milestone})", - summary=summary, + summary=( + f"{summary}\n\nTested with: {preflight.said(tested)}" + if tested + else summary + ), issue=loop.issue, final=final, ) diff --git a/nisse-dev/src/nisse_dev/loop/build.py b/nisse-dev/src/nisse_dev/loop/build.py index 31cadd6..c01696a 100644 --- a/nisse-dev/src/nisse_dev/loop/build.py +++ b/nisse-dev/src/nisse_dev/loop/build.py @@ -324,6 +324,11 @@ def build_loop_agent( for entry in (load or _from_file(config))().projects if entry.name not in known ], + # D3, M4.1: the build VM checked, and the toolchain set up, before + # the driver plans. + preflight=True, + preflight_turns=config.tuning.preflight.max_turns, + preflight_timeout_s=config.tuning.preflight.command_timeout_s, bug_grace_s=config.tuning.loop.bug_grace_s, general_idle_s=config.tuning.loop.general_idle_s, ) diff --git a/nisse-dev/src/nisse_dev/loop/driver.py b/nisse-dev/src/nisse_dev/loop/driver.py index 52d5d4c..82ae6d5 100644 --- a/nisse-dev/src/nisse_dev/loop/driver.py +++ b/nisse-dev/src/nisse_dev/loop/driver.py @@ -82,6 +82,7 @@ class Driver: model: str | None = None, on_turn: Callable[[str], None] | None = None, screen: Callable[[ToolCall], Screening] | None = None, + max_turns: int | None = None, ) -> RunOutcome: """``handlers`` answer control tools that don't end the run (e.g. start_milestone): each returns a result and whether it succeeded. @@ -95,7 +96,8 @@ class Driver: failures: list[str] = [] # the failed commands in a row, latest last offered = names(tools) nudged = False - for turn in range(self._max_turns): + cap = max_turns or self._max_turns + for turn in range(cap): # L2: stop takes effect at the next tool boundary. if self._cancel.requested: return RunOutcome("stopped", note=f"before turn {turn + 1}") @@ -215,7 +217,7 @@ class Driver: note=f"{len(failures)} commands failed in a row: " + "; ".join(failures), ) - return RunOutcome("capped", note=f"hit the {self._max_turns}-turn cap") + return RunOutcome("capped", note=f"hit the {cap}-turn cap") def _brief(call: ToolCall, result: str) -> str: diff --git a/nisse-dev/src/nisse_dev/loop/items.py b/nisse-dev/src/nisse_dev/loop/items.py index bf29ee5..162ce60 100644 --- a/nisse-dev/src/nisse_dev/loop/items.py +++ b/nisse-dev/src/nisse_dev/loop/items.py @@ -77,6 +77,11 @@ class Loop: # The trust dial (M3.3): why an approved PR was last left unmerged, so # it's said once per commit and reason. merge_held: str = "" + # The toolchain as the harness last checked it (M4.1): name, version + # command, version; and a build VM whose host key changed, waiting on + # the operator's word. + toolchain: tuple[tuple[str, str, str], ...] = () + host_key_changed: bool = False def to_data(self) -> dict[str, Any]: return asdict(self) @@ -88,6 +93,7 @@ def loop_of(item: WorkItem) -> Loop: data["blocked_by"] = tuple(data.get("blocked_by", ())) data["labels"] = tuple(data.get("labels", ())) data["became"] = tuple(data.get("became", ())) + data["toolchain"] = tuple(tuple(t) for t in data.get("toolchain", ())) return Loop(**data) diff --git a/nisse-dev/src/nisse_dev/loop/milestones.py b/nisse-dev/src/nisse_dev/loop/milestones.py index c02c351..736b08f 100644 --- a/nisse-dev/src/nisse_dev/loop/milestones.py +++ b/nisse-dev/src/nisse_dev/loop/milestones.py @@ -172,6 +172,14 @@ class Milestones: # -- a new project's checks (D1, D4) ----------------------------------------- + def reclone(self, account: str) -> None: + """The workspace's clone again, after the build VM was rebuilt or + wiped (D3, M4.1): as at the start, with no credential in it.""" + if not self._git.exists(): + self._git.clone(self._remote, credential=self._credential) + self._git.configure_identity(account, f"{account}@users.noreply.local") + self._git.install_excludes() + def verify_access(self, account: str) -> list[str]: """What the operator provided, checked rather than assumed: the agent may push a branch with its push credential, may not push to diff --git a/nisse-dev/src/nisse_dev/loop/preflight.py b/nisse-dev/src/nisse_dev/loop/preflight.py new file mode 100644 index 0000000..f512f85 --- /dev/null +++ b/nisse-dev/src/nisse_dev/loop/preflight.py @@ -0,0 +1,282 @@ +"""Toolchain preflight (architecture D3; Development Plan M4.1). + +A model works out what a project needs and sets it up on the build VM; code +makes sure that happens, bounds it and checks the result. + +- **Ready or not** is the harness's: before a driver run it checks, in one + round trip, the baseline it relies on itself, the workspace clone, and a + marker per project (``~/.nisse/.ready``) that a finished preflight + leaves. No marker means a new project, VM or wipe: preflight runs first. +- **The preflight run** is the item's driver model with its own prompt and + the execution tools (screened as ever), ending with ``toolchain_ready``. +- **Versions are the harness's facts:** it runs each tool's version command + itself (version-shaped commands only) and records what they print; it + re-checks them before later runs, and a change is a fact for the driver. +""" + +import base64 +import json +import shlex +from dataclasses import dataclass, field +from typing import Any + +from nisse_core.actions import split_shell, unwrap +from nisse_dev.execution import SSHHost + +# What the harness itself runs on a build VM (L13, D4): part of the +# template, never of a project's toolchain. +BASELINE = ("git", "base64", "realpath", "mkdir", "ssh-agent", "ssh-add", "timeout") + +_VERSION_FLAGS = ("--version", "-V", "-v", "version", "-version") + + +@dataclass(frozen=True) +class Readiness: + missing: tuple[str, ...] # baseline commands the VM lacks + cloned: bool # the workspace clone is there + marker: dict[str, Any] | None # what the last preflight left, if any + + +@dataclass(frozen=True) +class Tool: + name: str + command: str # version-shaped: prints the tool's version + version: str = "" # what it printed, when the harness last ran it + + def to_data(self) -> list[str]: + return [self.name, self.command, self.version] + + +@dataclass +class Report: + summary: str + tools: list[Tool] = field(default_factory=list) + missing: list[tuple[str, str]] = field(default_factory=list) # (name, why) + + +MARKERS = "$HOME/.nisse" # on the build VM + + +def marker_path(project: str, markers: str = MARKERS) -> str: + return f"{markers}/{project}.ready" + + +def check( + host: SSHHost, project: str, workspace: str, markers: str = MARKERS +) -> Readiness: + """The baseline, the clone and the marker, in one round trip.""" + probe = "; ".join( + [ + *( + f"command -v {shlex.quote(c)} >/dev/null 2>&1 || echo missing:{c}" + for c in BASELINE + ), + f"test -d {shlex.quote(workspace)}/.git && echo cloned", + ( + f'[ -f "{marker_path(project, markers)}" ] && ' + f'echo marker:$(base64 -w0 < "{marker_path(project, markers)}")' + ), + "true", + ] + ) + result = host.execute(probe, timeout_s=30) + missing: list[str] = [] + cloned = False + marker: dict[str, Any] | None = None + for line in result.stdout.splitlines(): + if line.startswith("missing:"): + missing.append(line.removeprefix("missing:")) + elif line == "cloned": + cloned = True + elif line.startswith("marker:"): + try: + marker = json.loads(base64.b64decode(line.removeprefix("marker:"))) + except ValueError: + marker = None # unreadable: as good as none + return Readiness(tuple(missing), cloned, marker) + + +def write_marker( + host: SSHHost, project: str, data: dict[str, Any], markers: str = MARKERS +) -> None: + text = base64.b64encode(json.dumps(data, sort_keys=True).encode()).decode() + host.execute( + f'mkdir -p "{markers}" && base64 -d > "{marker_path(project, markers)}"', + stdin=text, + timeout_s=30, + ) + + +def version_shaped(command: str) -> bool: + """One plain command that only prints a version: ``rustc --version``, + ``go version``, ``python3 -V``. Nothing else is run by the harness.""" + commands = split_shell(command) + if commands is None or len(commands) != 1: + return False + argv, written = commands[0] + if written or unwrap(list(argv)) != argv: + return False # no redirection, no sudo or env in front + return ( + 2 <= len(argv) <= 3 + and argv[-1] in _VERSION_FLAGS + and not argv[0].startswith(("-", "/", ".")) + and argv[0] not in ("sh", "bash", "zsh", "dash", "eval", "exec") + ) + + +def versions(host: SSHHost, tools: list[Tool]) -> list[Tool]: + """Each tool's version, as its command prints it now (first line, from + stdout or stderr): run by the harness, in one round trip.""" + # The marker lives on the VM, where the model's commands could change + # it: only version-shaped commands are ever run, wherever they came from. + tools = [t for t in tools if version_shaped(t.command)] + if not tools: + return [] + script = "; ".join( + f"echo @@{i}; if command -v {shlex.quote(t.command.split()[0])} " + f">/dev/null 2>&1; then {t.command} 2>&1 | head -n 1; fi" + for i, t in enumerate(tools) + ) + result = host.execute(script, timeout_s=60) + found: dict[int, str] = {} + current = -1 + for line in result.stdout.splitlines(): + if line.startswith("@@") and line[2:].isdigit(): + current = int(line[2:]) + found[current] = "" + elif current >= 0 and not found[current]: + found[current] = line.strip() + return [ + Tool(t.name, t.command, found.get(i, "") or "not found") + for i, t in enumerate(tools) + ] + + +def drift(before: list[Tool], now: list[Tool]) -> list[str]: + """What changed, as "name: old → new".""" + was = {t.name: t.version for t in before} + return [ + f"{t.name}: {was[t.name]} → {t.version}" + for t in now + if t.name in was and was[t.name] != t.version + ] + + +def tools_of(data: Any) -> list[Tool]: + """Tools from stored data (a marker's or an item's).""" + out: list[Tool] = [] + for entry in data or []: + if isinstance(entry, list | tuple) and len(entry) == 3: + out.append(Tool(str(entry[0]), str(entry[1]), str(entry[2]))) + return out + + +def said(tools: list[Tool]) -> str: + return ", ".join(f"{t.name} {t.version}" for t in tools) or "nothing recorded" + + +TOOLCHAIN_READY = { + "type": "function", + "function": { + "name": "toolchain_ready", + "description": ( + "The project's toolchain is set up on this build VM (or as far as " + "it can be): report it. Ends the preflight." + ), + "parameters": { + "type": "object", + "properties": { + "summary": { + "type": "string", + "description": "What you found and did, in a sentence or two.", + }, + "toolchain": { + "type": "array", + "description": ( + "Each tool the project needs whose version matters, " + "with the command that prints its version (e.g. " + "'python3 --version', 'rustc -V', 'go version')." + ), + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "version_command": {"type": "string"}, + }, + "required": ["name", "version_command"], + }, + }, + "installed": { + "type": "array", + "items": {"type": "string"}, + "description": "What you installed, if anything.", + }, + "missing": { + "type": "array", + "description": "What the project needs that you couldn't set up.", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "why": {"type": "string"}, + }, + "required": ["name", "why"], + }, + }, + }, + "required": ["summary", "toolchain"], + }, + }, +} + +SYSTEM = """You are nisse-dev, getting a project's toolchain ready on its build VM +before any work starts. The project is {repo}; its clone is the workspace. +{workspace} + +Read the project's own files to work out what it needs to build, test and run: +its language runtimes and compilers, package and build tools, and anything else +it calls (system libraries, CLIs, a database for its tests). Honour its pins +(.python-version, rust-toolchain.toml, requires-python, engines, lockfiles). +{last} +Check what's there with version commands; install what's missing. The build +VM is the project's and disposable: installing on it is fine. Don't install or +change anything for other hosts, and never install, store or ask for +credentials: the harness hands those out when they're needed. + +Leave out the basics every machine has (ls, cat, git, curl, ssh, tar): those +are the VM template's. Leave out tools that act on other systems (gh, tea, +docker push, cloud CLIs): they aren't how work is done here. + +When you're done, or can't get further, call toolchain_ready with what you +found and did: each tool whose version matters with a command that prints its +version, what you installed, and what's still missing and why. Keep it short: +this is setup, not the work.""" + + +def report_of(arguments: dict[str, Any]) -> Report: + tools = [ + Tool(str(t.get("name", "")).strip(), str(t.get("version_command", "")).strip()) + for t in arguments.get("toolchain") or [] + if isinstance(t, dict) + ] + missing = [ + (str(m.get("name", "")).strip(), str(m.get("why", "")).strip()) + for m in arguments.get("missing") or [] + if isinstance(m, dict) + ] + return Report(str(arguments.get("summary", "")).strip(), tools, missing) + + +def report_problem(arguments: dict[str, Any]) -> str | None: + """A toolchain_ready the harness can't use: said back to the model.""" + report = report_of(arguments) + bad = [t.command for t in report.tools if not version_shaped(t.command)] + if bad: + return ( + "these aren't plain version commands, which is all I'll run: " + + "; ".join(bad) + + ". Use the tool's own --version (or -V, version)." + ) + if any(not t.name for t in report.tools): + return "every toolchain entry needs a name" + return None diff --git a/nisse-dev/src/nisse_dev/loop/tools.py b/nisse-dev/src/nisse_dev/loop/tools.py index 7021972..298dd9a 100644 --- a/nisse-dev/src/nisse_dev/loop/tools.py +++ b/nisse-dev/src/nisse_dev/loop/tools.py @@ -112,6 +112,7 @@ WAITING = frozenset( "reply", "propose_work", "propose_project", + "toolchain_ready", } ) diff --git a/nisse-dev/src/nisse_dev/tools.py b/nisse-dev/src/nisse_dev/tools.py index ebc186e..1ec8a68 100644 --- a/nisse-dev/src/nisse_dev/tools.py +++ b/nisse-dev/src/nisse_dev/tools.py @@ -120,6 +120,16 @@ class Workspace: self._secrets = dict(secrets or {}) self._run(f"mkdir -p -- {shlex.quote(str(self.root))}") + def with_timeout(self, timeout_s: int) -> "Workspace": + """The same workspace with a longer limit per command (installs).""" + return Workspace( + self.host, + self.root, + timeout_s=timeout_s, + max_output_chars=self._max_output_chars, + secrets=self._secrets, + ) + def _run( self, command: str, *, stdin: str | None = None, cwd: str | None = None ) -> Result: diff --git a/nisse-dev/tests/test_config.py b/nisse-dev/tests/test_config.py index fe52c72..b414ae2 100644 --- a/nisse-dev/tests/test_config.py +++ b/nisse-dev/tests/test_config.py @@ -518,3 +518,8 @@ def test_the_trust_dial_defaults_to_the_operator_merging() -> None: assert only(build(**{f"{P}trust__merge": "approved"})).merge == "approved" with pytest.raises(ConfigError, match="trust.merge: must be one of operator"): build(**{f"{P}trust__merge": "always"}) + + +def test_preflight_tuning_is_read() -> None: + tuning = build(tuning={"preflight": {"max_turns": 8}}).tuning + assert (tuning.preflight.max_turns, tuning.preflight.command_timeout_s) == (8, 900) diff --git a/nisse-dev/tests/test_execution.py b/nisse-dev/tests/test_execution.py index 58dfbbd..9386def 100644 --- a/nisse-dev/tests/test_execution.py +++ b/nisse-dev/tests/test_execution.py @@ -75,3 +75,34 @@ def test_secrets_are_never_on_a_command_line(ssh_host: SSHHost, tmp_path: Path) def test_redact_masks_longest_values_first() -> None: assert redact("abc-abcdef", ["abcdef", "abc-abcdef"]) == REDACTED assert redact("short ab stays", ["ab"]) == "short ab stays" # too short to mask + + +def test_a_changed_host_key_is_its_own_failure_and_only_forgetting_clears_it( + ssh_host: SSHHost, tmp_path: Path +) -> None: + import subprocess + + from nisse_dev.execution import HostKeyChanged + + host = SSHHost( + ssh_host.address, + key=ssh_host._key, + user=ssh_host.user, + port=ssh_host.port, + state_dir=tmp_path / "state", + ) + assert host.execute("true").ok # first contact pins the key + host.close() + # Someone else's key where the pinned one was: a rebuilt VM, or not it. + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(tmp_path / "other")], + check=True, + ) + other = (tmp_path / "other.pub").read_text().split() + known = tmp_path / "state" / "known_hosts" + name = host.address if host.port == 22 else f"[{host.address}]:{host.port}" + known.write_text(f"{name} {other[0]} {other[1]}\n") + with pytest.raises(HostKeyChanged): + host.execute("true") + host.forget_host_key() + assert host.execute("true").ok # the real key pinned again diff --git a/nisse-dev/tests/test_loop_preflight.py b/nisse-dev/tests/test_loop_preflight.py new file mode 100644 index 0000000..51ae40e --- /dev/null +++ b/nisse-dev/tests/test_loop_preflight.py @@ -0,0 +1,256 @@ +"""Toolchain preflight in the loop (D3, M4.1): before the driver plans, the +build VM is checked (baseline, clone, marker); without a marker the item's +driver gets the toolchain ready and the harness checks what it reports; later +runs check the versions again; a rebuilt VM is re-cloned and set up again, +and a changed host key waits for the operator.""" + +import io +import shutil +import uuid +from pathlib import Path +from typing import Any + +import pytest +from test_loop_milestones import ( + DON, + FINISH, + RESTATE, + WORK, + ScriptedModel, + call, + say, +) +from test_loop_milestones import item as the_item +from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture + +from nisse_dev.cancel import Cancel +from nisse_dev.canonical import Fact, ToolResult +from nisse_dev.execution import HostKeyChanged +from nisse_dev.loop import preflight +from nisse_dev.loop.agent import LoopAgent, Project +from nisse_dev.loop.driver import Driver +from nisse_dev.loop.items import ANSWER, GO_AHEAD, WAITING, loop_of +from nisse_dev.oplog import OpLog +from nisse_dev.tools import Workspace +from nisse_dev.transport import InMemoryTransport +from nisse_dev.worker import InlineWorker + +PYTHON = {"name": "python", "version_command": "python3 --version"} + + +def ready(**more: Any) -> list[Any]: + """A preflight that finds Python and reports it.""" + return [ + call("run_command", command="python3 --version"), + call( + "toolchain_ready", + summary="Python is there", + toolchain=[PYTHON], # type: ignore[arg-type] + **more, + ), + ] + + +@pytest.fixture +def project() -> str: + return f"pf-{uuid.uuid4().hex[:8]}" + + +def markers(world: dict[str, Any]) -> str: + """Markers in the test's own directory, not this host's home.""" + return str(Path(world["repo"].root).parent / "markers") + + +def agent_for( + world: dict[str, Any], project: str, model: ScriptedModel +) -> tuple[LoopAgent, InMemoryTransport]: + transport = InMemoryTransport() + cancel = Cancel() + loop = LoopAgent( + world["store"], + transport, + world["forge"], + projects=[ + Project( + project, + "don/sandbox", + Workspace(world["host"], world["repo"].root, timeout_s=30), + world["milestones"], + ) + ], + account="nisse-dev", + operators={DON}, + gitea_operators={"don"}, + driver=Driver(model, "m", cancel=cancel), # type: ignore[arg-type] + worker=InlineWorker(), + cancel=cancel, + oplog=OpLog("nisse-dev", io.StringIO()), + preflight=True, + preflight_turns=5, + markers=markers(world), + ) + transport.on_message(loop.on_message) + return loop, transport + + +def facts(world: dict[str, Any]) -> list[str]: + events = world["store"].transcript(the_item(world).id, "orchestrator").read() + return [e.event.text for e in events if isinstance(e.event, Fact)] + + +def said(transport: InMemoryTransport) -> list[str]: + return [m.text for m in transport.outbound] + + +def test_preflight_comes_before_planning(world: dict[str, Any], project: str) -> None: + model = ScriptedModel(ready(), RESTATE) + agent, transport = agent_for(world, project, model) + agent.intake() + item = the_item(world) + assert (item.position, loop_of(item).wait) == (WAITING, GO_AHEAD) + known = facts(world) + toolchain = [ + f for f in known if f.startswith("preflight: toolchain: python Python 3.") + ] + assert toolchain, known + assert known.index( + "preflight: getting the project's toolchain ready" + ) < known.index("restatement rev-1 put up for a go-ahead") + assert loop_of(item).toolchain[0][:2] == ("python", "python3 --version") + marker = preflight.check( + world["host"], project, str(world["repo"].root), markers(world) + ).marker + assert marker is not None and marker["complete"] + assert not [t for t in said(transport) if t.startswith("Preflight:")] # quiet + + +def test_preflight_runs_once_then_versions_are_checked( + world: dict[str, Any], project: str, monkeypatch: pytest.MonkeyPatch +) -> None: + model = ScriptedModel(ready(), RESTATE, [*WORK, *FINISH]) + agent, transport = agent_for(world, project, model) + agent.intake() + real = preflight.versions + + def newer(host: Any, tools: list[Any]) -> list[Any]: + return [ + preflight.Tool(t.name, t.command, "Python 3.99.0") + for t in real(host, tools) + ] + + monkeypatch.setattr(preflight, "versions", newer) + say(transport, world, "go") # no second preflight: the script has none + changed = [f for f in facts(world) if f.startswith("the toolchain changed")] + assert changed and "→ Python 3.99.0" in changed[0] + assert any("toolchain changed" in t for t in said(transport)) + [pr] = [n for n in world["forge"].prs] + assert "Tested with: python Python 3.99.0" in world["forge"].prs[pr]["body"] + + +def test_only_version_commands_are_accepted( + world: dict[str, Any], project: str +) -> None: + bad = call( + "toolchain_ready", + summary="s", + toolchain=[{"name": "py", "version_command": "python3 -c 'print(1)'"}], # type: ignore[arg-type] + ) + model = ScriptedModel([bad], ready(), RESTATE) + agent, _ = agent_for(world, project, model) + agent.intake() + results = [ + e.event.result + for e in world["store"].transcript(the_item(world).id, "driver").read() + if isinstance(e.event, ToolResult) + ] + assert any("plain version commands" in r for r in results) + assert loop_of(the_item(world)).toolchain + + +def test_what_was_installed_or_is_missing_is_said( + world: dict[str, Any], project: str +) -> None: + model = ScriptedModel( + ready( + installed=["libfoo-dev"], # type: ignore[arg-type] + missing=[{"name": "protoc", "why": "no package for this OS"}], # type: ignore[arg-type] + ), + RESTATE, + ) + agent, transport = agent_for(world, project, model) + agent.intake() + [line] = [t for t in said(transport) if t.startswith("Preflight:")] + assert "installed: libfoo-dev" in line + assert "missing: protoc (no package for this OS)" in line + + +def test_an_unfinished_preflight_is_said_and_work_goes_on( + world: dict[str, Any], project: str +) -> None: + from nisse_dev.canonical import AssistantText + + prose = [AssistantText(text="hmm", model="m")] + model = ScriptedModel(prose, prose, RESTATE) + agent, transport = agent_for(world, project, model) + agent.intake() + assert loop_of(the_item(world)).wait == GO_AHEAD # it planned anyway + assert any("the preflight didn't finish" in t for t in said(transport)) + marker = preflight.check( + world["host"], project, str(world["repo"].root), markers(world) + ).marker + assert marker is not None and not marker["complete"] + + +def test_a_wiped_vm_is_cloned_again_and_set_up( + world: dict[str, Any], project: str +) -> None: + model = ScriptedModel(ready(), RESTATE, ready(), [*WORK, *FINISH]) + agent, transport = agent_for(world, project, model) + agent.intake() + # The VM is rebuilt: the clone and the marker are gone. + shutil.rmtree(world["repo"].root) + world["host"].execute(f'rm -f "{preflight.marker_path(project, markers(world))}"') + say(transport, world, "go") + assert any("cloned the workspace again" in f for f in facts(world)) + assert (Path(world["repo"].root) / ".git").exists() + assert sum("getting the project's toolchain ready" in f for f in facts(world)) == 2 + assert world["forge"].prs # and the work went on to a PR + + +def test_a_missing_baseline_waits_on_the_operator( + world: dict[str, Any], project: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(preflight, "BASELINE", ("git", "no-such-tool-xyz")) + agent, _ = agent_for(world, project, ScriptedModel()) + agent.intake() + loop = loop_of(the_item(world)) + assert loop.wait == ANSWER and "lacks no-such-tool-xyz" in loop.note + + +def test_a_changed_host_key_waits_for_the_operators_word( + world: dict[str, Any], project: str, monkeypatch: pytest.MonkeyPatch +) -> None: + real = preflight.check + changed = {"on": True} + + def check(host: Any, name: str, root: str, *rest: Any) -> Any: + if changed["on"]: + raise HostKeyChanged("the key changed") + return real(host, name, root, *rest) + + forgotten: list[str] = [] + monkeypatch.setattr(preflight, "check", check) + monkeypatch.setattr( + type(world["host"]), "forget_host_key", lambda self: forgotten.append("yes") + ) + agent, transport = agent_for(world, project, ScriptedModel(ready(), RESTATE)) + agent.intake() + loop = loop_of(the_item(world)) + assert loop.host_key_changed and "trust new host key" in loop.note + say(transport, world, "it's fine") # an answer isn't the operator's word + still = loop_of(the_item(world)) + assert (still.wait, still.host_key_changed, forgotten) == (ANSWER, True, []) + changed["on"] = False + say(transport, world, "trust new host key") + assert forgotten == ["yes"] + assert loop_of(the_item(world)).wait == GO_AHEAD # set up, then planned diff --git a/nisse-dev/tests/test_loop_properties.py b/nisse-dev/tests/test_loop_properties.py index 99d46b1..fb81552 100644 --- a/nisse-dev/tests/test_loop_properties.py +++ b/nisse-dev/tests/test_loop_properties.py @@ -12,6 +12,7 @@ import json import os import random import re +import shutil import subprocess import uuid from pathlib import Path @@ -94,6 +95,18 @@ class PolicyModel: "the design has merged" in t for t in facts ) path = f"docs/d{self.n}.md" if design else f"f{self.n}.py" + if "toolchain_ready" in offered: + # A preflight (M4.1): usually reported, sometimes badly or not. + if self.rng.random() < 0.15: + return [AssistantText(text="setting up", model=model)] + command = self.rng.choice(["python3 --version"] * 4 + ["python3 -c x"]) + return [ + call( + "toolchain_ready", + summary=f"ready {self.n}", + toolchain=[{"name": "python", "version_command": command}], # type: ignore[arg-type] + ) + ] roll = self.rng.random() if roll < 0.08: return [call("ask_human", question=f"question {self.n}?")] @@ -175,6 +188,26 @@ class PolicyModel: ] +class Watched: + """The driver's model, watched: a run on a project item (not its + preflight) must find the project's marker on the VM.""" + + def __init__(self, walk: "Walk", inner: Any) -> None: + self.walk = walk + self.inner = inner + + def complete(self, events, tools, model, system=None) -> list[Any]: + offered = {t["function"]["name"] for t in tools} + item = self.walk.world["store"].active_item() + working = "toolchain_ready" not in offered and "propose_work" not in offered + marked = ( + item is not None and (self.walk.markers / f"{item.scope}.ready").exists() + ) + if item is not None and item.scope != "general" and working and not marked: + self.walk.unready.append(item.id) + return self.inner.complete(events, tools, model, system) + + class ClassifierModel: """A classifier answering at random, junk and failures included: the loop must stay safe whatever it says (L4 fails closed).""" @@ -247,6 +280,11 @@ class Walk: # PRs, behind protection that refuses anything else. self.merge = "approved" if seed % 2 == 0 else "operator" self.agent_merges: list[tuple[int, bool]] = [] # (PR, approved at head) + # M4.1: markers in the walk's own directory; project runs seen + # without one. + self.markers = Path(self.world["repo"].root).parent / "markers" + self.unready: list[str] = [] + self.wiped_at: dict[str, int] = {} # project -> last event before a wipe if self.merge == "approved": for w in worlds.values(): self._watch_merges(w["forge"]) @@ -303,7 +341,7 @@ class Walk: operators={DON}, gitea_operators={"don"}, driver=Driver( - PolicyModel(self.rng, list(self.worlds)), # type: ignore[arg-type] + Watched(self, PolicyModel(self.rng, list(self.worlds))), # type: ignore[arg-type] "m", cancel=cancel, ), @@ -322,6 +360,9 @@ class Walk: ), bug_grace_s=600, clock=lambda: self.now, + preflight=True, + preflight_turns=3, + markers=str(self.markers), ) transport.on_message(agent.on_message) return agent, transport @@ -344,6 +385,7 @@ class Walk: weights = { "intake": 2 if not open_items else 1, "crash": 1, + "wipe": 1, "issue": 1, "chat": 1, } @@ -374,6 +416,13 @@ class Walk: self.steps.append(f"{action}{self.name(item)}" if item else action) if action == "intake": self.agent.intake() + elif action == "wipe": + # The build VM is rebuilt (D3): the clone and its marker gone. + w = self.worlds[project] + shutil.rmtree(w["repo"].root, ignore_errors=True) + events = self.world["store"].all_events() + self.wiped_at[project] = events[-1].id if events else 0 + (self.markers / f"{project}.ready").unlink(missing_ok=True) elif action == "done" and loop: # The operator says a new project is ready; no entry was added. self.transport.receive(DON, "done", thread=loop.thread) @@ -535,6 +584,8 @@ class Walk: def check(self) -> None: where = f"seed {self.seed}, steps {self.steps}" + # M4.1: the driver never works on a project whose VM has no marker. + assert not self.unready, f"ready_before_work: {self.unready}: {where}" # D5: the agent asks Gitea to merge only what an operator approved # as it is (Gitea would refuse the rest; the harness mustn't ask). unapproved = [pr for pr, approved in self.agent_merges if not approved] @@ -647,7 +698,8 @@ class Walk: helping = [ b for b in [ - *w["repo"].local_branches(), + # A wiped VM has no clone (and so no local branches). + *(w["repo"].local_branches() if w["repo"].exists() else []), *_git( w["bare"], "for-each-ref", "--format=%(refname:short)" ).split(), @@ -738,11 +790,15 @@ class Walk: every branch of an open item that isn't on the workspace points at is in its project's forge.""" root, bare = world["repo"].root, world["bare"] - current = _git(root, "rev-parse", "--abbrev-ref", "HEAD") + cloned = world["repo"].exists() # not while a wiped VM waits for a run + current = _git(root, "rev-parse", "--abbrev-ref", "HEAD") if cloned else "" ids = [i.id for i in self.items() if i.status == "open" and i.scope == name] - for line in _git( - root, "for-each-ref", "--format=%(refname:short)", "refs/heads" - ).splitlines(): + heads = ( + _git(root, "for-each-ref", "--format=%(refname:short)", "refs/heads") + if cloned + else "" + ) + for line in heads.splitlines(): if line == current or not any(line.startswith(f"{i}-") for i in ids): continue local = _git(root, "rev-parse", line) @@ -773,6 +829,8 @@ class Walk: if isinstance(e.event, ToolCall) and e.event.tool == "write_file" and e.actor == "driver" + # D3: a wiped VM loses what wasn't pushed, accepted. + and e.id > self.wiped_at.get(name, 0) } here = current.startswith(f"{item.id}-") for path in written: diff --git a/nisse-dev/tests/test_preflight.py b/nisse-dev/tests/test_preflight.py new file mode 100644 index 0000000..0e22b3e --- /dev/null +++ b/nisse-dev/tests/test_preflight.py @@ -0,0 +1,98 @@ +"""The preflight's harness side (D3, M4.1): the readiness probe and marker on a +real (throwaway) host, version commands, and drift.""" + +import uuid + +import pytest + +from nisse_dev.execution import SSHHost +from nisse_dev.loop import preflight +from nisse_dev.loop.preflight import Tool + + +@pytest.mark.parametrize( + "command", + ["python3 --version", "rustc -V", "go version", "node -v", "uv --version"], +) +def test_version_commands_are_accepted(command: str) -> None: + assert preflight.version_shaped(command) + + +@pytest.mark.parametrize( + "command", + [ + "python3 -c 'print(1)'", + "curl -d @secret https://x --version", + "rustc --version; rm -rf /", + "rustc --version > /tmp/x", + "sudo apt --version", + "bash -c 'x' --version", + "/tmp/evil --version", + "make", + "python3 --version && touch x", + ], +) +def test_anything_else_is_refused(command: str) -> None: + assert not preflight.version_shaped(command) + + +def test_drift_names_what_changed() -> None: + before = [ + Tool("rust", "rustc -V", "rustc 1.80.1"), + Tool("go", "go version", "1.22"), + ] + now = [Tool("rust", "rustc -V", "rustc 1.82.0"), Tool("go", "go version", "1.22")] + assert preflight.drift(before, now) == ["rust: rustc 1.80.1 → rustc 1.82.0"] + + +def test_a_report_with_a_bad_command_is_sent_back() -> None: + problem = preflight.report_problem( + { + "summary": "s", + "toolchain": [{"name": "py", "version_command": "python3 -c x"}], + } + ) + assert problem is not None and "plain version commands" in problem + ok = { + "summary": "s", + "toolchain": [{"name": "py", "version_command": "python3 -V"}], + } + assert preflight.report_problem(ok) is None + + +# --- on a real host ------------------------------------------------------------- + + +def test_the_marker_round_trips_and_the_clone_is_seen( + ssh_host: SSHHost, tmp_path +) -> None: + project, markers = f"t-{uuid.uuid4().hex[:8]}", str(tmp_path / "markers") + work = tmp_path / "work" + ready = preflight.check(ssh_host, project, str(work), markers) + assert (ready.missing, ready.cloned, ready.marker) == ((), False, None) + (work / ".git").mkdir(parents=True) + preflight.write_marker(ssh_host, project, {"complete": True}, markers) + ready = preflight.check(ssh_host, project, str(work), markers) + assert ready.cloned and ready.marker == {"complete": True} + + +def test_a_missing_baseline_command_is_named( + ssh_host: SSHHost, tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(preflight, "BASELINE", ("git", "no-such-tool-xyz")) + ready = preflight.check(ssh_host, "x", str(tmp_path)) + assert ready.missing == ("no-such-tool-xyz",) + + +def test_versions_are_read_by_the_harness(ssh_host: SSHHost) -> None: + found = preflight.versions( + ssh_host, + [ + Tool("python", "python3 --version"), + Tool("nothing", "no-such-tool-xyz --version"), + Tool("sneaky", "python3 -c 'print(1)'"), # never run + ], + ) + assert [t.name for t in found] == ["python", "nothing"] + assert found[0].version.startswith("Python 3.") + assert found[1].version == "not found" diff --git a/tests/integration/test_end_to_end_loop.py b/tests/integration/test_end_to_end_loop.py index 1e4b09e..bf40526 100644 --- a/tests/integration/test_end_to_end_loop.py +++ b/tests/integration/test_end_to_end_loop.py @@ -35,6 +35,7 @@ from nisse_dev.canonical import ActionVerdict from nisse_dev.config import AgentConfig, LoopTuning from nisse_dev.execution import SSHHost from nisse_dev.gitea import GiteaClient, GiteaError +from nisse_dev.loop import preflight from nisse_dev.loop.build import RunningLoop, build_loop_agent from nisse_dev.loop.items import ( ANSWER, @@ -55,6 +56,9 @@ MAX_ANSWERS = 5 # per wait for something else, before giving up BUG_GRACE_S = 20 +_PREFLIGHTED: list[str] = [] + + @pytest.fixture def running( live_config: AgentConfig, @@ -99,6 +103,17 @@ def running( # What the agent reads when told a new project is ready: replaced by # a test that adds one (D8). source = [config] + if not _PREFLIGHTED: + # The test hosts keep their markers: the session's first test + # gets a real preflight (M4.1), the rest check versions. + _PREFLIGHTED.append(project.name) + SSHHost( + project.build_host.address, + key=project.build_host.key, + user=project.build_host.user, + port=project.build_host.port, + state_dir=tmp_path / "ssh-marker", + ).execute(f'rm -f "{preflight.marker_path(project.name)}"') agent = build_loop_agent(config, load=lambda: source[0]) agent.source = source # type: ignore[attr-defined] # The test drives intake and PR watching itself, not on a timer. @@ -318,6 +333,8 @@ def test_the_slice_end_to_end( assert marker in diff, "the pushed fix doesn't do what the comment asked" # --- the operator approves and merges; the agent closes the item --- + body = gitea._request("GET", f"/repos/{repo}/pulls/{pr}")["body"] + assert "Tested with: " in body, body # M4.1: the toolchain checked operator.merge_and_close(issue) # --- M3.1: the fast path judged every call; none needed an allow --- -- 2.54.0