M4.3: a broken build VM blocks its projects until it's ready

The driver can request_reimage(reason) when its build VM is broken past the
usual fixes (before the go-ahead too: it asks, it doesn't act). The harness
parks the work in progress (pushed to Gitea), then blocks the VM in the state
store (schema 5, vm_blocks): no item on any project on it is taken up, across
restarts. The operator hears once at top level, with the command to clear it,
and each blocked item's thread gets a line. `vm ready <project>` lifts the
block and trusts the VM's new host key; a reimaged VM is set up again by the
toolchain step, and the item that found it goes first.

loop.qnt gains the block (blocked_vm_runs_nothing) and a scenario; it found
that a reimage lost work in progress, hence parking first. The walk requests
reimages and recovers them, reimaged or not. The unreachable case reuses the
block in #137. Architecture P5 is softer, and D3 describes the block.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk

nisse-dev/tests/test_loop_properties.py

@@ -107,6 +107,9 @@ class PolicyModel:
                     toolchain=[{"name": "python", "version_command": command}],  # type: ignore[arg-type]
                 )
             ]
+        if "request_reimage" in offered and self.rng.random() < 0.03:
+            # M4.3: the build VM seems broken past the usual fixes.
+            return [call("request_reimage", reason=f"broken {self.n}")]
         roll = self.rng.random()
         if roll < 0.08:
             return [call("ask_human", question=f"question {self.n}?")]
@@ -196,7 +199,8 @@ class PolicyModel:
 
 class Watched:
     """The driver's model, watched: a run on a project item (not its
-    preflight) must find the project's marker on the VM."""
+    preflight) must find the project's marker on the VM, and no model runs
+    for an item whose build VM is blocked."""
 
     def __init__(self, walk: "Walk", inner: Any) -> None:
         self.walk = walk
@@ -211,6 +215,10 @@ class Watched:
         )
         if item is not None and item.scope != "general" and working and not marked:
             self.walk.unready.append(item.id)
+        if item is not None and item.scope in self.walk.worlds:
+            host = self.walk.worlds[item.scope]["host"].describe()
+            if host in self.walk.world["store"].vm_blocks():
+                self.walk.on_blocked.append(item.id)
         return self.inner.complete(events, tools, model, system)
 
 
@@ -291,6 +299,7 @@ class Walk:
         self.markers = Path(self.world["repo"].root).parent / "markers"
         self.unready: list[str] = []
         self.wiped_at: dict[str, int] = {}  # project -> last event before a wipe
+        self.on_blocked: list[str] = []  # items run while their VM was blocked
         if self.merge == "approved":
             for w in worlds.values():
                 self._watch_merges(w["forge"])
@@ -395,6 +404,8 @@ class Walk:
             "issue": 1,
             "chat": 1,
         }
+        if self.world["store"].vm_blocks():
+            weights.update({"vm-ready": 3})
         item = self.rng.choice(open_items) if open_items else None
         loop = loop_of(item) if item else None
         # The forge an item's project lives on; a new issue picks one.
@@ -423,12 +434,20 @@ class Walk:
         if action == "intake":
             self.agent.intake()
         elif action == "wipe":
-            # The build VM is rebuilt (D3): the clone and its marker gone.
-            w = self.worlds[project]
-            shutil.rmtree(w["repo"].root, ignore_errors=True)
-            events = self.world["store"].all_events()
-            self.wiped_at[project] = events[-1].id if events else 0
-            (self.markers / f"{project}.ready").unlink(missing_ok=True)
+            self._wipe(project)
+        elif action == "vm-ready":
+            # M4.3: the operator recovers a blocked VM, reimaged or not; a
+            # reimage wipes every project on it.
+            blocks = self.world["store"].vm_blocks()
+            on_it = [
+                name
+                for name, w in self.worlds.items()
+                if w["host"].describe() in blocks
+            ]
+            if self.rng.random() < 0.5:
+                for name in on_it:
+                    self._wipe(name)
+            self.transport.receive(DON, f"vm ready {self.rng.choice(on_it)}")
         elif action == "done" and loop:
             # The operator says a new project is ready; no entry was added.
             self.transport.receive(DON, "done", thread=loop.thread)
@@ -557,6 +576,14 @@ class Walk:
             self.agent.watch()
             self.agent._schedule()
 
+    def _wipe(self, project: str) -> None:
+        """The build VM is rebuilt (D3): the clone and its marker gone."""
+        w = self.worlds[project]
+        shutil.rmtree(w["repo"].root, ignore_errors=True)
+        events = self.world["store"].all_events()
+        self.wiped_at[project] = events[-1].id if events else 0
+        (self.markers / f"{project}.ready").unlink(missing_ok=True)
+
     # -- the properties (specs/loop.qnt) -------------------------------------
 
     def _dangerous_only_when_allowed(self, where: str) -> None:
@@ -592,6 +619,10 @@ class Walk:
         where = f"seed {self.seed}, steps {self.steps}"
         # M4.1: the driver never works on a project whose VM has no marker.
         assert not self.unready, f"ready_before_work: {self.unready}: {where}"
+        # M4.3: nothing runs for a project whose build VM is blocked.
+        assert not self.on_blocked, (
+            f"blocked_vm_runs_nothing: {self.on_blocked}: {where}"
+        )
         # D5: the agent asks Gitea to merge only what an operator approved
         # as it is (Gitea would refuse the rest; the harness mustn't ask).
         unapproved = [pr for pr, approved in self.agent_merges if not approved]

nisse-dev/tests/test_loop_toolchain.py

@@ -3,7 +3,9 @@ the build VM is checked (baseline, clone, marker); without a marker the harness
 starts a toolchain step that sets the VM up from the repository and edits
 nothing there; the driver can ask for one once work is agreed, whose edits are
 part of the work; later runs check versions again; a rebuilt VM is re-cloned
-and set up again, and a changed host key waits for the operator."""
+and set up again, and a changed host key waits for the operator. A VM the
+driver finds broken blocks its projects until the operator says it's ready
+(M4.3)."""
 
 import io
 import shutil
@@ -27,6 +29,7 @@ from test_loop_milestones import world as world  # noqa: PLC0414 -- the fixture
 from nisse_dev.cancel import Cancel
 from nisse_dev.canonical import Fact, ToolResult
 from nisse_dev.execution import HostKeyChanged
+from nisse_dev.gitea import Issue
 from nisse_dev.loop import preflight
 from nisse_dev.loop.agent import LoopAgent, Project
 from nisse_dev.loop.driver import Driver
@@ -338,3 +341,130 @@ def test_before_the_go_ahead_the_driver_only_looks(
     assert "Python 3." in results[0]  # looking runs
     assert results[1].startswith("not run: not before the go-ahead")
     assert loop_of(the_item(world)).wait == GO_AHEAD
+
+
+# --- M4.3: a broken VM blocks its projects until it's ready ---------------------------
+
+
+def reimage(reason: str = "pip crashes in every venv, even a new one") -> list[Any]:
+    return [call("request_reimage", reason=reason)]
+
+
+def by_issue(world: dict[str, Any], number: int) -> Any:
+    [found] = [i for i in world["store"].work_items() if loop_of(i).issue == number]
+    return found
+
+
+def facts_of(world: dict[str, Any], item: Any) -> list[str]:
+    events = world["store"].transcript(item.id, "orchestrator").read()
+    return [e.event.text for e in events if isinstance(e.event, Fact)]
+
+
+def test_a_broken_vm_blocks_its_project_until_the_operator_says_ready(
+    world: dict[str, Any], project: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    world["forge"].issues.append(Issue(15, "Add count_lines", "count lines", "don", ()))
+    forgotten: list[str] = []
+    monkeypatch.setattr(
+        type(world["host"]), "forget_host_key", lambda self: forgotten.append("yes")
+    )
+    # Asked for before the go-ahead: it's a request, not an action.
+    model = ScriptedModel(ready(), reimage(), ready(), RESTATE, RESTATE)
+    agent, transport = agent_for(world, project, model)
+    agent.intake()
+    first, second = by_issue(world, 14), by_issue(world, 15)
+    assert "build VM reimage requested: pip crashes in every venv, even a new one" in (
+        facts_of(world, first)
+    )
+    [asked] = [
+        m for m in transport.outbound if "reimage it from its template" in m.text
+    ]
+    assert asked.thread is None  # top level: it can cover several projects
+    assert "pip crashes in every venv" in asked.text
+    assert f"`vm ready {project}`" in asked.text
+    assert any(f.startswith("Blocked: the build VM") for f in facts_of(world, second))
+    # Neither runs while the VM is blocked, even after a restart.
+    again, _ = agent_for(world, project, ScriptedModel())
+    assert again._queue() == []
+    assert "its build VM needs recovery" in again.status()
+    # The operator reimages it: the clone and the marker are gone.
+    shutil.rmtree(world["repo"].root)
+    world["host"].execute(f'rm -f "{preflight.marker_path(project, markers(world))}"')
+    transport.receive(DON, f"vm ready {project}")
+    assert forgotten == ["yes"]  # its new host key is trusted
+    assert world["store"].vm_blocks() == {}
+    assert any("cloned the workspace again" in f for f in facts_of(world, first))
+    # The item that found it went first, and set the VM up again.
+    assert loop_of(by_issue(world, 14)).wait == GO_AHEAD
+    assert loop_of(by_issue(world, 15)).wait == GO_AHEAD
+    setups = [
+        f
+        for item in (first, second)
+        for f in facts_of(world, item)
+        if "setting the build VM up from the repository" in f
+    ]
+    assert len(setups) == 2  # the first, and after the reimage
+    assert setups == [f for f in facts_of(world, first) if "setting the build VM" in f]
+
+
+def test_a_vm_said_ready_without_a_reimage_goes_straight_on(
+    world: dict[str, Any], project: str
+) -> None:
+    model = ScriptedModel(
+        ready(),
+        RESTATE,
+        [call("start_milestone", name="m"), *reimage("the build hangs")],
+        [*WORK[1:], *FINISH],
+    )
+    agent, transport = agent_for(world, project, model)
+    agent.intake()
+    say(transport, world, "go")
+    assert world["store"].vm_blocks()
+    transport.receive(DON, "vm ready")  # the only blocked VM: no name needed
+    # Not reimaged: the marker is still there, so no new setup step.
+    assert (
+        sum("setting the build VM up from the repository" in f for f in facts(world))
+        == 1
+    )
+    assert world["forge"].prs  # the work went on to a PR
+
+
+def test_vm_ready_with_nothing_blocked_says_so(
+    world: dict[str, Any], project: str
+) -> None:
+    _, transport = agent_for(world, project, ScriptedModel())
+    transport.receive(DON, "vm ready")
+    transport.receive(DON, f"vm ready {project}")
+    transport.receive(DON, "vm ready nowhere")
+    assert said(transport) == [
+        "No build VM is blocked.",
+        f"Nothing is blocking {project}'s build VM.",
+        "I don't know a project called nowhere.",
+    ]
+
+
+def test_work_in_progress_is_pushed_before_a_reimage(
+    world: dict[str, Any], project: str
+) -> None:
+    model = ScriptedModel(
+        ready(),
+        RESTATE,
+        [
+            call("start_milestone", name="m"),
+            call("write_file", path="w.py", content="x = 1\n"),
+            *reimage(),
+        ],
+        ready(),  # set up again after the reimage
+        [call("finish_milestone", summary="w.py, tested", final="yes")],
+    )
+    agent, transport = agent_for(world, project, model)
+    agent.intake()
+    say(transport, world, "go")
+    [asked] = [t for t in said(transport) if "reimage it from its template" in t]
+    assert "The work in progress is pushed" in asked
+    shutil.rmtree(world["repo"].root)
+    world["host"].execute(f'rm -f "{preflight.marker_path(project, markers(world))}"')
+    transport.receive(DON, f"vm ready {project}")
+    forge = world["forge"]
+    [pr] = [n for n, p in forge.prs.items() if not p["head"].startswith("nisse-check")]
+    assert "w.py" in forge.files(pr)  # it survived the reimage

nisse-dev/tests/test_state.py

@@ -121,3 +121,28 @@ def test_newer_schema_is_refused(db: Path) -> None:
         pass
     with pytest.raises(StateSchemaError, match="newer"):
         StateStore(db, schema_version=1)
+
+
+def test_a_vm_block_persists_across_reopen(db: Path) -> None:
+    with StateStore(db) as store:
+        store.block_vm("root@vm1", cause="broken", reason="pip crashes", work_item="x")
+        store.block_vm("root@vm1", cause="broken", reason="later", work_item="y")
+    with StateStore(db) as store:
+        [block] = store.vm_blocks().values()
+        assert (block.host, block.reason, block.work_item) == (
+            "root@vm1",
+            "pip crashes",  # the first block is kept
+            "x",
+        )
+        assert store.unblock_vm("root@vm1", cause="test") == block
+        assert store.unblock_vm("root@vm1", cause="test") is None
+        assert store.vm_blocks() == {}
+
+
+def test_a_version_4_store_gains_vm_blocks(db: Path) -> None:
+    with StateStore(db, schema_version=4) as store:
+        store._conn.execute("DROP TABLE vm_blocks")  # as a real version 4 store
+    with StateStore(db) as store:
+        assert store.schema_version == 5
+        store.block_vm("root@vm1", cause="broken", reason="r", work_item=None)
+        assert list(store.vm_blocks()) == ["root@vm1"]