M4.3: a broken build VM blocks its projects until it's ready

The driver can request_reimage(reason) when its build VM is broken past the
usual fixes (before the go-ahead too: it asks, it doesn't act). The harness
parks the work in progress (pushed to Gitea), then blocks the VM in the state
store (schema 5, vm_blocks): no item on any project on it is taken up, across
restarts. The operator hears once at top level, with the command to clear it,
and each blocked item's thread gets a line. `vm ready <project>` lifts the
block and trusts the VM's new host key; a reimaged VM is set up again by the
toolchain step, and the item that found it goes first.

loop.qnt gains the block (blocked_vm_runs_nothing) and a scenario; it found
that a reimage lost work in progress, hence parking first. The walk requests
reimages and recovers them, reimaged or not. The unreachable case reuses the
block in #137. Architecture P5 is softer, and D3 describes the block.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk

nisse-dev/src/nisse_dev/loop/agent.py

@@ -84,6 +84,7 @@ from nisse_dev.loop.tools import (
     PROPOSE_PROJECT,
     PROPOSE_WORK,
     REPLY,
+    REQUEST_REIMAGE,
     RESTATE,
     START_MILESTONE,
     TOOLCHAIN_STEP,
@@ -109,6 +110,8 @@ _COMMITTED = (
     "finish_milestone, which pushes the milestone's branch and opens or "
     "updates its PR. There is no PR until then."
 )
+# The operator's word that a blocked build VM is ready again (D3).
+_VM_READY = re.compile(r"vm ready(?: ([a-z0-9][a-z0-9-]*))?")
 _NEEDS_ITEM = frozenset({"go", "approve", "abandon", "allow", "trust new host key"})
 # Near-misses for abandon: asked about, never taken for it or for a reply.
 _NEAR_ABANDON = frozenset(
@@ -125,6 +128,8 @@ _NEAR_ABANDON = frozenset(
 )
 BUG_GRACE_S = 600.0
 GENERAL_IDLE_S = 86400.0
+_SAVED = "The work in progress is pushed; anything else on it will be lost."
+BROKEN = "broken"  # a VM block's cause: the driver found it broken (D3)
 PROGRESS_EVERY_S = 60.0  # progress in prose at a readable rate (L5, #42)
 PROGRESS_CHARS = 400
 DETAILS = 10  # tool calls shown by `details`
@@ -162,6 +167,11 @@ docs and CI configuration in step, as part of the work. Before the go-ahead,
 you only look: say what the work needs in your restatement.
 Never change branches yourself.
 
+If the build VM itself seems broken, fix it the usual ways first: reinstall
+the project's dependencies, or a toolchain_step if the toolchain is wrong. If
+that doesn't help, don't spend the work diagnosing it: request_reimage, and it
+is rebuilt from its template.
+
 Before each group of tool calls, say in a sentence what you're doing and why:
 the operator sees it as your progress.
 
@@ -1273,6 +1283,16 @@ class LoopAgent:
             )
             self._schedule()
             return
+        ready = _VM_READY.fullmatch(command)
+        if ready is not None:
+            self._store.transcript(item.id if item else None, "operator").append(
+                UserMessage(text=text, source=f"transport:{msg.sender}")
+            )
+            self._transport.post(
+                self._vm_ready(ready.group(1), msg.sender), thread=msg.thread
+            )
+            self._schedule()
+            return
         named = _NAMED.fullmatch(command)
         if named is not None:
             # L5: a command that names its item applies to it.
@@ -1814,9 +1834,21 @@ class LoopAgent:
         """Ready items in the order they run (L1): the operator's order
         first, then the oldest."""
         items = self._store.work_items("open")  # oldest first
-        ready = [i for i in items if i.position == READY]
+        blocks = self._store.vm_blocks()
+        ready = [
+            i
+            for i in items
+            if i.position == READY and self._host_of(i.scope) not in blocks
+        ]
         return sorted(ready, key=lambda i: -loop_of(i).first)
 
+    def _host_of(self, project: str | None) -> str | None:
+        """The build VM a project's work runs on, as blocks name it (D3)."""
+        found = self._projects.get(project or "")
+        if found is None or found.workspace is None:
+            return None
+        return found.workspace.host.describe()
+
     def status(self) -> str:
         items = self._store.work_items("open")
         if not items:
@@ -1839,6 +1871,10 @@ class LoopAgent:
                 what += f", going ahead in {left} min unless you reply"
         elif item.position == BLOCKED:
             what = f"blocked by {', '.join(loop.blocked_by)}"
+        elif item.position == READY and item.id not in queue:
+            block = self._store.vm_blocks().get(self._host_of(item.scope) or "")
+            why = block.reason if block else "unknown"
+            what = f"blocked: its build VM needs recovery ({why})"
         elif item.position == READY:
             what = f"ready, {queue.index(item.id) + 1} of {len(queue)} in line"
         name = (
@@ -2425,6 +2461,7 @@ class LoopAgent:
         tools = [
             *([] if hold else [RESTATE]),
             ASK_HUMAN,
+            *([REQUEST_REIMAGE] if checked else []),
             *(
                 _LOOK
                 if not loop.started
@@ -2656,6 +2693,8 @@ class LoopAgent:
             question = str(args.get("question", "")).strip()
             self._answer_where_asked(item, loop, question)
             self._wait(item, replace(loop, wait=ANSWER, note=question), None)
+        elif outcome.kind == "wait" and outcome.tool == "request_reimage":
+            self._reimage(item, loop, str(args.get("reason", "")).strip())
         elif outcome.kind == "wait" and outcome.tool == "wait_for_review":
             note = str(args.get("note", "")).strip() or f"waiting on PR #{loop.pr}"
             if loop.pr:
@@ -2699,6 +2738,105 @@ class LoopAgent:
             self._wait(item, replace(loop, wait=ANSWER, note=note), note)
         self._store.set_active(None, cause="item waiting")
 
+    def _reimage(self, item: WorkItem, loop: Loop, reason: str) -> None:
+        """P5, D3: the driver found its build VM broken past the usual fixes.
+        The VM is blocked, so nothing runs on it, until the operator says
+        `vm ready`; the item that found it goes first after that."""
+        project = self._at(item)
+        host = self._host_of(project.name)
+        if host is None:
+            note = f"The build VM seems broken: {reason}. Reply once it's fixed."
+            self._wait(item, replace(loop, wait=ANSWER, note=note), note)
+            return
+        self._fact(item, f"build VM reimage requested: {reason}")
+        # Nothing of the work may live only on a VM about to be wiped: park
+        # it, as for a switch (pushed to Gitea). The VM may be too broken.
+        risk = ""
+        if project.milestones is not None:
+            try:
+                project.milestones.park(keep=self._keep(project.name))
+            except Exception as exc:  # noqa: BLE001 -- said; the reimage loses it
+                risk = (
+                    f"I couldn't push the work in progress first ({exc}): it "
+                    "will be lost with everything else on it."
+                )
+                self._fact(item, f"couldn't park the work before a reimage: {exc}")
+        first = max(loop_of(i).first for i in self._store.work_items("open")) + 1
+        save(
+            self._store,
+            item.id,
+            READY,
+            replace(loop, first=first, question=False, back_to="", back_note=""),
+            cause="build VM blocked",
+        )
+        self._block_vm(host, BROKEN, reason, item, risk)
+
+    def _block_vm(
+        self, host: str, cause: str, reason: str, by: WorkItem, risk: str = ""
+    ) -> None:
+        """Block a build VM (D3): every project on it waits until it is
+        recovered. Said once at top level, and in each waiting item's thread.
+        The recovery is the operator's for now; nisse-ops' later (M7.6)."""
+        self._store.block_vm(host, cause=cause, reason=reason, work_item=by.id)
+        names = sorted(p for p in self._projects if self._host_of(p) == host)
+        blocked = [
+            i
+            for i in self._store.work_items("open")
+            if i.scope in names and i.scope != GENERAL
+        ]
+        for other in blocked:
+            line = (
+                f"Blocked: the build VM ({host}) needs reimaging. "
+                f"Waiting for `vm ready {other.scope}`."
+            )
+            self._fact(other, line)
+            self._transport.post(line, thread=loop_of(other).thread)
+        self._transport.post(
+            f"The build VM {host} seems broken, found working on "
+            f"{self._label(by)}: {reason}\n\nThe projects on it are blocked "
+            f"until it's fixed: {', '.join(names)}. Could you reimage it from "
+            f"its template? {risk or _SAVED} If its "
+            "address changes, update `build_host` in my configuration and "
+            "restart me first.\n\nOnce it's ready, say "
+            + " or ".join(f"`vm ready {n}`" for n in names)
+            + "; that also trusts its new host key."
+        )
+
+    def _vm_ready(self, name: str | None, sender: str) -> str:
+        """The operator's `vm ready`: the build VM's block lifts and its new
+        host key is trusted (D3). A reimaged VM has no marker, so its first
+        run sets it up again from the repository."""
+        blocks = self._store.vm_blocks()
+        if name is None:
+            names = sorted(p for p in self._projects if self._host_of(p) in blocks)
+            if len(names) != 1:
+                return (
+                    "No build VM is blocked."
+                    if not names
+                    else "Which one? Say "
+                    + " or ".join(f"`vm ready {n}`" for n in names)
+                )
+            [name] = names
+        if name not in self._projects:
+            return f"I don't know a project called {name}."
+        host = self._host_of(name)
+        workspace = self._projects[name].workspace
+        if host is None or workspace is None or host not in blocks:
+            return f"Nothing is blocking {name}'s build VM."
+        workspace.host.forget_host_key()
+        block = self._store.unblock_vm(host, cause=f"vm ready from {sender}")
+        names = sorted(p for p in self._projects if self._host_of(p) == host)
+        for other in self._store.work_items("open"):
+            if other.scope in names:
+                self._fact(
+                    other,
+                    f"the operator ({sender}) said the build VM ({host}) is "
+                    "ready: its new host key is trusted, and it is set up again "
+                    "if it was reimaged",
+                )
+        assert block is not None  # checked above, on the worker
+        return f"{host} is unblocked: {', '.join(names)} can go on."
+
     # -- milestones (D4) -----------------------------------------------------
 
     def _start(self, item_id: str, args: dict[str, object]) -> tuple[str, bool]: