M4.3: a broken build VM blocks its projects until it's ready

The driver can request_reimage(reason) when its build VM is broken past the
usual fixes (before the go-ahead too: it asks, it doesn't act). The harness
parks the work in progress (pushed to Gitea), then blocks the VM in the state
store (schema 5, vm_blocks): no item on any project on it is taken up, across
restarts. The operator hears once at top level, with the command to clear it,
and each blocked item's thread gets a line. `vm ready <project>` lifts the
block and trusts the VM's new host key; a reimaged VM is set up again by the
toolchain step, and the item that found it goes first.

loop.qnt gains the block (blocked_vm_runs_nothing) and a scenario; it found
that a reimage lost work in progress, hence parking first. The walk requests
reimages and recovers them, reimaged or not. The unreachable case reuses the
block in #137. Architecture P5 is softer, and D3 describes the block.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk

nisse-dev/tests/test_loop_toolchain.py

@@ -3,7 +3,9 @@ the build VM is checked (baseline, clone, marker); without a marker the harness
 starts a toolchain step that sets the VM up from the repository and edits
 nothing there; the driver can ask for one once work is agreed, whose edits are
 part of the work; later runs check versions again; a rebuilt VM is re-cloned
-and set up again, and a changed host key waits for the operator."""
+and set up again, and a changed host key waits for the operator. A VM the
+driver finds broken blocks its projects until the operator says it's ready
+(M4.3)."""
 
 import io
 import shutil
@@ -27,6 +29,7 @@ from test_loop_milestones import world as world  # noqa: PLC0414 -- the fixture
 from nisse_dev.cancel import Cancel
 from nisse_dev.canonical import Fact, ToolResult
 from nisse_dev.execution import HostKeyChanged
+from nisse_dev.gitea import Issue
 from nisse_dev.loop import preflight
 from nisse_dev.loop.agent import LoopAgent, Project
 from nisse_dev.loop.driver import Driver
@@ -338,3 +341,130 @@ def test_before_the_go_ahead_the_driver_only_looks(
     assert "Python 3." in results[0]  # looking runs
     assert results[1].startswith("not run: not before the go-ahead")
     assert loop_of(the_item(world)).wait == GO_AHEAD
+
+
+# --- M4.3: a broken VM blocks its projects until it's ready ---------------------------
+
+
+def reimage(reason: str = "pip crashes in every venv, even a new one") -> list[Any]:
+    return [call("request_reimage", reason=reason)]
+
+
+def by_issue(world: dict[str, Any], number: int) -> Any:
+    [found] = [i for i in world["store"].work_items() if loop_of(i).issue == number]
+    return found
+
+
+def facts_of(world: dict[str, Any], item: Any) -> list[str]:
+    events = world["store"].transcript(item.id, "orchestrator").read()
+    return [e.event.text for e in events if isinstance(e.event, Fact)]
+
+
+def test_a_broken_vm_blocks_its_project_until_the_operator_says_ready(
+    world: dict[str, Any], project: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    world["forge"].issues.append(Issue(15, "Add count_lines", "count lines", "don", ()))
+    forgotten: list[str] = []
+    monkeypatch.setattr(
+        type(world["host"]), "forget_host_key", lambda self: forgotten.append("yes")
+    )
+    # Asked for before the go-ahead: it's a request, not an action.
+    model = ScriptedModel(ready(), reimage(), ready(), RESTATE, RESTATE)
+    agent, transport = agent_for(world, project, model)
+    agent.intake()
+    first, second = by_issue(world, 14), by_issue(world, 15)
+    assert "build VM reimage requested: pip crashes in every venv, even a new one" in (
+        facts_of(world, first)
+    )
+    [asked] = [
+        m for m in transport.outbound if "reimage it from its template" in m.text
+    ]
+    assert asked.thread is None  # top level: it can cover several projects
+    assert "pip crashes in every venv" in asked.text
+    assert f"`vm ready {project}`" in asked.text
+    assert any(f.startswith("Blocked: the build VM") for f in facts_of(world, second))
+    # Neither runs while the VM is blocked, even after a restart.
+    again, _ = agent_for(world, project, ScriptedModel())
+    assert again._queue() == []
+    assert "its build VM needs recovery" in again.status()
+    # The operator reimages it: the clone and the marker are gone.
+    shutil.rmtree(world["repo"].root)
+    world["host"].execute(f'rm -f "{preflight.marker_path(project, markers(world))}"')
+    transport.receive(DON, f"vm ready {project}")
+    assert forgotten == ["yes"]  # its new host key is trusted
+    assert world["store"].vm_blocks() == {}
+    assert any("cloned the workspace again" in f for f in facts_of(world, first))
+    # The item that found it went first, and set the VM up again.
+    assert loop_of(by_issue(world, 14)).wait == GO_AHEAD
+    assert loop_of(by_issue(world, 15)).wait == GO_AHEAD
+    setups = [
+        f
+        for item in (first, second)
+        for f in facts_of(world, item)
+        if "setting the build VM up from the repository" in f
+    ]
+    assert len(setups) == 2  # the first, and after the reimage
+    assert setups == [f for f in facts_of(world, first) if "setting the build VM" in f]
+
+
+def test_a_vm_said_ready_without_a_reimage_goes_straight_on(
+    world: dict[str, Any], project: str
+) -> None:
+    model = ScriptedModel(
+        ready(),
+        RESTATE,
+        [call("start_milestone", name="m"), *reimage("the build hangs")],
+        [*WORK[1:], *FINISH],
+    )
+    agent, transport = agent_for(world, project, model)
+    agent.intake()
+    say(transport, world, "go")
+    assert world["store"].vm_blocks()
+    transport.receive(DON, "vm ready")  # the only blocked VM: no name needed
+    # Not reimaged: the marker is still there, so no new setup step.
+    assert (
+        sum("setting the build VM up from the repository" in f for f in facts(world))
+        == 1
+    )
+    assert world["forge"].prs  # the work went on to a PR
+
+
+def test_vm_ready_with_nothing_blocked_says_so(
+    world: dict[str, Any], project: str
+) -> None:
+    _, transport = agent_for(world, project, ScriptedModel())
+    transport.receive(DON, "vm ready")
+    transport.receive(DON, f"vm ready {project}")
+    transport.receive(DON, "vm ready nowhere")
+    assert said(transport) == [
+        "No build VM is blocked.",
+        f"Nothing is blocking {project}'s build VM.",
+        "I don't know a project called nowhere.",
+    ]
+
+
+def test_work_in_progress_is_pushed_before_a_reimage(
+    world: dict[str, Any], project: str
+) -> None:
+    model = ScriptedModel(
+        ready(),
+        RESTATE,
+        [
+            call("start_milestone", name="m"),
+            call("write_file", path="w.py", content="x = 1\n"),
+            *reimage(),
+        ],
+        ready(),  # set up again after the reimage
+        [call("finish_milestone", summary="w.py, tested", final="yes")],
+    )
+    agent, transport = agent_for(world, project, model)
+    agent.intake()
+    say(transport, world, "go")
+    [asked] = [t for t in said(transport) if "reimage it from its template" in t]
+    assert "The work in progress is pushed" in asked
+    shutil.rmtree(world["repo"].root)
+    world["host"].execute(f'rm -f "{preflight.marker_path(project, markers(world))}"')
+    transport.receive(DON, f"vm ready {project}")
+    forge = world["forge"]
+    [pr] = [n for n, p in forge.prs.items() if not p["head"].startswith("nisse-check")]
+    assert "w.py" in forge.files(pr)  # it survived the reimage