M4.4: an unreachable build VM blocks its projects until it's back (#137)

M4.3's block, with new triggers and a way out. A VM out of reach at the
check before a run, in a run (which ends there: the driver, a toolchain
step and a requested step), or while preparing the workspace blocks every
project on it; the item stays ready, waiting on no one. On the intake round
it is tried again every tuning.build_vm.retry_s (300), and once it answers
it must still answer settle_s (180) later; then the work a lost run left
there is parked (pushed), the VM unblocked, and that said once.

A changed host key, at the check or on a retry, blocks the VM too, until
the operator's `vm ready`, which trusts the new key: `trust new host key`
is retired, and the message says it's a key problem that may be an
impersonation. Questions are answered while a VM is blocked, without it.
Starting the agent with a VM down no longer fails.

The walk breaks the VM's SSH, now or partway through a run, and checks that
no model works on a VM that is down; loop.qnt models the VM going and
coming back (unblocked_only_when_up). Together they found: a lost run's
work set aside by the next item there; a question left behind a block; a
lost VM swallowed while the toolchain step checked or recorded it; and a
remark waiting to be answered that outlived its PR's merge. All fixed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk

nisse-dev/tests/test_loop_properties.py

@@ -35,7 +35,7 @@ from nisse_dev.canonical import (
     UserMessage,
 )
 from nisse_dev.delegation import DELEGATION_INTENT, Caps
-from nisse_dev.execution import SSHHost
+from nisse_dev.execution import HostUnavailable, SSHHost
 from nisse_dev.gitea import Comment, Issue
 from nisse_dev.llm import LLMError, RoleMap
 from nisse_dev.loop.agent import LoopAgent, Project
@@ -209,13 +209,30 @@ class Watched:
     def complete(self, events, tools, model, system=None) -> list[Any]:
         offered = {t["function"]["name"] for t in tools}
         item = self.walk.world["store"].active_item()
-        working = "toolchain_ready" not in offered and "propose_work" not in offered
+        # Working on the VM: offered its tools (an answer given while the VM
+        # is blocked isn't, #137), and not setting it up.
+        working = (
+            "read_file" in offered
+            and "toolchain_ready" not in offered
+            and "propose_work" not in offered
+        )
         marked = (
             item is not None and (self.walk.markers / f"{item.scope}.ready").exists()
         )
         if item is not None and item.scope != "general" and working and not marked:
             self.walk.unready.append(item.id)
-        if item is not None and item.scope in self.walk.worlds:
+        if (
+            item is not None
+            and item.scope != "general"
+            and "read_file" in offered
+            and self.walk.down
+        ):
+            self.walk.while_down.append(item.id)
+        if (
+            item is not None
+            and "read_file" in offered
+            and item.scope in self.walk.worlds
+        ):
             host = self.walk.worlds[item.scope]["host"].describe()
             if host in self.walk.world["store"].vm_blocks():
                 self.walk.on_blocked.append(item.id)
@@ -282,7 +299,12 @@ class Walk:
     """``worlds``: project name -> its world (forge, bare repo, clone,
     milestones), all over one store."""
 
-    def __init__(self, worlds: dict[str, dict[str, Any]], seed: int) -> None:
+    def __init__(
+        self,
+        worlds: dict[str, dict[str, Any]],
+        seed: int,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
         self.worlds = worlds
         self.world = next(iter(worlds.values()))  # the store's home
         self.rng = random.Random(seed)
@@ -298,6 +320,26 @@ class Walk:
         # without one.
         self.markers = Path(self.world["repo"].root).parent / "markers"
         self.unready: list[str] = []
+        # #137: the build VM out of reach, now or after a few more commands
+        # (so mid-run); model calls made while it was.
+        self.down = False
+        self.drop_in = 0
+        self.while_down: list[str] = []
+        self.checking = False  # the walk's own checks see the real VM
+        real = SSHHost.execute
+        walk = self
+
+        def execute(host: SSHHost, command: str, **kw: Any) -> Any:
+            if walk.checking:
+                return real(host, command, **kw)
+            if walk.drop_in:
+                walk.drop_in -= 1
+                walk.down = walk.down or walk.drop_in == 0
+            if walk.down:
+                raise HostUnavailable("ssh: connect to host: Connection timed out")
+            return real(host, command, **kw)
+
+        monkeypatch.setattr(SSHHost, "execute", execute)
         self.wiped_at: dict[str, int] = {}  # project -> last event before a wipe
         self.on_blocked: list[str] = []  # items run while their VM was blocked
         if self.merge == "approved":
@@ -406,6 +448,7 @@ class Walk:
         }
         if self.world["store"].vm_blocks():
             weights.update({"vm-ready": 3})
+        weights.update({"vm-up": 3} if self.down else {"vm-down": 1})
         item = self.rng.choice(open_items) if open_items else None
         loop = loop_of(item) if item else None
         # The forge an item's project lives on; a new issue picks one.
@@ -435,6 +478,14 @@ class Walk:
             self.agent.intake()
         elif action == "wipe":
             self._wipe(project)
+        elif action == "vm-down":
+            # #137: out of reach now, or partway through what comes next.
+            if self.rng.random() < 0.5:
+                self.down = True
+            else:
+                self.drop_in = self.rng.randint(1, 30)
+        elif action == "vm-up":
+            self.down, self.drop_in = False, 0
         elif action == "vm-ready":
             # M4.3: the operator recovers a blocked VM, reimaged or not; a
             # reimage wipes every project on it.
@@ -616,9 +667,18 @@ class Walk:
                     )
 
     def check(self) -> None:
+        self.checking = True
+        try:
+            self._check()
+        finally:
+            self.checking = False
+
+    def _check(self) -> None:
         where = f"seed {self.seed}, steps {self.steps}"
         # M4.1: the driver never works on a project whose VM has no marker.
         assert not self.unready, f"ready_before_work: {self.unready}: {where}"
+        # #137: no model works on a project whose build VM is out of reach.
+        assert not self.while_down, f"no_model_on_a_lost_vm: {self.while_down}: {where}"
         # M4.3: nothing runs for a project whose build VM is blocked.
         assert not self.on_blocked, (
             f"blocked_vm_runs_nothing: {self.on_blocked}: {where}"
@@ -859,13 +919,21 @@ class Walk:
         for item in self.items():
             if item.status != "open" or item.scope != name:
                 continue
+            events = self.world["store"].transcript(item.id, "driver").read()
+            # Only writes that happened: one cut off by a lost VM (#137) didn't.
+            done = {
+                e.event.call_id
+                for e in events
+                if isinstance(e.event, ToolResult) and e.event.success
+            }
             written = {
                 str(e.event.arguments.get("path"))
-                for e in self.world["store"].transcript(item.id, "driver").read()
+                for e in events
                 # The driver's own: a failed helper's are discarded on purpose.
                 if isinstance(e.event, ToolCall)
                 and e.event.tool == "write_file"
                 and e.actor == "driver"
+                and e.event.call_id in done
                 # D3: a wiped VM loses what wasn't pushed, accepted.
                 and e.id > self.wiped_at.get(name, 0)
             }
@@ -932,7 +1000,7 @@ def test_the_loop_keeps_its_properties(
     world: dict[str, Any], seed: int, monkeypatch: pytest.MonkeyPatch
 ) -> None:
     _seeded_ids(monkeypatch, seed)
-    walk = Walk({"sandbox": world}, seed)
+    walk = Walk({"sandbox": world}, seed, monkeypatch)
     for _ in range(STEPS):
         walk.step()
         walk.check()
@@ -948,7 +1016,7 @@ def test_two_projects_keep_the_properties(
     other = make_world(
         ssh_host, tmp_path / "other", store=sandbox["store"], repo_name="don/other"
     )
-    walk = Walk({"sandbox": sandbox, "other": other}, seed)
+    walk = Walk({"sandbox": sandbox, "other": other}, seed, monkeypatch)
     for _ in range(STEPS):
         walk.step()
         walk.check()