Fix the run-breaking tool-result order; e2e test runs the slice (M2.1, PR D)

The first live run of the new loop's e2e test found that a fact recorded
while start_milestone was in flight landed between the tool call and its
result; both API formats reject that, so every later request in the run
failed. Views now keep each result right after its call. Scripted test
models check the APIs' ordering rules, so this can't pass unit tests
again.

The e2e tests no longer depend on the sandbox's history: each run asks for
a uniquely named function and works against a base branch of its own,
deleted afterwards. The review step asks for a change that can't already
be there. The scripted operator gives up after five fruitless answers, and
each run keeps its state store for inspection. The driver's prompt says
review fixes stay in the current milestone.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE

tests/integration/e2e_support.py

@@ -0,0 +1,42 @@
+"""Shared by the end-to-end tests: each run gets work that can't already be
+done, and a base branch of its own, so runs never depend on one another or
+on what earlier runs merged."""
+
+import contextlib
+import os
+from collections.abc import Iterator
+
+from nisse_dev.gitea import GiteaClient, GiteaError
+
+
+def task(run: str) -> str:
+    """A request no earlier run can have fulfilled."""
+    return (
+        f"Add a function vowels_{run}(text) to the sandbox package that returns "
+        "how many of a, e, i, o, u (either case) the text contains, with tests."
+    )
+
+
+def operator_gitea(url: str) -> GiteaClient:
+    token = os.environ.get("NISSE_OPERATOR_TOKEN", "")
+    assert token.strip(), (
+        "NISSE_OPERATOR_TOKEN is not set: the test operator's Gitea token"
+    )
+    return GiteaClient(url, token)
+
+
+@contextlib.contextmanager
+def base_branch(operator: GiteaClient, repo: str, run: str) -> Iterator[str]:
+    """A branch off main for this run's PRs to target and merge into, so
+    nothing accumulates in main; deleted afterwards."""
+    name = f"e2e-base-{run}"
+    operator._request(
+        "POST",
+        f"/repos/{repo}/branches",
+        json={"new_branch_name": name, "old_branch_name": "main"},
+    )
+    try:
+        yield name
+    finally:
+        with contextlib.suppress(GiteaError):
+            operator._request("DELETE", f"/repos/{repo}/branches/{name}")

tests/integration/test_end_to_end.py

@@ -25,6 +25,8 @@ from pathlib import Path
 from typing import Any
 
 import pytest
+from e2e_support import base_branch, operator_gitea
+from e2e_support import task as make_task
 
 from nisse_dev.agent import Agent, build_agent
 from nisse_dev.canonical import UserMessage
@@ -43,10 +45,6 @@ pytestmark = [pytest.mark.integration, pytest.mark.e2e]
 
 OPERATOR = "@dongar:dongar.net"  # an allowlisted operator in nisse-test.yaml
 SETTLE_S = 1200  # a milestone, with real models, inside one message
-TASK = (
-    "Add a function `count_vowels(text)` to the sandbox package that returns "
-    "how many of a, e, i, o, u (either case) the text contains, with tests."
-)
 
 
 def build_host(ssh_host: SSHHost) -> BuildHost:
@@ -70,10 +68,14 @@ def agent(
     ssh_host: SSHHost,
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
-) -> Iterator[tuple[Agent, InMemoryTransport]]:
+) -> Iterator[tuple[Agent, InMemoryTransport, str]]:
     run = uuid.uuid4().hex[:8]
+    stack = contextlib.ExitStack()
+    operator = operator_gitea(live_config.gitea_url)
+    base = stack.enter_context(base_branch(operator, live_config.gitea_repo, run))
     config = replace(
         live_config,
+        base_branch=base,
         state_db=tmp_path / "state.db",
         workspace=Path(f"/tmp/nisse-e2e-{run}/sandbox"),
         build_host=build_host(ssh_host),
@@ -84,7 +86,7 @@ def agent(
     try:
         agent = build_agent(config)
         agent.start()
-        yield agent, transport
+        yield agent, transport, run
     finally:
         if agent is not None:
             agent.stop()
@@ -97,6 +99,7 @@ def agent(
             state_dir=tmp_path / "ssh",
         )
         host.execute(f"rm -rf /tmp/nisse-e2e-{run}")
+        stack.close()  # the run's base branch
 
 
 class Operator:
@@ -148,14 +151,16 @@ def is_artifact(path: str) -> bool:
     )
 
 
-def test_a_milestone_end_to_end(agent: tuple[Agent, InMemoryTransport]) -> None:
-    running, transport = agent
+def test_a_milestone_end_to_end(
+    agent: tuple[Agent, InMemoryTransport, str],
+) -> None:
+    running, transport, run = agent
     operator = Operator(running, transport)
     gitea, repo = running.gitea, running.config.gitea_repo
     pr: dict[str, Any] = {}
     try:
         # The task first, before any project: it must be kept, not lost.
-        operator.say(TASK)
+        operator.say(make_task(run))
         operator.say(f"use {running.config.project}")
         state = operator.drive_to_pr()
         pr = state.data["pr"]

tests/integration/test_end_to_end_loop.py

@@ -19,10 +19,11 @@ from pathlib import Path
 from typing import Any
 
 import pytest
+from e2e_support import base_branch, operator_gitea, task
 
 from nisse_dev.config import AgentConfig
 from nisse_dev.execution import SSHHost
-from nisse_dev.gitea import GiteaClient, GiteaError
+from nisse_dev.gitea import GiteaError
 from nisse_dev.loop.build import RunningLoop, build_loop_agent
 from nisse_dev.loop.items import ANSWER, GO_AHEAD, MERGE, WAITING, loop_of
 from nisse_dev.transport import InMemoryTransport
@@ -30,19 +31,8 @@ from nisse_dev.transport import InMemoryTransport
 pytestmark = [pytest.mark.integration, pytest.mark.e2e]
 
 OPERATOR = "@dongar:dongar.net"  # an allowlisted Matrix operator in nisse-test.yaml
-TASK = (
-    "Add a function count_vowels(text) to the sandbox package that returns how "
-    "many of a, e, i, o, u (either case) the text contains, with tests."
-)
 DEADLINE_S = 1200
-
-
-def operator_gitea(config: AgentConfig) -> GiteaClient:
-    token = os.environ.get("NISSE_OPERATOR_TOKEN", "")
-    assert token.strip(), (
-        "NISSE_OPERATOR_TOKEN is not set: the test operator's Gitea token"
-    )
-    return GiteaClient(config.gitea_url, token)
+MAX_ANSWERS = 5  # per wait for something else, before giving up
 
 
 @pytest.fixture
@@ -51,14 +41,23 @@ def running(
     ssh_host: SSHHost,
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
-) -> Iterator[tuple[RunningLoop, InMemoryTransport]]:
+) -> Iterator[tuple[RunningLoop, InMemoryTransport, str]]:
     from test_end_to_end import build_host
 
     run = uuid.uuid4().hex[:8]
+    stack = contextlib.ExitStack()
+    # Kept outside pytest's temp directories, which pytest prunes, so a
+    # failed run's transcript can be read afterwards (nisse_dev.dump).
+    keep = Path(os.environ.get("NISSE_E2E_RUNS", "/tmp/nisse-e2e-runs")) / run
+    keep.mkdir(parents=True, exist_ok=True)
+    print(f"\ne2e run {run}: state store in {keep}")
+    operator = operator_gitea(live_config.gitea_url)
+    base = stack.enter_context(base_branch(operator, live_config.gitea_repo, run))
     config = replace(
         live_config,
         loop="new",
-        state_db=tmp_path / "state.db",
+        base_branch=base,
+        state_db=keep / "state.db",
         workspace=Path(f"/tmp/nisse-e2e-{run}/sandbox"),
         build_host=build_host(ssh_host),
     )
@@ -70,7 +69,7 @@ def running(
         # The test drives intake and PR watching itself, not on a timer.
         agent.loop._interval = 3600
         agent.start()
-        yield agent, transport
+        yield agent, transport, run
     finally:
         if agent is not None:
             agent.stop()
@@ -82,6 +81,7 @@ def running(
             state_dir=tmp_path / "ssh",
         )
         host.execute(f"rm -rf /tmp/nisse-e2e-{run}")
+        stack.close()  # the run's base branch
 
 
 def is_artifact(path: str) -> bool:
@@ -116,6 +116,7 @@ class Operator:
     def drive_until(self, wait: str) -> Any:
         """Answer whatever the agent asks until it waits for ``wait``."""
         deadline = time.monotonic() + DEADLINE_S
+        answers = 0
         while time.monotonic() < deadline:
             item = self.item()
             assert item is not None, "no work item"
@@ -127,6 +128,9 @@ class Operator:
             if item.position == WAITING and loop.wait == GO_AHEAD:
                 self.say("go")
             elif item.position == WAITING and loop.wait == ANSWER:
+                answers += 1
+                # Answering forever is how a stuck agent looks: give up.
+                assert answers <= MAX_ANSWERS, f"stuck asking: {self.last()}"
                 self.say("Use your best judgement and proceed.")
             else:
                 self.settle()
@@ -136,18 +140,18 @@ class Operator:
         return self.transport.outbound[-1].text if self.transport.outbound else ""
 
 
-def test_the_slice_end_to_end(running: tuple[RunningLoop, InMemoryTransport]) -> None:
-    agent, transport = running
+def test_the_slice_end_to_end(
+    running: tuple[RunningLoop, InMemoryTransport, str],
+) -> None:
+    agent, transport, run = running
     operator = Operator(agent, transport)
     gitea, repo = agent.gitea, agent.config.gitea_repo
-    as_operator = operator_gitea(agent.config)
+    as_operator = operator_gitea(agent.config.gitea_url)
     issue = None
     pr = 0
     try:
         # --- the operator files an issue and assigns it to the agent (D1) ---
-        issue = as_operator.create_issue(
-            repo, f"count_vowels ({uuid.uuid4().hex[:6]})", TASK
-        )
+        issue = as_operator.create_issue(repo, f"vowels_{run}", task(run))
         as_operator._request(
             "PATCH",
             f"/repos/{repo}/issues/{issue}",
@@ -167,14 +171,18 @@ def test_the_slice_end_to_end(running: tuple[RunningLoop, InMemoryTransport]) ->
         assert f"Closes #{issue}" in body
 
         # --- a review comment is answered with a fix (D5) ---
+        # A request that can't already be met, so a fix must follow.
+        marker = f"reviewed-{uuid.uuid4().hex[:8]}"
         before = gitea._request("GET", f"/repos/{repo}/pulls/{pr}")["head"]["sha"]
         as_operator.post_comment(
-            repo, pr, "Please add a one-line docstring to count_vowels."
+            repo, pr, f"Please add the comment line `# {marker}` above vowels_{run}."
         )
         operator.tick()
         operator.drive_until(MERGE)
         after = gitea._request("GET", f"/repos/{repo}/pulls/{pr}")["head"]["sha"]
         assert after != before, "no fix was pushed for the review comment"
+        diff = gitea._http.get(f"/repos/{repo}/pulls/{pr}.diff").text
+        assert marker in diff, "the pushed fix doesn't do what the comment asked"
 
         # --- the operator approves and merges; the agent closes the item ---
         with contextlib.suppress(GiteaError):  # approval may not be required
@@ -188,8 +196,11 @@ def test_the_slice_end_to_end(running: tuple[RunningLoop, InMemoryTransport]) ->
         )
         operator.tick()
         deadline = time.monotonic() + DEADLINE_S
+        nudges = 0
         while operator.item().status != "closed" and time.monotonic() < deadline:
             if loop_of(operator.item()).wait == ANSWER:
+                nudges += 1
+                assert nudges <= MAX_ANSWERS, f"never closed: {operator.last()}"
                 operator.say("It's merged; please close the item.")
             operator.settle()
         assert operator.item().outcome == "completed"