nisse/nisse-dev/tests/test_no_stranding.py

"""The orchestrator's resting-state invariant, checked under injected faults.

Working states (planning, executing) are only right while work runs. After
any message is handled the agent must be somewhere it can wait for the
operator, and some reply must lead on from there. The first live run found
four ways to break this one at a time; this drives a whole project with a
fault at each step instead.
"""

import io
from pathlib import Path
from typing import Any

import pytest

from nisse_dev.control import (
    AWAIT_ARCH_SIGNOFF,
    AWAIT_INPUT,
    AWAIT_MILESTONE_SIGNOFF,
    EXECUTING,
    IDLE,
    MILESTONE_PLANNING,
    PLANNING_ARCH,
    PR_OPEN,
    AskHuman,
    Orchestrator,
    PlanningError,
    SubmitPlan,
)
from nisse_dev.executor import ExecutorOutcome
from nisse_dev.gitea import GiteaError, PullRequest
from nisse_dev.oplog import OpLog
from nisse_dev.publish import PublishResult
from nisse_dev.state import StateStore
from nisse_dev.transport import InMemoryTransport

WORKING = {PLANNING_ARCH, MILESTONE_PLANNING, EXECUTING}


class Faults:
    """One fault, fired once, at the nth call to one site."""

    def __init__(self, site: str, nth: int, fault: Any) -> None:
        self._site, self._nth, self._fault = site, nth, fault
        self._calls: dict[str, int] = {}
        self.fired = False

    def check(self, site: str) -> Any:
        """Raise or return the fault if this call is the one; else None."""
        self._calls[site] = self._calls.get(site, 0) + 1
        if self.fired or site != self._site or self._calls[site] != self._nth:
            return None
        self.fired = True
        if isinstance(self._fault, Exception):
            raise self._fault
        return self._fault


class Planner:
    def __init__(self, faults: Faults) -> None:
        self._faults = faults

    def plan(
        self,
        transcript: object,
        gate: str,
        project: str,
        feedback: str | None = None,
        answer: str | None = None,
    ) -> SubmitPlan | AskHuman:
        return self._faults.check("planner") or SubmitPlan(f"{gate} plan")


class Executor:
    def __init__(self, faults: Faults) -> None:
        self._faults = faults

    def run(self, transcript: object, milestone: int) -> ExecutorOutcome:
        return self._faults.check("executor") or ExecutorOutcome("completed", "done")


class Publisher:
    def __init__(self, faults: Faults) -> None:
        self._faults = faults
        self._opened = 0

    def reconcilers(self) -> dict[str, Any]:
        return {}

    def publish(self, **kwargs: Any) -> PublishResult:
        self._faults.check("publisher")
        self._opened += 1
        pr = PullRequest(number=self._opened, state="open", head="h", url="u")
        return PublishResult(pr, f"opened PR #{self._opened}")


def next_reply(store: StateStore, used_project: bool) -> str:
    """What an operator following the prompts would say next."""
    state = store.get_state()
    assert state is not None
    data = state.data
    match state.state:
        case "IDLE":
            return "status" if used_project else "use demo"
        case "AWAIT_ARCH_SIGNOFF":
            return "approve"
        case "AWAIT_MILESTONE_SIGNOFF":
            if data["gate"]["approved"]:
                return f"start milestone {data['milestone']}"
            return "approve"
        case "PR_OPEN":
            return "next milestone" if data["milestone"] < 2 else "project complete"
        case "AWAIT_INPUT":
            return "continue"
    raise AssertionError(f"stranded in {state.state}")


FAULTS = [
    ("planner", 1, RuntimeError("bug in planning")),
    ("planner", 1, PlanningError("planner returned 400")),
    ("planner", 1, AskHuman("which language?")),
    ("planner", 2, RuntimeError("bug in planning")),
    ("planner", 3, OSError("disk full")),
    ("executor", 1, RuntimeError("bug in the executor")),
    ("executor", 1, ExecutorOutcome("escalate", "stuck")),
    ("executor", 1, ExecutorOutcome("capped", "max turns")),
    ("executor", 2, ExecutorOutcome("stopped", "operator")),
    ("publisher", 1, RuntimeError("bug in publishing")),
    ("publisher", 1, GiteaError("gitea is down")),
    ("publisher", 2, RuntimeError("bug in publishing")),
]


@pytest.mark.parametrize(
    ("site", "nth", "fault"), FAULTS, ids=[f"{s}{n}-{f!r}" for s, n, f in FAULTS]
)
def test_a_fault_anywhere_never_strands_the_agent(
    tmp_path: Path, site: str, nth: int, fault: Any
) -> None:
    faults = Faults(site, nth, fault)
    store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
    transport = InMemoryTransport()
    Orchestrator(
        store,
        transport,
        Planner(faults),
        operators={"don"},
        executor=Executor(faults),
        publisher=Publisher(faults),
    ).start()

    used_project = False
    for _ in range(40):
        reply = next_reply(store, used_project)
        used_project = used_project or reply.startswith("use ")
        transport.receive("don", reply)
        state = store.get_state()
        assert state is not None
        assert state.state not in WORKING, (
            f"after {reply!r}: stranded in {state.state}; "
            f"last said: {transport.outbound[-1].text!r}"
        )
        if reply == "project complete":
            break
    else:
        raise AssertionError("never reached project complete")
    assert faults.fired
    assert store.get_state().state == IDLE


def test_a_signoff_state_without_its_gate_offers_a_way_out(tmp_path: Path) -> None:
    faults = Faults("none", 0, None)
    store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
    transport = InMemoryTransport()
    orch = Orchestrator(store, transport, Planner(faults), operators={"don"})
    orch.start()
    transport.receive("don", "use demo")
    state = store.get_state()
    store.transition(
        AWAIT_ARCH_SIGNOFF,
        cause="test",
        project="demo",
        data={**state.data, "gate": None},
    )
    orch._data["gate"] = None  # as a restart would load it
    transport.receive("don", "approve")
    assert store.get_state().state == AWAIT_INPUT
    transport.receive("don", "continue")
    assert store.get_state().state == AWAIT_ARCH_SIGNOFF


class RaisingReviewExecutor:
    def run(self, transcript: object, milestone: int) -> ExecutorOutcome:
        raise RuntimeError("bug while addressing review")


def test_a_failing_review_response_keeps_pr_open_and_says_so(
    tmp_path: Path,
) -> None:
    faults = Faults("none", 0, None)
    store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
    transport = InMemoryTransport()
    orch = Orchestrator(
        store,
        transport,
        Planner(faults),
        operators={"don"},
        gitea_operators={"don"},
        executor=Executor(faults),
        publisher=Publisher(faults),
    )
    orch.start()
    for reply in ("use demo", "approve", "approve", "start milestone 1"):
        transport.receive("don", reply)
    assert store.get_state().state == PR_OPEN
    orch._executor = RaisingReviewExecutor()

    class Comment:
        id, author, body = 1, "don", "add a test"

    orch.handle_review_comment(Comment())
    assert store.get_state().state == PR_OPEN
    assert "could not handle a review comment" in transport.outbound[-1].text
    transport.receive("don", "project complete")
    assert store.get_state().state == IDLE


def test_the_stub_configuration_reaches_every_state(tmp_path: Path) -> None:
    store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
    transport = InMemoryTransport()
    Orchestrator(
        store, transport, Planner(Faults("none", 0, None)), operators={"don"}
    ).start()
    for reply in ("use demo", "approve", "approve", "start milestone 1"):
        transport.receive("don", reply)
    assert store.get_state().state == PR_OPEN
    transport.receive("don", "next milestone")
    assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF