nisse/nisse-dev/tests/test_planner.py

import io
from pathlib import Path
from typing import Any

import pytest

from nisse_dev.canonical import (
    AssistantText,
    CanonicalEvent,
    PlanSubmitted,
    StoredEvent,
    ToolCall,
    UserMessage,
)
from nisse_dev.control import (
    AWAIT_ARCH_SIGNOFF,
    AWAIT_MILESTONE_SIGNOFF,
    AskHuman,
    Orchestrator,
    PlanningError,
    SubmitPlan,
)
from nisse_dev.llm import BudgetExhausted, RoleMap
from nisse_dev.oplog import OpLog
from nisse_dev.planner import PLANNER_TOOLS, LLMPlanningAgent
from nisse_dev.state import StateStore, Transcript
from nisse_dev.transport import InMemoryTransport

ROLES = RoleMap(
    executor="local/qwen",
    planner="anthropic/claude-opus-5",
    helper="anthropic/claude-opus-5",
    expert="anthropic/claude-opus-5",
)


class ScriptedClient:
    """Returns canned canonical events per call, recording what it was asked."""

    def __init__(self, turns: list[list[CanonicalEvent]]) -> None:
        self._turns = turns
        self.calls: list[dict[str, Any]] = []

    def complete(
        self,
        events: list[StoredEvent],
        tools: list[dict[str, Any]] | None,
        model: str,
        system: str | None = None,
    ) -> list[CanonicalEvent]:
        self.calls.append(
            {"events": events, "tools": tools, "model": model, "system": system}
        )
        if not self._turns:
            return []
        return self._turns.pop(0)


def plan_call(content: str, gate: str = "arch") -> ToolCall:
    return ToolCall(
        call_id="call-1",
        tool="submit_plan",
        arguments={"type": gate, "content": content},
    )


ITEM = "t"  # the work item the unit tests plan in


def log(store: StateStore) -> Transcript:
    return store.transcript(ITEM, "planner")


@pytest.fixture
def store(tmp_path: Path) -> StateStore:
    return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))


def test_submit_plan_tool_call_becomes_submit_plan(store: StateStore) -> None:
    client = ScriptedClient([[plan_call("step 1: create src/app.py")]])
    agent = LLMPlanningAgent(client, ROLES)
    action = agent.plan(log(store), "arch", "demo")
    assert action == SubmitPlan("step 1: create src/app.py")


def test_ask_human_tool_call_becomes_ask_human(store: StateStore) -> None:
    client = ScriptedClient(
        [
            [
                ToolCall(
                    call_id="call-1",
                    tool="ask_human",
                    arguments={"question": "postgres or sqlite?"},
                )
            ]
        ]
    )
    action = LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
    assert action == AskHuman("postgres or sqlite?")


def test_planner_role_and_tools_are_used(store: StateStore) -> None:
    client = ScriptedClient([[plan_call("plan")]])
    LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
    call = client.calls[0]
    assert call["model"] == "anthropic/claude-opus-5"  # never the executor
    assert [t["function"]["name"] for t in call["tools"]] == [
        "submit_plan",
        "ask_human",
    ]
    assert "demo" in call["system"] and "arch" in call["system"]


def test_no_proceed_tool_is_offered() -> None:
    names = {tool["function"]["name"] for tool in PLANNER_TOOLS}
    assert names == {"submit_plan", "ask_human"}  # no self-advancement path


def test_model_output_is_appended_to_the_log(store: StateStore) -> None:
    client = ScriptedClient(
        [[AssistantText(text="thinking out loud", model="m"), plan_call("the plan")]]
    )
    LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
    kinds = [type(e.event).__name__ for e in log(store).read()]
    assert kinds == ["AssistantText", "ToolCall", "ToolResult"]


def test_gate_tool_call_is_answered_in_the_log(store: StateStore) -> None:
    # submit_plan ends the turn and has no natural result, but a tool call
    # left dangling makes the next render invalid for the provider.
    client = ScriptedClient([[plan_call("the plan")]])
    LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
    events = [e.event for e in log(store).read()]
    assert [type(e).__name__ for e in events] == ["ToolCall", "ToolResult"]
    assert events[1].call_id == events[0].call_id
    assert "posted for approval" in events[1].result


def test_prose_only_turn_retries_once_then_raises(store: StateStore) -> None:
    client = ScriptedClient(
        [
            [AssistantText(text="here's what I think", model="m")],
            [AssistantText(text="still prose", model="m")],
        ]
    )
    with pytest.raises(PlanningError, match="no submit_plan or ask_human"):
        LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
    assert len(client.calls) == 2
    assert "did not call a tool" in client.calls[1]["system"]


def test_retry_succeeds_after_prose(store: StateStore) -> None:
    client = ScriptedClient(
        [[AssistantText(text="prose", model="m")], [plan_call("the real plan")]]
    )
    action = LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
    assert action == SubmitPlan("the real plan")


def test_malformed_tool_arguments_are_rejected(store: StateStore) -> None:
    blank = ToolCall(call_id="c", tool="submit_plan", arguments={"content": "   "})
    client = ScriptedClient([[blank], [blank]])
    with pytest.raises(PlanningError):
        LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")


def test_planner_sees_prior_transcript(store: StateStore) -> None:
    store.transcript(ITEM, "operator").append(
        UserMessage(text="use demo", source="transport:@don")
    )
    client = ScriptedClient([[plan_call("plan")]])
    LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
    seen = [getattr(e.event, "text", None) for e in client.calls[0]["events"]]
    assert "[operator, Matrix] use demo" in seen


# --- orchestrator integration (still no network) -------------------------


def build(tmp_path: Path, client: ScriptedClient) -> tuple[Orchestrator, StateStore]:
    store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
    transport = InMemoryTransport()
    agent = LLMPlanningAgent(client, ROLES)
    orch = Orchestrator(store, transport, agent, operators={"don"}, roles=ROLES)
    orch.start()
    return orch, store, transport


def test_use_project_produces_llm_plan_and_transcript(tmp_path: Path) -> None:
    client = ScriptedClient([[plan_call("1. create app.py\n2. add a test")]])
    _, store, transport = build(tmp_path, client)
    transport.receive("don", "use demo")

    request = transport.outbound[-1]
    assert request.kind == "approval_request"
    assert "create app.py" in request.text

    # The work item's own events: `use demo` was said while idle, and is
    # carried into the item it opened (D1).
    item = store.active_item()
    events = [e.event for e in store.transcript(item.id, "operator").read()]
    kinds = [type(e).__name__ for e in events]
    assert kinds == [
        "UserMessage",
        "Fact",
        "ToolCall",
        "ToolResult",
        "PlanSubmitted",
        "GateOpened",
    ]
    submitted = next(e for e in events if isinstance(e, PlanSubmitted))
    assert submitted.model == "anthropic/claude-opus-5"  # pinned planner


def test_roles_pinned_at_task_start(tmp_path: Path) -> None:
    client = ScriptedClient([[plan_call("plan")]])
    _, store, transport = build(tmp_path, client)
    transport.receive("don", "use demo")
    state = store.get_state()
    assert state is not None
    assert state.data["roles"]["planner"] == "anthropic/claude-opus-5"


def test_planning_failure_gates_to_await_input(tmp_path: Path) -> None:
    prose = [AssistantText(text="prose", model="m")]
    client = ScriptedClient([prose, prose])
    _, store, transport = build(tmp_path, client)
    transport.receive("don", "use demo")
    state = store.get_state()
    assert state is not None
    assert state.state == "AWAIT_INPUT"
    assert transport.outbound[-1].kind == "question"


def test_operator_can_recover_from_a_planning_failure(tmp_path: Path) -> None:
    # Regression: AWAIT_INPUT reached by any path other than ask_human used to
    # record no pending item, so every operator reply was refused with "no
    # pending question" and the agent was stuck for good.
    prose = [AssistantText(text="prose", model="m")]
    client = ScriptedClient([prose, prose, [plan_call("recovered plan")]])
    _, store, transport = build(tmp_path, client)
    transport.receive("don", "use demo")
    assert store.get_state().state == "AWAIT_INPUT"

    transport.receive("don", "try again, keep it short")

    assert store.get_state().state == AWAIT_ARCH_SIGNOFF
    assert "recovered plan" in transport.outbound[-1].text


def test_operator_can_recover_after_a_stopped_milestone(tmp_path: Path) -> None:
    class StoppedExecutor:
        def run(self, transcript: object, milestone: int) -> Any:
            from nisse_dev.executor import ExecutorOutcome

            return ExecutorOutcome("escalate", "tests failed twice")

    store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
    transport = InMemoryTransport()
    client = ScriptedClient([[plan_call("p1")], [plan_call("p2")]])
    orch = Orchestrator(
        store,
        transport,
        LLMPlanningAgent(client, ROLES),
        operators={"don"},
        roles=ROLES,
        executor=StoppedExecutor(),
    )
    orch.start()
    transport.receive("don", "use demo")
    transport.receive("don", "approve")
    transport.receive("don", "approve")
    transport.receive("don", "start milestone 1")
    assert store.get_state().state == "AWAIT_INPUT"

    transport.receive("don", "ok, I fixed the dependency")

    assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF
    assert "start milestone 1" in transport.outbound[-1].text


def test_rejection_feedback_reaches_the_planner_via_the_log(
    tmp_path: Path,
) -> None:
    client = ScriptedClient([[plan_call("v1")], [plan_call("v2")]])
    _, _store, transport = build(tmp_path, client)
    transport.receive("don", "use demo")
    transport.receive("don", "too vague, name the files")

    replan_context = [e.event for e in client.calls[1]["events"]]
    assert any(
        isinstance(e, UserMessage) and "too vague" in e.text for e in replan_context
    )
    assert "v2" in transport.outbound[-1].text


def test_budget_exhaustion_is_not_retried(store: StateStore) -> None:
    """A budget will not reappear within a turn, so a retry buys nothing and
    delays telling the operator."""

    class BrokeClient:
        def __init__(self) -> None:
            self.calls = 0

        def complete(self, *args: Any, **kwargs: Any):
            self.calls += 1
            raise BudgetExhausted("daily budget exhausted")

    client = BrokeClient()
    with pytest.raises(BudgetExhausted):
        LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
    assert client.calls == 1  # not two