nisse/nisse-dev/tests/test_delegation.py

import io
from pathlib import Path
from typing import Any

import pytest

from nisse_core.execution.ssh import SSHHost
from nisse_core.log.canonical import (
    AssistantText,
    CanonicalEvent,
    DelegationCompleted,
    DelegationStarted,
    StoredEvent,
    ToolCall,
    ToolResult,
)
from nisse_core.models.llm import BudgetExhausted, LLMError, UpstreamUnavailable
from nisse_core.oplog import OpLog
from nisse_core.state.cancel import STOPPED_RESULT, Cancel
from nisse_core.state.store import Transcript
from nisse_dev.delegation import (
    DELEGATE_TOOLS,
    DELEGATION_INTENT,
    Caps,
    Delegation,
    DelegationResult,
    delegation_reconciler,
)
from nisse_dev.git import GitRepo
from nisse_dev.state import StateStore
from nisse_dev.tools import Workspace


class ScriptedClient:
    def __init__(self, turns: list[list[CanonicalEvent]]) -> None:
        self._turns = turns
        self.calls: list[dict[str, Any]] = []

    def complete(
        self,
        events: list[StoredEvent],
        tools: list[dict[str, Any]] | None,
        model: str,
        system: str | None = None,
    ) -> list[CanonicalEvent]:
        self.calls.append({"tools": tools, "model": model, "system": system})
        if not self._turns:
            return [AssistantText(text="still thinking", model=model)]
        return self._turns.pop(0)


class FailingClient:
    def complete(self, *args: Any, **kwargs: Any) -> list[CanonicalEvent]:
        raise LLMError("frontier unavailable")


def call(tool: str, cid: str = "c1", **arguments: Any) -> ToolCall:
    return ToolCall(call_id=cid, tool=tool, arguments=arguments)


def hand_back(succeeded: bool = True, cid: str = "hb", **overrides: Any) -> ToolCall:
    args = {
        "changed": "added greet.py",
        "now_true": "greet() returns a greeting and its test passes",
        "next": "open the PR",
        "succeeded": succeeded,
    }
    args.update(overrides)
    return ToolCall(call_id=cid, tool="hand_back", arguments=args)


@pytest.fixture
def store(tmp_path: Path) -> StateStore:
    return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))


@pytest.fixture
def repo(ssh_host: SSHHost, tmp_path: Path) -> GitRepo:
    r = GitRepo(ssh_host, tmp_path / "work")
    r.init()
    r.configure_identity("bot", "bot@example.test")
    (r.root / "seed.txt").write_text("seed")
    r.commit_all("seed")
    return r


@pytest.fixture
def ws(repo: GitRepo) -> Workspace:
    return Workspace(repo.host, repo.root, timeout_s=10)


ITEM = "t"  # the work item these delegations run in


def log(store: StateStore) -> Transcript:
    return store.transcript(ITEM, "helper")


class Bound:
    """A delegation run in the test's work item, as the executor runs it."""

    def __init__(self, delegation: Delegation, store: StateStore) -> None:
        self.delegation = delegation
        self.store = store

    def run(self, **kw: Any) -> DelegationResult:
        return self.delegation.run(log(self.store), **kw)

    def resume(self, intent: Any) -> DelegationResult:
        return self.delegation.resume(intent)

    def recover(self, notify: Any) -> DelegationResult | None:
        return self.delegation.recover(notify)


def build(client: Any, store: StateStore, ws: Workspace, repo: GitRepo, **kw: Any):
    return Bound(
        Delegation(client, store, ws, repo, model="claude-haiku-4-5", **kw), store
    )


def test_successful_delegation_returns_a_structured_handback(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    client = ScriptedClient(
        [
            [call("write_file", path="greet.py", content="def greet(): ...")],
            [hand_back()],
        ]
    )
    result = build(client, store, ws, repo).run(brief="add greet", branch="d1")

    assert result.usable
    assert result.handback is not None
    assert "greet.py" in result.handback.changed
    assert "what to do next" in result.handback.render().lower()
    assert result.artifacts == ["greet.py"]


def test_delegation_works_on_its_own_branch(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    client = ScriptedClient(
        [[call("write_file", path="a.py", content="x")], [hand_back()]]
    )
    build(client, store, ws, repo).run(brief="b", branch="deleg-1")
    assert repo.current_branch() == "deleg-1"


def test_abort_discards_the_branch_and_its_work(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    # L9: abort must leave nothing to resume on top of.
    client = ScriptedClient(
        [
            [call("write_file", path="half-done.py", content="oops")],
            [hand_back(succeeded=False, changed="could not finish")],
        ]
    )
    result = build(client, store, ws, repo).run(brief="b", branch="deleg-2")

    assert not result.usable
    assert repo.current_branch() == "main"
    assert not (repo.root / "half-done.py").exists()


def test_stop_discards_the_branch_and_settles_the_intent(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    # L2's stop is treated like a failure, not like an outage: the operator
    # stopped it for a reason, so nothing is kept for resuming.
    cancel = Cancel()
    client = ScriptedClient(
        [
            [call("write_file", path="half-done.py", content="oops")],
            [call("list_dir", path=".")],
        ]
    )
    original = client.complete

    def complete_then_stop(*args: Any, **kwargs: Any) -> list[CanonicalEvent]:
        produced = original(*args, **kwargs)
        if len(client.calls) == 2:
            cancel.set()  # the operator says stop during turn 2's model call
        return produced

    client.complete = complete_then_stop  # type: ignore[method-assign]
    result = build(client, store, ws, repo, cancel=cancel).run(
        brief="b", branch="deleg-stop"
    )

    assert result.status == "stopped"
    assert not result.usable
    assert repo.current_branch() == "main"
    assert not repo.branch_exists("deleg-stop")
    assert not (repo.root / "half-done.py").exists()
    assert store.pending_intents() == []
    # The unrun call still has a result, so the transcript renders.
    skipped = [
        e.event
        for e in log(store).read()
        if isinstance(e.event, ToolResult) and e.event.result == STOPPED_RESULT
    ]
    assert len(skipped) == 1


def test_turn_cap_is_hard(store: StateStore, ws: Workspace, repo: GitRepo) -> None:
    forever = [[call("list_dir", path=".")] for _ in range(20)]
    client = ScriptedClient(forever)
    result = build(client, store, ws, repo, caps=Caps(turns=3)).run(
        brief="b", branch="d"
    )

    assert result.status == "capped"
    assert "turn cap of 3" in result.note
    assert len(client.calls) == 3


def test_wall_clock_cap_is_hard(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    ticks = iter([0.0, 1.0, 99.0, 99.0, 99.0])
    client = ScriptedClient([[call("list_dir", path=".")] for _ in range(10)])
    result = build(
        client,
        store,
        ws,
        repo,
        caps=Caps(turns=10, wall_clock_s=60.0),
        now=lambda: next(ticks),
    ).run(brief="b", branch="d")

    assert result.status == "capped"
    assert "wall-clock cap" in result.note


def test_cap_breach_is_not_usable_and_discards_work(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    client = ScriptedClient([[call("write_file", path="partial.py", content="x")]] * 5)
    result = build(client, store, ws, repo, caps=Caps(turns=2)).run(
        brief="b", branch="d"
    )

    assert not result.usable  # the caller must gate, not resume (L9)
    assert not (repo.root / "partial.py").exists()


def test_events_are_checkpointed_as_they_arrive(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    client = ScriptedClient(
        [[call("write_file", path="a.py", content="x")], [hand_back()]]
    )
    build(client, store, ws, repo).run(brief="add a", branch="d")

    kinds = [type(e.event).__name__ for e in log(store).read()]
    assert kinds == [
        "DelegationStarted",
        "ToolCall",
        "ToolResult",
        "ToolCall",  # hand_back
        "ToolResult",
        "DelegationCompleted",
    ]


def test_composite_events_record_the_brief_caps_and_outcome(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    client = ScriptedClient(
        [[call("write_file", path="a.py", content="x")], [hand_back()]]
    )
    caps = Caps(turns=7, tokens=1234, wall_clock_s=42.0)
    build(client, store, ws, repo, caps=caps).run(brief="the brief", branch="d")

    events = [e.event for e in log(store).read()]
    started = next(e for e in events if isinstance(e, DelegationStarted))
    completed = next(e for e in events if isinstance(e, DelegationCompleted))
    assert started.brief == "the brief"
    assert started.model == "claude-haiku-4-5"
    assert started.caps == {"turns": 7, "tokens": 1234, "wall_clock_s": 42.0}
    # L9: this pair is the eval record, so it must carry the outcome.
    assert completed.exit_status == "handback"
    assert "greet()" in completed.handback


def test_hand_back_tool_is_offered_alongside_the_executor_tools() -> None:
    names = [t["function"]["name"] for t in DELEGATE_TOOLS]
    assert names[-1] == "hand_back"
    assert "run_command" in names  # full tool access, not a consult (L9)


def test_malformed_handback_is_rejected_and_the_delegation_continues(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    client = ScriptedClient(
        [
            [hand_back(cid="bad", changed="   ")],  # blank field
            [call("write_file", path="a.py", content="x")],
            [hand_back(cid="good")],
        ]
    )
    result = build(client, store, ws, repo).run(brief="b", branch="d")

    assert result.usable  # recovered rather than accepting a useless note
    assert len(client.calls) == 3


def test_non_boolean_success_flag_is_rejected(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    client = ScriptedClient(
        [
            [call("write_file", path="a.py", content="x")],
            [hand_back(succeeded="yes")],  # not a bool: unparseable
            [hand_back(cid="ok")],
        ]
    )
    result = build(client, store, ws, repo).run(brief="b", branch="d")
    assert result.usable
    assert len(client.calls) == 3


def test_client_failure_is_a_defined_return_not_a_crash(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    result = build(FailingClient(), store, ws, repo).run(brief="b", branch="d")
    assert result.status == "failed"
    assert "frontier unavailable" in result.note
    assert not result.usable


def test_a_delegate_that_never_hands_back_is_capped_not_believed(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    # Prose forever: the delegate never reports, so its tree state is unknown.
    client = ScriptedClient([])
    result = build(client, store, ws, repo, caps=Caps(turns=3)).run(
        brief="b", branch="d"
    )
    assert result.status == "capped"
    assert not result.usable


def test_success_claimed_without_any_change_is_refused(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    # The handback plus file state is the whole interface back, so a note
    # describing work that was never done is worse than no note.
    client = ScriptedClient(
        [
            [hand_back(cid="early")],
            [call("write_file", path="real.py", content="x")],
            [hand_back(cid="earned")],
        ]
    )
    result = build(client, store, ws, repo).run(brief="b", branch="d")

    assert result.usable
    assert len(client.calls) == 3  # the first claim was sent back
    assert (repo.root / "real.py").exists()


def test_failure_may_be_reported_without_doing_work(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    # "This sub-task is impossible" is a legitimate finding that requires no
    # changes; only a claim of success needs evidence.
    client = ScriptedClient(
        [[hand_back(succeeded=False, changed="nothing: the API does not exist")]]
    )
    result = build(client, store, ws, repo).run(brief="b", branch="d")

    assert result.status == "failed"
    assert not result.usable
    assert len(client.calls) == 1  # believed immediately, not challenged


# --- L8 boot step 5: the resume rule -----------------------------------------


def pending_delegation_intent(store: StateStore, branch: str = "d") -> Any:
    store.begin_intent(
        DELEGATION_INTENT,
        {"branch": branch, "base": "main", "brief": "finish it"},
        cause="test",
        work_item=ITEM,  # owned by the work item it ran in
    )
    return store.pending_intents()[-1]


def test_intent_is_recorded_and_settled_by_a_completed_delegation(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    client = ScriptedClient(
        [[call("write_file", path="a.py", content="x")], [hand_back()]]
    )
    build(client, store, ws, repo).run(brief="b", branch="d")
    # Nothing left in flight: an aborted delegation is a finished one too.
    assert store.pending_intents() == []


def test_aborted_delegation_also_settles_its_intent(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    client = ScriptedClient([[hand_back(succeeded=False, changed="cannot")]])
    build(client, store, ws, repo).run(brief="b", branch="d")
    assert store.pending_intents() == []


def test_clean_cut_after_a_tool_result_is_resumable(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    repo.checkout_new_branch("d")
    (repo.root / "partial.py").write_text("x")
    repo.commit_all("partial work")
    log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={}))
    log(store).append(ToolCall(call_id="c", tool="write_file", arguments={}))
    log(store).append(ToolResult(call_id="c", result="wrote", success=True))
    intent = pending_delegation_intent(store)

    assert delegation_reconciler(store, repo)(intent) is True


def test_dangling_tool_call_gates(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    # The effect of the call is unknown and not verifiable by inspection.
    repo.checkout_new_branch("d")
    log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={}))
    log(store).append(ToolCall(call_id="c", tool="run_command", arguments={}))
    intent = pending_delegation_intent(store)

    assert delegation_reconciler(store, repo)(intent) is None


def test_missing_branch_means_nothing_started(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={}))
    log(store).append(ToolResult(call_id="c", result="ok", success=True))
    intent = pending_delegation_intent(store, branch="never-created")

    assert delegation_reconciler(store, repo)(intent) is False


def test_a_finished_delegation_settles_rather_than_resumes(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    repo.checkout_new_branch("d")
    log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={}))
    log(store).append(
        DelegationCompleted(
            handback="done",
            artifacts=[],
            tokens=0,
            duration_s=1.0,
            exit_status="handback",
        )
    )
    intent = pending_delegation_intent(store)

    assert delegation_reconciler(store, repo)(intent) is True


def test_resume_continues_the_same_delegation(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    # Interrupt after one turn, then resume: no second delegation_started,
    # because L7 defines a session as the delegation, not the process.
    repo.checkout_new_branch("d")
    log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={}))
    log(store).append(ToolResult(call_id="c", result="wrote a.py", success=True))
    intent = pending_delegation_intent(store)

    client = ScriptedClient(
        [[call("write_file", path="b.py", content="x")], [hand_back()]]
    )
    result = build(client, store, ws, repo).resume(intent)

    assert result.usable
    starts = [e for e in log(store).read() if isinstance(e.event, DelegationStarted)]
    assert len(starts) == 1
    assert store.pending_intents() == []
    assert (repo.root / "b.py").exists()


def test_resume_onto_a_missing_branch_fails_cleanly(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    intent = pending_delegation_intent(store, branch="gone")
    result = build(ScriptedClient([]), store, ws, repo).resume(intent)

    assert result.status == "failed"
    assert not result.usable
    assert store.pending_intents() == []  # not left dangling for the next boot


# --- boot recovery sweep ------------------------------------------------


def test_recover_resumes_a_clean_cut_with_a_notice(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    repo.checkout_new_branch("d")
    log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={}))
    log(store).append(ToolResult(call_id="c", result="wrote a.py", success=True))
    pending_delegation_intent(store)
    notices: list[str] = []

    client = ScriptedClient(
        [[call("write_file", path="b.py", content="x")], [hand_back()]]
    )
    result = build(client, store, ws, repo).recover(notices.append)

    assert result is not None and result.usable
    # L8: a notice, not a question — a clean cut needs no operator decision.
    assert notices and "resuming the delegation" in notices[0]


def test_recover_settles_a_finished_delegation_without_resuming(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    repo.checkout_new_branch("d")
    log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={}))
    log(store).append(
        DelegationCompleted(
            handback="done",
            artifacts=[],
            tokens=0,
            duration_s=1.0,
            exit_status="handback",
        )
    )
    pending_delegation_intent(store)
    client = ScriptedClient([])

    assert build(client, store, ws, repo).recover(lambda _m: None) is None
    assert store.pending_intents() == []
    assert client.calls == []  # the model was never asked


def test_recover_discards_an_intent_whose_branch_never_existed(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    log(store).append(ToolResult(call_id="c", result="ok", success=True))
    pending_delegation_intent(store, branch="never-made")

    assert build(ScriptedClient([]), store, ws, repo).recover(lambda _m: None) is None
    assert store.pending_intents() == []


def test_recover_leaves_an_unclean_cut_for_the_boot_gate(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    repo.checkout_new_branch("d")
    log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={}))
    log(store).append(ToolCall(call_id="c", tool="run_command", arguments={}))
    pending_delegation_intent(store)

    assert build(ScriptedClient([]), store, ws, repo).recover(lambda _m: None) is None
    # Still pending, so reconcile_on_boot gates rather than blind-resuming.
    assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT]


# --- the token cap, which L9 calls hard and was decoration -------------


class MeteredClient(ScriptedClient):
    """A client that reports usage, as the real ones do."""

    def __init__(self, turns: list, per_call: int = 1000) -> None:
        super().__init__(turns)
        self._per_call = per_call
        self.last_usage: dict[str, int] = {}

    def complete(self, *args: Any, **kwargs: Any):
        produced = super().complete(*args, **kwargs)
        self.last_usage = {"input": self._per_call, "output": self._per_call}
        return produced


def test_token_cap_stops_a_delegation(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    # An unbounded delegation can exhaust a daily budget and take external
    # models away until it resets, so the cap has to be real.
    client = MeteredClient(
        [[call("list_dir", path=".")] for _ in range(20)], per_call=600
    )
    result = build(client, store, ws, repo, caps=Caps(turns=20, tokens=2000)).run(
        brief="b", branch="d"
    )

    assert result.status == "capped"
    assert "token cap of 2000" in result.note
    # 600 in + 600 out per call: stops on the second turn, not the twentieth.
    assert len(client.calls) == 2


def test_tokens_spent_are_reported_not_left_at_zero(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    client = MeteredClient(
        [[call("write_file", path="a.py", content="x")], [hand_back()]], per_call=500
    )
    result = build(client, store, ws, repo).run(brief="b", branch="d")

    assert result.usable
    assert result.tokens == 2000  # two calls, 1000 each

    completed = next(
        e.event for e in log(store).read() if isinstance(e.event, DelegationCompleted)
    )
    # L9 evaluates cost against outcome, which needs a real number.
    assert completed.tokens == 2000


def test_a_client_without_usage_still_works(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    # Scripted and third-party clients need not report usage; the cap simply
    # does not bind rather than crashing the delegation.
    client = ScriptedClient(
        [[call("write_file", path="a.py", content="x")], [hand_back()]]
    )
    assert build(client, store, ws, repo).run(brief="b", branch="d").usable


def test_budget_exhaustion_preserves_the_work_for_later(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    """An outage is not a bad delegate.

    The branch may hold perfectly good work, so it is kept and the intent
    left pending — the next boot treats it exactly like a crash mid-delegation
    and resumes if the cut was clean. Discarding would throw away work
    because the budget ran out.
    """

    class BrokeClient(ScriptedClient):
        def complete(self, *args: Any, **kwargs: Any):
            if self.calls:
                raise BudgetExhausted("daily budget exhausted")
            return super().complete(*args, **kwargs)

    client = BrokeClient([[call("write_file", path="progress.py", content="real")]])
    result = build(client, store, ws, repo).run(brief="b", branch="d")

    assert result.status == "interrupted"
    assert result.interrupted and not result.usable
    # The work survives, on its branch.
    assert repo.current_branch() == "d"
    assert (repo.root / "progress.py").exists()
    # And the intent stays pending, which is what L8's resume rule keys on.
    assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT]


def test_an_ordinary_failure_still_discards(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    # Only external interruptions are preserved; a delegate that made a mess
    # is still cleaned up (L9).
    client = ScriptedClient([[hand_back(succeeded=False, changed="cannot")]])
    result = build(client, store, ws, repo).run(brief="b", branch="d")

    assert result.status == "failed"
    assert not result.interrupted
    assert repo.current_branch() == "main"
    assert store.pending_intents() == []


def test_a_network_outage_also_preserves_the_work(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    # Same reasoning as an exhausted budget: the branch may be fine, and the
    # outage says nothing about the quality of the work on it.
    class FlakyClient(ScriptedClient):
        def complete(self, *args: Any, **kwargs: Any):
            if self.calls:
                raise UpstreamUnavailable("LLM request failed: connection refused")
            return super().complete(*args, **kwargs)

    client = FlakyClient([[call("write_file", path="progress.py", content="real")]])
    result = build(client, store, ws, repo).run(brief="b", branch="d")

    assert result.status == "interrupted"
    assert (repo.root / "progress.py").exists()
    assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT]


def test_a_bad_request_is_not_given_the_benefit_of_the_doubt(
    store: StateStore, ws: Workspace, repo: GitRepo
) -> None:
    # A 4xx that is not a rate limit means we sent something wrong; treating
    # that as an outage would preserve branches and hide a real bug.
    class BadRequestClient(ScriptedClient):
        def complete(self, *args: Any, **kwargs: Any):
            raise LLMError("LLM request returned 400: malformed tool schema")

    result = build(BadRequestClient([]), store, ws, repo).run(brief="b", branch="d")
    assert result.status == "failed"
    assert not result.interrupted