nisse/nisse-dev/src/nisse_dev/loop/agent.py

"""The loop's scheduler (L1, L2, L5, D1): intake, messages and driver runs.

Everything durable happens on one worker thread, in order. Inputs (an
operator's message, an issue assigned to the agent) are handed to it as they
arrive; a message is stamped with the revision on screen in its thread first,
because by the time the worker reaches it a newer one may be up (L3).

One work item at a time is active. An item runs until its driver calls a
waiting tool; it then waits on the operator, the active pointer is cleared
(a safe point, L1), and the next ready item runs. Conversation is not work:
a message is recorded and changes its item's state, but never interrupts a
run (L1).
"""

import threading
import uuid
from collections.abc import Callable, Iterable
from dataclasses import replace

from nisse_dev.cancel import Cancel
from nisse_dev.canonical import Fact, PlanSubmitted, UserMessage
from nisse_dev.gates import command_form
from nisse_dev.gitea import GiteaClient, GiteaError, Issue
from nisse_dev.loop.driver import Driver, RunOutcome
from nisse_dev.loop.items import (
    ANSWER,
    CLOSED,
    GO_AHEAD,
    MERGE,
    READY,
    RUNNING,
    WAITING,
    Loop,
    item_for_issue,
    item_for_thread,
    loop_of,
    save,
)
from nisse_dev.loop.milestones import OPEN_PR, Milestones
from nisse_dev.loop.tools import (
    ASK_HUMAN,
    CLOSE_ITEM,
    FINISH_MILESTONE,
    RESTATE,
    START_MILESTONE,
)
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore, WorkItem
from nisse_dev.tools import EXECUTOR_TOOLS, Workspace
from nisse_dev.transport import InboundMessage, Transport
from nisse_dev.worker import Worker

INTAKE_INTERVAL_S = 30.0
_GO = frozenset({"go", "approve"})

# Before the go-ahead the driver may look around (read, list, run a command
# to reproduce a bug) but not change anything (D1); after it, everything.
_LOOK = [t for t in EXECUTOR_TOOLS if t["function"]["name"] != "write_file"]

SYSTEM = """You are nisse-dev, a software agent working with the operator on
the repository {repo}. You are on one work item: issue #{issue}, "{title}".
The issue and the conversation about it are below.

Before any work, find out what you need, then use restate to put your
understanding to the operator: what you take the request to mean, how it fits
the project, and your first step. Wait for their go-ahead. If the request is
unclear, ask_human instead of guessing.

After the go-ahead, work in a milestone: call start_milestone before
changing anything (never create or switch branches yourself), make the
change, run the project's tests, then call finish_milestone with a summary.
The harness commits, pushes and opens a PR, and you wait for review. Review
comments arrive as messages: make the changes within the same milestone (don't
start another), run the tests, and call finish_milestone again to push the fix. When the PR has merged, call
close_item with what was done, what was verified, and what is left open. If
the PR is closed without merging, ask the operator what next.

{workspace}"""


class LoopAgent:
    def __init__(
        self,
        store: StateStore,
        transport: Transport,
        gitea: GiteaClient,
        *,
        repo: str,
        project: str,
        account: str,
        operators: Iterable[str],
        gitea_operators: Iterable[str],
        driver: Driver,
        workspace: Workspace | None,
        worker: Worker,
        cancel: Cancel,
        oplog: OpLog,
        roles: dict[str, str] | None = None,
        milestones: Milestones | None = None,
        intake_interval_s: float = INTAKE_INTERVAL_S,
    ) -> None:
        self._store = store
        self._transport = transport
        self._gitea = gitea
        self._repo = repo
        self._project = project
        self._account = account
        self._operators = frozenset(operators)
        self._gitea_operators = frozenset(gitea_operators)
        self._driver = driver
        self._workspace = workspace
        self._worker = worker
        self._cancel = cancel
        self._oplog = oplog
        self._roles = roles or {}
        self._milestones = milestones
        self._interval = intake_interval_s
        # thread root -> revision on screen there, read on the transport's
        # thread to stamp replies on arrival (L3).
        self._shown: dict[str, int] = {}
        self._running = False
        self._stop = threading.Event()
        self._intake_thread: threading.Thread | None = None

    # -- lifecycle ----------------------------------------------------------

    def start(self) -> None:
        self._worker.call(self.boot)
        self._transport.on_message(self.on_message)
        start = getattr(self._transport, "start", None)
        if start is not None:
            start()
        self._intake_thread = threading.Thread(target=self._intake_loop, daemon=True)
        self._intake_thread.start()

    def stop(self) -> None:
        self._stop.set()
        if self._intake_thread is not None:
            self._intake_thread.join(timeout=5)

    def _intake_loop(self) -> None:
        while not self._stop.is_set():
            self._worker.submit(self._settled("intake", self.intake))
            self._stop.wait(self._interval)

    def _settled(self, what: str, job: Callable[[], None]) -> Callable[[], None]:
        """A worker job that reports a failure instead of losing it."""

        def run() -> None:
            try:
                job()
            except Exception as exc:  # noqa: BLE001 -- reported, the loop goes on
                self._oplog.emit("loop.job_failed", cause=what, error=repr(exc))
                self._transport.post(f"something went wrong with {what}: {exc}")

        return run

    # -- boot (L8) --------------------------------------------------------------

    def boot(self) -> None:
        """Resume waiting points as they were, and never resume interrupted
        work blindly: an item cut off mid-run waits on the operator."""
        self._store.set_active(None, cause="boot")
        self._reconcile_publishing()
        for item in self._store.work_items("open"):
            loop = loop_of(item)
            if item.position == RUNNING:
                note = "I restarted while working on this. Reply to continue."
                self._wait(item, replace(loop, wait=ANSWER, note=note), note)
            elif item.position == WAITING and loop.thread:
                if loop.wait == GO_AHEAD:
                    self._shown[loop.thread] = loop.rev
                    self._transport.post(
                        f"(after a restart) rev-{loop.rev} is still up; say go "
                        "to start, or reply with changes.",
                        thread=loop.thread,
                    )
                else:
                    self._transport.post(
                        f"(after a restart) still waiting: {loop.note}",
                        thread=loop.thread,
                    )

    def _reconcile_publishing(self) -> None:
        """L8 step 2: a publish cut off by a crash is settled against Gitea.
        An open PR for the branch is adopted; otherwise the intent is aborted
        and the operator is told, never a blind retry."""
        for intent in self._store.pending_intents():
            if intent.kind != OPEN_PR or not intent.work_item:
                continue
            item = self._store.work_item(intent.work_item)
            branch = str(intent.payload.get("branch", ""))
            try:
                pr = self._gitea.find_pull_request(self._repo, branch)
            except GiteaError:
                continue  # unverifiable: left pending, asked about below
            if item is None or item.status != "open":
                self._store.abort_intent(intent.id, cause="item no longer open")
            elif pr is not None and pr.state == "open":
                self._store.commit_intent(intent.id, cause=f"adopted PR #{pr.number}")
                loop = replace(loop_of(item), pr=pr.number, pr_url=pr.url)
                self._fact(item, f"after a restart, adopted PR #{pr.number} ({pr.url})")
                save(
                    self._store,
                    item.id,
                    WAITING,
                    replace(loop, wait=MERGE, note=f"PR #{pr.number}"),
                    cause="adopted PR after restart",
                )
            else:
                self._store.abort_intent(intent.id, cause="no PR was opened")
                note = "publishing was cut off by a restart; reply to try again."
                self._wait(item, replace(loop_of(item), wait=ANSWER, note=note), note)

    # -- intake (D1) ---------------------------------------------------------

    def intake(self) -> None:
        """Issues assigned to the agent become work items, once each."""
        try:
            issues = self._gitea.assigned_issues(self._repo, self._account)
        except GiteaError as exc:
            # L5: an outage delays intake, never fails anything.
            self._oplog.emit("loop.intake_unavailable", cause=str(exc))
            return
        for issue in issues:
            if item_for_issue(self._store, issue.number) is not None:
                continue
            if not self._from_an_operator(issue):
                self._oplog.emit(
                    "loop.intake_ignored",
                    cause="not filed or assigned by an operator",
                    issue=issue.number,
                )
                continue
            self._open(issue)
        self.watch()
        self._schedule()

    # -- PR review and merge (D4, D5) -------------------------------------------

    def watch(self) -> None:
        """New operator comments on an item's PR, and its merge or close,
        reach the item; a waiting item becomes ready to act on them."""
        for item in self._store.work_items("open"):
            loop = loop_of(item)
            if not loop.pr:
                continue
            try:
                pr = self._gitea.pull_request(self._repo, loop.pr)
                comments = self._gitea.list_comments(self._repo, loop.pr)
                notes = self._gitea.review_notes(self._repo, loop.pr)
            except GiteaError as exc:
                self._oplog.emit("loop.watch_unavailable", cause=str(exc))
                continue
            log = self._store.transcript(item.id, "operator")
            new = False
            for comment in comments:
                if comment.id <= loop.comments:
                    continue
                loop = replace(loop, comments=comment.id)
                if comment.author in self._gitea_operators:
                    log.append(
                        UserMessage(
                            text=comment.body,
                            source=f"gitea:pr:{loop.pr}:comment:{comment.id}",
                        )
                    )
                    new = True
            for note in notes:
                if note.key in loop.notes:
                    continue
                loop = replace(loop, notes=(*loop.notes, note.key))
                if note.author in self._gitea_operators:
                    where = f"on {note.where}: " if note.where else ""
                    log.append(
                        UserMessage(
                            text=f"{where}{note.body}",
                            source=f"gitea:pr:{loop.pr}:review:{note.key}",
                        )
                    )
                    new = True
            position = item.position
            if pr.merged:
                self._fact(item, f"PR #{loop.pr} merged")
                loop, position = replace(loop, pr=0, pr_url="", branch=""), READY
            elif pr.state != "open":
                self._fact(item, f"PR #{loop.pr} was closed without merging")
                loop, position = replace(loop, pr=0, pr_url="", branch=""), READY
            elif new and item.position == WAITING:
                position = READY
            if position != item.position or loop != loop_of(item):
                if position == READY:
                    loop = replace(loop, wait="", note="")
                save(self._store, item.id, position, loop, cause="PR update")

    def _from_an_operator(self, issue: Issue) -> bool:
        """L11: only an operator's issue, or one an operator assigned, is
        an instruction."""
        if issue.author in self._gitea_operators:
            return True
        try:
            who = self._gitea.assigned_by(self._repo, issue.number, self._account)
        except GiteaError:
            return False
        return who in self._gitea_operators

    def _open(self, issue: Issue) -> None:
        item_id = f"{self._project}-{issue.number}-{uuid.uuid4().hex[:6]}"
        cause = f"assigned {self._repo}#{issue.number}"
        root = self._transport.post(
            f"Queued {self._repo}#{issue.number}: {issue.title}"
        )
        loop = Loop(thread=root, issue=issue.number)
        self._store.open_work_item(
            item_id,
            title=issue.title,
            scope=self._project,
            position=READY,
            data={"loop": loop.to_data(), "roles": self._roles},
            cause=cause,
        )
        log = self._store.transcript(item_id, "orchestrator")
        log.append(
            Fact(
                text=f"work item opened from {self._repo}#{issue.number} "
                f"({issue.title}), filed by {issue.author}"
                + (f", labels: {', '.join(issue.labels)}" if issue.labels else "")
            )
        )
        text = f"{issue.title}\n\n{issue.body}".strip()
        if issue.author in self._gitea_operators:
            log.as_actor("operator").append(
                UserMessage(text=text, source=f"gitea:issue-body:{issue.number}")
            )
        else:
            # L11: someone else's text is data; the operator's assignment is
            # the instruction.
            log.append(Fact(text=f"the issue as written by {issue.author}:\n{text}"))

    # -- messages (L5) -------------------------------------------------------

    def on_message(self, msg: InboundMessage) -> None:
        """On the transport's thread: stamp, then hand to the worker."""
        if msg.sender not in self._operators:
            return
        if command_form(msg.text) == "stop" and self._running:
            # Straight to the flag: the run is what it has to reach (L2).
            self._cancel.set()
        about = self._shown.get(msg.thread) if msg.thread else None
        self._worker.submit(
            self._settled("your message", lambda: self.handle(msg, about))
        )

    def handle(self, msg: InboundMessage, about: int | None) -> None:
        item = item_for_thread(self._store, msg.thread) if msg.thread else None
        text = (msg.original or msg.text).strip()
        command = command_form(msg.text)
        if item is None:
            self._store.transcript(None, "operator").append(
                UserMessage(text=text, source=f"transport:{msg.sender}")
            )
            if command == "status":
                self._transport.post(self.status(), thread=msg.thread)
            elif command != "stop":
                self._transport.post(
                    f"I take work from issues assigned to me in {self._repo}: "
                    "assign one and I'll pick it up. Reply in its thread to "
                    "talk about it.",
                    thread=msg.thread,
                )
            return
        self._store.transcript(item.id, "operator").append(
            UserMessage(text=text, source=f"transport:{msg.sender}")
        )
        loop = loop_of(item)
        if command == "abandon":
            self._close(item, loop, "abandoned", f"abandoned by {msg.sender}")
        elif item.position == WAITING and loop.wait == GO_AHEAD and command in _GO:
            if about != loop.rev:
                # L3: a go-ahead is about what was on screen when it arrived.
                self._transport.post(
                    f"that go was for rev-{about}, but rev-{loop.rev} is up now; "
                    "say go again if it still stands.",
                    thread=loop.thread,
                )
                return
            self._fact(item, f"go-ahead on rev-{loop.rev} from {msg.sender}")
            save(
                self._store,
                item.id,
                READY,
                replace(loop, started=True, wait=""),
                cause=f"go-ahead on rev-{loop.rev}",
            )
        elif item.position == WAITING:
            # An answer, or changes to a restatement: the driver reads it.
            save(
                self._store,
                item.id,
                READY,
                replace(loop, wait="", note=""),
                cause="the operator answered",
            )
        self._schedule()

    def status(self) -> str:
        items = self._store.work_items("open")
        if not items:
            return f"idle: no open work items. I take issues assigned to me in {self._repo}."
        lines = []
        for item in items:
            loop = loop_of(item)
            what = item.position
            if item.position == WAITING:
                what = f"waiting for {loop.wait} ({loop.note or f'rev-{loop.rev}'})"
            lines.append(f"#{loop.issue} {item.title}: {what}")
        return "\n".join(lines)

    # -- running (L1, L2) ------------------------------------------------------

    def _schedule(self) -> None:
        """Run ready items until none is left. Only on the worker."""
        if self._running:
            return
        while True:
            ready = [i for i in self._store.work_items("open") if i.position == READY]
            if not ready:
                self._store.set_active(None, cause="nothing ready")
                return
            self._run(ready[0])  # oldest first (L1)

    def _run(self, item: WorkItem) -> None:
        loop = loop_of(item)
        self._store.set_active(item.id, cause="ready")
        if not loop.prepared and self._milestones is not None:
            # D3: a fresh workspace when a work item starts.
            try:
                notes = self._milestones.prepare(f"{item.id}-work")
            except Exception as exc:  # noqa: BLE001 -- the item waits, not the loop
                note = f"I couldn't prepare the workspace: {exc}. Reply to retry."
                self._wait(item, replace(loop, wait=ANSWER, note=note), note)
                self._store.set_active(None, cause="workspace failed")
                return
            self._fact(item, "workspace prepared from the latest default branch")
            if notes:
                said = "Set aside from the workspace: " + "; ".join(notes)
                self._fact(item, said)
                self._transport.post(said, thread=loop.thread)
            loop = replace(loop, prepared=True)
        save(self._store, item.id, RUNNING, loop, cause="driver run")
        tools = [
            RESTATE,
            ASK_HUMAN,
            *(
                _LOOK
                if not loop.started
                else [START_MILESTONE, FINISH_MILESTONE, CLOSE_ITEM, *EXECUTOR_TOOLS]
            ),
        ]
        system = SYSTEM.format(
            repo=self._repo,
            issue=loop.issue,
            title=item.title,
            workspace=self._workspace.describe() if self._workspace else "",
        )
        self._running = True
        try:
            outcome = self._driver.run(
                self._store.transcript(item.id, "driver"),
                system=system,
                tools=tools,
                workspace=self._workspace,
                handlers={"start_milestone": lambda args: self._start(item.id, args)},
            )
        finally:
            self._running = False
            self._cancel.clear()
        self._settle(item.id, outcome)

    def _settle(self, item_id: str, outcome: RunOutcome) -> None:
        item = self._store.work_item(item_id)
        assert item is not None
        loop = loop_of(item)
        args = outcome.arguments or {}
        if outcome.kind == "wait" and outcome.tool == "restate":
            text = str(args.get("text", "")).strip()
            rev = loop.rev + 1
            self._store.transcript(item.id, "driver").append(
                PlanSubmitted(plan=text, model=self._roles.get("planner", ""))
            )
            self._fact(item, f"restatement rev-{rev} put up for a go-ahead")
            self._shown[loop.thread] = rev
            self._transport.post(
                f"rev-{rev}: {text}\n\nSay go to start, or reply with changes.",
                thread=loop.thread,
            )
            self._wait(item, replace(loop, rev=rev, wait=GO_AHEAD, note=""), None)
        elif outcome.kind == "wait" and outcome.tool == "ask_human":
            question = str(args.get("question", "")).strip()
            self._wait(item, replace(loop, wait=ANSWER, note=question), question)
        elif outcome.kind == "wait" and outcome.tool == "finish_milestone":
            self._finish(item, loop, str(args.get("summary", "")).strip())
        elif outcome.kind == "wait" and outcome.tool == "close_item":
            summary = str(args.get("summary", "")).strip()
            self._transport.post(f"Done: {summary}", thread=loop.thread)
            self._close(item, loop, "completed", summary)
        elif outcome.kind == "stopped":
            note = f"stopped ({outcome.note}). Why did you stop it? Reply to continue."
            self._wait(item, replace(loop, wait=ANSWER, note=note), note)
        else:
            note = f"I stopped short: {outcome.note}. Reply to continue."
            self._wait(item, replace(loop, wait=ANSWER, note=note), note)
        self._store.set_active(None, cause="item waiting")

    # -- milestones (D4) -----------------------------------------------------

    def _start(self, item_id: str, args: dict[str, object]) -> tuple[str, bool]:
        """start_milestone, mid-run: cut the milestone's branch."""
        item = self._store.work_item(item_id)
        assert item is not None
        loop = loop_of(item)
        if loop.branch:
            busy = f"milestone {loop.milestone} is still open; finish it first"
            return busy, False
        if self._milestones is None:
            return "milestones aren't available here", False
        number = loop.milestone + 1
        branch = f"{item.id}-milestone-{number}-{uuid.uuid4().hex[:4]}"
        self._milestones.start(branch)
        name = str(args.get("name", "")).strip()
        self._fact(item, f"milestone {number} started ({name}) on branch {branch}")
        save(
            self._store,
            item.id,
            RUNNING,
            replace(loop, milestone=number, branch=branch),
            cause=f"milestone {number} started",
        )
        return f"started milestone {number} on branch {branch}", True

    def _finish(self, item: WorkItem, loop: Loop, summary: str) -> None:
        """finish_milestone: publish the milestone, or push a review fix."""
        if self._milestones is None:
            note = "milestones aren't available here"
            self._wait(item, replace(loop, wait=ANSWER, note=note), note)
            return
        if loop.pr:
            fixed = self._milestones.push_fix(
                item_id=item.id,
                branch=loop.branch,
                pr=loop.pr,
                message=summary or "Address review comments",
                attempt=loop.fixes + 1,
            )
            self._fact(item, f"review fix: {fixed.note}")
            if fixed.kind in ("pushed", "nothing"):
                self._gitea_comment(loop.pr, fixed.note)
                self._wait(
                    item,
                    replace(loop, wait=MERGE, note=f"PR #{loop.pr}"),
                    f"PR #{loop.pr}: {fixed.note}",
                )
            elif fixed.kind == "new-pr" and fixed.pr is not None:
                # Later fixes go to the new PR's branch, never the merged one.
                loop = replace(
                    loop,
                    pr=fixed.pr.number,
                    pr_url=fixed.pr.url,
                    branch=fixed.branch,
                    fixes=loop.fixes + 1,
                )
                self._wait(
                    item,
                    replace(loop, wait=MERGE, note=f"PR #{fixed.pr.number}"),
                    f"The PR merged before the fix landed, so it's a new PR: "
                    f"#{fixed.pr.number} {fixed.pr.url}",
                )
            else:
                self._wait(
                    item,
                    replace(loop, wait=ANSWER, note=fixed.note),
                    f"{fixed.note}. What next?",
                )
            return
        if not loop.branch:
            # D4: finishing without a start cuts the branch now.
            number = loop.milestone + 1
            branch = f"{item.id}-milestone-{number}-{uuid.uuid4().hex[:4]}"
            self._milestones.start(branch)
            loop = replace(loop, milestone=number, branch=branch)
        published = self._milestones.publish(
            item_id=item.id,
            branch=loop.branch,
            title=f"{item.title} (milestone {loop.milestone})",
            summary=summary,
            issue=loop.issue,
        )
        left = (
            f" (left out {len(published.left_out)} build artifact(s))"
            if published.left_out
            else ""
        )
        if published.pr is not None and published.kind in ("opened", "adopted"):
            pr = published.pr
            self._fact(
                item,
                f"milestone {loop.milestone} published: PR #{pr.number} "
                f"({pr.url}){left}",
            )
            self._wait(
                item,
                replace(
                    loop,
                    pr=pr.number,
                    pr_url=pr.url,
                    wait=MERGE,
                    note=f"PR #{pr.number}",
                ),
                f"Milestone {loop.milestone} is up for review: {pr.url}{left}",
            )
        else:
            self._fact(
                item, f"milestone {loop.milestone} not published: {published.note}"
            )
            self._wait(
                item,
                replace(loop, wait=ANSWER, note=published.note),
                f"Milestone {loop.milestone}: {published.note}. What next?",
            )

    def _gitea_comment(self, pr: int, text: str) -> None:
        try:
            self._gitea.post_comment(self._repo, pr, text)
        except GiteaError:
            pass  # courtesy only; the pushed commit is the answer

    def _wait(self, item: WorkItem, loop: Loop, say: str | None) -> None:
        if say:
            self._transport.post(say, thread=loop.thread)
        save(self._store, item.id, WAITING, loop, cause=f"waiting for {loop.wait}")

    def _close(self, item: WorkItem, loop: Loop, outcome: str, note: str) -> None:
        self._fact(item, f"work item {outcome}: {note}")
        if loop.pr and outcome == "abandoned":
            # L2: abandoning closes the item's open PR with a note.
            try:
                self._gitea.post_comment(self._repo, loop.pr, f"abandoned: {note}")
                self._gitea.close_pull_request(self._repo, loop.pr)
            except GiteaError as exc:
                self._transport.post(
                    f"could not close PR #{loop.pr}: {exc}", thread=loop.thread
                )
        save(self._store, item.id, CLOSED, loop, cause=outcome)
        self._store.close_work_item(item.id, outcome=outcome, cause=note)
        if loop.issue:
            try:
                self._gitea.post_comment(self._repo, loop.issue, f"{outcome}: {note}")
                self._gitea.close_issue(self._repo, loop.issue)
            except GiteaError as exc:
                self._transport.post(
                    f"closed here, but could not close #{loop.issue}: {exc}",
                    thread=loop.thread,
                )
        if outcome == "abandoned":
            self._transport.post(f"abandoned: {note}", thread=loop.thread)

    def _fact(self, item: WorkItem, text: str) -> None:
        self._store.transcript(item.id, "orchestrator").append(Fact(text=text))