"""The loop's scheduler (L1, L2, L5, D1): intake, messages and driver runs. Everything durable happens on one worker thread, in order. Inputs (an operator's message, an issue assigned to the agent) are handed to it as they arrive; a message is stamped with the revision on screen in its thread first, because by the time the worker reaches it a newer one may be up (L3). One work item at a time is active. An item runs until its driver calls a waiting tool; it then waits on the operator, the active pointer is cleared (a safe point, L1), and the next ready item runs. Conversation is not work: a message is recorded and changes its item's state, but never interrupts a run (L1). """ import threading import uuid from collections.abc import Callable, Iterable from dataclasses import replace from nisse_dev.cancel import Cancel from nisse_dev.canonical import Fact, PlanSubmitted, UserMessage from nisse_dev.gates import command_form from nisse_dev.gitea import GiteaClient, GiteaError, Issue from nisse_dev.loop.driver import Driver, RunOutcome from nisse_dev.loop.items import ( ANSWER, CLOSED, GO_AHEAD, MERGE, READY, RUNNING, WAITING, Loop, item_for_issue, item_for_thread, loop_of, save, ) from nisse_dev.loop.milestones import OPEN_PR, Milestones from nisse_dev.loop.tools import ( ASK_HUMAN, CLOSE_ITEM, FINISH_MILESTONE, RESTATE, START_MILESTONE, ) from nisse_dev.oplog import OpLog from nisse_dev.state import StateStore, WorkItem from nisse_dev.tools import EXECUTOR_TOOLS, Workspace from nisse_dev.transport import InboundMessage, Transport from nisse_dev.worker import Worker INTAKE_INTERVAL_S = 30.0 _GO = frozenset({"go", "approve"}) # Before the go-ahead the driver may look around (read, list, run a command # to reproduce a bug) but not change anything (D1); after it, everything. _LOOK = [t for t in EXECUTOR_TOOLS if t["function"]["name"] != "write_file"] SYSTEM = """You are nisse-dev, a software agent working with the operator on the repository {repo}. You are on one work item: issue #{issue}, "{title}". The issue and the conversation about it are below. Before any work, find out what you need, then use restate to put your understanding to the operator: what you take the request to mean, how it fits the project, and your first step. Wait for their go-ahead. If the request is unclear, ask_human instead of guessing. After the go-ahead, work in a milestone: call start_milestone before changing anything (never create or switch branches yourself), make the change, run the project's tests, then call finish_milestone with a summary. The harness commits, pushes and opens a PR, and you wait for review. Review comments arrive as messages: make the changes, run the tests, and call finish_milestone again to push the fix. When the PR has merged, call close_item with what was done, what was verified, and what is left open. If the PR is closed without merging, ask the operator what next. {workspace}""" class LoopAgent: def __init__( self, store: StateStore, transport: Transport, gitea: GiteaClient, *, repo: str, project: str, account: str, operators: Iterable[str], gitea_operators: Iterable[str], driver: Driver, workspace: Workspace | None, worker: Worker, cancel: Cancel, oplog: OpLog, roles: dict[str, str] | None = None, milestones: Milestones | None = None, intake_interval_s: float = INTAKE_INTERVAL_S, ) -> None: self._store = store self._transport = transport self._gitea = gitea self._repo = repo self._project = project self._account = account self._operators = frozenset(operators) self._gitea_operators = frozenset(gitea_operators) self._driver = driver self._workspace = workspace self._worker = worker self._cancel = cancel self._oplog = oplog self._roles = roles or {} self._milestones = milestones self._interval = intake_interval_s # thread root -> revision on screen there, read on the transport's # thread to stamp replies on arrival (L3). self._shown: dict[str, int] = {} self._running = False self._stop = threading.Event() self._intake_thread: threading.Thread | None = None # -- lifecycle ---------------------------------------------------------- def start(self) -> None: self._worker.call(self.boot) self._transport.on_message(self.on_message) start = getattr(self._transport, "start", None) if start is not None: start() self._intake_thread = threading.Thread(target=self._intake_loop, daemon=True) self._intake_thread.start() def stop(self) -> None: self._stop.set() if self._intake_thread is not None: self._intake_thread.join(timeout=5) def _intake_loop(self) -> None: while not self._stop.is_set(): self._worker.submit(self._settled("intake", self.intake)) self._stop.wait(self._interval) def _settled(self, what: str, job: Callable[[], None]) -> Callable[[], None]: """A worker job that reports a failure instead of losing it.""" def run() -> None: try: job() except Exception as exc: # noqa: BLE001 -- reported, the loop goes on self._oplog.emit("loop.job_failed", cause=what, error=repr(exc)) self._transport.post(f"something went wrong with {what}: {exc}") return run # -- boot (L8) -------------------------------------------------------------- def boot(self) -> None: """Resume waiting points as they were, and never resume interrupted work blindly: an item cut off mid-run waits on the operator.""" self._store.set_active(None, cause="boot") self._reconcile_publishing() for item in self._store.work_items("open"): loop = loop_of(item) if item.position == RUNNING: note = "I restarted while working on this. Reply to continue." self._wait(item, replace(loop, wait=ANSWER, note=note), note) elif item.position == WAITING and loop.thread: if loop.wait == GO_AHEAD: self._shown[loop.thread] = loop.rev self._transport.post( f"(after a restart) rev-{loop.rev} is still up; say go " "to start, or reply with changes.", thread=loop.thread, ) else: self._transport.post( f"(after a restart) still waiting: {loop.note}", thread=loop.thread, ) def _reconcile_publishing(self) -> None: """L8 step 2: a publish cut off by a crash is settled against Gitea. An open PR for the branch is adopted; otherwise the intent is aborted and the operator is told, never a blind retry.""" for intent in self._store.pending_intents(): if intent.kind != OPEN_PR or not intent.work_item: continue item = self._store.work_item(intent.work_item) branch = str(intent.payload.get("branch", "")) try: pr = self._gitea.find_pull_request(self._repo, branch) except GiteaError: continue # unverifiable: left pending, asked about below if item is None or item.status != "open": self._store.abort_intent(intent.id, cause="item no longer open") elif pr is not None and pr.state == "open": self._store.commit_intent(intent.id, cause=f"adopted PR #{pr.number}") loop = replace(loop_of(item), pr=pr.number, pr_url=pr.url) self._fact(item, f"after a restart, adopted PR #{pr.number} ({pr.url})") save( self._store, item.id, WAITING, replace(loop, wait=MERGE, note=f"PR #{pr.number}"), cause="adopted PR after restart", ) else: self._store.abort_intent(intent.id, cause="no PR was opened") note = "publishing was cut off by a restart; reply to try again." self._wait(item, replace(loop_of(item), wait=ANSWER, note=note), note) # -- intake (D1) --------------------------------------------------------- def intake(self) -> None: """Issues assigned to the agent become work items, once each.""" try: issues = self._gitea.assigned_issues(self._repo, self._account) except GiteaError as exc: # L5: an outage delays intake, never fails anything. self._oplog.emit("loop.intake_unavailable", cause=str(exc)) return for issue in issues: if item_for_issue(self._store, issue.number) is not None: continue if not self._from_an_operator(issue): self._oplog.emit( "loop.intake_ignored", cause="not filed or assigned by an operator", issue=issue.number, ) continue self._open(issue) self.watch() self._schedule() # -- PR review and merge (D4, D5) ------------------------------------------- def watch(self) -> None: """New operator comments on an item's PR, and its merge or close, reach the item; a waiting item becomes ready to act on them.""" for item in self._store.work_items("open"): loop = loop_of(item) if not loop.pr: continue try: pr = self._gitea.pull_request(self._repo, loop.pr) comments = self._gitea.list_comments(self._repo, loop.pr) notes = self._gitea.review_notes(self._repo, loop.pr) except GiteaError as exc: self._oplog.emit("loop.watch_unavailable", cause=str(exc)) continue log = self._store.transcript(item.id, "operator") new = False for comment in comments: if comment.id <= loop.comments: continue loop = replace(loop, comments=comment.id) if comment.author in self._gitea_operators: log.append( UserMessage( text=comment.body, source=f"gitea:pr:{loop.pr}:comment:{comment.id}", ) ) new = True for note in notes: if note.key in loop.notes: continue loop = replace(loop, notes=(*loop.notes, note.key)) if note.author in self._gitea_operators: where = f"on {note.where}: " if note.where else "" log.append( UserMessage( text=f"{where}{note.body}", source=f"gitea:pr:{loop.pr}:review:{note.key}", ) ) new = True position = item.position if pr.merged: self._fact(item, f"PR #{loop.pr} merged") loop, position = replace(loop, pr=0, pr_url="", branch=""), READY elif pr.state != "open": self._fact(item, f"PR #{loop.pr} was closed without merging") loop, position = replace(loop, pr=0, pr_url="", branch=""), READY elif new and item.position == WAITING: position = READY if position != item.position or loop != loop_of(item): if position == READY: loop = replace(loop, wait="", note="") save(self._store, item.id, position, loop, cause="PR update") def _from_an_operator(self, issue: Issue) -> bool: """L11: only an operator's issue, or one an operator assigned, is an instruction.""" if issue.author in self._gitea_operators: return True try: who = self._gitea.assigned_by(self._repo, issue.number, self._account) except GiteaError: return False return who in self._gitea_operators def _open(self, issue: Issue) -> None: item_id = f"{self._project}-{issue.number}-{uuid.uuid4().hex[:6]}" cause = f"assigned {self._repo}#{issue.number}" root = self._transport.post( f"Queued {self._repo}#{issue.number}: {issue.title}" ) loop = Loop(thread=root, issue=issue.number) self._store.open_work_item( item_id, title=issue.title, scope=self._project, position=READY, data={"loop": loop.to_data(), "roles": self._roles}, cause=cause, ) log = self._store.transcript(item_id, "orchestrator") log.append( Fact( text=f"work item opened from {self._repo}#{issue.number} " f"({issue.title}), filed by {issue.author}" + (f", labels: {', '.join(issue.labels)}" if issue.labels else "") ) ) text = f"{issue.title}\n\n{issue.body}".strip() if issue.author in self._gitea_operators: log.as_actor("operator").append( UserMessage(text=text, source=f"gitea:issue-body:{issue.number}") ) else: # L11: someone else's text is data; the operator's assignment is # the instruction. log.append(Fact(text=f"the issue as written by {issue.author}:\n{text}")) # -- messages (L5) ------------------------------------------------------- def on_message(self, msg: InboundMessage) -> None: """On the transport's thread: stamp, then hand to the worker.""" if msg.sender not in self._operators: return if command_form(msg.text) == "stop" and self._running: # Straight to the flag: the run is what it has to reach (L2). self._cancel.set() about = self._shown.get(msg.thread) if msg.thread else None self._worker.submit( self._settled("your message", lambda: self.handle(msg, about)) ) def handle(self, msg: InboundMessage, about: int | None) -> None: item = item_for_thread(self._store, msg.thread) if msg.thread else None text = (msg.original or msg.text).strip() command = command_form(msg.text) if item is None: self._store.transcript(None, "operator").append( UserMessage(text=text, source=f"transport:{msg.sender}") ) if command == "status": self._transport.post(self.status(), thread=msg.thread) elif command != "stop": self._transport.post( f"I take work from issues assigned to me in {self._repo}: " "assign one and I'll pick it up. Reply in its thread to " "talk about it.", thread=msg.thread, ) return self._store.transcript(item.id, "operator").append( UserMessage(text=text, source=f"transport:{msg.sender}") ) loop = loop_of(item) if command == "abandon": self._close(item, loop, "abandoned", f"abandoned by {msg.sender}") elif item.position == WAITING and loop.wait == GO_AHEAD and command in _GO: if about != loop.rev: # L3: a go-ahead is about what was on screen when it arrived. self._transport.post( f"that go was for rev-{about}, but rev-{loop.rev} is up now; " "say go again if it still stands.", thread=loop.thread, ) return self._fact(item, f"go-ahead on rev-{loop.rev} from {msg.sender}") save( self._store, item.id, READY, replace(loop, started=True, wait=""), cause=f"go-ahead on rev-{loop.rev}", ) elif item.position == WAITING: # An answer, or changes to a restatement: the driver reads it. save( self._store, item.id, READY, replace(loop, wait="", note=""), cause="the operator answered", ) self._schedule() def status(self) -> str: items = self._store.work_items("open") if not items: return f"idle: no open work items. I take issues assigned to me in {self._repo}." lines = [] for item in items: loop = loop_of(item) what = item.position if item.position == WAITING: what = f"waiting for {loop.wait} ({loop.note or f'rev-{loop.rev}'})" lines.append(f"#{loop.issue} {item.title}: {what}") return "\n".join(lines) # -- running (L1, L2) ------------------------------------------------------ def _schedule(self) -> None: """Run ready items until none is left. Only on the worker.""" if self._running: return while True: ready = [i for i in self._store.work_items("open") if i.position == READY] if not ready: self._store.set_active(None, cause="nothing ready") return self._run(ready[0]) # oldest first (L1) def _run(self, item: WorkItem) -> None: loop = loop_of(item) self._store.set_active(item.id, cause="ready") if not loop.prepared and self._milestones is not None: # D3: a fresh workspace when a work item starts. try: notes = self._milestones.prepare(f"{item.id}-work") except Exception as exc: # noqa: BLE001 -- the item waits, not the loop note = f"I couldn't prepare the workspace: {exc}. Reply to retry." self._wait(item, replace(loop, wait=ANSWER, note=note), note) self._store.set_active(None, cause="workspace failed") return self._fact(item, "workspace prepared from the latest default branch") if notes: said = "Set aside from the workspace: " + "; ".join(notes) self._fact(item, said) self._transport.post(said, thread=loop.thread) loop = replace(loop, prepared=True) save(self._store, item.id, RUNNING, loop, cause="driver run") tools = [ RESTATE, ASK_HUMAN, *( _LOOK if not loop.started else [START_MILESTONE, FINISH_MILESTONE, CLOSE_ITEM, *EXECUTOR_TOOLS] ), ] system = SYSTEM.format( repo=self._repo, issue=loop.issue, title=item.title, workspace=self._workspace.describe() if self._workspace else "", ) self._running = True try: outcome = self._driver.run( self._store.transcript(item.id, "driver"), system=system, tools=tools, workspace=self._workspace, handlers={"start_milestone": lambda args: self._start(item.id, args)}, ) finally: self._running = False self._cancel.clear() self._settle(item.id, outcome) def _settle(self, item_id: str, outcome: RunOutcome) -> None: item = self._store.work_item(item_id) assert item is not None loop = loop_of(item) args = outcome.arguments or {} if outcome.kind == "wait" and outcome.tool == "restate": text = str(args.get("text", "")).strip() rev = loop.rev + 1 self._store.transcript(item.id, "driver").append( PlanSubmitted(plan=text, model=self._roles.get("planner", "")) ) self._fact(item, f"restatement rev-{rev} put up for a go-ahead") self._shown[loop.thread] = rev self._transport.post( f"rev-{rev}: {text}\n\nSay go to start, or reply with changes.", thread=loop.thread, ) self._wait(item, replace(loop, rev=rev, wait=GO_AHEAD, note=""), None) elif outcome.kind == "wait" and outcome.tool == "ask_human": question = str(args.get("question", "")).strip() self._wait(item, replace(loop, wait=ANSWER, note=question), question) elif outcome.kind == "wait" and outcome.tool == "finish_milestone": self._finish(item, loop, str(args.get("summary", "")).strip()) elif outcome.kind == "wait" and outcome.tool == "close_item": summary = str(args.get("summary", "")).strip() self._transport.post(f"Done: {summary}", thread=loop.thread) self._close(item, loop, "completed", summary) elif outcome.kind == "stopped": note = f"stopped ({outcome.note}). Why did you stop it? Reply to continue." self._wait(item, replace(loop, wait=ANSWER, note=note), note) else: note = f"I stopped short: {outcome.note}. Reply to continue." self._wait(item, replace(loop, wait=ANSWER, note=note), note) self._store.set_active(None, cause="item waiting") # -- milestones (D4) ----------------------------------------------------- def _start(self, item_id: str, args: dict[str, object]) -> tuple[str, bool]: """start_milestone, mid-run: cut the milestone's branch.""" item = self._store.work_item(item_id) assert item is not None loop = loop_of(item) if loop.branch: busy = f"milestone {loop.milestone} is still open; finish it first" return busy, False if self._milestones is None: return "milestones aren't available here", False number = loop.milestone + 1 branch = f"{item.id}-milestone-{number}-{uuid.uuid4().hex[:4]}" self._milestones.start(branch) name = str(args.get("name", "")).strip() self._fact(item, f"milestone {number} started ({name}) on branch {branch}") save( self._store, item.id, RUNNING, replace(loop, milestone=number, branch=branch), cause=f"milestone {number} started", ) return f"started milestone {number} on branch {branch}", True def _finish(self, item: WorkItem, loop: Loop, summary: str) -> None: """finish_milestone: publish the milestone, or push a review fix.""" if self._milestones is None: note = "milestones aren't available here" self._wait(item, replace(loop, wait=ANSWER, note=note), note) return if loop.pr: fixed = self._milestones.push_fix( item_id=item.id, branch=loop.branch, pr=loop.pr, message=summary or "Address review comments", attempt=loop.fixes + 1, ) self._fact(item, f"review fix: {fixed.note}") if fixed.kind in ("pushed", "nothing"): self._gitea_comment(loop.pr, fixed.note) self._wait( item, replace(loop, wait=MERGE, note=f"PR #{loop.pr}"), f"PR #{loop.pr}: {fixed.note}", ) elif fixed.kind == "new-pr" and fixed.pr is not None: loop = replace( loop, pr=fixed.pr.number, pr_url=fixed.pr.url, fixes=loop.fixes + 1 ) self._wait( item, replace(loop, wait=MERGE, note=f"PR #{fixed.pr.number}"), f"The PR merged before the fix landed, so it's a new PR: " f"#{fixed.pr.number} {fixed.pr.url}", ) else: self._wait( item, replace(loop, wait=ANSWER, note=fixed.note), f"{fixed.note}. What next?", ) return if not loop.branch: # D4: finishing without a start cuts the branch now. number = loop.milestone + 1 branch = f"{item.id}-milestone-{number}-{uuid.uuid4().hex[:4]}" self._milestones.start(branch) loop = replace(loop, milestone=number, branch=branch) published = self._milestones.publish( item_id=item.id, branch=loop.branch, title=f"{item.title} (milestone {loop.milestone})", summary=summary, issue=loop.issue, ) left = ( f" (left out {len(published.left_out)} build artifact(s))" if published.left_out else "" ) if published.pr is not None and published.kind in ("opened", "adopted"): pr = published.pr self._fact( item, f"milestone {loop.milestone} published: PR #{pr.number} " f"({pr.url}){left}", ) self._wait( item, replace( loop, pr=pr.number, pr_url=pr.url, wait=MERGE, note=f"PR #{pr.number}", ), f"Milestone {loop.milestone} is up for review: {pr.url}{left}", ) else: self._fact( item, f"milestone {loop.milestone} not published: {published.note}" ) self._wait( item, replace(loop, wait=ANSWER, note=published.note), f"Milestone {loop.milestone}: {published.note}. What next?", ) def _gitea_comment(self, pr: int, text: str) -> None: try: self._gitea.post_comment(self._repo, pr, text) except GiteaError: pass # courtesy only; the pushed commit is the answer def _wait(self, item: WorkItem, loop: Loop, say: str | None) -> None: if say: self._transport.post(say, thread=loop.thread) save(self._store, item.id, WAITING, loop, cause=f"waiting for {loop.wait}") def _close(self, item: WorkItem, loop: Loop, outcome: str, note: str) -> None: self._fact(item, f"work item {outcome}: {note}") save(self._store, item.id, CLOSED, loop, cause=outcome) self._store.close_work_item(item.id, outcome=outcome, cause=note) if loop.issue: try: self._gitea.post_comment(self._repo, loop.issue, f"{outcome}: {note}") self._gitea.close_issue(self._repo, loop.issue) except GiteaError as exc: self._transport.post( f"closed here, but could not close #{loop.issue}: {exc}", thread=loop.thread, ) if outcome == "abandoned": self._transport.post(f"abandoned: {note}", thread=loop.thread) def _fact(self, item: WorkItem, text: str) -> None: self._store.transcript(item.id, "orchestrator").append(Fact(text=text))