dongarandClaude Opus 5.5
abb7fa0216
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 49s
`stop` could not have worked by adding the command alone. Planning turns, milestones and review responses ran synchronously on the thread that delivered the message, so the Matrix sync loop read nothing until the work finished. The state store was also opened on the main thread and used from the transport and poller threads, which SQLite refuses with its default same-thread check; the in-memory test transport never showed it. A single worker thread now runs every handler and owns the store: - The worker opens the store, so the same-thread check stays on and fails loudly if anything touches the store from elsewhere - Matrix messages are handed to the worker; pollers hand over and wait, so their cursor never passes an unhandled comment - A message that arrives during long work gets an immediate "busy; say 'stop' to halt" reply and is recorded, not applied later in a state the operator was not answering - Tests keep running inline through InlineWorker, which defers a job submitted from inside another as the real worker would queue it `stop` sets a shared Cancel flag straight from the transport's thread. The executor and delegation loops check it before each model call and between tool calls; the planner checks it before each turn. Calls left unrun get a result saying so, so the transcript still renders. A stopped delegation is discarded like a failed one. The orchestrator moves to AWAIT_INPUT and asks why the work was stopped. The reply is recorded for the model, and a stopped planning turn resumes with it as the answer. `stop` with nothing running says so and changes nothing; at a gate it is no longer taken as rejection feedback. Closes #40 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
139 lines
4.5 KiB
Python
139 lines
4.5 KiB
Python
"""The one thread that runs orchestrator work and owns the state store.
|
|
|
|
Planning turns, milestones and review responses can each run for minutes.
|
|
Running them on the thread that delivered the message blocked that thread for
|
|
as long: the Matrix sync loop read nothing new until the work finished, so an
|
|
operator's `stop` (A4) would only have arrived after the work it was meant to
|
|
stop. Every handler now runs here instead, and transports only hand work over.
|
|
|
|
Owning the store is the other half. SQLite connections are bound to the thread
|
|
that opened them, and the store keeps that check on: it is opened on this
|
|
thread, used only from it, and any future code that touches it from elsewhere
|
|
fails loudly instead of racing.
|
|
|
|
Jobs run one at a time, in order, so state changes never interleave — the
|
|
guarantee the orchestrator's lock used to give.
|
|
"""
|
|
|
|
import sys
|
|
import threading
|
|
import traceback
|
|
from collections import deque
|
|
from collections.abc import Callable
|
|
from concurrent.futures import Future
|
|
from queue import Queue
|
|
from typing import Protocol, TypeVar
|
|
|
|
T = TypeVar("T")
|
|
|
|
|
|
class Worker(Protocol):
|
|
@property
|
|
def busy(self) -> bool: ...
|
|
|
|
def submit(self, job: Callable[[], object]) -> None: ...
|
|
|
|
def call(self, job: Callable[[], T], timeout: float | None = None) -> T: ...
|
|
|
|
|
|
class InlineWorker:
|
|
"""Runs jobs on the caller's thread, for tests and single-threaded use.
|
|
|
|
A job submitted from inside a running job waits until that one finishes,
|
|
exactly as it would queue behind it on the real worker. Without that, a
|
|
test that says `stop` from inside a fake milestone would run the stop
|
|
handling in the middle of the milestone, which production never does.
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
self._pending: deque[Callable[[], object]] = deque()
|
|
self._running = False
|
|
|
|
@property
|
|
def busy(self) -> bool:
|
|
return self._running
|
|
|
|
def submit(self, job: Callable[[], object]) -> None:
|
|
self._pending.append(job)
|
|
if self._running:
|
|
return
|
|
self._running = True
|
|
try:
|
|
while self._pending:
|
|
self._pending.popleft()()
|
|
finally:
|
|
self._running = False
|
|
|
|
def call(self, job: Callable[[], T], timeout: float | None = None) -> T:
|
|
if self._running:
|
|
return job() # already on the worker, as a nested call would be
|
|
box: list[T] = []
|
|
self.submit(lambda: box.append(job()))
|
|
return box[0]
|
|
|
|
|
|
_STOP = object()
|
|
|
|
|
|
def _report(exc: BaseException) -> None:
|
|
traceback.print_exception(exc, file=sys.stderr)
|
|
|
|
|
|
class SerialWorker:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
on_error: Callable[[BaseException], None] = _report,
|
|
name: str = "nisse-worker",
|
|
) -> None:
|
|
self._queue: Queue[object] = Queue()
|
|
self._on_error = on_error
|
|
self._busy = False
|
|
self._thread = threading.Thread(target=self._run, name=name, daemon=True)
|
|
|
|
@property
|
|
def busy(self) -> bool:
|
|
return self._busy
|
|
|
|
def start(self) -> None:
|
|
self._thread.start()
|
|
|
|
def submit(self, job: Callable[[], object]) -> None:
|
|
self._queue.put((job, None))
|
|
|
|
def call(self, job: Callable[[], T], timeout: float | None = None) -> T:
|
|
"""Run a job here and wait for its result. Raises TimeoutError if it
|
|
has not finished in time; the job itself still runs."""
|
|
if threading.current_thread() is self._thread:
|
|
return job()
|
|
future: Future[T] = Future()
|
|
self._queue.put((job, future))
|
|
return future.result(timeout)
|
|
|
|
def stop(self, timeout: float | None = None) -> None:
|
|
self._queue.put(_STOP)
|
|
self._thread.join(timeout)
|
|
|
|
def _run(self) -> None:
|
|
while True:
|
|
item = self._queue.get()
|
|
if item is _STOP:
|
|
return
|
|
job, future = item # type: ignore[misc]
|
|
self._busy = True
|
|
try:
|
|
result = job()
|
|
except Exception as exc: # noqa: BLE001 -- see below
|
|
# One failed job must not take the agent's only worker with
|
|
# it. A caller that waits gets the exception; otherwise it is
|
|
# reported and the next job runs.
|
|
if future is not None:
|
|
future.set_exception(exc)
|
|
else:
|
|
self._on_error(exc)
|
|
else:
|
|
if future is not None:
|
|
future.set_result(result)
|
|
finally:
|
|
self._busy = False
|