dev/nisse
2
1 Fork 0
Code Issues 23 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

M2.8: switch over to the new loop
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 2m16s
Details
CI / e2e (pull_request) Successful in 10m51s
Details

Browse Source 
nisse-dev now runs only the conversational loop. The gated state machine
it was built alongside is gone:

- Removed control.py, planner.py, executor.py, approvals.py, gates.py
  (command_form moves to loop/reader.py), publish.py and poller.py, the
  old assembly in agent.py (its shared helpers move to loop/build.py),
  the store's state-machine view (get_state, transition), their tests,
  the old end-to-end and live planning/milestone/gate tests, and
  specs/orchestrator.qnt with its scenarios.
- Roles are driver, helper, expert and classifier; driver is required.
  loop:, roles.planner, roles.executor and tuning.executor are refused
  with what to use instead; tuning.driver takes the turn cap and failure
  limit.
- The benchmark and the capability matrix measure the driver: the
  loop's own system prompt and tools, the task's plan as the agreed
  restatement. Task plans end with finish_milestone, so earlier runs
  show as (old); stored runs' executor tuning reads as the driver's.
- A delegation on the loop records what its helper's alias resolved to
  (L9), as the old path's did.
- The dump shows a work item's revision and what it waits for.
- Behaviour the old tests covered and the loop shares stays covered:
  driver runs that fail on a model call, stop mid-turn, reset the
  failure count; go-aheads ignoring case and punctuation; a request too
  big to fit is never sent; the assembly tests, now on build_loop_agent.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
This commit is contained in:
dongarandClaude Opus 5.5 committed 2026-10-08 03:12:58 +00:00
1 parent 09232dcbdd
commit 6ae2375194
89 files changed
+951 -8592
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

nisse-dev/tests/test_abandon.py
-186
View File
@@ -1,186 +0,0 @@
"""The `abandon` command: ending a task from any waiting state (L2)."""
import io
from pathlib import Path
from typing import Any
import pytest
from nisse_dev.control import (
AWAIT_ARCH_SIGNOFF,
AWAIT_INPUT,
AWAIT_MILESTONE_SIGNOFF,
IDLE,
PR_OPEN,
AskHuman,
Orchestrator,
SubmitPlan,
)
from nisse_dev.executor import ExecutorOutcome
from nisse_dev.gitea import GiteaError, PullRequest
from nisse_dev.oplog import OpLog
from nisse_dev.publish import PublishResult
from nisse_dev.state import StateStore
from nisse_dev.transport import InMemoryTransport
class Planner:
def __init__(self, ask_first: bool = False) -> None:
self._ask = ask_first
def plan(
self,
transcript: object,
gate: str,
project: str,
feedback: str | None = None,
answer: str | None = None,
) -> SubmitPlan | AskHuman:
if self._ask and answer is None:
return AskHuman("which language?")
return SubmitPlan(f"{gate} plan")
class Board:
def __init__(self, *, fail_close: bool = False) -> None:
self.closed: list[tuple[int, str]] = []
self._fail_close = fail_close
def open_project_issue(self, project: str, task_id: str) -> int:
return 10
def post_plan(self, issue: int, gate: str, rev: int, plan: str) -> Any:
class Posted:
comment_id = 1
return Posted()
def echo(self, issue: int, text: str) -> None:
pass
def close(self, issue: int, note: str) -> None:
if self._fail_close:
raise GiteaError("gitea is down")
self.closed.append((issue, note))
class Executor:
def run(self, transcript: object, milestone: int) -> ExecutorOutcome:
return ExecutorOutcome("completed", "done")
class Publisher:
def __init__(self) -> None:
self.closed: list[tuple[int, str]] = []
def reconcilers(self) -> dict[str, Any]:
return {}
def publish(self, **kwargs: Any) -> PublishResult:
pr = PullRequest(number=20, state="open", head="h", url="u")
return PublishResult(pr, "opened PR #20")
def close_pull_request(self, index: int, note: str) -> None:
self.closed.append((index, note))
class Harness:
def __init__(
self, tmp_path: Path, *, ask_first: bool = False, board: Board | None = None
) -> None:
self.store = StateStore(
tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())
)
self.transport = InMemoryTransport()
self.board = board or Board()
self.publisher = Publisher()
Orchestrator(
self.store,
self.transport,
Planner(ask_first),
operators={"don"},
board=self.board,
executor=Executor(),
publisher=self.publisher,
).start()
@property
def state(self) -> str:
state = self.store.get_state()
assert state is not None
return state.state
def say(self, *replies: str) -> None:
for reply in replies:
self.transport.receive("don", reply)
WAITING = [
("arch sign-off", ["use demo"], AWAIT_ARCH_SIGNOFF),
("milestone sign-off", ["use demo", "approve"], AWAIT_MILESTONE_SIGNOFF),
(
"approved, not started",
["use demo", "approve", "approve"],
AWAIT_MILESTONE_SIGNOFF,
),
("PR open", ["use demo", "approve", "approve", "start milestone 1"], PR_OPEN),
]
@pytest.mark.parametrize(
("replies", "waiting"),
[(r, w) for _, r, w in WAITING],
ids=[n for n, _, _ in WAITING],
)
def test_abandon_ends_the_task_from_any_waiting_state(
tmp_path: Path, replies: list[str], waiting: str
) -> None:
h = Harness(tmp_path)
h.say(*replies)
assert h.state == waiting
h.say("abandon")
state = h.store.get_state()
assert state.state == IDLE
assert state.project is None
assert "abandoned" in h.transport.outbound[-1].text
assert h.board.closed == [(10, "abandoned by don")]
# A new task starts cleanly afterwards.
h.say("use other")
assert h.state == AWAIT_ARCH_SIGNOFF
def test_abandon_answers_a_question_by_ending_the_task(tmp_path: Path) -> None:
h = Harness(tmp_path, ask_first=True)
h.say("use demo")
assert h.state == AWAIT_INPUT
h.say("abandon")
assert h.state == IDLE
def test_abandon_in_pr_open_closes_the_pr(tmp_path: Path) -> None:
h = Harness(tmp_path)
h.say("use demo", "approve", "approve", "start milestone 1")
h.say("abandon")
assert h.publisher.closed == [(20, "abandoned by don")]
def test_abandon_aborts_a_pending_publish_intent(tmp_path: Path) -> None:
h = Harness(tmp_path)
h.say("use demo")
h.store.begin_intent("open_pr", {"branch": "b"}, cause="publish cut short")
h.say("abandon")
assert h.store.pending_intents() == []
def test_abandon_still_ends_the_task_when_gitea_is_down(tmp_path: Path) -> None:
h = Harness(tmp_path, board=Board(fail_close=True))
h.say("use demo", "abandon")
assert h.state == IDLE
assert "could not close plan issue #10" in h.transport.outbound[-1].text
def test_abandon_with_no_task_says_so(tmp_path: Path) -> None:
h = Harness(tmp_path)
h.say("abandon")
assert h.state == IDLE
assert "nothing to abandon" in h.transport.outbound[-1].text
nisse-dev/tests/test_agent_build.py
+134 -194
View File
@@ -11,10 +11,12 @@ import httpx
import pytest
from test_config import SECRETS, P, tree
from nisse_dev.agent import build_agent, discover_models, prepare_workspace
from nisse_dev.catalog import ModelCatalog, UnknownModel
from nisse_dev.config import AgentConfig
from nisse_dev.execution import SSHHost
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient
from nisse_dev.loop.build import RunningLoop, build_loop_agent, discover_models
from nisse_dev.loop.build import prepare_workspace as prepare
from nisse_dev.oplog import OpLog
# What the test roles resolve to, as LiteLLM would report them.
@@ -92,124 +94,115 @@ def origin(tmp_path: Path) -> Path:
return remote
def prepare_workspace(config: AgentConfig, origin: Path) -> Any:
[project] = config.projects
return prepare(HOST, config, "nisse-dev-testing", str(origin), None, project)
def test_workspace_is_cloned_on_the_build_host_when_absent(
tmp_path: Path, origin: Path
) -> None:
config = config_for(tmp_path)
repo = prepare_workspace(HOST, config, "nisse-dev-testing", str(origin), None)
assert (config.workspace / "README.md").exists()
repo = prepare_workspace(config, origin)
workspace = config.projects[0].workspace
assert (workspace / "README.md").exists()
assert repo.current_branch() == "main"
# The clone's configuration holds the plain URL it was given (#89).
assert str(origin) in (config.workspace / ".git" / "config").read_text()
assert str(origin) in (workspace / ".git" / "config").read_text()
def test_workspace_is_reused_when_already_cloned(tmp_path: Path, origin: Path) -> None:
config = config_for(tmp_path)
prepare_workspace(HOST, config, "nisse-dev-testing", str(origin), None)
(config.workspace / "local-work.txt").write_text("in progress")
workspace = config.projects[0].workspace
prepare_workspace(config, origin)
(workspace / "local-work.txt").write_text("in progress")
prepare_workspace(HOST, config, "nisse-dev-testing", str(origin), None)
prepare_workspace(config, origin)
# Re-cloning would silently destroy uncommitted work.
assert (config.workspace / "local-work.txt").exists()
assert (workspace / "local-work.txt").exists()
class FakeTransport:
def __init__(self) -> None:
self.sent: list[str] = []
self.started = False
self.closed = False
def send_message(self, text: str) -> None:
self.sent.append(text)
def send_approval_request(self, text: str) -> None:
self.sent.append(text)
def ask_question(self, text: str) -> None:
def post(self, text: str, thread: str | None = None) -> str:
self.sent.append(text)
return f"$event{len(self.sent)}"
def on_message(self, callback: Any) -> None:
self.callback = callback
def start(self) -> None:
self.started = True
def close(self) -> None:
self.closed = True
def build_offline(tmp_path: Path, origin: Path, monkeypatch: Any) -> Any:
"""Assemble the real graph with only the network boundaries faked."""
config = config_for(tmp_path)
def offline(monkeypatch: Any, origins: dict[str, str] | str, catalog=None) -> None:
"""Only the network boundaries faked: Gitea, LiteLLM's model list, Matrix."""
# The proxy here doesn't exist: a call to it fails at once rather than
# waiting out an outage's backoff (L5).
monkeypatch.setattr("nisse_dev.llm.OUTAGE_BACKOFF_S", ())
monkeypatch.setattr(
"nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing"
)
monkeypatch.setattr(
"nisse_dev.loop.build.GiteaClient.clone_url",
lambda self, repo: origins if isinstance(origins, str) else origins[repo],
)
monkeypatch.setattr(
"nisse_dev.loop.build.ModelCatalog.discover",
classmethod(lambda cls, *a, **kw: catalog or TEST_CATALOG),
)
monkeypatch.setattr(
"nisse_dev.loop.build.MatrixTransport", lambda cfg: FakeTransport()
)
monkeypatch.setattr(
"nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing"
)
monkeypatch.setattr(
"nisse_dev.agent.ModelCatalog.discover",
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
)
monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport())
monkeypatch.setattr(
"nisse_dev.agent.GiteaClient.clone_url", lambda self, repo: str(origin)
)
return build_agent(config)
def build_offline(
tmp_path: Path, origin: Path, monkeypatch: Any, config: AgentConfig | None = None
) -> RunningLoop:
offline(monkeypatch, str(origin))
return build_loop_agent(config or config_for(tmp_path))
def test_the_whole_graph_assembles(tmp_path: Path, origin: Path, monkeypatch) -> None:
agent = build_offline(tmp_path, origin, monkeypatch)
orchestrator = agent.orchestrator
# Every component that was previously unreachable is now connected.
assert orchestrator._executor is not None
assert orchestrator._publisher is not None
assert orchestrator._board is not None
assert orchestrator._delegate is not None
assert orchestrator._poller_factory is not None
assert orchestrator._executor._delegate is not None # L9 escalation path
assert orchestrator._executor._expert is not None # L9 consult path
agent.stop()
running = build_offline(tmp_path, origin, monkeypatch)
loop = running.loop
try:
assert running.account == "nisse-dev-testing"
[project] = loop._projects.values()
assert project.repo == "don/demo"
assert (tmp_path / "work" / "README.md").exists() # cloned on the build host
assert project.milestones is not None
assert loop._driver is not None
assert loop._reader is not None # plain-language go-aheads (L5)
assert loop._helpers is not None # delegation and ask_expert (L9)
finally:
running.stop()
def test_start_announces_where_it_resumed(
def test_start_announces_the_account_and_projects(
tmp_path: Path, origin: Path, monkeypatch
) -> None:
agent = build_offline(tmp_path, origin, monkeypatch)
agent.start()
assert agent.transport.started
readiness = agent.transport.sent[-1]
assert "nisse-dev up as nisse-dev-testing" in readiness
assert "greet" in readiness # the project
assert "claude-opus-5" in readiness # the pinned planner
agent.stop()
running = build_offline(tmp_path, origin, monkeypatch)
running.loop.start = lambda: None # type: ignore[method-assign] # no polling
running.start()
readiness = running.transport.sent[-1] # type: ignore[attr-defined]
assert "nisse-dev up as nisse-dev-testing on don/demo" in readiness
running.stop()
def test_stop_closes_everything_and_releases_the_wait(
tmp_path: Path, origin: Path, monkeypatch
) -> None:
agent = build_offline(tmp_path, origin, monkeypatch)
agent.start()
agent.stop()
running = build_offline(tmp_path, origin, monkeypatch)
running.stop()
assert agent.transport.closed
agent.wait() # returns immediately rather than hanging
assert agent.store.closed
def test_poller_factory_ignores_the_agents_own_comments(
tmp_path: Path, origin: Path, monkeypatch
) -> None:
# Without this the agent answers its own PR replies in a loop.
agent = build_offline(tmp_path, origin, monkeypatch)
poller = agent.orchestrator._poller_factory(lambda c: None)
assert "nisse-dev-testing" in poller._ignore
agent.stop()
assert running.transport.closed # type: ignore[attr-defined]
running.wait() # returns immediately rather than hanging
assert running.store.closed
def test_delegation_uses_the_passthrough_route(
@@ -217,24 +210,46 @@ def test_delegation_uses_the_passthrough_route(
) -> None:
# L9: delegations run vendor-native; the normalized route is for the
# places where fidelity is cheap.
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient
agent = build_offline(tmp_path, origin, monkeypatch)
delegation = agent.orchestrator._delegate
assert isinstance(delegation._client._inner, AnthropicPassthroughClient)
assert isinstance(agent.orchestrator._executor._client._inner, LiteLLMClient)
agent.stop()
running = build_offline(tmp_path, origin, monkeypatch)
helpers = running.loop._helpers
assert helpers is not None
helper = helpers.delegate_client("claude-haiku-4-5")
assert isinstance(helper._inner, AnthropicPassthroughClient) # type: ignore[attr-defined]
assert isinstance(running.loop._driver._client._inner, LiteLLMClient) # type: ignore[attr-defined]
running.stop()
def test_httpx_clients_are_not_shared_between_routes(
tmp_path: Path, origin: Path, monkeypatch
) -> None:
agent = build_offline(tmp_path, origin, monkeypatch)
normalized = agent.orchestrator._executor._client._inner._http
passthrough = agent.orchestrator._delegate._client._inner._http
running = build_offline(tmp_path, origin, monkeypatch)
helpers = running.loop._helpers
assert helpers is not None
normalized = running.loop._driver._client._inner._http # type: ignore[attr-defined]
passthrough = helpers.delegate_client("claude-haiku-4-5")._inner._http # type: ignore[attr-defined]
assert isinstance(normalized, httpx.Client)
assert normalized is not passthrough # different base URLs and headers
agent.stop()
running.stop()
def test_delegations_record_what_the_alias_resolved_to(
tmp_path: Path, origin: Path, monkeypatch
) -> None:
running = build_offline(tmp_path, origin, monkeypatch)
helpers = running.loop._helpers
assert helpers is not None and helpers.backing is not None
assert helpers.backing("claude-haiku-4-5") == "anthropic/claude-haiku-4-5"
running.stop()
def test_the_configured_roles_are_what_items_pin(
tmp_path: Path, origin: Path, monkeypatch
) -> None:
running = build_offline(tmp_path, origin, monkeypatch)
pinned = running.loop._roles
assert pinned == running.config.roles.pinned()
assert set(pinned) == {"driver", "helper", "expert", "classifier"}
running.stop()
# --- startup model discovery (L9) ------------------------------------
@@ -243,12 +258,12 @@ def test_httpx_clients_are_not_shared_between_routes(
def test_a_typod_role_refuses_to_boot(tmp_path: Path, monkeypatch) -> None:
# Misconfiguration should cost a second at startup, not be discovered ten
# minutes into a milestone.
config = config_for(tmp_path, roles__planner="clause-opus-5")
config = config_for(tmp_path, roles__driver="clause-opus-5")
monkeypatch.setattr(
"nisse_dev.agent.ModelCatalog.discover",
"nisse_dev.loop.build.ModelCatalog.discover",
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
)
with pytest.raises(UnknownModel, match="planner='clause-opus-5'"):
with pytest.raises(UnknownModel, match="driver='clause-opus-5'"):
discover_models(config, OpLog("test"))
@@ -257,12 +272,12 @@ def test_a_role_this_key_cannot_reach_refuses_to_boot(
) -> None:
# /model/info is scoped per key, so the same check catches a role pointed
# at a real model that this token has no access to.
config = config_for(tmp_path, roles__executor="local-fast")
config = config_for(tmp_path, roles__helper="local-fast")
monkeypatch.setattr(
"nisse_dev.agent.ModelCatalog.discover",
"nisse_dev.loop.build.ModelCatalog.discover",
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
)
with pytest.raises(UnknownModel, match="executor='local-fast'"):
with pytest.raises(UnknownModel, match="helper='local-fast'"):
discover_models(config, OpLog("test"))
@@ -288,7 +303,7 @@ def test_a_tool_less_model_warns_but_still_boots(tmp_path: Path, monkeypatch) ->
}
)
monkeypatch.setattr(
"nisse_dev.agent.ModelCatalog.discover",
"nisse_dev.loop.build.ModelCatalog.discover",
classmethod(lambda cls, *a, **kw: tool_less),
)
stream = io.StringIO()
@@ -296,36 +311,10 @@ def test_a_tool_less_model_warns_but_still_boots(tmp_path: Path, monkeypatch) ->
logged = stream.getvalue()
assert "model_tools_unsupported" in logged
assert '"role": "executor"' in logged
assert '"role": "driver"' in logged
assert '"role": "expert"' not in logged # ask_expert drives no tools
def test_the_resolved_model_is_pinned_at_task_start(
tmp_path: Path, origin: Path, monkeypatch
) -> None:
agent = build_offline(tmp_path, origin, monkeypatch)
# The store belongs to the worker thread, so the work has to run there.
agent.worker.call(lambda: agent.orchestrator._begin_project("greet"))
pinned = agent.orchestrator._data["models"]
assert pinned["planner"] == {
"alias": "claude-opus-5",
"backing": "anthropic/claude-opus-5",
"provider": "anthropic",
}
agent.stop()
def test_delegation_records_what_the_alias_resolved_to(
tmp_path: Path, origin: Path, monkeypatch
) -> None:
# L9 eval history keyed on an alias alone would average across whatever
# that alias pointed at over time.
agent = build_offline(tmp_path, origin, monkeypatch)
assert agent.orchestrator._delegate._backing_model == "anthropic/claude-opus-5"
agent.stop()
def test_tuning_reaches_every_component(
tmp_path: Path, origin: Path, monkeypatch
) -> None:
@@ -333,8 +322,9 @@ def test_tuning_reaches_every_component(
from nisse_dev.config import (
DelegationTuning,
ExecutorTuning,
DriverTuning,
ExpertTuning,
LoopTuning,
ToolsTuning,
Tuning,
)
@@ -343,83 +333,47 @@ def test_tuning_reaches_every_component(
config_for(tmp_path),
max_output_tokens=8192,
tuning=Tuning(
executor=ExecutorTuning(max_turns=40, failure_limit=3),
driver=DriverTuning(max_turns=40, failure_limit=3),
expert=ExpertTuning(consult_budget=5),
delegation=DelegationTuning(
max_turns=12, max_tokens=50_000, wall_clock_s=60
),
tools=ToolsTuning(command_timeout_s=300, max_output_chars=2000),
loop=LoopTuning(bug_grace_s=120, general_idle_s=900),
),
)
monkeypatch.setattr(
"nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing"
)
monkeypatch.setattr(
"nisse_dev.agent.ModelCatalog.discover",
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
)
monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport())
monkeypatch.setattr(
"nisse_dev.agent.GiteaClient.clone_url", lambda self, repo: str(origin)
)
agent = build_agent(config)
executor = agent.orchestrator._executor
delegation = agent.orchestrator._delegate
running = build_offline(tmp_path, origin, monkeypatch, config)
loop = running.loop
driver = loop._driver
helpers = loop._helpers
assert helpers is not None
[project] = loop._projects.values()
assert (executor._max_turns, executor._failure_limit) == (40, 3)
assert executor._expert._budget == 5
assert delegation._caps.turns == 12
assert delegation._caps.tokens == 50_000
assert delegation._caps.wall_clock_s == 60
assert executor._workspace._timeout_s == 300
assert executor._workspace._max_output_chars == 2000
assert executor._client._inner._max_tokens == 8192
assert delegation._client._inner._max_tokens == 8192
assert executor._client._max_output_tokens == 8192
assert (driver._max_turns, driver._failure_limit) == (40, 3)
assert helpers.expert_budget == 5
assert (helpers.caps.turns, helpers.caps.tokens) == (12, 50_000)
assert helpers.caps.wall_clock_s == 60
assert project.workspace is not None
assert project.workspace._timeout_s == 300
assert project.workspace._max_output_chars == 2000
assert (loop._grace, loop._idle) == (120, 900)
client = driver._client
assert client._inner._max_tokens == 8192 # type: ignore[attr-defined]
assert client._max_output_tokens == 8192 # type: ignore[attr-defined]
helper = helpers.delegate_client("claude-haiku-4-5")
assert helper._inner._max_tokens == 8192 # type: ignore[attr-defined]
# tool_choice is asked of exactly the models LiteLLM declares enforce it.
assert executor._client._enforces_tool_choice("claude-opus-5") is False
agent.stop()
def test_the_new_loop_assembles(tmp_path: Path, origin: Path, monkeypatch) -> None:
"""`loop: new` builds the LoopAgent over the same services."""
from nisse_dev.loop.build import build_loop_agent
config = config_for(tmp_path, loop="new")
monkeypatch.setattr(
"nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing"
)
monkeypatch.setattr(
"nisse_dev.loop.build.GiteaClient.clone_url", lambda self, repo: str(origin)
)
monkeypatch.setattr(
"nisse_dev.agent.ModelCatalog.discover",
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
)
monkeypatch.setattr(
"nisse_dev.loop.build.MatrixTransport", lambda cfg: FakeTransport()
)
running = build_loop_agent(config)
try:
assert running.account == "nisse-dev-testing"
[project] = running.loop._projects.values()
assert project.repo == config.gitea_repo
assert (config.workspace / "README.md").exists() # cloned on the build host
finally:
running.worker.call(running.store.close)
running.worker.stop(timeout=5)
assert client._enforces_tool_choice("claude-opus-5") is False # type: ignore[attr-defined]
running.stop()
def test_each_project_gets_its_own_clone_and_secrets(
tmp_path: Path, origin: Path, monkeypatch
) -> None:
"""D8: a second project is only a second entry."""
from nisse_dev.loop.build import build_loop_agent
second = tmp_path / "second"
config = config_for(
tmp_path,
loop="new",
projects__other={
"repo": "don/other",
"workspace": str(tmp_path / "other-work"),
@@ -436,20 +390,7 @@ def test_each_project_gets_its_own_clone_and_secrets(
},
)
git("clone", "-q", "--bare", str(origin), str(second))
origins = {"don/demo": str(origin), "don/other": str(second)}
monkeypatch.setattr(
"nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing"
)
monkeypatch.setattr(
"nisse_dev.loop.build.GiteaClient.clone_url", lambda self, repo: origins[repo]
)
monkeypatch.setattr(
"nisse_dev.agent.ModelCatalog.discover",
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
)
monkeypatch.setattr(
"nisse_dev.loop.build.MatrixTransport", lambda cfg: FakeTransport()
)
offline(monkeypatch, {"don/demo": str(origin), "don/other": str(second)})
running = build_loop_agent(config)
try:
projects = running.loop._projects
@@ -460,5 +401,4 @@ def test_each_project_gets_its_own_clone_and_secrets(
assert projects["other"].workspace._secrets == {"API_KEY": "sk-x"}
assert projects["greet"].workspace._secrets == {}
finally:
running.worker.call(running.store.close)
running.worker.stop(timeout=5)
running.stop()
nisse-dev/tests/test_approval_binding.py
-147
View File
@@ -1,147 +0,0 @@
"""Approvals and feedback bind to the plan revision they were about (L3)."""
import io
from pathlib import Path
from nisse_dev.canonical import UserMessage
from nisse_dev.control import (
AWAIT_MILESTONE_SIGNOFF,
Orchestrator,
StubAgent,
)
from nisse_dev.gitea import Comment
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore
from nisse_dev.transport import InMemoryTransport
class Held:
"""A worker that queues jobs until released, like one under load."""
def __init__(self) -> None:
self.jobs: list = []
self.busy = False
def submit(self, job) -> None:
self.jobs.append(job)
def call(self, job, timeout=None):
return job()
def run(self) -> None:
while self.jobs:
self.jobs.pop(0)()
class Board:
def __init__(self) -> None:
self.next_id = 100
def open_project_issue(self, project: str, task_id: str) -> int:
return 10
def post_plan(self, issue: int, gate: str, rev: int, plan: str):
self.next_id += 1
class Posted:
comment_id = self.next_id
return Posted()
def echo(self, issue: int, text: str) -> None:
pass
def build(tmp_path: Path, worker: Held | None = None, board: Board | None = None):
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
transport = InMemoryTransport()
orch = Orchestrator(
store,
transport,
StubAgent(),
operators={"don"},
gitea_operators={"don"},
board=board,
**({"worker": worker} if worker else {}),
)
orch.start()
return store, transport, orch
def gate(store: StateStore) -> dict:
return store.get_state().data.get("gate") or {}
def test_revisions_run_across_the_task(tmp_path: Path) -> None:
store, transport, _ = build(tmp_path)
transport.receive("don", "use demo")
assert gate(store)["rev"] == 1
transport.receive("don", "approve")
assert gate(store) == {**gate(store), "name": "milestone-1", "rev": 2}
assert "milestone-1 rev-2" in transport.outbound[-1].text
def test_a_stale_approval_is_answered_and_a_fresh_one_works(tmp_path: Path) -> None:
worker = Held()
store, transport, _ = build(tmp_path, worker)
transport.receive("don", "use demo")
worker.run()
transport.receive("don", "approve")
transport.receive("don", "approve") # both while arch rev-1 was up
worker.run()
assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF
assert not gate(store)["approved"]
assert "arrived while rev-1 was up" in transport.outbound[-1].text
transport.receive("don", "approve") # now about milestone-1 rev-2
worker.run()
assert gate(store)["approved"]
def test_stale_feedback_is_recorded_but_rejects_nothing(tmp_path: Path) -> None:
worker = Held()
store, transport, _ = build(tmp_path, worker)
transport.receive("don", "use demo")
worker.run()
transport.receive("don", "approve")
transport.receive("don", "also, keep it in one file")
worker.run()
assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF
assert gate(store)["rev"] == 2
assert "not taken as feedback" in transport.outbound[-1].text
said = [
e.event.text for e in store.all_events() if isinstance(e.event, UserMessage)
]
assert "also, keep it in one file" in said # the planner still hears it
def test_a_gitea_reply_binds_to_the_latest_plan_posted_before_it(
tmp_path: Path,
) -> None:
board = Board()
store, transport, orch = build(tmp_path, board=board)
transport.receive("don", "use demo") # arch rev-1 posted as comment 101
orch._handle_plan_comment_now(
Comment(id=102, author="don", body="approve", created_at="t")
)
# Approved arch rev-1; milestone-1 rev-2 is now posted as comment 103.
assert gate(store) == {**gate(store), "name": "milestone-1", "approved": False}
# A reply older than that post was about arch rev-1: not applied.
orch._handle_plan_comment_now(
Comment(id=102, author="don", body="approve", created_at="t")
)
assert not gate(store)["approved"]
# One after it is about milestone-1 rev-2.
orch._handle_plan_comment_now(
Comment(id=104, author="don", body="approve", created_at="t")
)
assert gate(store)["approved"]
def test_the_plan_up_survives_a_restart(tmp_path: Path) -> None:
_, transport, _ = build(tmp_path)
transport.receive("don", "use demo") # arch rev-1 up; no board, so unsaved
store, transport, _ = build(tmp_path) # restart at the open gate
transport.receive("don", "approve") # stamped with the restored rev-1
assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF
assert gate(store)["name"] == "milestone-1"
assert gate(store)["rev"] == 2
nisse-dev/tests/test_approvals.py
-210
View File
@@ -1,210 +0,0 @@
"""Plan externalisation to Gitea and approvals arriving on either channel
(architecture L3, D2)."""
import io
from pathlib import Path
from typing import Any
from nisse_dev.approvals import PlanBoard
from nisse_dev.canonical import CanonicalEvent, ToolCall
from nisse_dev.control import (
AWAIT_ARCH_SIGNOFF,
AWAIT_MILESTONE_SIGNOFF,
MILESTONE_PLANNING,
Orchestrator,
)
from nisse_dev.gitea import Comment, GiteaError
from nisse_dev.llm import RoleMap
from nisse_dev.oplog import OpLog
from nisse_dev.planner import LLMPlanningAgent
from nisse_dev.state import StateStore
from nisse_dev.transport import InMemoryTransport
ROLES = RoleMap(executor="x", planner="y", helper="y", expert="y")
class FakeGitea:
def __init__(self, *, fail: bool = False) -> None:
self.issues: list[dict[str, str]] = []
self.comments: list[tuple[int, str]] = []
self._fail = fail
self._next_id = 100
def create_issue(self, repo: str, title: str, body: str) -> int:
if self._fail:
raise GiteaError("gitea is down")
self.issues.append({"title": title, "body": body})
return len(self.issues)
def post_comment(self, repo: str, index: int, body: str) -> Comment:
if self._fail:
raise GiteaError("gitea is down")
self.comments.append((index, body))
self._next_id += 1
return Comment(
id=self._next_id, author="nisse-dev-testing", body=body, created_at="t"
)
def list_comments(self, repo: str, index: int, since: str | None = None):
return []
class ScriptedClient:
def __init__(self) -> None:
self.calls = 0
def complete(self, events, tools, model, system=None) -> list[CanonicalEvent]:
self.calls += 1
return [
ToolCall(
call_id=f"c{self.calls}",
tool="submit_plan",
arguments={"type": "x", "content": f"plan {self.calls}"},
)
]
def build(
tmp_path: Path, gitea: Any
) -> tuple[Orchestrator, StateStore, InMemoryTransport, PlanBoard]:
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
transport = InMemoryTransport()
board = PlanBoard(gitea, "o/sandbox")
orch = Orchestrator(
store,
transport,
LLMPlanningAgent(ScriptedClient(), ROLES),
operators={"don"},
roles=ROLES,
gitea_operators={"dongar"},
board=board,
)
orch.start()
return orch, store, transport, board
def comment(body: str, author: str = "dongar", cid: int = 900) -> Comment:
return Comment(id=cid, author=author, body=body, created_at="t")
def test_project_start_opens_an_issue_and_posts_the_plan(tmp_path: Path) -> None:
gitea = FakeGitea()
_, store, transport, _ = build(tmp_path, gitea)
transport.receive("don", "use demo")
assert gitea.issues[0]["title"] == "nisse-dev: demo"
issue, body = gitea.comments[0]
assert issue == 1
assert "plan 1" in body
assert "approve rev-1" in body # the tag is printed for copy-paste (L3)
# The store keeps a pointer, not the plan record itself (D2).
assert store.get_state().data["board"]["issue"] == 1
def test_approval_by_gitea_comment_satisfies_the_gate(tmp_path: Path) -> None:
gitea = FakeGitea()
orch, store, _, _ = build(tmp_path, gitea)
orch._transport.receive("don", "use demo")
assert store.get_state().state == AWAIT_ARCH_SIGNOFF
orch.handle_plan_comment(comment("approve rev-1"))
assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF
def test_stale_revision_from_gitea_is_refused(tmp_path: Path) -> None:
gitea = FakeGitea()
orch, store, transport, _ = build(tmp_path, gitea)
transport.receive("don", "use demo")
transport.receive("don", "too vague") # now at rev-2
orch.handle_plan_comment(comment("approve rev-1"))
assert store.get_state().state == AWAIT_ARCH_SIGNOFF # not advanced
orch.handle_plan_comment(comment("approve rev-2", cid=901))
assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF
def test_non_operator_gitea_comment_cannot_approve(tmp_path: Path) -> None:
gitea = FakeGitea()
orch, store, transport, _ = build(tmp_path, gitea)
transport.receive("don", "use demo")
orch.handle_plan_comment(comment("approve rev-1", author="drive-by"))
assert store.get_state().state == AWAIT_ARCH_SIGNOFF
def test_agents_own_comment_cannot_approve_its_plan(tmp_path: Path) -> None:
# The agent posts plans as nisse-dev-testing; that login is not on the
# operator allowlist, so its own comments can never satisfy its gate (L3).
gitea = FakeGitea()
orch, store, transport, _ = build(tmp_path, gitea)
transport.receive("don", "use demo")
orch.handle_plan_comment(comment("approve rev-1", author="nisse-dev-testing"))
assert store.get_state().state == AWAIT_ARCH_SIGNOFF
def test_rejection_by_gitea_comment_replans(tmp_path: Path) -> None:
gitea = FakeGitea()
orch, store, transport, _ = build(tmp_path, gitea)
transport.receive("don", "use demo")
orch.handle_plan_comment(comment("needs more detail on the schema"))
assert store.get_state().state == AWAIT_ARCH_SIGNOFF
assert "plan 2" in transport.outbound[-1].text # replanned
def test_matrix_approval_is_echoed_to_gitea(tmp_path: Path) -> None:
gitea = FakeGitea()
_, _, transport, _ = build(tmp_path, gitea)
transport.receive("don", "use demo")
transport.receive("don", "approve")
echoes = [body for _, body in gitea.comments if "approved by don" in body]
assert echoes, "the Gitea record must not silently disagree with Matrix"
def test_gitea_approval_is_echoed_to_matrix(tmp_path: Path) -> None:
gitea = FakeGitea()
orch, _, transport, _ = build(tmp_path, gitea)
transport.receive("don", "use demo")
orch.handle_plan_comment(comment("approve rev-1"))
assert any("approved by dongar via gitea" in m.text for m in transport.outbound)
def test_gitea_outage_leaves_matrix_a_complete_channel(tmp_path: Path) -> None:
gitea = FakeGitea(fail=True)
_, store, transport, _ = build(tmp_path, gitea)
transport.receive("don", "use demo")
# Planning proceeded and the gate is usable despite Gitea being down.
assert store.get_state().state == AWAIT_ARCH_SIGNOFF
assert any("Matrix-only" in m.text for m in transport.outbound)
transport.receive("don", "approve")
assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF
def test_comments_are_ignored_outside_a_gate(tmp_path: Path) -> None:
gitea = FakeGitea()
orch, store, transport, _ = build(tmp_path, gitea)
transport.receive("don", "use demo")
transport.receive("don", "approve") # arch approved; now at milestone gate
transport.receive("don", "approve") # milestone approved, awaiting start
before = store.get_state().state
orch.handle_plan_comment(comment("approve rev-1", cid=902))
assert store.get_state().state == before
def test_board_records_the_milestone_plan_too(tmp_path: Path) -> None:
gitea = FakeGitea()
_, store, transport, _ = build(tmp_path, gitea)
transport.receive("don", "use demo")
transport.receive("don", "approve")
assert store.get_state().state in (MILESTONE_PLANNING, AWAIT_MILESTONE_SIGNOFF)
posted = [body for _, body in gitea.comments if "milestone-1" in body]
assert posted, "the milestone plan must be externalised as well"
nisse-dev/tests/test_bench.py
+44 -10
View File
@@ -1,4 +1,4 @@
"""The executor benchmark."""
"""The driver benchmark."""
import json
import shutil
@@ -19,7 +19,7 @@ from nisse_dev.bench.runner import MeteringClient, run_task
from nisse_dev.bench.tasks import CHECK_DIR, load_tasks
from nisse_dev.canonical import AssistantText, ToolCall
from nisse_dev.catalog import ModelFacts
from nisse_dev.config import ExecutorTuning, Tuning
from nisse_dev.config import DriverTuning, Tuning
from nisse_dev.execution import SSHHost
# The build VM for every test here: the session's throwaway sshd.
@@ -79,7 +79,7 @@ def test_the_seed_fails_its_check_and_the_solution_passes(task, tmp_path: Path)
def test_the_hidden_check_is_not_in_the_seed(task, tmp_path: Path) -> None:
task.seed_into(tmp_path)
assert not (tmp_path / CHECK_DIR).exists()
assert "Call finish" in task.plan # plans end the way the executor must
assert "Call finish_milestone" in task.plan # as the driver must end
def test_long_output_really_is_long(tmp_path: Path) -> None:
@@ -192,7 +192,7 @@ SOLVE_SLUGIFY = [
)
],
[call("run_command", "r", command="python -c 'import slugify'")],
[call("finish", "f", summary="added slugify")],
[call("finish_milestone", "f", summary="added slugify", final="yes")],
]
@@ -208,7 +208,7 @@ def test_a_solved_task_is_a_success(tmp_path: Path) -> None:
assert record.tokens_in == 3000
assert record.task_hash == task.content_hash
assert record.model_info["supports_tool_choice"] is True
assert record.tuning["executor"]["max_turns"] == 40
assert record.tuning["driver"]["max_turns"] == 40
def test_saying_finished_is_not_success(tmp_path: Path) -> None:
@@ -223,7 +223,7 @@ def test_saying_finished_is_not_success(tmp_path: Path) -> None:
content="def slugify(t):\n return t\n",
)
],
[call("finish", "f", summary="done")],
[call("finish_milestone", "f", summary="done", final="yes")],
]
record = run_task(task, FACTS, Scripted(wrong), Tuning(), host=HOST)
assert record.outcome == "completed"
@@ -239,7 +239,7 @@ def test_misbehaviour_is_counted(tmp_path: Path) -> None:
]
# A higher failure limit, so the invented tool and the failed command
# don't escalate before the prose turn is reached.
tuning = Tuning(executor=ExecutorTuning(failure_limit=5))
tuning = Tuning(driver=DriverTuning(failure_limit=5))
record = run_task(task, FACTS, Scripted(turns), tuning, host=HOST)
assert record.invented_tools == 1
assert record.failed_commands == 1
@@ -247,6 +247,31 @@ def test_misbehaviour_is_counted(tmp_path: Path) -> None:
assert not record.success
def test_asking_the_operator_is_an_escalation(tmp_path: Path) -> None:
[task] = load_tasks(["write-function"])
asks = [[call("ask_human", "q", question="which file?")]]
record = run_task(task, FACTS, Scripted(asks), Tuning(), host=HOST)
assert (record.outcome, record.success) == ("escalated", False)
def test_the_driver_gets_the_loops_own_prompt_and_tools(tmp_path: Path) -> None:
from nisse_dev.loop.agent import SYSTEM
seen: list[tuple[str, set[str]]] = []
class Spy(Scripted):
def complete(self, events, tools, model, system=None): # type: ignore[no-untyped-def]
seen.append((system or "", {t["function"]["name"] for t in tools}))
return super().complete(events, tools, model, system)
[task] = load_tasks(["write-function"])
run_task(task, FACTS, Spy(list(SOLVE_SLUGIFY)), Tuning(), host=HOST)
system, tools = seen[0]
assert system.startswith(SYSTEM.split("{")[0])
assert {"start_milestone", "finish_milestone", "run_command"} <= tools
assert not {"delegate", "ask_expert", "restate"} & tools
def test_a_fallback_answer_can_never_be_a_success(tmp_path: Path) -> None:
[task] = load_tasks(["write-function"])
client = Scripted(list(SOLVE_SLUGIFY), answered="claude-sonnet-5")
@@ -383,11 +408,11 @@ def test_results_stored_before_waits_were_recorded_still_load() -> None:
def test_runs_with_different_tuning_are_kept_apart() -> None:
current = {t.name: t.content_hash for t in TASKS}
capped = record_json()
capped["tuning"]["executor"]["max_turns"] = 7
capped["tuning"]["driver"]["max_turns"] = 7
rows = summarise([record_json(), capped], current)
assert sorted(row.tuning for row in rows) == ["", "executor.max_turns=7"]
assert sorted(row.tuning for row in rows) == ["", "driver.max_turns=7"]
table = render(rows)
assert "executor.max_turns=7" in table
assert "driver.max_turns=7" in table
assert "tuning = settings that differ" in table
@@ -395,3 +420,12 @@ def test_a_report_at_default_tuning_has_no_tuning_column() -> None:
current = {t.name: t.content_hash for t in TASKS}
table = render(summarise([record_json()], current))
assert "tuning" not in table
def test_runs_from_before_the_driver_read_as_its_defaults() -> None:
# Stored before M2.8, the driver's section was called executor.
current = {t.name: t.content_hash for t in TASKS}
old = record_json()
old["tuning"]["executor"] = old["tuning"].pop("driver")
[row] = summarise([old], current)
assert row.tuning == ""
nisse-dev/tests/test_config.py
+61 -54
View File
@@ -4,7 +4,7 @@ from typing import Any
import pytest
from nisse_dev.config import AgentConfig, ConfigError
from nisse_dev.config import AgentConfig, ConfigError, Project
from nisse_dev.llm import DEFAULT_REQUEST_TIMEOUT_S
# A complete config as the YAML file would give it, with secrets by reference.
@@ -19,9 +19,8 @@ COMPLETE: dict[str, Any] = {
},
"llm": {"base_url": "https://litellm.test", "token": "${TEST_LLM_TOKEN}"},
"roles": {
"executor": "claude-haiku-4-5",
"planner": "claude-opus-5",
"helper": "claude-opus-5",
"driver": "claude-opus-5",
"helper": "claude-haiku-4-5",
"expert": "claude-haiku-4-5",
},
"matrix": {
@@ -64,22 +63,27 @@ def build(**changes: Any) -> AgentConfig:
return AgentConfig.from_tree(tree(**changes), SECRETS)
def only(config: AgentConfig) -> Project:
[project] = config.projects
return project
def test_complete_config_builds_with_secrets_by_reference() -> None:
config = build()
assert config.project == "greet"
assert only(config).name == "greet"
assert config.llm_token == "sk-x"
assert config.gitea_token == "gt"
assert config.roles.planner == "claude-opus-5"
assert config.roles.driver == "claude-opus-5"
assert config.matrix.room_id == "!room:test"
assert config.matrix.homeserver == "https://matrix.test"
assert config.gitea_repo == "don/demo"
assert config.base_branch == "main"
assert config.tuning.executor.max_turns == 40 # absent section: defaults
assert only(config).repo == "don/demo"
assert only(config).base_branch == "main"
assert config.tuning.driver.max_turns == 40 # absent section: defaults
def test_paths_expand_the_home_directory() -> None:
config = build()
assert config.workspace.is_absolute()
assert only(config).workspace.is_absolute()
assert "~" not in str(config.state_db)
@@ -119,7 +123,7 @@ def test_an_operator_list_of_only_blanks_is_missing() -> None:
def test_base_branch_is_configurable() -> None:
assert build(**{f"{P}base_branch": "trunk"}).base_branch == "trunk"
assert only(build(**{f"{P}base_branch": "trunk"})).base_branch == "trunk"
def test_max_output_tokens_is_configurable_and_checked() -> None:
@@ -137,8 +141,8 @@ def test_request_timeout_is_configurable_and_checked() -> None:
def test_the_request_timeout_reaches_both_clients() -> None:
from nisse_dev.agent import make_llm_clients
from nisse_dev.catalog import ModelCatalog
from nisse_dev.loop.build import make_llm_clients
normalized, passthrough = make_llm_clients(
build(llm__request_timeout_s=42), ModelCatalog({})
@@ -158,7 +162,7 @@ def test_redacted_summary_carries_no_secrets() -> None:
def test_tuning_is_read() -> None:
config = build(
tuning={
"executor": {"max_turns": 40, "failure_limit": 3},
"driver": {"max_turns": 40, "failure_limit": 3},
"expert": {"consult_budget": 5},
"delegation": {"max_turns": 20, "max_tokens": 100000, "wall_clock_s": 600},
"tools": {"command_timeout_s": 300, "max_output_chars": 2000},
@@ -166,7 +170,7 @@ def test_tuning_is_read() -> None:
}
)
tuning = config.tuning
assert (tuning.executor.max_turns, tuning.executor.failure_limit) == (40, 3)
assert (tuning.driver.max_turns, tuning.driver.failure_limit) == (40, 3)
assert tuning.expert.consult_budget == 5
assert tuning.delegation.max_tokens == 100_000
assert tuning.delegation.wall_clock_s == 600.0
@@ -175,8 +179,8 @@ def test_tuning_is_read() -> None:
def test_a_partly_given_tuning_section_keeps_the_other_defaults() -> None:
tuning = build(tuning={"executor": {"max_turns": 40}}).tuning
assert tuning.executor.failure_limit == 2
tuning = build(tuning={"driver": {"max_turns": 40}}).tuning
assert tuning.driver.failure_limit == 2
def test_every_problem_is_reported_at_once() -> None:
@@ -184,7 +188,7 @@ def test_every_problem_is_reported_at_once() -> None:
# with no effect, so unknown keys are errors.
bad = tree(llm_extra=1, **{f"{P}workspace": None})
bad["tuning"] = {
"executor": {"max_turn": 40, "failure_limit": 0},
"driver": {"max_turn": 40, "failure_limit": 0},
"expert": {"consult_budget": True},
"bogus": {"x": 1},
}
@@ -194,8 +198,8 @@ def test_every_problem_is_reported_at_once() -> None:
for expected in (
"projects.greet: unset or empty — workspace",
"llm_extra: unknown setting",
"tuning.executor.max_turn: unknown setting",
"tuning.executor.failure_limit: must be a positive number",
"tuning.driver.max_turn: unknown setting",
"tuning.driver.failure_limit: must be a positive number",
"tuning.expert.consult_budget: must be a positive number",
"tuning.bogus: unknown section",
):
@@ -203,8 +207,8 @@ def test_every_problem_is_reported_at_once() -> None:
def test_a_whole_number_is_required_where_one_is_meant() -> None:
with pytest.raises(ConfigError, match="tuning.executor.max_turns"):
build(tuning={"executor": {"max_turns": 2.5}})
with pytest.raises(ConfigError, match="tuning.driver.max_turns"):
build(tuning={"driver": {"max_turns": 2.5}})
# --- the file --------------------------------------------------------------
@@ -220,8 +224,7 @@ projects:
key: ~/.ssh/nisse-dev
llm: {base_url: "https://litellm.test", token: "${TEST_LLM_TOKEN}"}
roles:
executor: local-fast
planner: claude-opus-5
driver: local-fast
helper: claude-opus-5
expert: claude-haiku-4-5
matrix:
@@ -245,7 +248,7 @@ def write(directory: Path, text: str = FILE) -> Path:
def test_a_file_is_read(tmp_path: Path) -> None:
config = AgentConfig.from_file(write(tmp_path), SECRETS)
assert config.roles.executor == "local-fast"
assert config.roles.driver == "local-fast"
assert config.config_file == tmp_path / "nisse-dev.yaml"
@@ -256,7 +259,9 @@ def test_invalid_yaml_is_reported(tmp_path: Path) -> None:
def test_load_uses_nisse_config(tmp_path: Path) -> None:
path = write(tmp_path)
assert AgentConfig.load({**SECRETS, "NISSE_CONFIG": str(path)}).project == "greet"
assert (
only(AgentConfig.load({**SECRETS, "NISSE_CONFIG": str(path)})).name == "greet"
)
def test_load_defaults_to_the_working_directory(
@@ -264,7 +269,7 @@ def test_load_defaults_to_the_working_directory(
) -> None:
write(tmp_path)
monkeypatch.chdir(tmp_path)
assert AgentConfig.load(SECRETS).project == "greet"
assert only(AgentConfig.load(SECRETS)).name == "greet"
def test_load_without_a_file_says_how_to_make_one(
@@ -284,8 +289,8 @@ def test_the_example_file_is_valid() -> None:
"NISSE_GITEA_TOKEN": "gt",
}
config = AgentConfig.from_file(example, env)
assert config.tuning.executor.max_turns == 40
assert config.roles.executor == "qwen3.8-27b"
assert config.tuning.driver.max_turns == 40
assert config.roles.driver == "claude-opus-5-5"
# --- nothing that carries secrets may sit inside the clone -----------------
@@ -294,7 +299,7 @@ def test_the_example_file_is_valid() -> None:
def test_state_db_and_workspace_are_separate() -> None:
# L8: build artefacts must not share a volume with agent state.
config = build()
assert config.state_db.parent != config.workspace
assert config.state_db.parent != only(config).workspace
def test_state_db_inside_the_clone_is_refused(tmp_path: Path) -> None:
@@ -333,7 +338,7 @@ def test_sibling_paths_are_fine(tmp_path: Path) -> None:
),
{**SECRETS, "NISSE_ENV": str(tmp_path / ".env")},
)
assert config.workspace.name == "project"
assert only(config).workspace.name == "project"
def test_context_tuning_is_read_and_checked() -> None:
@@ -346,10 +351,10 @@ def test_context_tuning_is_read_and_checked() -> None:
def test_the_build_host_is_required_and_defaults_to_root_on_22() -> None:
config = build()
assert (config.build_host.address, config.build_host.user) == ("build.test", "root")
assert config.build_host.port == 22
assert config.build_host.key.is_absolute()
host = only(build()).build_host
assert (host.address, host.user) == ("build.test", "root")
assert host.port == 22
assert host.key.is_absolute()
with pytest.raises(ConfigError, match="build_host.address"):
build(**{f"{P}build_host__address": None})
@@ -360,14 +365,14 @@ def test_a_bad_build_host_port_is_reported() -> None:
def test_the_push_key_is_optional_and_a_path() -> None:
assert build().push_key is None
assert build(**{f"{P}push_key": "~/k"}).push_key == Path("~/k").expanduser()
assert only(build()).push_key is None
assert only(build(**{f"{P}push_key": "~/k"})).push_key == Path("~/k").expanduser()
def test_a_push_token_needs_its_user_and_excludes_a_key() -> None:
token = {f"{P}push_user": "pusher", f"{P}push_token": "${TEST_GITEA_TOKEN}"}
config = build(**token)
assert (config.push_user, config.push_token) == ("pusher", "gt")
project = only(build(**token))
assert (project.push_user, project.push_token) == ("pusher", "gt")
with pytest.raises(ConfigError, match="set both or neither"):
build(**{f"{P}push_user": "pusher"})
with pytest.raises(ConfigError, match="not both"):
@@ -386,11 +391,21 @@ def test_no_credential_appears_in_the_config_repr() -> None:
assert secret not in text
def test_the_loop_defaults_to_old_and_takes_new() -> None:
assert build().loop == "old"
assert build(loop="new").loop == "new"
with pytest.raises(ConfigError, match="loop: must be old or new"):
build(loop="newer")
def test_the_old_loops_settings_are_refused_with_what_to_do_instead() -> None:
old = tree(loop="new", roles__planner="claude-opus-5", roles__driver=None)
old["roles"]["executor"] = "local-fast"
old["tuning"] = {"executor": {"max_turns": 40}}
with pytest.raises(ConfigError) as excinfo:
AgentConfig.from_tree(old, SECRETS)
message = str(excinfo.value)
for expected in (
"loop: retired — the loop is the only one now; remove it",
"roles.planner: retired — use roles.driver",
"roles.executor: retired",
"tuning.executor: retired — it is tuning.driver now",
"unset or empty — roles.driver",
):
assert expected in message
# --- projects (D8) -----------------------------------------------------------
@@ -410,14 +425,6 @@ def test_a_second_project_is_a_second_entry() -> None:
assert other.base_branch == "main" and other.labels == () and other.secrets == {}
def test_the_old_loop_refuses_more_than_one_project() -> None:
config = build(projects__other=SECOND)
with pytest.raises(ConfigError, match="exactly one project"):
config.only_project()
with pytest.raises(ConfigError, match="exactly one project"):
_ = config.gitea_repo
def test_at_least_one_project_is_needed() -> None:
with pytest.raises(ConfigError, match="projects: must name at least one"):
build(projects={})
@@ -439,8 +446,8 @@ def test_a_project_name_is_a_slug() -> None:
def test_labels_and_egress_are_lists_of_names() -> None:
config = build(**{f"{P}labels": ["discuss"], f"{P}egress": ["pypi.org"]})
assert config.only_project().labels == ("discuss",)
assert config.only_project().egress == ("pypi.org",)
assert only(config).labels == ("discuss",)
assert only(config).egress == ("pypi.org",)
with pytest.raises(ConfigError, match="projects.greet.labels: must be a list"):
build(**{f"{P}labels": "discuss"})
@@ -454,7 +461,7 @@ def test_secrets_are_by_reference_with_a_visibility() -> None:
}
}
)
project = config.only_project()
project = only(config)
assert project.command_secrets() == {"DB_URL": "sk-x"}
assert project.secrets["DEPLOY"].visible == "harness"
nisse-dev/tests/test_context.py
+15 -61
View File
@@ -9,8 +9,6 @@ import pytest
from nisse_dev.canonical import (
AssistantText,
CanonicalEvent,
GateOpened,
PlanSubmitted,
StoredEvent,
ToolCall,
ToolResult,
@@ -25,8 +23,7 @@ from nisse_dev.context import (
fit,
head_and_tail,
)
from nisse_dev.executor import LLMExecutor
from nisse_dev.llm import RoleMap
from nisse_dev.loop.driver import Driver
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore
@@ -230,70 +227,27 @@ def test_the_catalog_reads_each_models_limits() -> None:
def test_a_role_on_a_model_without_a_window_is_refused() -> None:
CATALOG.require_limits({"executor": "llama3.2", "planner": "claude-haiku-4-5"})
with pytest.raises(UnknownLimits, match="executor='undeclared'"):
CATALOG.require_limits({"executor": "undeclared", "planner": "llama3.2"})
CATALOG.require_limits({"driver": "llama3.2", "helper": "claude-haiku-4-5"})
with pytest.raises(UnknownLimits, match="driver='undeclared'"):
CATALOG.require_limits({"driver": "undeclared", "helper": "llama3.2"})
# --- the executor sees the approved plan and what came after ---------------
def test_the_executor_sees_only_its_milestone(tmp_path: Path) -> None:
def test_a_request_that_cannot_fit_ends_the_run(tmp_path: Path) -> None:
# Never sent: the driver's run fails and the item waits on the operator,
# like any failed call, rather than a server getting a prompt it would
# silently cut.
store = StateStore(tmp_path / "state.db", oplog=OpLog("t", io.StringIO()))
for event in (
UserMessage(text="use greet: a long planning conversation", source="t"),
PlanSubmitted(plan="arch plan", model="m"),
UserMessage(text="approve", source="t"),
PlanSubmitted(plan="milestone 1: write greet.py", model="m"),
GateOpened(gate="milestone-1", reason="plan submitted"),
UserMessage(text="start milestone 1", source="t"),
):
store.transcript("t", "executor").append(event)
class Workspace:
def describe(self) -> str:
return "workspace"
seen: dict[str, Any] = {}
class Client:
def complete(self, events, tools, model, system=None):
seen["events"] = [e.event for e in events]
seen["system"] = system
return [AssistantText(text="done", model=model)]
roles = RoleMap(executor="m", planner="m", helper="m", expert="m")
LLMExecutor(Client(), roles, Workspace()).run(store.transcript("t", "executor"), 1)
texts = [getattr(e, "text", None) for e in seen["events"]]
assert "start milestone 1" in texts
assert "use greet: a long planning conversation" not in texts
assert not any(isinstance(e, PlanSubmitted) for e in seen["events"])
# The plan rides in the system prompt, which the budget never drops.
assert "The approved plan:\nmilestone 1: write greet.py" in seen["system"]
assert "arch plan" not in seen["system"]
def test_a_request_that_cannot_fit_stops_the_milestone(tmp_path: Path) -> None:
# Never sent: the executor stops and asks the operator, like any failed
# call, rather than handing a server a prompt it would silently cut.
store = StateStore(tmp_path / "state.db", oplog=OpLog("t", io.StringIO()))
store.transcript("t", "operator").append(
UserMessage(text="start milestone 1", source="t")
)
class Workspace:
def describe(self) -> str:
return "workspace " + "w" * 20_000 # a system prompt too big to fit
store.transcript("t", "operator").append(UserMessage(text="go", source="t"))
inner = RecordingClient()
client = BudgetedClient(inner, lambda m: (4096, 1024))
roles = RoleMap(executor="m", planner="m", helper="m", expert="m")
outcome = LLMExecutor(client, roles, Workspace()).run(
store.transcript("t", "executor"), 1
outcome = Driver(client, "m").run(
store.transcript("t", "driver"),
system="s" * 20_000, # a system prompt too big to fit
tools=[],
workspace=None,
)
assert outcome.status == "escalate"
assert outcome.kind == "failed"
assert "fit" in outcome.note
assert inner.calls == []
nisse-dev/tests/test_control.py
-289
View File
@@ -1,289 +0,0 @@
import io
import json
from pathlib import Path
import pytest
from nisse_dev.canonical import UserMessage
from nisse_dev.control import (
AWAIT_ARCH_SIGNOFF,
AWAIT_INPUT,
AWAIT_MILESTONE_SIGNOFF,
EXECUTING,
IDLE,
PLANNING_ARCH,
PR_OPEN,
AskHuman,
Orchestrator,
PlanningError,
StubAgent,
SubmitPlan,
)
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore
from nisse_dev.transport import InMemoryTransport
class Harness:
def __init__(self, tmp_path: Path, agent: object | None = None) -> None:
self.stream = io.StringIO()
self.store = StateStore(
tmp_path / "state.db", oplog=OpLog("nisse-dev", self.stream)
)
self.transport = InMemoryTransport()
self.orch = Orchestrator(
self.store, self.transport, agent or StubAgent(), operators={"don"}
)
self.orch.start()
@property
def state(self) -> str:
state = self.store.get_state()
assert state is not None
return state.state
def events(self, kind: str) -> list[dict]:
return [
e
for e in map(json.loads, self.stream.getvalue().splitlines())
if e["event"] == kind
]
def outbound_texts(self) -> list[str]:
return [m.text for m in self.transport.outbound]
def executed(h: Harness) -> bool:
return any(e["to"] == EXECUTING for e in h.events("state.transition"))
@pytest.fixture
def h(tmp_path: Path) -> Harness:
return Harness(tmp_path)
def test_boots_idle_and_greets(h: Harness) -> None:
assert h.state == IDLE
assert "ready" in h.outbound_texts()[0]
def test_use_project_submits_arch_plan_and_opens_gate(h: Harness) -> None:
h.transport.receive("don", "use demo")
assert h.state == AWAIT_ARCH_SIGNOFF
request = h.transport.outbound[-1]
assert request.kind == "approval_request"
assert "rev-1" in request.text
[opened] = h.events("gate.open")
assert opened["gate"] == "arch"
assert opened["rev"] == "rev-1"
assert opened["task_id"] is not None
def test_full_happy_path_through_execution(h: Harness) -> None:
h.transport.receive("don", "use demo")
h.transport.receive("don", "approve")
assert h.state == AWAIT_MILESTONE_SIGNOFF
h.transport.receive("don", "approve")
assert h.state == AWAIT_MILESTONE_SIGNOFF # approved, awaiting start
h.transport.receive("don", "start milestone 1")
assert executed(h)
# The stub completes at once and waits in PR_OPEN, never in EXECUTING.
assert h.state == PR_OPEN
approvals = h.events("gate.approve")
assert [e["gate"] for e in approvals] == ["arch", "milestone-1"]
def test_reject_replans_with_feedback_and_bumps_revision(h: Harness) -> None:
h.transport.receive("don", "use demo")
h.transport.receive("don", "too vague, add tests")
assert h.state == AWAIT_ARCH_SIGNOFF
request = h.transport.outbound[-1]
assert "rev-2" in request.text
assert "too vague, add tests" in request.text
[rejected] = h.events("gate.reject")
assert rejected["rev"] == "rev-1"
def test_stale_approval_is_refused_then_current_works(h: Harness) -> None:
h.transport.receive("don", "use demo")
h.transport.receive("don", "needs work") # now at rev-2
h.transport.receive("don", "approve rev-1")
assert h.state == AWAIT_ARCH_SIGNOFF
assert "not applied" in h.outbound_texts()[-1]
assert h.events("gate.approve") == []
h.transport.receive("don", "approve rev-2")
assert h.state == AWAIT_MILESTONE_SIGNOFF
def test_non_operator_messages_are_ignored(h: Harness) -> None:
h.transport.receive("don", "use demo")
before = len(h.transport.outbound)
h.transport.receive("mallory", "approve")
assert h.state == AWAIT_ARCH_SIGNOFF
assert len(h.transport.outbound) == before
def test_start_milestone_requires_approval_first(h: Harness) -> None:
h.transport.receive("don", "use demo")
h.transport.receive("don", "approve")
h.transport.receive("don", "start milestone 1")
assert h.state == AWAIT_MILESTONE_SIGNOFF # it was rejection feedback
# Revisions run across the task: arch rev-1, milestone-1 rev-2, and
# after that rejection milestone-1 rev-3.
h.transport.receive("don", "approve rev-3")
h.transport.receive("don", "start milestone 1")
assert executed(h)
def test_a_message_is_matched_as_stripped_but_recorded_as_sent(
h: Harness,
) -> None:
h.transport.receive("don", "use demo")
h.transport.receive("don", "approve", original="Nisse Dev: approve")
assert h.state == AWAIT_MILESTONE_SIGNOFF
recorded = [
e.event.text for e in h.store.all_events() if isinstance(e.event, UserMessage)
]
assert "Nisse Dev: approve" in recorded
assert "approve" not in recorded
def test_status_reports_state_and_gate(h: Harness) -> None:
h.transport.receive("don", "use demo")
h.transport.receive("don", "status")
status = h.outbound_texts()[-1]
assert "AWAIT_ARCH_SIGNOFF" in status
assert "demo" in status
assert "rev-1" in status
class QuestionThenPlan:
def plan(
self,
transcript: object,
gate: str,
project: str,
feedback: str | None = None,
answer: str | None = None,
) -> SubmitPlan | AskHuman:
if answer is None:
return AskHuman("monorepo or polyrepo?")
return SubmitPlan(f"[stub] {gate} plan using {answer}")
def test_ask_human_pauses_and_answer_resumes(tmp_path: Path) -> None:
h = Harness(tmp_path, agent=QuestionThenPlan())
h.transport.receive("don", "use demo")
assert h.state == AWAIT_INPUT
assert h.transport.outbound[-1].kind == "question"
h.transport.receive("don", "monorepo")
assert h.state == AWAIT_ARCH_SIGNOFF
assert "monorepo" in h.transport.outbound[-1].text
# --- restarts (L8 boot steps 4 and 5) --------------------------------------------
class FailsOnce:
"""A planner whose first turn fails, as Opus 5.5's did on its first live
run when asked for forced tool use."""
def __init__(self) -> None:
self.calls = 0
def plan(
self,
transcript: object,
gate: str,
project: str,
feedback: str | None = None,
answer: str | None = None,
) -> SubmitPlan | AskHuman:
self.calls += 1
if self.calls == 1:
raise PlanningError("planner returned 400")
return SubmitPlan(f"[stub] {gate} plan for {project}")
def test_restart_while_awaiting_input_asks_the_same_question(
tmp_path: Path,
) -> None:
agent = FailsOnce()
h = Harness(tmp_path, agent=agent)
h.transport.receive("don", "use demo")
assert h.state == AWAIT_INPUT
h = Harness(tmp_path, agent=agent) # restart on the same state
question = h.transport.outbound[-1]
assert question.kind == "question"
assert "restarted while waiting on you" in question.text
assert "planning for arch failed" in question.text
h.transport.receive("don", "Please retry.")
assert h.state == AWAIT_ARCH_SIGNOFF
def test_restart_mid_planning_offers_continue_and_it_replans(
tmp_path: Path,
) -> None:
h = Harness(tmp_path)
h.store.transition(PLANNING_ARCH, cause="test", project="demo", data={})
h = Harness(tmp_path)
assert h.state == AWAIT_INPUT
assert "'continue'" in h.outbound_texts()[-1]
h.transport.receive("don", "Please retry.") # neither choice: asked again
assert h.state == AWAIT_INPUT
assert h.transport.outbound[-1].kind == "question"
h.transport.receive("don", "continue")
assert h.state == AWAIT_ARCH_SIGNOFF
def test_restart_mid_execution_continue_returns_to_the_approved_plan(
h: Harness, tmp_path: Path
) -> None:
h.transport.receive("don", "use demo")
h.transport.receive("don", "approve")
h.transport.receive("don", "approve")
# A crash mid-milestone: the store still says EXECUTING.
state = h.store.get_state()
h.store.transition(EXECUTING, cause="test", project="demo", data=state.data)
h = Harness(tmp_path)
assert h.state == AWAIT_INPUT
h.transport.receive("don", "continue")
assert h.state == AWAIT_MILESTONE_SIGNOFF
assert "start milestone 1" in h.outbound_texts()[-1]
h.transport.receive("don", "start milestone 1")
assert executed(h)
def test_restart_question_without_its_interrupted_state_still_replans(
tmp_path: Path,
) -> None:
# What the first live run left behind: a restart question recorded
# before the question kept its prompt or the state it interrupted.
h = Harness(tmp_path)
h.store.transition(
AWAIT_INPUT, cause="test", project="demo", data={"pending": {"kind": "boot"}}
)
h = Harness(tmp_path)
assert "my last question still stands" in h.outbound_texts()[-1]
h.transport.receive("don", "re-plan")
assert h.state == AWAIT_ARCH_SIGNOFF
def test_use_starts_over_when_nothing_is_pending(h: Harness) -> None:
h.store.transition(AWAIT_INPUT, cause="forced", project="demo", data={})
h.transport.receive("don", "use demo")
assert h.state == AWAIT_ARCH_SIGNOFF
def test_commands_ignore_case_but_the_project_name_keeps_it(h: Harness) -> None:
h.transport.receive("don", "Use Demo.")
assert h.state == AWAIT_ARCH_SIGNOFF
assert h.store.get_state().project == "Demo"
h.transport.receive("don", "Status")
assert "AWAIT_ARCH_SIGNOFF" in h.outbound_texts()[-1]
h.transport.receive("don", "Approve")
h.transport.receive("don", "Approve.")
h.transport.receive("don", "Start Milestone 1")
assert h.state == PR_OPEN
h.transport.receive("don", "Project Complete!")
assert h.state == IDLE
nisse-dev/tests/test_dump.py
+15 -12
View File
@@ -12,20 +12,23 @@ def store_with_a_run(path: Path) -> None:
store.transcript(None, "operator").append(
UserMessage(text="hello before use", source="matrix")
)
store.transition(
"AWAIT_ARCH_SIGNOFF",
store.open_work_item(
"demo-1",
title="Add a truncate helper",
scope="demo",
position="WAITING",
data={"loop": {"wait": "go-ahead", "rev": 1}},
cause="test",
project="demo",
data={"task_id": "demo-1", "gate": {"name": "arch", "rev": 1}},
)
store.set_active("demo-1", cause="test")
log = store.transcript("demo-1", "operator")
log.append(UserMessage(text="Add a truncate helper", source="matrix"))
log.as_actor("planner").append(PlanSubmitted(plan="x" * 500, model="opus"))
executor = log.as_actor("executor")
executor.append(
driver = log.as_actor("driver")
driver.append(PlanSubmitted(plan="x" * 500, model="opus"))
driver.append(
ToolCall(call_id="c1", tool="run_command", arguments={"command": "ls"})
)
executor.append(ToolResult(call_id="c1", result="README.md", success=False))
driver.append(ToolResult(call_id="c1", result="README.md", success=False))
log.begin_intent("open_pr", {"branch": "b"}, cause="test")
store.close()
@@ -38,14 +41,14 @@ def test_the_dump_shows_items_intents_and_each_event_with_its_actor(
text = render(db)
assert "active work item: demo-1" in text
assert (
"demo-1 open position: AWAIT_ARCH_SIGNOFF scope: demo gate: arch rev-1"
in (text)
"demo-1 open position: WAITING scope: demo rev-1 waits for: go-ahead"
in text
)
assert "pending intent #1 open_pr [demo-1]" in text
assert "[agent/operator] USER [matrix]: hello before use" in text
assert "[demo-1/operator] USER [matrix]: Add a truncate helper" in text
assert "[demo-1/planner] PLAN (opus, 500 chars): " in text and "[340 more]" in text
assert '[demo-1/executor] CALL run_command {"command": "ls"}' in text
assert "[demo-1/driver] PLAN (opus, 500 chars): " in text and "[340 more]" in text
assert '[demo-1/driver] CALL run_command {"command": "ls"}' in text
assert "result (FAILED): README.md" in text
nisse-dev/tests/test_executor.py
-397
View File
@@ -1,397 +0,0 @@
import io
from pathlib import Path
from typing import Any
import pytest
from nisse_dev.canonical import (
AssistantText,
CanonicalEvent,
StoredEvent,
ToolCall,
ToolResult,
)
from nisse_dev.execution import SSHHost
from nisse_dev.executor import LLMExecutor
from nisse_dev.llm import LLMError, RoleMap
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore, Transcript
from nisse_dev.tools import Workspace
ROLES = RoleMap(
executor="llama3.2",
planner="claude-haiku-4-5",
helper="claude-haiku-4-5",
expert="claude-haiku-4-5",
)
class ScriptedClient:
def __init__(self, turns: list[list[CanonicalEvent]]) -> None:
self._turns = turns
self.calls: list[dict[str, Any]] = []
def complete(
self,
events: list[StoredEvent],
tools: list[dict[str, Any]] | None,
model: str,
system: str | None = None,
) -> list[CanonicalEvent]:
self.calls.append({"tools": tools, "model": model, "system": system})
if not self._turns:
return [finish(call_id=f"fin-{len(self.calls)}")]
return self._turns.pop(0)
class FailingClient:
def complete(self, *args: Any, **kwargs: Any) -> list[CanonicalEvent]:
raise LLMError("proxy exploded")
def call(tool: str, call_id: str = "c1", **arguments: Any) -> ToolCall:
return ToolCall(call_id=call_id, tool=tool, arguments=arguments)
def finish(summary: str = "done", call_id: str = "fin") -> ToolCall:
return ToolCall(call_id=call_id, tool="finish", arguments={"summary": summary})
@pytest.fixture
def store(tmp_path: Path) -> StateStore:
return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
@pytest.fixture
def ws(ssh_host: SSHHost, tmp_path: Path) -> Workspace:
return Workspace(ssh_host, tmp_path / "work", timeout_s=10)
ITEM = "t" # the work item these tests run in
def log(store: StateStore) -> Transcript:
return store.transcript(ITEM, "executor")
class Bound:
"""An executor run in the test's work item, as the orchestrator runs it."""
def __init__(self, executor: LLMExecutor, store: StateStore) -> None:
self.executor = executor
self.store = store
def run(self, milestone: int) -> Any:
return self.executor.run(log(self.store), milestone)
def build(client: Any, store: StateStore, ws: Workspace, **kw: Any) -> Bound:
return Bound(LLMExecutor(client, ROLES, ws, **kw), store)
def test_tool_calls_are_executed_and_results_logged(
store: StateStore, ws: Workspace
) -> None:
client = ScriptedClient(
[
[call("write_file", path="app.py", content="print(1)\n")],
[finish("wrote the file")],
]
)
outcome = build(client, store, ws).run(milestone=1)
assert outcome.status == "completed"
assert outcome.note == "wrote the file"
assert (ws.root / "app.py").read_text() == "print(1)\n"
kinds = [type(e.event).__name__ for e in log(store).read()]
assert kinds == ["ToolCall", "ToolResult", "ToolCall", "ToolResult"]
def test_uses_executor_role_and_reduced_manifest(
store: StateStore, ws: Workspace
) -> None:
client = ScriptedClient([])
build(client, store, ws).run(milestone=2)
assert client.calls[0]["model"] == "llama3.2" # never the planner
names = [t["function"]["name"] for t in client.calls[0]["tools"]]
assert names == ["read_file", "write_file", "list_dir", "run_command", "finish"]
assert "milestone 2" in client.calls[0]["system"]
def test_prose_no_longer_ends_the_milestone(store: StateStore, ws: Workspace) -> None:
# Only finish ends a milestone. Prose breaks the tool contract: the model
# is told once, and here it gets back to work.
client = ScriptedClient(
[
[call("write_file", path="a.py", content="x")],
[AssistantText(text="all steps done", model="m")],
[finish()],
]
)
assert build(client, store, ws).run(milestone=1).status == "completed"
assert "Every reply must be a tool call" in client.calls[2]["system"]
def test_persisting_in_prose_escalates(store: StateStore, ws: Workspace) -> None:
prose = [AssistantText(text="let me just chat", model="m")]
client = ScriptedClient([prose, prose])
outcome = build(client, store, ws).run(milestone=1)
assert outcome.status == "escalate"
assert "prose" in outcome.note
def test_finishing_without_any_work_is_refused_once(
store: StateStore, ws: Workspace
) -> None:
# A model that finishes on turn one has done nothing; believing it would
# record an empty milestone as a success.
client = ScriptedClient(
[
[finish("looks done to me", call_id="f1")],
[call("write_file", path="a.py", content="x")],
[finish("now it is done", call_id="f2")],
]
)
outcome = build(client, store, ws).run(milestone=1)
assert outcome.status == "completed"
assert (ws.root / "a.py").exists() # the refusal produced actual work
[refusal] = [
e.event
for e in log(store).read()
if isinstance(e.event, ToolResult) and e.event.call_id == "f1"
]
assert not refusal.success
assert "not made any change yet" in refusal.result
def test_persisting_in_an_empty_finish_escalates(
store: StateStore, ws: Workspace
) -> None:
client = ScriptedClient([[finish(call_id="f1")], [finish(call_id="f2")]])
outcome = build(client, store, ws).run(milestone=1)
assert outcome.status == "escalate"
assert "without making any change" in outcome.note
def test_calls_after_finish_are_not_run(store: StateStore, ws: Workspace) -> None:
client = ScriptedClient(
[
[call("write_file", path="a.py", content="x")],
[finish(), call("write_file", call_id="late", path="b.py", content="y")],
]
)
assert build(client, store, ws).run(milestone=1).status == "completed"
assert not (ws.root / "b.py").exists()
[late] = [
e.event
for e in log(store).read()
if isinstance(e.event, ToolResult) and e.event.call_id == "late"
]
assert "finish had already ended the milestone" in late.result
def test_an_invented_tool_counts_toward_escalation(
store: StateStore, ws: Workspace
) -> None:
# Measured: a model on a server that doesn't enforce tool_choice will
# call tools that were never offered ("chat", "respond"). Without this it
# would burn every turn up to the cap.
client = ScriptedClient(
[[call("chat", call_id="x1")], [call("respond_user", call_id="x2")]]
)
outcome = build(client, store, ws, failure_limit=2).run(milestone=1)
assert outcome.status == "escalate"
assert len(client.calls) == 2
results = [e.event for e in log(store).read() if isinstance(e.event, ToolResult)]
assert "no such tool: chat" in results[0].result
assert "finish" in results[0].result # it is told what does exist
def test_two_consecutive_command_failures_escalate(
store: StateStore, ws: Workspace
) -> None:
client = ScriptedClient(
[
[call("run_command", command="exit 1")],
[call("run_command", command="exit 1")],
[call("run_command", command="exit 1")],
]
)
outcome = build(client, store, ws, failure_limit=2).run(milestone=1)
assert outcome.status == "escalate"
assert "2 consecutive command failures" in outcome.note
assert len(client.calls) == 2 # stopped, did not keep trying
def test_a_success_resets_the_failure_count(store: StateStore, ws: Workspace) -> None:
client = ScriptedClient(
[
[call("run_command", command="exit 1")],
[call("run_command", command="true")],
[call("run_command", command="exit 1")],
[finish()],
]
)
assert build(client, store, ws, failure_limit=2).run(milestone=1).status == (
"completed"
)
def test_failed_reads_do_not_count_toward_escalation(
store: StateStore, ws: Workspace
) -> None:
# Two failed reads in a row must not trip the failure limit: looking for
# a file that is not there is the model orienting itself, not failing.
client = ScriptedClient(
[
[call("read_file", path="missing-a.txt")],
[call("read_file", path="missing-b.txt")],
[call("write_file", path="a.py", content="x")],
[finish()],
]
)
assert build(client, store, ws, failure_limit=2).run(milestone=1).status == (
"completed"
)
def test_reading_alone_is_not_doing_the_work(store: StateStore, ws: Workspace) -> None:
# Exploration is not evidence a milestone was carried out, so a
# completion claim backed only by reads is still challenged.
client = ScriptedClient(
[
[call("list_dir", path=".")],
[finish("looks done to me", call_id="f1")],
[finish("really, done", call_id="f2")],
]
)
outcome = build(client, store, ws).run(milestone=1)
assert outcome.status == "escalate"
def test_turn_cap_is_enforced(store: StateStore, ws: Workspace) -> None:
forever = [[call("list_dir", path=".")] for _ in range(50)]
client = ScriptedClient(forever)
outcome = build(client, store, ws, max_turns=3).run(milestone=1)
assert outcome.status == "capped"
assert "3-turn cap" in outcome.note
assert len(client.calls) == 3
def test_llm_error_escalates_rather_than_crashing(
store: StateStore, ws: Workspace
) -> None:
outcome = build(FailingClient(), store, ws).run(milestone=1)
assert outcome.status == "escalate"
assert "proxy exploded" in outcome.note
def test_multiple_calls_in_one_turn_all_run(store: StateStore, ws: Workspace) -> None:
client = ScriptedClient(
[
[
call("write_file", call_id="a", path="one.txt", content="1"),
call("write_file", call_id="b", path="two.txt", content="2"),
],
[finish()],
]
)
build(client, store, ws).run(milestone=1)
assert (ws.root / "one.txt").exists() and (ws.root / "two.txt").exists()
results = [
e.event for e in log(store).read() if type(e.event).__name__ == "ToolResult"
]
assert {r.call_id for r in results} == {"a", "b", "fin"}
class StubDesk:
"""Stands in for ExpertDesk without a model behind it."""
def __init__(self, answer: str = "use sqlite", error: Exception | None = None):
self.asked: list[str] = []
self._answer = answer
self._error = error
def ask(self, transcript: Any, question: str, **kw: Any) -> Any:
self.asked.append(question)
if self._error is not None:
raise self._error
class _A:
def render(self_inner) -> str:
return self._answer
return _A()
def test_ask_expert_is_offered_only_when_a_desk_exists(
store: StateStore, ws: Workspace
) -> None:
client = ScriptedClient([])
build(client, store, ws).run(milestone=1)
assert all(t["function"]["name"] != "ask_expert" for t in client.calls[0]["tools"])
client2 = ScriptedClient([])
build(client2, store, ws, expert=StubDesk()).run(milestone=1)
names = [t["function"]["name"] for t in client2.calls[0]["tools"]]
assert "ask_expert" in names
assert names[-1] == "finish"
# L9: in addition to the observable triggers, not instead of them.
assert "run_command" in names
def test_ask_expert_answer_comes_back_as_a_tool_result(
store: StateStore, ws: Workspace
) -> None:
desk = StubDesk(answer="prefer sqlite here")
client = ScriptedClient(
[
[call("ask_expert", question="sqlite or postgres?")],
[call("write_file", path="a.py", content="x")],
[finish()],
]
)
build(client, store, ws, expert=desk).run(milestone=1)
assert desk.asked == ["sqlite or postgres?"]
results = [
e.event for e in log(store).read() if type(e.event).__name__ == "ToolResult"
]
assert "prefer sqlite here" in results[0].result
def test_consulting_is_not_doing_the_work(store: StateStore, ws: Workspace) -> None:
# Asking is not changing anything, so a completion claim backed only by
# consults is still challenged.
client = ScriptedClient(
[
[call("ask_expert", question="what should I do?")],
[finish("I think it is done", call_id="f1")],
[finish("really done", call_id="f2")],
]
)
outcome = build(client, store, ws, expert=StubDesk()).run(milestone=1)
assert outcome.status == "escalate"
def test_exhausted_budget_is_reported_to_the_model_not_raised(
store: StateStore, ws: Workspace
) -> None:
from nisse_dev.expert import BudgetExhausted
desk = StubDesk(error=BudgetExhausted("consult budget of 3 is spent"))
client = ScriptedClient(
[
[call("ask_expert", question="again?")],
[call("write_file", path="a.py", content="x")],
[finish()],
]
)
outcome = build(client, store, ws, expert=desk).run(milestone=1)
assert outcome.status == "completed" # it got on with the work
results = [
e.event for e in log(store).read() if type(e.event).__name__ == "ToolResult"
]
assert not results[0].success
assert "budget" in results[0].result
nisse-dev/tests/test_expert.py
+1 -2
View File
@@ -23,8 +23,7 @@ from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore, Transcript
ROLES = RoleMap(
executor="llama3.2",
planner="haiku",
driver="haiku",
helper="haiku",
expert="claude-haiku-4-5",
)
nisse-dev/tests/test_gates.py
-105
View File
@@ -1,105 +0,0 @@
import pytest
from nisse_dev.gates import Approved, Gate, Ignored, PlanRevision, Rejected
REVISION = PlanRevision(gate="arch", rev=2, content="the plan")
OPERATORS = {"don"}
def gate() -> Gate:
return Gate(REVISION, OPERATORS)
def test_bare_approve_binds_to_current_revision() -> None:
result = gate().evaluate("don", "approve", channel="transport")
assert isinstance(result, Approved)
assert result.revision.tag == "rev-2"
assert result.approver == "don"
def test_tagged_approve_matching_current_revision() -> None:
result = gate().evaluate("don", "approve rev-2", channel="gitea")
assert isinstance(result, Approved)
assert result.channel == "gitea"
def test_stale_revision_tag_is_ignored_with_notice() -> None:
result = gate().evaluate("don", "approve rev-1", channel="transport")
assert isinstance(result, Ignored)
assert result.notice is not None
assert "rev-1" in result.notice
assert "rev-2" in result.notice
def test_any_other_reply_is_rejection_with_feedback() -> None:
result = gate().evaluate("don", "too vague, split milestone 1", "transport")
assert isinstance(result, Rejected)
assert result.feedback == "too vague, split milestone 1"
def test_unknown_sender_is_silently_ignored() -> None:
result = gate().evaluate("mallory", "approve", channel="transport")
assert isinstance(result, Ignored)
assert result.notice is None
def test_approve_with_trailing_words_is_rejection() -> None:
# "approve, but rename the module" is feedback, not approval (L3: the
# typed word approve, nothing else).
result = gate().evaluate("don", "approve, but rename the module", "transport")
assert isinstance(result, Rejected)
@pytest.mark.parametrize(
"text", ["Approve", "APPROVE", "Approve.", "approve!", " Approve rev-2 "]
)
def test_approval_ignores_case_and_trailing_punctuation(text: str) -> None:
# A phone keyboard's "Approve" used to count as rejection feedback (#64).
assert isinstance(gate().evaluate("don", text, channel="transport"), Approved)
def test_a_capitalised_stale_approval_is_still_refused() -> None:
result = gate().evaluate("don", "Approve REV-1", channel="transport")
assert isinstance(result, Ignored)
def test_a_sentence_containing_approve_is_still_feedback() -> None:
result = gate().evaluate("don", "I approve of most of it", channel="transport")
assert isinstance(result, Rejected)
assert result.feedback == "I approve of most of it"
# --- binding by arrival (L3): a reply that names no revision is about the one
# that was up when it arrived.
def test_a_bare_approval_about_an_earlier_revision_is_not_applied() -> None:
result = gate().evaluate("don", "approve", channel="transport", about=1)
assert isinstance(result, Ignored)
assert "arrived while rev-1 was up" in result.notice
assert "arch rev-2" in result.notice
def test_feedback_about_an_earlier_revision_is_not_a_rejection() -> None:
result = gate().evaluate("don", "also add docs", channel="transport", about=1)
assert isinstance(result, Ignored)
assert "not taken as feedback on arch rev-2" in result.notice
def test_a_reply_before_any_plan_was_up_is_not_applied() -> None:
result = gate().evaluate("don", "approve", channel="transport", about=0)
assert isinstance(result, Ignored)
assert "no plan" in result.notice
def test_replies_about_the_current_revision_apply() -> None:
assert isinstance(gate().evaluate("don", "approve", "transport", about=2), Approved)
result = gate().evaluate("don", "add docs", "transport", about=2)
assert isinstance(result, Rejected)
def test_a_named_revision_binds_by_its_number() -> None:
# "approve rev-2" says which plan it means, whenever it arrived.
assert isinstance(
gate().evaluate("don", "approve rev-2", "gitea", about=1), Approved
)
nisse-dev/tests/test_gitea.py
+5 -13
View File
@@ -197,10 +197,7 @@ def test_intent_without_a_branch_is_unverifiable() -> None:
assert pull_request_reconciler(client(handler), "o/r")(blank) is None
def test_closing_comments_first_then_closes_issue_and_pr() -> None:
from nisse_dev.approvals import PlanBoard
from nisse_dev.publish import MilestonePublisher
def test_closing_an_issue_and_a_pr() -> None:
seen: list[tuple[str, str, dict]] = []
def handler(request: httpx.Request) -> httpx.Response:
@@ -213,15 +210,10 @@ def test_closing_comments_first_then_closes_issue_and_pr() -> None:
return httpx.Response(201, json={})
gitea = client(handler)
PlanBoard(gitea, "o/r").close(10, "abandoned by don")
publisher = MilestonePublisher(
None, # type: ignore[arg-type] -- closing touches neither store nor git
gitea,
"o/r",
None, # type: ignore[arg-type]
remote_url="ssh://git@gitea.test/o/r.git",
)
publisher.close_pull_request(20, "abandoned by don")
gitea.post_comment("o/r", 10, "abandoned by don")
gitea.close_issue("o/r", 10)
gitea.post_comment("o/r", 20, "abandoned by don")
gitea.close_pull_request("o/r", 20)
assert seen == [
("POST", "/api/v1/repos/o/r/issues/10/comments", {"body": "abandoned by don"}),
nisse-dev/tests/test_llm.py
+13 -10
View File
@@ -183,28 +183,31 @@ def test_malformed_response_raises_llm_error() -> None:
ROLES = RoleMap(
executor="local/qwen",
planner="anthropic/claude-opus-5",
helper="anthropic/claude-opus-5",
driver="local/qwen",
helper="anthropic/claude-sonnet-5",
expert="anthropic/claude-opus-5",
)
def test_role_map_resolves_and_pins() -> None:
roles = ROLES
assert roles.resolve("executor") == "local/qwen"
assert roles.resolve("driver") == "local/qwen"
assert roles.pinned() == {
"executor": "local/qwen",
"planner": "anthropic/claude-opus-5",
"helper": "anthropic/claude-opus-5",
"driver": "local/qwen",
"helper": "anthropic/claude-sonnet-5",
"expert": "anthropic/claude-opus-5",
# Unset, the new loop's roles fall back: driver to the planner's
# model, classifier to the expert's (L9).
"driver": "anthropic/claude-opus-5",
# Unset, the classifier falls back to the expert's model (L9).
"classifier": "anthropic/claude-opus-5",
}
def test_a_pinned_map_from_before_the_driver_reads_its_own_roles() -> None:
# Pinned before M2.8: the old loop's roles are there, and are ignored.
old = {"planner": "p", "executor": "e", "driver": "d", "helper": "h"}
roles = RoleMap.from_pinned(old, ROLES)
assert (roles.driver, roles.helper, roles.expert) == ("d", "h", ROLES.expert)
def test_unknown_role_raises() -> None:
with pytest.raises(LLMError, match="unknown role"):
ROLES.resolve("boss")
nisse-dev/tests/test_loop.py
+80 -1
View File
@@ -9,11 +9,19 @@ import pytest
from model_contract import assert_renderable
from nisse_dev.cancel import Cancel
from nisse_dev.canonical import AssistantText, CanonicalEvent, Fact, ToolCall
from nisse_dev.canonical import (
AssistantText,
CanonicalEvent,
Fact,
ToolCall,
ToolResult,
)
from nisse_dev.gitea import Issue
from nisse_dev.llm import LLMError
from nisse_dev.loop.agent import LoopAgent
from nisse_dev.loop.driver import Driver
from nisse_dev.loop.items import CLOSED, GO_AHEAD, READY, RUNNING, WAITING, loop_of
from nisse_dev.loop.reader import command_form
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore
from nisse_dev.transport import InMemoryTransport
@@ -205,6 +213,22 @@ def test_before_the_go_ahead_nothing_can_change_or_close(h: Harness) -> None:
assert {"restate", "ask_human", "read_file", "run_command"} <= h.model.offered[0]
@pytest.mark.parametrize("text", ["Go", "GO", "Go.", "go!", " go "])
def test_a_go_ahead_ignores_case_and_trailing_punctuation(
h: Harness, text: str
) -> None:
# A phone keyboard's "Approve" once counted as feedback, not approval (#64).
h.model.turns.append([call("close_item", summary="added and tested")])
h.agent.intake()
h.say(text)
assert h.store.work_items()[0].status == "closed"
def test_the_command_form_keeps_inner_words_apart() -> None:
assert command_form(" Do NEXT.") == "do next"
assert command_form("go ahead, but carefully") == "go ahead, but carefully"
def test_go_starts_the_work_and_close_item_closes_the_issue(h: Harness) -> None:
h.model.turns.append([call("close_item", summary="added and tested")])
h.agent.intake()
@@ -368,6 +392,61 @@ def test_stop_ends_a_run_at_the_next_boundary(tmp_path: Path) -> None:
assert outcome.kind == "stopped"
def test_a_failed_model_call_fails_the_run(tmp_path: Path) -> None:
class Down:
def complete(self, events, tools, model, system=None): # type: ignore[no-untyped-def]
raise LLMError("proxy down")
outcome = Driver(Down(), "m").run( # type: ignore[arg-type]
driver_log(tmp_path), system="s", tools=_tools("read_file"), workspace=None
)
assert outcome.kind == "failed" and "proxy down" in outcome.note
def test_stop_mid_turn_skips_the_rest_of_its_calls(tmp_path: Path) -> None:
cancel = Cancel()
def start(arguments: dict[str, Any]) -> tuple[str, bool]:
cancel.set() # the operator's stop lands while this call runs
return "started", True
model = ScriptedModel(
[call("start_milestone", name="m"), call("read_file", path="a")]
)
log = driver_log(tmp_path)
outcome = Driver(model, "m", cancel=cancel).run(
log,
system="s",
tools=_tools("start_milestone", "read_file"),
workspace=None,
handlers={"start_milestone": start},
)
assert outcome.kind == "stopped"
results = [e.event for e in log.read() if isinstance(e.event, ToolResult)]
assert [r.success for r in results] == [True, False] # the read never ran
def test_a_command_that_works_resets_the_failure_count(tmp_path: Path) -> None:
class Commands:
def dispatch(self, tool: str, arguments: dict[str, Any]) -> Any:
ok = arguments["command"] == "good"
return type("Outcome", (), {"result": "out", "success": ok})()
def run(command: str) -> list[Any]:
return [call("run_command", command=command)]
model = ScriptedModel(
run("bad"), run("good"), run("bad"), [call("ask_human", question="?")]
)
outcome = Driver(model, "m", failure_limit=2).run(
driver_log(tmp_path),
system="s",
tools=_tools("run_command", "ask_human"),
workspace=Commands(), # type: ignore[arg-type]
)
assert outcome.kind == "wait" # never two failures in a row
def _tools(*names: str) -> list[dict[str, Any]]:
return [
{"type": "function", "function": {"name": n, "parameters": {}}} for n in names
nisse-dev/tests/test_loop_help.py
+36 -2
View File
@@ -18,14 +18,20 @@ from test_loop_milestones import (
)
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from nisse_dev.canonical import AssistantText, Fact, ToolCall, ToolResult
from nisse_dev.canonical import (
AssistantText,
DelegationStarted,
Fact,
ToolCall,
ToolResult,
)
from nisse_dev.config import Profile
from nisse_dev.delegation import DELEGATION_INTENT, Caps
from nisse_dev.llm import RoleMap
from nisse_dev.loop.help import Helpers
from nisse_dev.loop.items import ANSWER, MERGE, loop_of
ROLES = RoleMap(executor="", planner="drv", helper="hlp", expert="exp", driver="drv")
ROLES = RoleMap(driver="drv", helper="hlp", expert="exp")
class Spy:
@@ -115,6 +121,34 @@ def test_a_helpers_work_lands_on_the_milestone(world: dict[str, Any]) -> None:
assert loop_of(item(world)).wait == MERGE
def test_a_delegation_records_what_the_alias_resolved_to(
world: dict[str, Any],
) -> None:
# L9 eval history keyed on an alias alone would average across whatever
# that alias pointed at over time.
helper = Spy(
[call("write_file", path="helped.py", content="H = 1\n")], [hand_back()]
)
model = ScriptedModel(
RESTATE,
[
call("start_milestone", name="words"),
call("delegate", task="write helped.py"),
call("finish_milestone", summary="with help", final="yes"),
],
)
backed = helpers(helper, backing=lambda alias: f"anthropic/{alias}-backing")
agent, transport = loop_agent(world, model, helpers=backed)
agent.intake()
say(transport, world, "go")
[started] = [
e.event
for e in world["store"].transcript(item(world).id, "helper").read()
if isinstance(e.event, DelegationStarted)
]
assert started.backing_model == "anthropic/hlp-backing"
def test_a_failed_helpers_work_is_discarded(world: dict[str, Any]) -> None:
helper = Spy(
[call("write_file", path="half.py", content="half\n")],
nisse-dev/tests/test_loop_properties.py
+1 -1
View File
@@ -223,7 +223,7 @@ class Walk:
reader=self, # type: ignore[arg-type]
helpers=Helpers(
client=HelperModel(self.rng), # type: ignore[arg-type]
roles=RoleMap(executor="", planner="m", helper="h", expert="x"),
roles=RoleMap(driver="m", helper="h", expert="x"),
caps=Caps(turns=4),
),
bug_grace_s=600,
nisse-dev/tests/test_no_stranding.py
-238
View File
@@ -1,238 +0,0 @@
"""The orchestrator's resting-state invariant, checked under injected faults.
Working states (planning, executing) are only right while work runs. After
any message is handled the agent must be somewhere it can wait for the
operator, and some reply must lead on from there. The first live run found
four ways to break this one at a time; this drives a whole project with a
fault at each step instead.
"""
import io
from pathlib import Path
from typing import Any
import pytest
from nisse_dev.control import (
AWAIT_ARCH_SIGNOFF,
AWAIT_INPUT,
AWAIT_MILESTONE_SIGNOFF,
EXECUTING,
IDLE,
MILESTONE_PLANNING,
PLANNING_ARCH,
PR_OPEN,
AskHuman,
Orchestrator,
PlanningError,
SubmitPlan,
)
from nisse_dev.executor import ExecutorOutcome
from nisse_dev.gitea import GiteaError, PullRequest
from nisse_dev.oplog import OpLog
from nisse_dev.publish import PublishResult
from nisse_dev.state import StateStore
from nisse_dev.transport import InMemoryTransport
WORKING = {PLANNING_ARCH, MILESTONE_PLANNING, EXECUTING}
class Faults:
"""One fault, fired once, at the nth call to one site."""
def __init__(self, site: str, nth: int, fault: Any) -> None:
self._site, self._nth, self._fault = site, nth, fault
self._calls: dict[str, int] = {}
self.fired = False
def check(self, site: str) -> Any:
"""Raise or return the fault if this call is the one; else None."""
self._calls[site] = self._calls.get(site, 0) + 1
if self.fired or site != self._site or self._calls[site] != self._nth:
return None
self.fired = True
if isinstance(self._fault, Exception):
raise self._fault
return self._fault
class Planner:
def __init__(self, faults: Faults) -> None:
self._faults = faults
def plan(
self,
transcript: object,
gate: str,
project: str,
feedback: str | None = None,
answer: str | None = None,
) -> SubmitPlan | AskHuman:
return self._faults.check("planner") or SubmitPlan(f"{gate} plan")
class Executor:
def __init__(self, faults: Faults) -> None:
self._faults = faults
def run(self, transcript: object, milestone: int) -> ExecutorOutcome:
return self._faults.check("executor") or ExecutorOutcome("completed", "done")
class Publisher:
def __init__(self, faults: Faults) -> None:
self._faults = faults
self._opened = 0
def reconcilers(self) -> dict[str, Any]:
return {}
def publish(self, **kwargs: Any) -> PublishResult:
self._faults.check("publisher")
self._opened += 1
pr = PullRequest(number=self._opened, state="open", head="h", url="u")
return PublishResult(pr, f"opened PR #{self._opened}")
def next_reply(store: StateStore, used_project: bool) -> str:
"""What an operator following the prompts would say next."""
state = store.get_state()
assert state is not None
data = state.data
match state.state:
case "IDLE":
return "status" if used_project else "use demo"
case "AWAIT_ARCH_SIGNOFF":
return "approve"
case "AWAIT_MILESTONE_SIGNOFF":
if data["gate"]["approved"]:
return f"start milestone {data['milestone']}"
return "approve"
case "PR_OPEN":
return "next milestone" if data["milestone"] < 2 else "project complete"
case "AWAIT_INPUT":
return "continue"
raise AssertionError(f"stranded in {state.state}")
FAULTS = [
("planner", 1, RuntimeError("bug in planning")),
("planner", 1, PlanningError("planner returned 400")),
("planner", 1, AskHuman("which language?")),
("planner", 2, RuntimeError("bug in planning")),
("planner", 3, OSError("disk full")),
("executor", 1, RuntimeError("bug in the executor")),
("executor", 1, ExecutorOutcome("escalate", "stuck")),
("executor", 1, ExecutorOutcome("capped", "max turns")),
("executor", 2, ExecutorOutcome("stopped", "operator")),
("publisher", 1, RuntimeError("bug in publishing")),
("publisher", 1, GiteaError("gitea is down")),
("publisher", 2, RuntimeError("bug in publishing")),
]
@pytest.mark.parametrize(
("site", "nth", "fault"), FAULTS, ids=[f"{s}{n}-{f!r}" for s, n, f in FAULTS]
)
def test_a_fault_anywhere_never_strands_the_agent(
tmp_path: Path, site: str, nth: int, fault: Any
) -> None:
faults = Faults(site, nth, fault)
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
transport = InMemoryTransport()
Orchestrator(
store,
transport,
Planner(faults),
operators={"don"},
executor=Executor(faults),
publisher=Publisher(faults),
).start()
used_project = False
for _ in range(40):
reply = next_reply(store, used_project)
used_project = used_project or reply.startswith("use ")
transport.receive("don", reply)
state = store.get_state()
assert state is not None
assert state.state not in WORKING, (
f"after {reply!r}: stranded in {state.state}; "
f"last said: {transport.outbound[-1].text!r}"
)
if reply == "project complete":
break
else:
raise AssertionError("never reached project complete")
assert faults.fired
assert store.get_state().state == IDLE
def test_a_signoff_state_without_its_gate_offers_a_way_out(tmp_path: Path) -> None:
faults = Faults("none", 0, None)
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
transport = InMemoryTransport()
orch = Orchestrator(store, transport, Planner(faults), operators={"don"})
orch.start()
transport.receive("don", "use demo")
state = store.get_state()
store.transition(
AWAIT_ARCH_SIGNOFF,
cause="test",
project="demo",
data={**state.data, "gate": None},
)
orch._data["gate"] = None # as a restart would load it
transport.receive("don", "approve")
assert store.get_state().state == AWAIT_INPUT
transport.receive("don", "continue")
assert store.get_state().state == AWAIT_ARCH_SIGNOFF
class RaisingReviewExecutor:
def run(self, transcript: object, milestone: int) -> ExecutorOutcome:
raise RuntimeError("bug while addressing review")
def test_a_failing_review_response_keeps_pr_open_and_says_so(
tmp_path: Path,
) -> None:
faults = Faults("none", 0, None)
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
transport = InMemoryTransport()
orch = Orchestrator(
store,
transport,
Planner(faults),
operators={"don"},
gitea_operators={"don"},
executor=Executor(faults),
publisher=Publisher(faults),
)
orch.start()
for reply in ("use demo", "approve", "approve", "start milestone 1"):
transport.receive("don", reply)
assert store.get_state().state == PR_OPEN
orch._executor = RaisingReviewExecutor()
class Comment:
id, author, body = 1, "don", "add a test"
orch.handle_review_comment(Comment())
assert store.get_state().state == PR_OPEN
assert "could not handle a review comment" in transport.outbound[-1].text
transport.receive("don", "project complete")
assert store.get_state().state == IDLE
def test_the_stub_configuration_reaches_every_state(tmp_path: Path) -> None:
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
transport = InMemoryTransport()
Orchestrator(
store, transport, Planner(Faults("none", 0, None)), operators={"don"}
).start()
for reply in ("use demo", "approve", "approve", "start milestone 1"):
transport.receive("don", reply)
assert store.get_state().state == PR_OPEN
transport.receive("don", "next milestone")
assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF
nisse-dev/tests/test_oplog.py
+29 -19
View File
@@ -29,10 +29,21 @@ def events(stream: io.StringIO) -> list[dict]:
return [json.loads(line) for line in stream.getvalue().splitlines()]
def start_item(
store: StateStore, item_id: str, *, scope: str | None = None, cause: str
) -> None:
"""Open a work item, make it the active one and set it running."""
store.open_work_item(
item_id, title=item_id, scope=scope, position="IDLE", cause=cause
)
store.set_active(item_id, cause=cause)
store.update_work_item(item_id, position="RUNNING", data={}, cause=cause)
def test_every_event_carries_the_envelope(
store: StateStore, stream: io.StringIO
) -> None:
store.transition("PLANNING_ARCH", cause="operator command: use demo")
start_item(store, "demo-1", cause="issue assigned")
intent_id = store.begin_intent("push_branch", {}, cause="milestone execution")
store.commit_intent(intent_id, cause="push verified")
for event in events(stream):
@@ -40,16 +51,20 @@ def test_every_event_carries_the_envelope(
assert event["agent"] == "nisse-dev"
def test_transition_logs_from_to_and_cause(
def test_a_position_change_logs_from_to_and_cause(
store: StateStore, stream: io.StringIO
) -> None:
store.transition("PLANNING_ARCH", cause="operator command: use demo", project="d")
store.transition("AWAIT_ARCH_SIGNOFF", cause="plan submitted", project="d")
start_item(store, "d-1", scope="d", cause="issue assigned")
store.update_work_item("d-1", position="WAITING", data={}, cause="restated")
# Opening the work item logs its own events first (work_item.open, .switch).
first, second = [e for e in events(stream) if e["event"] == "state.transition"]
assert (first["from"], first["to"]) == ("IDLE", "PLANNING_ARCH")
assert (second["from"], second["to"]) == ("PLANNING_ARCH", "AWAIT_ARCH_SIGNOFF")
assert second["cause"] == "plan submitted"
assert (first["from"], first["to"]) == ("IDLE", "RUNNING")
assert (second["from"], second["to"], second["project"]) == (
"RUNNING",
"WAITING",
"d",
)
assert second["cause"] == "restated"
def test_intent_lifecycle_events(store: StateStore, stream: io.StringIO) -> None:
@@ -63,9 +78,9 @@ def test_intent_lifecycle_events(store: StateStore, stream: io.StringIO) -> None
def test_task_id_threads_through_events(store: StateStore, stream: io.StringIO) -> None:
store.transition("IDLE", cause="boot")
store.open_work_item("general", title="general", position="IDLE", cause="boot")
store.oplog.task_id = "task-42"
store.transition("PLANNING_ARCH", cause="operator command")
start_item(store, "task-42", cause="issue assigned")
before, *during = events(stream)
assert before["task_id"] is None
assert {e["task_id"] for e in during} == {"task-42"}
@@ -119,12 +134,7 @@ def test_literal_log_lines_for_human_readability(
monkeypatch.setattr("nisse_dev.oplog.datetime", _FrozenDatetime)
store.transition(
"PLANNING_ARCH",
cause="operator command: use demo",
project="demo",
data={"task_id": "demo-m2"},
)
start_item(store, "demo-m2", scope="demo", cause="issue assigned: demo#2")
store.oplog.task_id = "demo-m2"
intent_id = store.begin_intent(
"push_branch", {"branch": "milestone-2"}, cause="milestone 2 execution"
@@ -135,20 +145,20 @@ def test_literal_log_lines_for_human_readability(
(
'{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",'
' "event": "work_item.open", "task_id": null,'
' "cause": "operator command: use demo",'
' "cause": "issue assigned: demo#2",'
' "work_item": "demo-m2", "scope": "demo"}'
),
(
'{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",'
' "event": "work_item.switch", "task_id": null,'
' "cause": "operator command: use demo",'
' "cause": "issue assigned: demo#2",'
' "from": null, "to": "demo-m2"}'
),
(
'{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",'
' "event": "state.transition", "task_id": null,'
' "cause": "operator command: use demo",'
' "from": "IDLE", "to": "PLANNING_ARCH", "project": "demo"}'
' "cause": "issue assigned: demo#2",'
' "from": "IDLE", "to": "RUNNING", "project": "demo"}'
),
(
'{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",'
nisse-dev/tests/test_planner.py
-312
View File
@@ -1,312 +0,0 @@
import io
from pathlib import Path
from typing import Any
import pytest
from nisse_dev.canonical import (
AssistantText,
CanonicalEvent,
PlanSubmitted,
StoredEvent,
ToolCall,
UserMessage,
)
from nisse_dev.control import (
AWAIT_ARCH_SIGNOFF,
AWAIT_MILESTONE_SIGNOFF,
AskHuman,
Orchestrator,
PlanningError,
SubmitPlan,
)
from nisse_dev.llm import BudgetExhausted, RoleMap
from nisse_dev.oplog import OpLog
from nisse_dev.planner import PLANNER_TOOLS, LLMPlanningAgent
from nisse_dev.state import StateStore, Transcript
from nisse_dev.transport import InMemoryTransport
ROLES = RoleMap(
executor="local/qwen",
planner="anthropic/claude-opus-5",
helper="anthropic/claude-opus-5",
expert="anthropic/claude-opus-5",
)
class ScriptedClient:
"""Returns canned canonical events per call, recording what it was asked."""
def __init__(self, turns: list[list[CanonicalEvent]]) -> None:
self._turns = turns
self.calls: list[dict[str, Any]] = []
def complete(
self,
events: list[StoredEvent],
tools: list[dict[str, Any]] | None,
model: str,
system: str | None = None,
) -> list[CanonicalEvent]:
self.calls.append(
{"events": events, "tools": tools, "model": model, "system": system}
)
if not self._turns:
return []
return self._turns.pop(0)
def plan_call(content: str, gate: str = "arch") -> ToolCall:
return ToolCall(
call_id="call-1",
tool="submit_plan",
arguments={"type": gate, "content": content},
)
ITEM = "t" # the work item the unit tests plan in
def log(store: StateStore) -> Transcript:
return store.transcript(ITEM, "planner")
@pytest.fixture
def store(tmp_path: Path) -> StateStore:
return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
def test_submit_plan_tool_call_becomes_submit_plan(store: StateStore) -> None:
client = ScriptedClient([[plan_call("step 1: create src/app.py")]])
agent = LLMPlanningAgent(client, ROLES)
action = agent.plan(log(store), "arch", "demo")
assert action == SubmitPlan("step 1: create src/app.py")
def test_ask_human_tool_call_becomes_ask_human(store: StateStore) -> None:
client = ScriptedClient(
[
[
ToolCall(
call_id="call-1",
tool="ask_human",
arguments={"question": "postgres or sqlite?"},
)
]
]
)
action = LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
assert action == AskHuman("postgres or sqlite?")
def test_planner_role_and_tools_are_used(store: StateStore) -> None:
client = ScriptedClient([[plan_call("plan")]])
LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
call = client.calls[0]
assert call["model"] == "anthropic/claude-opus-5" # never the executor
assert [t["function"]["name"] for t in call["tools"]] == [
"submit_plan",
"ask_human",
]
assert "demo" in call["system"] and "arch" in call["system"]
def test_no_proceed_tool_is_offered() -> None:
names = {tool["function"]["name"] for tool in PLANNER_TOOLS}
assert names == {"submit_plan", "ask_human"} # no self-advancement path
def test_model_output_is_appended_to_the_log(store: StateStore) -> None:
client = ScriptedClient(
[[AssistantText(text="thinking out loud", model="m"), plan_call("the plan")]]
)
LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
kinds = [type(e.event).__name__ for e in log(store).read()]
assert kinds == ["AssistantText", "ToolCall", "ToolResult"]
def test_gate_tool_call_is_answered_in_the_log(store: StateStore) -> None:
# submit_plan ends the turn and has no natural result, but a tool call
# left dangling makes the next render invalid for the provider.
client = ScriptedClient([[plan_call("the plan")]])
LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
events = [e.event for e in log(store).read()]
assert [type(e).__name__ for e in events] == ["ToolCall", "ToolResult"]
assert events[1].call_id == events[0].call_id
assert "posted for approval" in events[1].result
def test_prose_only_turn_retries_once_then_raises(store: StateStore) -> None:
client = ScriptedClient(
[
[AssistantText(text="here's what I think", model="m")],
[AssistantText(text="still prose", model="m")],
]
)
with pytest.raises(PlanningError, match="no submit_plan or ask_human"):
LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
assert len(client.calls) == 2
assert "did not call a tool" in client.calls[1]["system"]
def test_retry_succeeds_after_prose(store: StateStore) -> None:
client = ScriptedClient(
[[AssistantText(text="prose", model="m")], [plan_call("the real plan")]]
)
action = LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
assert action == SubmitPlan("the real plan")
def test_malformed_tool_arguments_are_rejected(store: StateStore) -> None:
blank = ToolCall(call_id="c", tool="submit_plan", arguments={"content": " "})
client = ScriptedClient([[blank], [blank]])
with pytest.raises(PlanningError):
LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
def test_planner_sees_prior_transcript(store: StateStore) -> None:
store.transcript(ITEM, "operator").append(
UserMessage(text="use demo", source="transport:@don")
)
client = ScriptedClient([[plan_call("plan")]])
LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
seen = [getattr(e.event, "text", None) for e in client.calls[0]["events"]]
assert "[operator, Matrix] use demo" in seen
# --- orchestrator integration (still no network) -------------------------
def build(tmp_path: Path, client: ScriptedClient) -> tuple[Orchestrator, StateStore]:
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
transport = InMemoryTransport()
agent = LLMPlanningAgent(client, ROLES)
orch = Orchestrator(store, transport, agent, operators={"don"}, roles=ROLES)
orch.start()
return orch, store, transport
def test_use_project_produces_llm_plan_and_transcript(tmp_path: Path) -> None:
client = ScriptedClient([[plan_call("1. create app.py\n2. add a test")]])
_, store, transport = build(tmp_path, client)
transport.receive("don", "use demo")
request = transport.outbound[-1]
assert request.kind == "approval_request"
assert "create app.py" in request.text
# The work item's own events: `use demo` was said while idle, and is
# carried into the item it opened (D1).
item = store.active_item()
events = [e.event for e in store.transcript(item.id, "operator").read()]
kinds = [type(e).__name__ for e in events]
assert kinds == [
"UserMessage",
"Fact",
"ToolCall",
"ToolResult",
"PlanSubmitted",
"GateOpened",
]
submitted = next(e for e in events if isinstance(e, PlanSubmitted))
assert submitted.model == "anthropic/claude-opus-5" # pinned planner
def test_roles_pinned_at_task_start(tmp_path: Path) -> None:
client = ScriptedClient([[plan_call("plan")]])
_, store, transport = build(tmp_path, client)
transport.receive("don", "use demo")
state = store.get_state()
assert state is not None
assert state.data["roles"]["planner"] == "anthropic/claude-opus-5"
def test_planning_failure_gates_to_await_input(tmp_path: Path) -> None:
prose = [AssistantText(text="prose", model="m")]
client = ScriptedClient([prose, prose])
_, store, transport = build(tmp_path, client)
transport.receive("don", "use demo")
state = store.get_state()
assert state is not None
assert state.state == "AWAIT_INPUT"
assert transport.outbound[-1].kind == "question"
def test_operator_can_recover_from_a_planning_failure(tmp_path: Path) -> None:
# Regression: AWAIT_INPUT reached by any path other than ask_human used to
# record no pending item, so every operator reply was refused with "no
# pending question" and the agent was stuck for good.
prose = [AssistantText(text="prose", model="m")]
client = ScriptedClient([prose, prose, [plan_call("recovered plan")]])
_, store, transport = build(tmp_path, client)
transport.receive("don", "use demo")
assert store.get_state().state == "AWAIT_INPUT"
transport.receive("don", "try again, keep it short")
assert store.get_state().state == AWAIT_ARCH_SIGNOFF
assert "recovered plan" in transport.outbound[-1].text
def test_operator_can_recover_after_a_stopped_milestone(tmp_path: Path) -> None:
class StoppedExecutor:
def run(self, transcript: object, milestone: int) -> Any:
from nisse_dev.executor import ExecutorOutcome
return ExecutorOutcome("escalate", "tests failed twice")
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
transport = InMemoryTransport()
client = ScriptedClient([[plan_call("p1")], [plan_call("p2")]])
orch = Orchestrator(
store,
transport,
LLMPlanningAgent(client, ROLES),
operators={"don"},
roles=ROLES,
executor=StoppedExecutor(),
)
orch.start()
transport.receive("don", "use demo")
transport.receive("don", "approve")
transport.receive("don", "approve")
transport.receive("don", "start milestone 1")
assert store.get_state().state == "AWAIT_INPUT"
transport.receive("don", "ok, I fixed the dependency")
assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF
assert "start milestone 1" in transport.outbound[-1].text
def test_rejection_feedback_reaches_the_planner_via_the_log(
tmp_path: Path,
) -> None:
client = ScriptedClient([[plan_call("v1")], [plan_call("v2")]])
_, _store, transport = build(tmp_path, client)
transport.receive("don", "use demo")
transport.receive("don", "too vague, name the files")
replan_context = [e.event for e in client.calls[1]["events"]]
assert any(
isinstance(e, UserMessage) and "too vague" in e.text for e in replan_context
)
assert "v2" in transport.outbound[-1].text
def test_budget_exhaustion_is_not_retried(store: StateStore) -> None:
"""A budget will not reappear within a turn, so a retry buys nothing and
delays telling the operator."""
class BrokeClient:
def __init__(self) -> None:
self.calls = 0
def complete(self, *args: Any, **kwargs: Any):
self.calls += 1
raise BudgetExhausted("daily budget exhausted")
client = BrokeClient()
with pytest.raises(BudgetExhausted):
LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo")
assert client.calls == 1 # not two
nisse-dev/tests/test_poller.py
-132
View File
@@ -1,132 +0,0 @@
import io
import json
import time
import pytest
from nisse_dev.gitea import Comment, GiteaError
from nisse_dev.oplog import OpLog
from nisse_dev.poller import MAX_BACKOFF_S, CommentPoller
class FakeGitea:
def __init__(self, pages: list[list[Comment]] | None = None) -> None:
self.pages = pages or []
self.calls = 0
self.fail_times = 0
def list_comments(self, repo: str, index: int, since: str | None = None):
self.calls += 1
if self.fail_times > 0:
self.fail_times -= 1
raise GiteaError("gitea unreachable")
if not self.pages:
return []
return self.pages.pop(0)
def comment(cid: int, author: str = "don", body: str = "please fix") -> Comment:
return Comment(id=cid, author=author, body=body, created_at="t")
def build(client, **kw) -> tuple[CommentPoller, list[Comment]]:
seen: list[Comment] = []
poller = CommentPoller(client, "o/r", seen.append, **kw)
return poller, seen
def test_new_comments_are_delivered_once() -> None:
client = FakeGitea([[comment(1)], [comment(1), comment(2)]])
poller, seen = build(client)
poller.poll_once(1)
poller.poll_once(1)
assert [c.id for c in seen] == [1, 2] # id 1 not redelivered
def test_prime_suppresses_the_existing_backlog() -> None:
# Starting a watch must not replay feedback already addressed.
client = FakeGitea([[comment(1), comment(2)], [comment(1), comment(2), comment(3)]])
poller, seen = build(client)
poller.prime(1)
poller.poll_once(1)
assert [c.id for c in seen] == [3]
def test_own_comments_are_ignored() -> None:
client = FakeGitea([[comment(1, author="nisse-dev-testing"), comment(2, "don")]])
poller, seen = build(client, ignore_authors={"nisse-dev-testing"})
poller.poll_once(1)
assert [c.id for c in seen] == [2]
def test_ignored_comments_are_not_redelivered_later() -> None:
client = FakeGitea(
[
[comment(1, author="bot")],
[comment(1, author="bot"), comment(2, "don")],
]
)
poller, seen = build(client, ignore_authors={"bot"})
poller.poll_once(1)
poller.poll_once(1)
assert [c.id for c in seen] == [2]
def test_outage_does_not_raise_out_of_the_loop() -> None:
stream = io.StringIO()
client = FakeGitea([[comment(1)]])
client.fail_times = 2
poller, seen = build(
client,
oplog=OpLog("nisse-dev", stream),
interval_s=0.01,
initial_backoff_s=0.01,
)
poller.start(1)
deadline = time.monotonic() + 5
while not seen and time.monotonic() < deadline:
time.sleep(0.05)
poller.stop()
# It recovered and delivered rather than dying on the outage (L5: an
# outage delays a reaction, never fails a task).
assert [c.id for c in seen] == [1]
assert "gitea.poll_failed" in stream.getvalue()
def test_prime_survives_an_outage() -> None:
client = FakeGitea()
client.fail_times = 1
poller, _ = build(client)
poller.prime(1) # must not raise
def test_backoff_grows_then_caps() -> None:
client = FakeGitea()
client.fail_times = 50
stream = io.StringIO()
poller, _ = build(
client,
oplog=OpLog("nisse-dev", stream),
interval_s=0.001,
initial_backoff_s=0.001,
max_backoff_s=0.008,
)
poller.start(1)
time.sleep(0.3)
poller.stop()
delays = [json.loads(line)["retry_in_s"] for line in stream.getvalue().splitlines()]
assert delays[:4] == [0.001, 0.002, 0.004, 0.008]
assert max(delays) == 0.008 # capped, and retried indefinitely (L5)
assert MAX_BACKOFF_S <= 300.0 # the shipped cap is "a few minutes"
@pytest.mark.parametrize("author", ["don", "someone-else"])
def test_poller_does_not_filter_by_author_itself(author: str) -> None:
# Authorisation belongs to the orchestrator's allowlist, not here: the
# poller is pure intake.
client = FakeGitea([[comment(1, author=author)]])
poller, seen = build(client)
poller.poll_once(1)
assert len(seen) == 1
nisse-dev/tests/test_pr_open.py
-202
View File
@@ -1,202 +0,0 @@
"""The loop past EXECUTING (D4, D5): publish, review gate, then next milestone or
project completion."""
import io
from pathlib import Path
from typing import Any
import pytest
from nisse_dev.canonical import AssistantText, CanonicalEvent, StoredEvent, ToolCall
from nisse_dev.control import (
AWAIT_INPUT,
AWAIT_MILESTONE_SIGNOFF,
IDLE,
PR_OPEN,
Orchestrator,
)
from nisse_dev.executor import ExecutorOutcome
from nisse_dev.gitea import GiteaError, PullRequest
from nisse_dev.llm import RoleMap
from nisse_dev.oplog import OpLog
from nisse_dev.planner import LLMPlanningAgent
from nisse_dev.publish import PublishResult
from nisse_dev.state import StateStore
from nisse_dev.transport import InMemoryTransport
ROLES = RoleMap(
executor="llama3.2",
planner="haiku",
helper="haiku",
expert="haiku",
)
class ScriptedClient:
def __init__(self, plans: list[str]) -> None:
self._plans = plans
self.calls: list[dict[str, Any]] = []
def complete(
self,
events: list[StoredEvent],
tools: list[dict[str, Any]] | None,
model: str,
system: str | None = None,
) -> list[CanonicalEvent]:
self.calls.append({"system": system})
if not self._plans:
return [AssistantText(text="no more plans", model=model)]
return [
ToolCall(
call_id=f"c{len(self.calls)}",
tool="submit_plan",
arguments={"type": "x", "content": self._plans.pop(0)},
)
]
class DoneExecutor:
def run(self, transcript: object, milestone: int) -> ExecutorOutcome:
return ExecutorOutcome("completed", f"milestone {milestone} done")
class FakePublisher:
def __init__(
self, *, result: PublishResult | None = None, error: Exception | None = None
) -> None:
self._result = result or PublishResult(
PullRequest(number=7, state="open", head="h", url="https://g/pulls/7"),
"opened PR #7: https://g/pulls/7",
)
self._error = error
self.calls: list[dict[str, Any]] = []
def reconcilers(self) -> dict[str, Any]:
return {"open_pr": lambda intent: True}
def publish(self, **kwargs: Any) -> PublishResult:
self.calls.append(kwargs)
if self._error is not None:
raise self._error
return self._result
def build(
tmp_path: Path, publisher: Any, plans: list[str] | None = None
) -> tuple[StateStore, InMemoryTransport]:
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
transport = InMemoryTransport()
client = ScriptedClient(plans or ["arch plan", "m1 plan", "m2 plan"])
orch = Orchestrator(
store,
transport,
LLMPlanningAgent(client, ROLES),
operators={"don"},
roles=ROLES,
executor=DoneExecutor(),
publisher=publisher,
)
orch.start()
return store, transport
def run_to_pr(tmp_path: Path, publisher: Any) -> tuple[StateStore, InMemoryTransport]:
store, transport = build(tmp_path, publisher)
transport.receive("don", "use demo")
transport.receive("don", "approve") # arch
transport.receive("don", "approve") # milestone plan
transport.receive("don", "start milestone 1")
return store, transport
def test_completed_milestone_publishes_and_enters_pr_open(tmp_path: Path) -> None:
publisher = FakePublisher()
store, transport = run_to_pr(tmp_path, publisher)
assert store.get_state().state == PR_OPEN
assert publisher.calls[0]["milestone"] == 1
assert publisher.calls[0]["project"] == "demo"
assert "m1 plan" in publisher.calls[0]["plan"] # the approved plan travels
assert "PR #7" in transport.outbound[-1].text
def test_pr_open_records_the_pull_request(tmp_path: Path) -> None:
store, _ = run_to_pr(tmp_path, FakePublisher())
assert store.get_state().data["pr"]["number"] == 7
def test_next_milestone_replans_and_bumps_the_counter(tmp_path: Path) -> None:
store, transport = run_to_pr(tmp_path, FakePublisher())
transport.receive("don", "next milestone")
state = store.get_state()
assert state.state != PR_OPEN
assert state.data["milestone"] == 2
assert "m2 plan" in transport.outbound[-1].text
def test_project_complete_returns_to_idle(tmp_path: Path) -> None:
store, transport = run_to_pr(tmp_path, FakePublisher())
transport.receive("don", "project complete")
state = store.get_state()
assert state.state == IDLE
assert state.project is None
assert "project closed" in transport.outbound[-1].text
def test_unknown_command_in_pr_open_explains_the_options(tmp_path: Path) -> None:
store, transport = run_to_pr(tmp_path, FakePublisher())
transport.receive("don", "looks good!")
assert store.get_state().state == PR_OPEN
assert "next milestone" in transport.outbound[-1].text
def test_no_changes_gates_to_the_operator_without_entering_pr_open(
tmp_path: Path,
) -> None:
publisher = FakePublisher(result=PublishResult(None, "no changes to publish"))
store, transport = run_to_pr(tmp_path, publisher)
# Not left in EXECUTING, where every message is answered "busy".
assert store.get_state().state == AWAIT_INPUT
assert "no changes to publish" in transport.outbound[-1].text
transport.receive("don", "ok")
assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF
assert "start milestone 1" in transport.outbound[-1].text
def test_publish_failure_gates_to_the_operator(tmp_path: Path) -> None:
publisher = FakePublisher(error=GiteaError("gitea is down"))
store, transport = run_to_pr(tmp_path, publisher)
assert store.get_state().state == AWAIT_INPUT
assert "could not publish" in transport.outbound[-1].text
# And the operator can still get back out (the deadlock regression).
transport.receive("don", "gitea is back")
assert store.get_state().state != AWAIT_INPUT
def test_publisher_reconciler_is_registered_at_boot(tmp_path: Path) -> None:
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
store.begin_intent("open_pr", {"branch": "b"}, cause="crash")
transport = InMemoryTransport()
orch = Orchestrator(
store,
transport,
LLMPlanningAgent(ScriptedClient([]), ROLES),
operators={"don"},
roles=ROLES,
publisher=FakePublisher(),
)
orch.start()
# The publisher's reconciler answered "already opened", so boot resumed
# instead of gating on an unknown intent kind.
assert store.pending_intents() == []
assert store.get_state().state != AWAIT_INPUT
@pytest.mark.parametrize("command", ["next milestone", "project complete"])
def test_pr_open_commands_are_operator_only(tmp_path: Path, command: str) -> None:
store, transport = run_to_pr(tmp_path, FakePublisher())
transport.receive("mallory", command)
assert store.get_state().state == PR_OPEN
nisse-dev/tests/test_pr_open_handling.py
-250
View File
@@ -1,250 +0,0 @@
"""Gitea threads across restarts and interruptions (#58, #59).
Uses the real CommentPoller against a fake Gitea, with its thread replaced
by explicit polls, so the cursor logic under test is the real one.
"""
import io
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from nisse_dev.control import (
AWAIT_ARCH_SIGNOFF,
AWAIT_INPUT,
AWAIT_MILESTONE_SIGNOFF,
MILESTONE_PLANNING,
PR_OPEN,
Orchestrator,
StubAgent,
)
from nisse_dev.executor import ExecutorOutcome
from nisse_dev.gitea import Comment, PullRequest
from nisse_dev.oplog import OpLog
from nisse_dev.poller import CommentPoller
from nisse_dev.publish import PublishResult
from nisse_dev.state import StateStore
from nisse_dev.transport import InMemoryTransport
AGENT = "nisse-bot"
PLAN_ISSUE = 10
PR = 20
class FakeGitea:
"""Comment threads by issue number; IDs grow across all threads."""
def __init__(self) -> None:
self.threads: dict[int, list[Comment]] = {}
self._next_id = 100
def add(self, index: int, author: str, body: str) -> int:
self._next_id += 1
self.threads.setdefault(index, []).append(
Comment(id=self._next_id, author=author, body=body, created_at="t")
)
return self._next_id
def list_comments(self, repo: str, index: int, since: str | None = None) -> list:
return list(self.threads.get(index, []))
@dataclass
class Posted:
comment_id: int
class Board:
def __init__(self, gitea: FakeGitea) -> None:
self._gitea = gitea
def open_project_issue(self, project: str, task_id: str) -> int:
return PLAN_ISSUE
def post_plan(self, issue: int, gate: str, rev: int, plan: str) -> Posted:
return Posted(self._gitea.add(issue, AGENT, f"{gate} rev-{rev}: {plan}"))
def echo(self, issue: int, text: str) -> None:
self._gitea.add(issue, AGENT, text)
class Executor:
def __init__(self) -> None:
self.runs = 0
self.outcome = ExecutorOutcome("completed", "done")
def run(self, transcript: object, milestone: int) -> ExecutorOutcome:
self.runs += 1
return self.outcome
class Publisher:
def __init__(self) -> None:
self.followups = 0
def reconcilers(self) -> dict[str, Any]:
return {}
def publish(self, **kwargs: Any) -> PublishResult:
return PublishResult(
PullRequest(number=PR, state="open", head="h", url="u"), f"opened PR #{PR}"
)
def push_followup(self, **kwargs: Any) -> str | None:
self.followups += 1
return "abcdef1234"
def comment(self, index: int, body: str) -> None:
pass
class Agent:
"""One orchestrator process: a restart builds a new one on the same
store and the same Gitea."""
def __init__(self, tmp_path: Path, gitea: FakeGitea, executor: Executor) -> None:
self.store = StateStore(
tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())
)
self.transport = InMemoryTransport()
self.pollers: dict[int, CommentPoller] = {}
self.publisher = Publisher()
def factory(handler: Any) -> CommentPoller:
poller = CommentPoller(gitea, "o/r", handler, ignore_authors=[AGENT])
agent = self
def start(index: int) -> None: # no thread: tests poll explicitly
agent.pollers[index] = poller
poller.start = start # type: ignore[method-assign]
poller.stop = lambda: None # type: ignore[method-assign]
return poller
self.orch = Orchestrator(
self.store,
self.transport,
StubAgent(),
operators={"don"},
gitea_operators={"don"},
board=Board(gitea),
executor=executor,
publisher=self.publisher,
poller_factory=factory,
)
self.orch.start()
@property
def state(self) -> str:
state = self.store.get_state()
assert state is not None
return state.state
def poll(self, index: int) -> list[Comment]:
return self.pollers[index].poll_once(index)
def to_pr_open(agent: Agent) -> None:
for reply in ("use demo", "approve", "approve", "start milestone 1"):
agent.transport.receive("don", reply)
assert agent.state == PR_OPEN
def test_an_approval_posted_while_down_is_applied_after_a_restart(
tmp_path: Path,
) -> None:
gitea = FakeGitea()
agent = Agent(tmp_path, gitea, Executor())
agent.transport.receive("don", "use demo")
assert agent.state == AWAIT_ARCH_SIGNOFF
gitea.add(PLAN_ISSUE, "don", "approve") # while the agent is down
agent = Agent(tmp_path, gitea, Executor())
[delivered] = agent.poll(PLAN_ISSUE)
assert delivered.body == "approve"
assert agent.state == AWAIT_MILESTONE_SIGNOFF
def test_a_handled_approval_is_not_replayed_after_a_restart(tmp_path: Path) -> None:
gitea = FakeGitea()
agent = Agent(tmp_path, gitea, Executor())
agent.transport.receive("don", "use demo")
gitea.add(PLAN_ISSUE, "don", "approve")
agent.poll(PLAN_ISSUE) # arch approved; the milestone plan is posted
assert agent.state == AWAIT_MILESTONE_SIGNOFF
agent = Agent(tmp_path, gitea, Executor())
assert agent.poll(PLAN_ISSUE) == []
assert agent.state == AWAIT_MILESTONE_SIGNOFF
def test_a_new_revision_ignores_an_approval_posted_before_it(tmp_path: Path) -> None:
gitea = FakeGitea()
agent = Agent(tmp_path, gitea, Executor())
agent.transport.receive("don", "use demo")
agent.transport.receive("don", "needs tests") # rejects: rev-2 is posted
assert agent.state == AWAIT_ARCH_SIGNOFF
stale = gitea.add(PLAN_ISSUE, "don", "approve")
agent.transport.receive("don", "and docs") # rev-3 posted after it
assert all(c.id != stale for c in agent.poll(PLAN_ISSUE))
assert agent.state == AWAIT_ARCH_SIGNOFF
def test_data_from_before_cursors_skips_old_comments_on_restart(
tmp_path: Path,
) -> None:
gitea = FakeGitea()
agent = Agent(tmp_path, gitea, Executor())
agent.transport.receive("don", "use demo")
gitea.add(PLAN_ISSUE, "don", "approve") # an old approval, say of rev-1
state = agent.store.get_state()
state.data["board"].pop("last_comment") # as saved before this change
agent.store.transition(
state.state, cause="test", project=state.project, data=state.data
)
agent = Agent(tmp_path, gitea, Executor())
assert agent.poll(PLAN_ISSUE) == []
assert agent.state == AWAIT_ARCH_SIGNOFF
def test_a_restart_in_pr_open_resumes_and_catches_up_on_review(
tmp_path: Path,
) -> None:
gitea = FakeGitea()
agent = Agent(tmp_path, gitea, Executor())
to_pr_open(agent)
gitea.add(PR, "don", "please add a test") # while the agent is down
agent = Agent(tmp_path, gitea, Executor())
assert agent.state == PR_OPEN # no restart question
assert "restarted with PR #20 open" in agent.transport.outbound[-1].text
[delivered] = agent.poll(PR)
assert delivered.body == "please add a test"
assert agent.publisher.followups == 1
assert agent.state == PR_OPEN
def test_a_failed_review_response_returns_to_the_open_pr(tmp_path: Path) -> None:
gitea = FakeGitea()
executor = Executor()
agent = Agent(tmp_path, gitea, executor)
to_pr_open(agent)
executor.outcome = ExecutorOutcome("escalate", "stuck")
gitea.add(PR, "don", "please add a test")
agent.poll(PR)
assert agent.state == AWAIT_INPUT
# A comment while it waits is not lost: back in PR_OPEN it is delivered.
executor.outcome = ExecutorOutcome("completed", "done")
gitea.add(PR, "don", "try again")
agent.poll(PR)
assert agent.state == AWAIT_INPUT
agent.transport.receive("don", "ok")
assert agent.state == PR_OPEN
[delivered] = agent.poll(PR)
assert delivered.body == "try again"
assert agent.publisher.followups == 1
agent.transport.receive("don", "next milestone")
assert agent.state in (MILESTONE_PLANNING, AWAIT_MILESTONE_SIGNOFF)
nisse-dev/tests/test_publish.py
-289
View File
@@ -1,289 +0,0 @@
import io
import subprocess
from pathlib import Path
from typing import Any
import pytest
from nisse_dev.execution import SSHHost
from nisse_dev.git import GitRepo, HTTPSToken
from nisse_dev.gitea import GiteaError, PullRequest
from nisse_dev.oplog import OpLog
from nisse_dev.publish import OPEN_PR_INTENT, MilestonePublisher
from nisse_dev.state import StateStore
# The build VM for every test here: the session's throwaway sshd.
HOST: SSHHost
@pytest.fixture(autouse=True)
def _build_host(ssh_host: SSHHost) -> None:
global HOST
HOST = ssh_host
class FakeGitea:
def __init__(self, *, fail: bool = False) -> None:
self.opened: list[dict[str, Any]] = []
self.existing: list[str] = []
self.closed: set[str] = set()
self._fail = fail
def open_pull_request(
self, repo: str, *, head: str, base: str, title: str, body: str
) -> PullRequest:
if self._fail:
raise GiteaError("gitea is down")
self.opened.append(
{"repo": repo, "head": head, "base": base, "title": title, "body": body}
)
self.existing.append(head)
return PullRequest(
number=len(self.opened), state="open", head=head, url=f"u/{head}"
)
def find_pull_request(self, repo: str, head: str) -> PullRequest | None:
if head in self.existing:
state = "closed" if head in self.closed else "open"
return PullRequest(number=1, state=state, head=head, url="u")
return None
class FakeGit:
"""A real repository whose origin/main is a local ref; pushes are
recorded, not sent."""
def __init__(
self, root: Path, *, changes: bool = True, committed: bool = False
) -> None:
self.repo = GitRepo(HOST, root)
self.repo.init()
self.repo.configure_identity("bot", "bot@example.test")
(root / "README.md").write_text("sandbox")
self.repo.commit_all("Initial commit")
git(root, "update-ref", "refs/remotes/origin/main", "HEAD")
if changes:
(root / "work.txt").write_text("done")
if committed:
# The executor committing its own work, as on the first live run.
self.repo.commit_all("Add the work")
self.pushed: list[str] = []
self.remote: dict[str, str] = {}
def head_sha(self) -> str:
return self.repo.head_sha()
def commits_since(self, ref: str) -> int:
return self.repo.commits_since(ref)
def remote_sha(
self, remote_url: str, branch: str, *, credential: object = None
) -> str | None:
return self.remote.get(branch)
def current_branch(self) -> str:
return self.repo.current_branch()
def checkout_new_branch(self, name: str) -> None:
self.repo.checkout_new_branch(name)
def commit_all(self, message: str) -> str | None:
return self.repo.commit_all(message)
def left_out(self) -> list[str]:
return self.repo.left_out()
def push(self, remote_url: str, branch: str, *, credential: object = None) -> None:
self.pushed.append(branch)
self.remote[branch] = self.repo.head_sha()
def git(root: Path, *args: str) -> None:
subprocess.run(["git", *args], cwd=root, check=True, capture_output=True)
@pytest.fixture
def store(tmp_path: Path) -> StateStore:
return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
def build(store: StateStore, git: Any, client: Any) -> MilestonePublisher:
return MilestonePublisher(
store,
client,
"o/sandbox",
git,
remote_url="ssh://git@gitea.test/o/sandbox.git",
push_credential=HTTPSToken("pusher", "tok"),
)
def test_publish_commits_pushes_and_opens_a_pr(
store: StateStore, tmp_path: Path
) -> None:
git = FakeGit(tmp_path / "work")
client = FakeGitea()
result = build(store, git, client).publish(
task_id="demo-abc", milestone=2, project="demo", plan="the plan"
)
assert git.pushed == ["demo-abc-milestone-2"]
assert client.opened[0]["head"] == "demo-abc-milestone-2"
assert client.opened[0]["base"] == "main"
assert "the plan" in client.opened[0]["body"]
assert result.pull_request is not None
assert "opened PR #1" in result.note
def test_a_branch_named_after_the_task_does_not_block_publishing(
store: StateStore, tmp_path: Path
) -> None:
# Live run 3: the executor ran `git checkout -b <task id>` itself. With
# "<task>/milestone-1" as the harness's branch, git refused it, since a
# branch name can't also be a directory of other branch names.
git = FakeGit(tmp_path / "work")
git.repo.checkout_new_branch("demo-abc")
result = build(store, git, FakeGitea()).publish(
task_id="demo-abc", milestone=1, project="demo", plan="p"
)
assert result.pull_request is not None
assert git.pushed == ["demo-abc-milestone-1"]
def test_left_out_artifacts_are_reported_not_published(
store: StateStore, tmp_path: Path
) -> None:
git = FakeGit(tmp_path / "work")
(tmp_path / "work" / "__pycache__").mkdir()
(tmp_path / "work" / "__pycache__" / "work.cpython-310.pyc").write_bytes(b"x")
client = FakeGitea()
result = build(store, git, client).publish(
task_id="t", milestone=1, project="demo", plan="p"
)
assert "left out 1 build artifact(s): __pycache__/work.cpython-310.pyc" in (
result.note
)
assert "## Left out" in client.opened[0]["body"]
assert "`__pycache__/work.cpython-310.pyc`" in client.opened[0]["body"]
def test_intent_is_recorded_then_committed(store: StateStore, tmp_path: Path) -> None:
build(store, FakeGit(tmp_path / "work"), FakeGitea()).publish(
task_id="t", milestone=1, project="demo", plan="p"
)
# Nothing left pending: the intent opened before the push was settled
# once the PR existed.
assert store.pending_intents() == []
def test_failure_after_intent_leaves_it_pending_for_reconciliation(
store: StateStore, tmp_path: Path
) -> None:
publisher = build(store, FakeGit(tmp_path / "work"), FakeGitea(fail=True))
with pytest.raises(GiteaError):
publisher.publish(task_id="t", milestone=1, project="demo", plan="p")
[pending] = store.pending_intents()
assert pending.kind == OPEN_PR_INTENT
assert pending.payload["branch"] == "t-milestone-1"
assert pending.payload["sha"]
def test_nothing_to_commit_is_reported_not_raised(
store: StateStore, tmp_path: Path
) -> None:
git = FakeGit(tmp_path / "work", changes=False)
client = FakeGitea()
result = build(store, git, client).publish(
task_id="t", milestone=1, project="demo", plan="p"
)
assert result.pull_request is None
assert result.note == "no changes to publish"
assert client.opened == []
# No externally visible act was attempted, so no intent was opened.
assert store.pending_intents() == []
def test_branch_is_namespaced_by_task(store: StateStore, tmp_path: Path) -> None:
publisher = build(store, FakeGit(tmp_path / "work"), FakeGitea())
assert publisher.branch_for("greet-9f2", 3) == "greet-9f2-milestone-3"
def test_reconciler_settles_an_intent_left_by_a_crash(
store: StateStore, tmp_path: Path
) -> None:
# Simulate the dangerous window: the PR was opened, but the process died
# before the intent could be committed. On the next boot the reconciler
# must say "already done" so the milestone is not published twice.
client = FakeGitea()
publisher = build(store, FakeGit(tmp_path / "work"), client)
intent_id = store.begin_intent(
OPEN_PR_INTENT, {"branch": "t-milestone-1"}, cause="crash simulation"
)
client.existing.append("t-milestone-1")
reconcilers = publisher.reconcilers()
assert set(reconcilers) == {OPEN_PR_INTENT}
[pending] = store.pending_intents()
assert reconcilers[OPEN_PR_INTENT](pending) is True
store.commit_intent(intent_id, cause="verified")
assert client.opened == [] # nothing re-published
def test_work_the_executor_committed_itself_is_published(
store: StateStore, tmp_path: Path
) -> None:
git = FakeGit(tmp_path / "work", committed=True)
client = FakeGitea()
result = build(store, git, client).publish(
task_id="t", milestone=1, project="demo", plan="p"
)
assert result.pull_request is not None
assert git.pushed == ["t-milestone-1"]
assert git.remote["t-milestone-1"] == git.head_sha()
def test_a_followup_the_executor_committed_itself_is_pushed(
store: StateStore, tmp_path: Path
) -> None:
git = FakeGit(tmp_path / "work")
publisher = build(store, git, FakeGitea())
publisher.publish(task_id="t", milestone=1, project="demo", plan="p")
(tmp_path / "work" / "work.txt").write_text("fixed")
git.repo.commit_all("Address review")
sha = publisher.push_followup(task_id="t", milestone=1, message="m")
assert sha == git.head_sha()
assert git.pushed == ["t-milestone-1", "t-milestone-1"]
# Nothing new since that push: no second push, and it says so.
assert publisher.push_followup(task_id="t", milestone=1, message="m") is None
assert len(git.pushed) == 2
def test_publishing_again_adopts_the_open_pr_instead_of_opening_another(
store: StateStore, tmp_path: Path
) -> None:
# E.g. Gitea opened the PR but the response was lost, and the milestone
# was published again: the PR is adopted and updated, never duplicated.
git = FakeGit(tmp_path / "work")
client = FakeGitea()
publisher = build(store, git, client)
publisher.publish(task_id="t", milestone=1, project="demo", plan="p")
(tmp_path / "work" / "more.txt").write_text("more")
result = publisher.publish(task_id="t", milestone=1, project="demo", plan="p")
assert len(client.opened) == 1
assert result.pull_request is not None and result.pull_request.number == 1
assert "already open" in result.note
assert git.pushed == ["t-milestone-1", "t-milestone-1"] # updated it
assert store.pending_intents() == []
def test_a_closed_pr_is_not_adopted(store: StateStore, tmp_path: Path) -> None:
git = FakeGit(tmp_path / "work")
client = FakeGitea()
client.existing.append("t-milestone-1")
client.closed.add("t-milestone-1")
publisher = build(store, git, client)
assert publisher.published_pull_request("t", 1) is None
publisher.publish(task_id="t", milestone=1, project="demo", plan="p")
assert len(client.opened) == 1 # the operator closed it; a new one opens
nisse-dev/tests/test_quint_counterexamples.py
-400
View File
@@ -1,400 +0,0 @@
"""The counterexamples found by specs/orchestrator.qnt, replayed against the
real orchestrator. Each names the scenario in specs/orchestrator_scenarios.qnt
it reproduces; all seven are fixed, and the tests stay as regressions.
The model is a hand-written reading of control.py; reproducing each trace
here is what shows the flaw is in the design, not in the model.
"""
import io
from pathlib import Path
from typing import Any
import pytest
from nisse_dev.control import (
AWAIT_INPUT,
AWAIT_MILESTONE_SIGNOFF,
EXECUTING,
IDLE,
MILESTONE_PLANNING,
PR_OPEN,
Orchestrator,
StubAgent,
)
from nisse_dev.executor import ExecutorOutcome
from nisse_dev.gitea import Comment, GiteaError, PullRequest
from nisse_dev.oplog import OpLog
from nisse_dev.poller import CommentPoller
from nisse_dev.publish import PublishResult
from nisse_dev.state import StateStore
from nisse_dev.transport import InMemoryTransport
from nisse_dev.worker import SerialWorker
AGENT = "nisse-bot"
class QueueingWorker(SerialWorker):
"""Holds submitted jobs until released, so a test can queue two
messages before either runs, as a real worker does under load."""
def __init__(self) -> None:
super().__init__()
self.held: list[Any] = []
self.holding = True
def submit(self, job: Any) -> None:
if self.holding:
self.held.append(job)
else:
job()
def call(self, job: Any, timeout: float | None = None) -> Any:
return job()
@property
def busy(self) -> bool:
return False
def release(self) -> None:
self.holding = False
jobs, self.held = self.held, []
for job in jobs:
job()
class FakeGitea:
def __init__(self) -> None:
self.threads: dict[int, list[Comment]] = {}
self.prs: set[str] = set()
self._next = 100
def add(self, index: int, author: str, body: str) -> int:
self._next += 1
self.threads.setdefault(index, []).append(
Comment(id=self._next, author=author, body=body, created_at="t")
)
return self._next
def list_comments(self, repo: str, index: int, since: str | None = None) -> list:
return list(self.threads.get(index, []))
class Board:
def __init__(self, gitea: FakeGitea) -> None:
self._gitea = gitea
def open_project_issue(self, project: str, task_id: str) -> int:
return 10
def post_plan(self, issue: int, gate: str, rev: int, plan: str) -> Any:
class Posted:
comment_id = self._gitea.add(issue, AGENT, f"{gate} rev-{rev}")
return Posted()
def echo(self, issue: int, text: str) -> None:
pass
def close(self, issue: int, note: str) -> None:
pass
class Executor:
def __init__(self) -> None:
self.outcome = ExecutorOutcome("completed", "done")
def run(self, transcript: object, milestone: int) -> ExecutorOutcome:
return self.outcome
class Publisher:
"""Opens one PR per branch, as Gitea does; a second attempt fails."""
def __init__(self, gitea: FakeGitea, store: StateStore) -> None:
self._gitea = gitea
self._store = store
self.crash_after_opening = False
self.attempts = 0
def reconcilers(self) -> dict[str, Any]:
return {"open_pr": lambda intent: intent.payload["branch"] in self._gitea.prs}
def publish(self, **kwargs: Any) -> PublishResult:
branch = f"{kwargs['task_id']}-milestone-{kwargs['milestone']}"
self.attempts += 1
intent = self._store.begin_intent("open_pr", {"branch": branch}, cause="t")
if branch in self._gitea.prs:
raise GiteaError("pull request already exists for these targets")
self._gitea.prs.add(branch)
if self.crash_after_opening:
raise SystemExit("crash") # the process dies here
self._store.commit_intent(intent, cause="opened")
return PublishResult(
PullRequest(number=20, state="open", head=branch, url="u"), "opened PR #20"
)
def published_pull_request(self, task_id: str, milestone: int) -> Any:
branch = f"{task_id}-milestone-{milestone}"
if branch in self._gitea.prs:
return PullRequest(number=20, state="open", head=branch, url="u")
return None
def push_followup(self, **kwargs: Any) -> str | None:
return "abc12345"
def comment(self, index: int, body: str) -> None:
pass
class Agent:
def __init__(self, tmp_path: Path, gitea: FakeGitea, *, worker: Any = None) -> None:
self.store = StateStore(
tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())
)
self.transport = InMemoryTransport()
self.gitea = gitea
self.executor = Executor()
self.publisher = Publisher(gitea, self.store)
self.pollers: dict[int, CommentPoller] = {}
def factory(handler: Any) -> CommentPoller:
poller = CommentPoller(gitea, "o/r", handler, ignore_authors=[AGENT])
def start(index: int) -> None:
self.pollers[index] = poller
poller.start = start # type: ignore[method-assign]
poller.stop = lambda: None # type: ignore[method-assign]
return poller
self.orch = Orchestrator(
self.store,
self.transport,
StubAgent(),
operators={"don"},
gitea_operators={"don"},
board=Board(gitea),
executor=self.executor,
publisher=self.publisher,
poller_factory=factory,
**({"worker": worker} if worker else {}),
)
self.orch.start()
@property
def state(self) -> str:
return self.store.get_state().state
def gate(self) -> dict:
return self.store.get_state().data.get("gate") or {}
def say(self, *texts: str) -> None:
for text in texts:
self.transport.receive("don", text)
def test_double_approve_approves_an_unseen_plan(tmp_path: Path) -> None:
"""doubleApproveApprovesAnUnseenPlanTest"""
worker = QueueingWorker()
agent = Agent(tmp_path, FakeGitea(), worker=worker)
worker.holding = False
agent.say("use demo") # arch rev-1 is up
worker.holding = True
agent.say("approve", "approve") # both queued, looking at arch rev-1
worker.release()
# The second approval must not approve the milestone plan, never seen.
assert not agent.gate().get("approved")
def test_named_revision_approves_the_wrong_gate(tmp_path: Path) -> None:
"""namedRevisionApprovesTheWrongGateTest"""
worker = QueueingWorker()
agent = Agent(tmp_path, FakeGitea(), worker=worker)
worker.holding = False
agent.say("use demo")
worker.holding = True
agent.say("approve", "approve rev-1") # "rev-1" meant the architecture
worker.release()
assert not agent.gate().get("approved")
def test_approval_on_both_channels_approves_the_next_plan(tmp_path: Path) -> None:
"""approvalOnBothChannelsApprovesTheNextPlanTest"""
gitea = FakeGitea()
worker = QueueingWorker()
agent = Agent(tmp_path, gitea, worker=worker)
worker.holding = False
agent.say("use demo")
worker.holding = True
agent.say("approve") # Matrix: queued
gitea.add(10, "don", "approve") # and on the issue, about the same plan
# The poller delivers it; it waits behind the Matrix approval.
delivered = agent.pollers[10]
worker.held.append(lambda: delivered.poll_once(10))
worker.release()
assert agent.state == AWAIT_MILESTONE_SIGNOFF
assert not agent.gate().get("approved")
def test_an_afterthought_rejects_the_next_plan(tmp_path: Path) -> None:
"""anAfterthoughtRejectsTheNextPlanTest"""
worker = QueueingWorker()
agent = Agent(tmp_path, FakeGitea(), worker=worker)
worker.holding = False
agent.say("use demo")
worker.holding = True
agent.say("approve", "also, keep it in one file")
worker.release()
# The note was about the architecture; the milestone plan was not
# rejected by it.
assert agent.state == AWAIT_MILESTONE_SIGNOFF
assert agent.gate().get("rev") == 2 # arch was rev-1; not re-planned
def test_crash_after_opening_the_pr_publishes_again(tmp_path: Path) -> None:
"""crashAfterOpeningThePrPublishesAgainTest"""
gitea = FakeGitea()
agent = Agent(tmp_path, gitea)
agent.say("use demo", "approve", "approve")
agent.publisher.crash_after_opening = True
with pytest.raises(SystemExit):
agent.say("start milestone 1")
assert agent.state == EXECUTING # the PR exists; PR_OPEN was never recorded
agent = Agent(tmp_path, gitea) # restart: the intent is verified
# There is a PR for this milestone: the agent adopts it on boot and ends
# up watching it, never trying to open another.
assert agent.state == PR_OPEN
assert "PR #20 is open" in agent.transport.outbound[-1].text
agent.say("continue", "start milestone 1") # no longer means anything
assert agent.publisher.attempts == 0
assert agent.state == PR_OPEN
def test_crash_during_a_review_loses_the_comment(tmp_path: Path) -> None:
"""crashDuringAReviewLosesTheCommentTest"""
gitea = FakeGitea()
agent = Agent(tmp_path, gitea)
agent.say("use demo", "approve", "approve", "start milestone 1")
assert agent.state == PR_OPEN
gitea.add(20, "don", "please add a test")
class Crash(BaseException):
pass
def dies(transcript: object, milestone: int) -> ExecutorOutcome:
raise Crash() # the process dies mid-review
agent.executor.run = dies # type: ignore[method-assign]
with pytest.raises(Crash):
agent.pollers[20].poll_once(20)
agent = Agent(tmp_path, gitea) # restart
assert agent.state == PR_OPEN
# The comment was taken but never answered: after the restart the
# operator is told, and it is delivered and answered again.
assert "answering it again" in agent.transport.outbound[-1].text
[delivered] = agent.pollers[20].poll_once(20)
assert delivered.body == "please add a test"
assert "pushed" in agent.transport.outbound[-1].text
assert "reviewing" not in agent.store.get_state().data["pr"]
def test_project_complete_leaves_the_plan_issue_open(tmp_path: Path) -> None:
"""projectCompleteLeavesThePlanIssueOpenTest"""
closed: list[int] = []
gitea = FakeGitea()
agent = Agent(tmp_path, gitea)
agent.orch._board.close = lambda issue, note: closed.append(issue) # type: ignore[attr-defined]
agent.say("use demo", "approve", "approve", "start milestone 1", "project complete")
assert agent.state == IDLE
assert closed == [10]
def test_the_harness_reaches_pr_open_without_faults(tmp_path: Path) -> None:
"""Guard for the harness itself: with nothing queued or crashed, the
same flow is correct."""
agent = Agent(tmp_path, FakeGitea())
agent.say("use demo", "approve")
assert agent.state == AWAIT_MILESTONE_SIGNOFF
agent.say("approve", "start milestone 1")
assert agent.state == PR_OPEN
assert agent.publisher.attempts == 1
_ = (AWAIT_INPUT, MILESTONE_PLANNING)
def test_boot_asks_as_before_when_gitea_cannot_say(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
gitea = FakeGitea()
agent = Agent(tmp_path, gitea)
agent.say("use demo", "approve", "approve")
agent.publisher.crash_after_opening = True
with pytest.raises(SystemExit):
agent.say("start milestone 1")
def unreachable(self: Publisher, task_id: str, milestone: int) -> Any:
raise GiteaError("gitea is down")
monkeypatch.setattr(Publisher, "published_pull_request", unreachable)
agent = Agent(tmp_path, gitea)
assert agent.state == AWAIT_INPUT
assert "restarted mid-EXECUTING" in agent.transport.outbound[-1].text
class Crash(BaseException):
"""The process dying: no handler runs."""
def crash_mid_review(agent: Agent, gitea: FakeGitea) -> None:
def dies(transcript: object, milestone: int) -> ExecutorOutcome:
raise Crash()
agent.executor.run = dies # type: ignore[method-assign]
with pytest.raises(Crash):
agent.pollers[20].poll_once(20)
def test_a_second_crash_while_answering_again_still_redelivers(tmp_path: Path) -> None:
gitea = FakeGitea()
agent = Agent(tmp_path, gitea)
agent.say("use demo", "approve", "approve", "start milestone 1")
gitea.add(20, "don", "please add a test")
crash_mid_review(agent, gitea)
agent = Agent(tmp_path, gitea)
crash_mid_review(agent, gitea) # dies again while answering it again
agent = Agent(tmp_path, gitea)
[delivered] = agent.pollers[20].poll_once(20)
assert delivered.body == "please add a test"
def test_an_executor_error_answers_the_comment_so_it_is_not_redelivered(
tmp_path: Path,
) -> None:
gitea = FakeGitea()
agent = Agent(tmp_path, gitea)
agent.say("use demo", "approve", "approve", "start milestone 1")
gitea.add(20, "don", "please add a test")
def fails(transcript: object, milestone: int) -> ExecutorOutcome:
raise RuntimeError("bug in the executor")
agent.executor.run = fails # type: ignore[method-assign]
agent.pollers[20].poll_once(20)
assert "could not handle a review comment" in agent.transport.outbound[-1].text
agent = Agent(tmp_path, gitea) # restart: nothing to answer again
assert "still watching" in agent.transport.outbound[-1].text
assert agent.pollers[20].poll_once(20) == []
def test_project_complete_still_completes_when_gitea_is_down(tmp_path: Path) -> None:
agent = Agent(tmp_path, FakeGitea())
def down(issue: int, note: str) -> None:
raise GiteaError("gitea is down")
agent.orch._board.close = down # type: ignore[attr-defined]
agent.say("use demo", "approve", "approve", "start milestone 1", "project complete")
assert agent.state == IDLE
assert "could not close plan issue #10" in agent.transport.outbound[-1].text
nisse-dev/tests/test_review_loop.py
-184
View File
@@ -1,184 +0,0 @@
"""Reacting to PR review comments while in PR_OPEN (architecture D5)."""
import io
from pathlib import Path
from typing import Any
from nisse_dev.canonical import CanonicalEvent, StoredEvent, ToolCall
from nisse_dev.control import AWAIT_INPUT, PR_OPEN, Orchestrator
from nisse_dev.executor import ExecutorOutcome
from nisse_dev.gitea import Comment, GiteaError, PullRequest
from nisse_dev.llm import RoleMap
from nisse_dev.oplog import OpLog
from nisse_dev.planner import LLMPlanningAgent
from nisse_dev.publish import PublishResult
from nisse_dev.state import StateStore
from nisse_dev.transport import InMemoryTransport
ROLES = RoleMap(executor="x", planner="y", helper="y", expert="y")
class ScriptedClient:
def __init__(self) -> None:
self.calls = 0
def complete(self, events, tools, model, system=None) -> list[CanonicalEvent]:
self.calls += 1
return [
ToolCall(
call_id=f"c{self.calls}",
tool="submit_plan",
arguments={"type": "x", "content": f"plan {self.calls}"},
)
]
class RecordingExecutor:
def __init__(self, outcomes: list[str] | None = None) -> None:
self.runs: list[int] = []
self.seen_events: list[list[StoredEvent]] = []
# One entry per run, so a test can let the milestone succeed and
# only the review response fail.
self._outcomes = outcomes or []
self.store: StateStore | None = None
def run(self, transcript: Any, milestone: int) -> ExecutorOutcome:
self.runs.append(milestone)
# What it is given: the work item's transcript, nothing wider.
self.seen_events.append(transcript.read())
status = self._outcomes.pop(0) if self._outcomes else "completed"
return ExecutorOutcome(status, f"executor said {status}")
class RecordingPublisher:
def __init__(self, *, sha: str | None = "abc12345", error: Exception | None = None):
self._sha = sha
self._error = error
self.followups: list[dict[str, Any]] = []
self.comments: list[tuple[int, str]] = []
def reconcilers(self) -> dict[str, Any]:
return {}
def publish(self, **kwargs: Any) -> PublishResult:
return PublishResult(
PullRequest(number=42, state="open", head="h", url="u"), "opened PR #42: u"
)
def push_followup(self, **kwargs: Any) -> str | None:
self.followups.append(kwargs)
if self._error is not None:
raise self._error
return self._sha
def comment(self, index: int, body: str) -> None:
self.comments.append((index, body))
def build(
tmp_path: Path,
executor: Any,
publisher: Any,
gitea_operators: set[str] | None = None,
) -> tuple[Orchestrator, StateStore, InMemoryTransport]:
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
if hasattr(executor, "store"):
executor.store = store
transport = InMemoryTransport()
orch = Orchestrator(
store,
transport,
LLMPlanningAgent(ScriptedClient(), ROLES),
operators={"don"},
roles=ROLES,
executor=executor,
publisher=publisher,
gitea_operators=gitea_operators if gitea_operators is not None else {"don"},
)
orch.start()
transport.receive("don", "use demo")
transport.receive("don", "approve")
transport.receive("don", "approve")
transport.receive("don", "start milestone 1")
return orch, store, transport
def review(
cid: int = 5, author: str = "don", body: str = "rename the module"
) -> Comment:
return Comment(id=cid, author=author, body=body, created_at="t")
def test_review_comment_triggers_a_fix_and_a_push(tmp_path: Path) -> None:
executor, publisher = RecordingExecutor(), RecordingPublisher()
orch, store, transport = build(tmp_path, executor, publisher)
assert store.get_state().state == PR_OPEN
orch.handle_review_comment(review())
assert executor.runs == [1, 1] # milestone run, then the review response
assert publisher.followups[0]["milestone"] == 1
assert "abc12345" in transport.outbound[-1].text
assert publisher.comments[0][0] == 42 # replied on the PR
def test_comment_enters_the_transcript_before_the_executor_runs(
tmp_path: Path,
) -> None:
executor, publisher = RecordingExecutor(), RecordingPublisher()
orch, _, _ = build(tmp_path, executor, publisher)
orch.handle_review_comment(review(body="please add a test"))
# The executor must see the feedback as conversation, not as an
# out-of-band instruction it was never shown.
texts = [getattr(e.event, "text", "") for e in executor.seen_events[-1]]
assert any("please add a test" in t for t in texts)
def test_non_operator_comments_are_ignored(tmp_path: Path) -> None:
executor, publisher = RecordingExecutor(), RecordingPublisher()
orch, store, _ = build(tmp_path, executor, publisher)
orch.handle_review_comment(review(author="drive-by"))
assert executor.runs == [1] # only the original milestone run
assert publisher.followups == []
assert store.get_state().state == PR_OPEN
def test_comments_outside_pr_open_are_ignored(tmp_path: Path) -> None:
executor, publisher = RecordingExecutor(), RecordingPublisher()
orch, _store, transport = build(tmp_path, executor, publisher)
transport.receive("don", "project complete") # back to IDLE
orch.handle_review_comment(review())
assert executor.runs == [1]
def test_failed_review_response_gates_to_the_operator(tmp_path: Path) -> None:
# Milestone succeeds; the response to the review comment does not.
executor = RecordingExecutor(outcomes=["completed", "escalate"])
publisher = RecordingPublisher()
orch, store, transport = build(tmp_path, executor, publisher)
orch.handle_review_comment(review())
assert store.get_state().state == AWAIT_INPUT
assert "could not address the review comment" in transport.outbound[-1].text
assert publisher.followups == []
def test_no_code_change_needed_is_reported(tmp_path: Path) -> None:
executor, publisher = RecordingExecutor(), RecordingPublisher(sha=None)
orch, _, transport = build(tmp_path, executor, publisher)
orch.handle_review_comment(review(body="just curious, why sqlite?"))
assert "no code change was needed" in transport.outbound[-1].text
def test_push_failure_is_reported_without_losing_pr_open(tmp_path: Path) -> None:
executor = RecordingExecutor()
publisher = RecordingPublisher(error=GiteaError("push rejected"))
orch, store, transport = build(tmp_path, executor, publisher)
orch.handle_review_comment(review())
assert "could not push the fix" in transport.outbound[-1].text
assert store.get_state().state == PR_OPEN # still reviewable, can retry
nisse-dev/tests/test_state.py
+13 -10
View File
@@ -24,22 +24,25 @@ def test_fresh_store_stamps_schema_version(db: Path) -> None:
assert store.schema_version == 1
def test_transition_persists_across_reopen(db: Path) -> None:
def test_a_work_item_persists_across_reopen(db: Path) -> None:
with StateStore(db) as store:
store.transition(
"EXECUTING", cause="test", project="demo", data={"milestone": 2}
store.open_work_item(
"demo-1", title="t", scope="demo", position="WAITING", cause="test"
)
store.set_active("demo-1", cause="test")
store.update_work_item(
"demo-1", position="RUNNING", data={"milestone": 2}, cause="test"
)
with StateStore(db) as store:
state = store.get_state()
assert state is not None
assert state.state == "EXECUTING"
assert state.project == "demo"
assert state.data == {"milestone": 2}
item = store.active_item()
assert item is not None
assert (item.id, item.scope, item.position) == ("demo-1", "demo", "RUNNING")
assert item.data == {"milestone": 2}
def test_get_state_on_fresh_store_is_none(db: Path) -> None:
def test_a_fresh_store_has_no_active_item(db: Path) -> None:
with StateStore(db) as store:
assert store.get_state() is None
assert store.active_item() is None
def test_intent_lifecycle(db: Path) -> None:
nisse-dev/tests/test_stop.py
-365
View File
@@ -1,365 +0,0 @@
"""The operator's kill switch, `stop` (architecture L2)."""
import io
import threading
from pathlib import Path
from typing import Any
import pytest
from nisse_dev.cancel import STOPPED_RESULT, Cancel
from nisse_dev.canonical import AssistantText, ToolCall, ToolResult, UserMessage
from nisse_dev.control import (
AWAIT_ARCH_SIGNOFF,
AWAIT_INPUT,
AWAIT_MILESTONE_SIGNOFF,
Orchestrator,
PlanningStopped,
StubAgent,
SubmitPlan,
)
from nisse_dev.delegation import DelegationResult
from nisse_dev.executor import ExecutorOutcome, LLMExecutor
from nisse_dev.llm import RoleMap
from nisse_dev.oplog import OpLog
from nisse_dev.planner import LLMPlanningAgent
from nisse_dev.state import StateStore, Transcript
from nisse_dev.tools import ToolOutcome
from nisse_dev.transport import InMemoryTransport
from nisse_dev.worker import SerialWorker
ROLES = RoleMap(
executor="llama3.2",
planner="claude-haiku-4-5",
helper="claude-haiku-4-5",
expert="claude-haiku-4-5",
)
def log(store: StateStore, actor: str = "executor") -> Transcript:
return store.transcript("t", actor)
def new_store(path: Path) -> StateStore:
return StateStore(path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
def user_messages(store: StateStore) -> list[str]:
return [
e.event.text for e in store.all_events() if isinstance(e.event, UserMessage)
]
# --- the orchestrator ------------------------------------------------------
class OperatorDuringWork:
"""An executor that hears from the operator mid-milestone, then does what
the real loop does at its next tool boundary."""
def __init__(self, cancel: Cancel, says: str) -> None:
self.cancel = cancel
self.says = says
self.transport: InMemoryTransport | None = None
def run(self, transcript: object, milestone: int) -> ExecutorOutcome:
assert self.transport is not None
self.transport.receive("don", self.says)
if self.cancel.requested:
return ExecutorOutcome("stopped", "during turn 3")
return ExecutorOutcome("completed", "done")
def at_start_of_milestone(
tmp_path: Path, says: str
) -> tuple[StateStore, InMemoryTransport, Cancel]:
store = new_store(tmp_path)
transport = InMemoryTransport()
cancel = Cancel()
executor = OperatorDuringWork(cancel, says)
executor.transport = transport
orch = Orchestrator(
store,
transport,
StubAgent(),
operators={"don"},
executor=executor,
cancel=cancel,
)
orch.start()
for text in ("use demo", "approve", "approve"):
transport.receive("don", text)
return store, transport, cancel
def test_stop_halts_a_running_milestone_and_asks_why(tmp_path: Path) -> None:
store, transport, cancel = at_start_of_milestone(tmp_path, "stop")
transport.receive("don", "start milestone 1")
assert store.get_state().state == AWAIT_INPUT
prompt = transport.outbound[-1]
assert prompt.kind == "question"
assert "stopped milestone 1 (during turn 3)" in prompt.text
assert "Why did you stop it?" in prompt.text
assert user_messages(store)[-1] == "stop"
assert not cancel.requested # the next piece of work is not pre-stopped
def test_the_reason_is_recorded_and_the_plan_stays_approved(tmp_path: Path) -> None:
store, transport, _ = at_start_of_milestone(tmp_path, "stop")
transport.receive("don", "start milestone 1")
transport.receive("don", "it was editing the wrong module")
assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF
# The next run renders this, so the model hears why it was stopped.
assert user_messages(store)[-1] == "it was editing the wrong module"
assert "start milestone 1" in transport.outbound[-1].text
def test_stop_with_nothing_running_says_so_and_changes_nothing(
tmp_path: Path,
) -> None:
store = new_store(tmp_path)
transport = InMemoryTransport()
orch = Orchestrator(store, transport, StubAgent(), operators={"don"})
orch.start()
transport.receive("don", "use demo")
transport.receive("don", "stop")
# At a gate, "stop" is the command, not rejection feedback.
assert store.get_state().state == AWAIT_ARCH_SIGNOFF
assert orch._data["revs"]["arch"] == 1
assert (
transport.outbound[-1].text == f"nothing to stop (state={AWAIT_ARCH_SIGNOFF})"
)
def test_stop_from_a_non_operator_is_ignored(tmp_path: Path) -> None:
_, transport, cancel = at_start_of_milestone(tmp_path, "approve")
before = len(transport.outbound)
transport.receive("mallory", "stop")
assert len(transport.outbound) == before
assert not cancel.requested
def test_other_messages_during_work_are_answered_not_acted_on(
tmp_path: Path,
) -> None:
store, transport, _ = at_start_of_milestone(tmp_path, "approve")
transport.receive("don", "start milestone 1")
texts = [m.text for m in transport.outbound]
assert "busy (EXECUTING); say 'stop' to halt" in texts
# Recorded, as every operator message is, but not applied to anything.
assert user_messages(store)[-1] == "approve"
class StopsBeforePlanning:
def __init__(self, cancel: Cancel) -> None:
self.cancel = cancel
self.transport: InMemoryTransport | None = None
self.answers: list[str | None] = []
def plan(
self,
transcript: object,
gate: str,
project: str,
feedback: str | None = None,
answer: str | None = None,
) -> SubmitPlan:
self.answers.append(answer)
if len(self.answers) == 1:
assert self.transport is not None
self.transport.receive("don", "stop")
if self.cancel.requested:
raise PlanningStopped("stopped by the operator")
return SubmitPlan(f"{gate} plan, after: {answer}")
def test_stop_during_planning_resumes_planning_with_the_reason(
tmp_path: Path,
) -> None:
store = new_store(tmp_path)
transport = InMemoryTransport()
cancel = Cancel()
planner = StopsBeforePlanning(cancel)
planner.transport = transport
orch = Orchestrator(store, transport, planner, operators={"don"}, cancel=cancel)
orch.start()
transport.receive("don", "use demo")
assert store.get_state().state == AWAIT_INPUT
assert "stopped planning arch" in transport.outbound[-1].text
transport.receive("don", "keep it to one module")
assert store.get_state().state == AWAIT_ARCH_SIGNOFF
assert planner.answers[-1] == "keep it to one module"
# --- on real threads -------------------------------------------------------
class BlocksUntilStopped:
def __init__(self, cancel: Cancel) -> None:
self.cancel = cancel
self.running = threading.Event()
def run(self, transcript: object, milestone: int) -> ExecutorOutcome:
self.running.set()
for _ in range(500):
if self.cancel.requested:
return ExecutorOutcome("stopped", "at a tool boundary")
threading.Event().wait(0.01)
return ExecutorOutcome("completed", "never stopped")
@pytest.mark.parametrize("word", ["stop", "Stop!", " STOP "])
def test_stop_reaches_work_running_on_the_worker(tmp_path: Path, word: str) -> None:
# The bug this design fixes: the work used to run on the transport's own
# thread, so no message, stop included, was read until it finished.
worker = SerialWorker()
worker.start()
store = worker.call(lambda: new_store(tmp_path))
transport = InMemoryTransport()
cancel = Cancel()
executor = BlocksUntilStopped(cancel)
orch = Orchestrator(
store,
transport,
StubAgent(),
operators={"don"},
executor=executor,
worker=worker,
cancel=cancel,
)
worker.call(orch.start)
for text in ("use demo", "approve", "approve"):
transport.receive("don", text)
# Reply only once the plan is up: a reply stamped with an earlier
# revision is not applied to a later one.
worker.call(lambda: None)
transport.receive("don", "start milestone 1")
assert executor.running.wait(5)
transport.receive("don", "status") # delivered while the milestone runs
transport.receive("don", word) # a phone may capitalise it (#64)
worker.call(lambda: None) # wait for the stop to be recorded
texts = [m.text for m in transport.outbound]
assert "busy (EXECUTING); say 'stop' to halt" in texts
assert worker.call(store.get_state).state == AWAIT_INPUT
assert "stopped milestone 1 (at a tool boundary)" in texts[-1]
worker.call(store.close)
worker.stop(timeout=5)
# --- the loops -------------------------------------------------------------
class ScriptedClient:
def __init__(self, turns: list[list[Any]], on_call: Any = None) -> None:
self.turns = turns
self.on_call = on_call
self.calls = 0
def complete(self, events: Any, tools: Any, model: str, system: Any = None):
self.calls += 1
if self.on_call:
self.on_call()
return self.turns.pop(0) if self.turns else [AssistantText("done", model)]
class StopsAfterFirstCall:
"""A workspace whose first tool call is where the operator says stop."""
def __init__(self, cancel: Cancel) -> None:
self.cancel = cancel
self.ran: list[str] = []
def describe(self) -> str:
return "workspace"
def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome:
self.ran.append(tool)
self.cancel.set()
return ToolOutcome("ok", success=True)
def test_executor_stops_before_a_turn_without_calling_the_model(
tmp_path: Path,
) -> None:
cancel = Cancel()
cancel.set()
client = ScriptedClient([])
executor = LLMExecutor(client, ROLES, StopsAfterFirstCall(cancel), cancel=cancel)
outcome = executor.run(log(new_store(tmp_path)), 1)
assert outcome == ExecutorOutcome("stopped", "before turn 1")
assert client.calls == 0
def test_executor_stops_between_tool_calls_and_records_the_skipped_ones(
tmp_path: Path,
) -> None:
cancel = Cancel()
store = new_store(tmp_path)
ws = StopsAfterFirstCall(cancel)
calls = [
ToolCall(call_id="a", tool="run_command", arguments={"command": "make"}),
ToolCall(call_id="b", tool="write_file", arguments={"path": "x"}),
]
executor = LLMExecutor(ScriptedClient([calls]), ROLES, ws, cancel=cancel)
outcome = executor.run(log(store), 1)
assert outcome == ExecutorOutcome("stopped", "during turn 1")
assert ws.ran == ["run_command"]
results = {
e.event.call_id: e.event.result
for e in log(store).read()
if isinstance(e.event, ToolResult)
}
# Every call has a result, so the transcript still renders.
assert results == {"a": "ok", "b": STOPPED_RESULT}
class StoppedDelegate:
def run(self, transcript: object, *, brief: str, branch: str) -> DelegationResult:
return DelegationResult("stopped", "stopped by the operator during turn 2")
def test_a_stopped_delegation_stops_the_milestone(tmp_path: Path) -> None:
failing = [
[ToolCall(call_id=f"c{n}", tool="run_command", arguments={"command": "x"})]
for n in range(2)
]
class Failing:
def describe(self) -> str:
return "workspace"
def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome:
return ToolOutcome("exit 1", success=False)
executor = LLMExecutor(
ScriptedClient(failing),
ROLES,
Failing(),
delegate=StoppedDelegate(),
)
outcome = executor.run(log(new_store(tmp_path)), 1)
assert outcome.status == "stopped"
assert "during a delegation" in outcome.note
def test_planner_does_not_start_a_turn_once_stopped(tmp_path: Path) -> None:
cancel = Cancel()
cancel.set()
client = ScriptedClient([])
planner = LLMPlanningAgent(client, ROLES, cancel=cancel)
try:
planner.plan(log(new_store(tmp_path), "planner"), "arch", "demo")
except PlanningStopped:
pass
else:
raise AssertionError("planning ran after stop")
assert client.calls == 0
nisse-dev/tests/test_tools.py
+3 -3
View File
@@ -5,7 +5,7 @@ from pathlib import Path
import pytest
from nisse_dev.execution import REDACTED, SSHHost
from nisse_dev.tools import EXECUTOR_TOOLS, MAX_OUTPUT_CHARS, Workspace
from nisse_dev.tools import EXECUTION_TOOLS, MAX_OUTPUT_CHARS, Workspace
@pytest.fixture
@@ -15,9 +15,9 @@ def ws(ssh_host: SSHHost, tmp_path: Path) -> Workspace:
def test_manifest_is_small_and_flat() -> None:
# L9: fewer tools, flatter arguments for the local executor.
names = [t["function"]["name"] for t in EXECUTOR_TOOLS]
names = [t["function"]["name"] for t in EXECUTION_TOOLS]
assert names == ["read_file", "write_file", "list_dir", "run_command"]
for tool in EXECUTOR_TOOLS:
for tool in EXECUTION_TOOLS:
props = tool["function"]["parameters"]["properties"]
assert all(p["type"] == "string" for p in props.values())
nisse-dev/tests/test_views.py
+13 -170
View File
@@ -2,11 +2,9 @@
import io
from pathlib import Path
from typing import Any
from nisse_dev.canonical import (
AssistantText,
CanonicalEvent,
DelegationCompleted,
DelegationStarted,
Fact,
@@ -16,19 +14,10 @@ from nisse_dev.canonical import (
ToolResult,
UserMessage,
)
from nisse_dev.control import MILESTONE_PLANNING, PR_OPEN, Orchestrator
from nisse_dev.executor import ExecutorOutcome
from nisse_dev.gitea import Comment, PullRequest
from nisse_dev.llm import RoleMap
from nisse_dev.oplog import OpLog
from nisse_dev.planner import LLMPlanningAgent
from nisse_dev.publish import PublishResult
from nisse_dev.render import OpenAIChatRenderer
from nisse_dev.render_anthropic import AnthropicMessagesRenderer
from nisse_dev.state import StateStore
from nisse_dev.transport import InMemoryTransport
ROLES = RoleMap(executor="x", planner="y", helper="y", expert="y")
def new_store(tmp_path: Path) -> StateStore:
@@ -39,155 +28,18 @@ def texts(events: list[StoredEvent]) -> list[str]:
return [getattr(e.event, "text", "") for e in events]
# --- the orchestrator: run 2's planner, after a milestone ----------------------
class PlanningClient:
"""Always submits a plan, recording the events of each request."""
def __init__(self) -> None:
self.requests: list[list[StoredEvent]] = []
def complete(self, events, tools, model, system=None) -> list[CanonicalEvent]:
self.requests.append(events)
n = len(self.requests)
return [
ToolCall(
call_id=f"plan-{n}",
tool="submit_plan",
arguments={"content": f"plan {n}"},
)
]
class WorkingExecutor:
"""Leaves the trail a real executor run does: text, calls, results."""
def __init__(self) -> None:
self.views: list[list[StoredEvent]] = []
def run(self, transcript: Any, milestone: int) -> ExecutorOutcome:
self.views.append(transcript.view())
transcript.append(AssistantText(text="I'll write the helper", model="x"))
call = ToolCall(
call_id=f"run-{len(self.views)}",
tool="run_command",
arguments={"command": "pytest"},
)
transcript.append(call)
transcript.append(
ToolResult(call_id=call.call_id, result="1 passed", success=True)
)
return ExecutorOutcome("completed", "added truncate() with a test")
class Publisher:
def reconcilers(self) -> dict[str, Any]:
return {}
def publish(self, **kwargs: Any) -> PublishResult:
return PublishResult(
PullRequest(number=165, state="open", head="h", url="https://g/pr/165"),
"opened PR #165",
)
def push_followup(self, **kwargs: Any) -> str | None:
return None
def comment(self, index: int, body: str) -> None:
pass
def run_one_milestone(
tmp_path: Path,
) -> tuple[Orchestrator, StateStore, InMemoryTransport, PlanningClient, Any]:
store = new_store(tmp_path)
transport = InMemoryTransport()
client = PlanningClient()
executor = WorkingExecutor()
orch = Orchestrator(
store,
transport,
LLMPlanningAgent(client, ROLES),
operators={"don"},
roles=ROLES,
executor=executor,
publisher=Publisher(),
gitea_operators={"don"},
)
orch.start()
for text in ("add a truncate helper", "use demo", "approve", "approve"):
transport.receive("don", text)
transport.receive("don", "start milestone 1")
assert store.get_state().state == PR_OPEN
return orch, store, transport, client, executor
def test_the_planner_after_a_milestone_sees_facts_not_executor_turns(
tmp_path: Path,
) -> None:
# Run 2: shown the executor's calls as its own turns, the planner took
# milestone 1's work for its own and planned around it (#78).
_, store, transport, client, _ = run_one_milestone(tmp_path)
transport.receive("don", "next milestone")
assert store.get_state().state != MILESTONE_PLANNING # it planned
events = client.requests[-1]
body = OpenAIChatRenderer().render(events, tools=None, model="y")
assistant = [m for m in body["messages"] if m["role"] == "assistant"]
called = [c["function"]["name"] for m in assistant for c in m["tool_calls"]]
assert set(called) == {"submit_plan"} # its own turns, and only those
seen = "\n".join(texts(events))
assert "I'll write the helper" not in seen
assert "[operator, Matrix, sent while idle] add a truncate helper" in seen
assert "[orchestrator] arch plan rev-1 approved by don" in seen
assert "milestone 1 finished: added truncate() with a test" in seen
assert "PR #165" in seen and "https://g/pr/165" in seen
assert "planning milestone 2" in seen
def test_the_executor_sees_the_milestone_start_and_the_comment_source(
tmp_path: Path,
) -> None:
orch, _, _, _, executor = run_one_milestone(tmp_path)
assert "[orchestrator] milestone 1 started" in texts(executor.views[0])
orch.handle_review_comment(
Comment(id=7, author="don", body="add a docstring", created_at="t")
)
review_view = texts(executor.views[-1])
assert "[operator, comment on PR #165] add a docstring" in review_view
# Its own earlier turn is still its own, not relabelled.
own = [e for e in executor.views[-1] if isinstance(e.event, AssistantText)]
assert [e.event.text for e in own] == ["I'll write the helper"]
def test_the_outcome_is_recorded_as_a_fact_when_the_task_closes(
tmp_path: Path,
) -> None:
_, store, transport, _, _ = run_one_milestone(tmp_path)
item = store.active_item()
transport.receive("don", "project complete")
facts = [
e.event.text
for e in store.transcript(item.id, "orchestrator").read()
if isinstance(e.event, Fact)
]
assert facts[-1] == "work item completed: don said project complete"
# --- the projection, per role ----------------------------------------------------
def delegated_milestone(store: StateStore) -> None:
"""A milestone in which the executor failed and handed off to a helper."""
store.transcript("a", "planner").append(PlanSubmitted(plan="1. fix it", model="y"))
executor = store.transcript("a", "executor")
executor.append(
"""A milestone in which the driver failed and handed off to a helper."""
store.transcript("a", "driver").append(PlanSubmitted(plan="1. fix it", model="y"))
driver = store.transcript("a", "driver")
driver.append(
ToolCall(call_id="e1", tool="run_command", arguments={"command": "make"})
)
executor.append(ToolResult(call_id="e1", result="error: missing ;", success=False))
helper = executor.as_actor("helper")
driver.append(ToolResult(call_id="e1", result="error: missing ;", success=False))
helper = driver.as_actor("helper")
helper.append(
DelegationStarted(brief="make fails", role="helper", model="y", caps={})
)
@@ -207,10 +59,10 @@ def delegated_milestone(store: StateStore) -> None:
)
def test_the_executor_sees_the_helpers_report_not_its_turns(tmp_path: Path) -> None:
def test_the_driver_sees_the_helpers_report_not_its_turns(tmp_path: Path) -> None:
store = new_store(tmp_path)
delegated_milestone(store)
view = store.transcript("a", "executor").view()
view = store.transcript("a", "driver").view()
assert [type(e.event).__name__ for e in view] == [
"ToolCall",
"ToolResult",
@@ -221,7 +73,7 @@ def test_the_executor_sees_the_helpers_report_not_its_turns(tmp_path: Path) -> N
)
def test_the_helper_sees_the_plan_and_the_executors_work_as_narration(
def test_the_helper_sees_the_plan_and_the_drivers_work_as_narration(
tmp_path: Path,
) -> None:
store = new_store(tmp_path)
@@ -229,7 +81,7 @@ def test_the_helper_sees_the_plan_and_the_executors_work_as_narration(
view = store.transcript("a", "helper").view()
said = texts(view)
assert said[0] == "[approved plan] 1. fix it"
assert said[1] == "[executor] run_command(make) → failed: error: missing ;"
assert said[1] == "[driver] run_command(make) → failed: error: missing ;"
# Its own events are unchanged, tool calls and all.
assert any(isinstance(e.event, ToolCall) and e.event.call_id == "h1" for e in view)
assert not any(
@@ -237,20 +89,11 @@ def test_the_helper_sees_the_plan_and_the_executors_work_as_narration(
)
def test_the_planner_sees_no_other_roles_work(tmp_path: Path) -> None:
store = new_store(tmp_path)
delegated_milestone(store)
store.transcript("a", "orchestrator").append(Fact(text="milestone 1 finished"))
assert texts(store.transcript("a", "planner").view()) == [
"[orchestrator] milestone 1 finished"
]
def test_a_view_keeps_log_ids_so_cursors_still_work(tmp_path: Path) -> None:
store = new_store(tmp_path)
delegated_milestone(store)
raw = store.transcript("a", "executor").read()
view = store.transcript("a", "executor").view(after=raw[2].id)
raw = store.transcript("a", "driver").read()
view = store.transcript("a", "driver").view(after=raw[2].id)
assert [e.id for e in view] == [raw[-1].id] # the helper's report
@@ -277,7 +120,7 @@ def test_operator_messages_carry_their_channel(tmp_path: Path) -> None:
operator = store.transcript("a", "operator")
for source in ("transport:@don", "gitea:issue:9:comment:4", "gitea:comment:5"):
operator.append(UserMessage(text="hi", source=source))
assert texts(store.transcript("a", "planner").view()) == [
assert texts(store.transcript("a", "driver").view()) == [
"[operator, Matrix] hi",
"[operator, comment on plan issue #9] hi",
"[operator, Gitea comment] hi",
nisse-dev/tests/test_work_items.py
-177
View File
@@ -1,177 +0,0 @@
"""Work items in the state store (architecture L1, L7; Development Plan M1.1)."""
import io
from pathlib import Path
from typing import Any
import pytest
from nisse_dev.canonical import AssistantText, UserMessage
from nisse_dev.control import (
AWAIT_ARCH_SIGNOFF,
PR_OPEN,
AskHuman,
Orchestrator,
SubmitPlan,
)
from nisse_dev.oplog import OpLog
from nisse_dev.state import IDLE, StateStore
from nisse_dev.transport import InMemoryTransport
def new_store(path: Path) -> StateStore:
return StateStore(path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
# --- the store ---------------------------------------------------------------
def test_a_transcript_reads_only_its_own_work_item(tmp_path: Path) -> None:
store = new_store(tmp_path)
store.transcript("a", "operator").append(UserMessage(text="for a", source="m"))
store.transcript("b", "operator").append(UserMessage(text="for b", source="m"))
store.transcript(None, "operator").append(UserMessage(text="idle", source="m"))
assert [e.event.text for e in store.transcript("a", "planner").read()] == ["for a"]
assert [e.event.text for e in store.transcript(None, "planner").read()] == ["idle"]
def test_every_event_carries_its_work_item_and_actor(tmp_path: Path) -> None:
store = new_store(tmp_path)
log = store.transcript("a", "executor")
log.append(AssistantText(text="working", model="m"))
log.as_actor("helper").append(AssistantText(text="helping", model="f"))
assert [(e.work_item, e.actor) for e in store.all_events()] == [
("a", "executor"),
("a", "helper"),
]
def test_an_unknown_actor_is_refused(tmp_path: Path) -> None:
with pytest.raises(ValueError, match="unknown actor"):
new_store(tmp_path).transcript("a", "someone")
def test_intents_are_owned_by_their_work_item(tmp_path: Path) -> None:
store = new_store(tmp_path)
store.transcript("a", "helper").begin_intent("delegation", {}, cause="t")
[intent] = store.pending_intents()
assert intent.work_item == "a"
def test_work_items_and_the_active_one_survive_a_restart(tmp_path: Path) -> None:
store = new_store(tmp_path)
store.open_work_item("a", title="demo", scope="demo", position=IDLE, cause="t")
store.set_active("a", cause="t")
store.update_work_item("a", position="PLANNING_ARCH", data={"x": 1}, cause="t")
store.close()
store = new_store(tmp_path)
item = store.active_item()
assert item is not None
assert (item.id, item.position, item.data, item.status) == (
"a",
"PLANNING_ARCH",
{"x": 1},
"open",
)
def test_closing_records_the_outcome_and_going_idle_clears_active(
tmp_path: Path,
) -> None:
store = new_store(tmp_path)
store.open_work_item("a", title="demo", position=IDLE, cause="t")
store.set_active("a", cause="t")
store.close_work_item("a", outcome="completed", cause="t")
store.set_active(None, cause="t")
assert store.active_item() is None
item = store.work_item("a")
assert item is not None and (item.status, item.outcome) == ("closed", "completed")
with pytest.raises(KeyError):
store.close_work_item("a", outcome="abandoned", cause="t") # already closed
def test_messages_sent_while_idle_are_carried_with_their_origin(
tmp_path: Path,
) -> None:
store = new_store(tmp_path)
store.set_active(None, cause="boot")
idle = store.transcript(None, "operator")
first = idle.append(UserMessage(text="add a truncate helper", source="m:1"))
store.open_work_item("a", title="demo", position=IDLE, cause="t")
assert store.carry_idle_messages("a") == 1
[carried] = store.transcript("a", "planner").read()
assert carried.event.text == "add a truncate helper"
assert f"#{first}" in carried.event.source # where it came from
assert len(store.all_events()) == 2 # a copy; the original is untouched
# --- the orchestrator ----------------------------------------------------------
class RecordingPlanner:
"""Records what each planning turn could read."""
def __init__(self) -> None:
self.seen: list[list[str]] = []
def plan(
self,
transcript: Any,
gate: str,
project: str,
feedback: str | None = None,
answer: str | None = None,
) -> SubmitPlan | AskHuman:
self.seen.append(
[
e.event.text
for e in transcript.read()
if isinstance(e.event, UserMessage)
]
)
return SubmitPlan(f"{gate} plan for {project}")
def build(tmp_path: Path) -> tuple[StateStore, InMemoryTransport, RecordingPlanner]:
store = new_store(tmp_path)
transport = InMemoryTransport()
planner = RecordingPlanner()
Orchestrator(store, transport, planner, operators={"don"}).start()
return store, transport, planner
def test_a_second_task_never_sees_the_first(tmp_path: Path) -> None:
# The second live run's planner read every earlier task's events (#78).
store, transport, planner = build(tmp_path)
for text in ("build the first thing", "use first", "approve", "approve"):
transport.receive("don", text)
transport.receive("don", "start milestone 1")
assert store.get_state().state == PR_OPEN
transport.receive("don", "project complete")
transport.receive("don", "build the second thing")
transport.receive("don", "use second")
assert store.get_state().state == AWAIT_ARCH_SIGNOFF
second_task_view = planner.seen[-1]
assert "build the second thing" in second_task_view
assert not any("first" in text for text in second_task_view)
def test_each_task_is_its_own_work_item_with_its_outcome(tmp_path: Path) -> None:
store, transport, _ = build(tmp_path)
for text in ("use first", "approve", "approve", "start milestone 1"):
transport.receive("don", text)
transport.receive("don", "project complete")
transport.receive("don", "use second")
transport.receive("don", "abandon")
outcomes = [(item.title, item.outcome) for item in store.work_items()]
assert outcomes == [("first", "completed"), ("second", "abandoned")]
assert store.active_item() is None
def test_the_task_message_before_use_reaches_the_planner(tmp_path: Path) -> None:
# Run 2's flow: the task, then `use`. It must still be planned from.
_, transport, planner = build(tmp_path)
transport.receive("don", "add a truncate helper")
transport.receive("don", "use demo")
assert "add a truncate helper" in planner.seen[0]
nisse-dev/tests/test_worker.py
+2 -2
View File
@@ -62,10 +62,10 @@ def test_the_store_is_usable_only_from_the_thread_that_owns_it(
store = worker.call(
lambda: StateStore(tmp_path / "state.db", oplog=OpLog("t", io.StringIO()))
)
assert worker.call(store.get_state) is None
assert worker.call(store.active_item) is None
# The tripwire the worker design keeps on: any other thread fails loudly.
with pytest.raises(Exception, match="same thread"):
store.get_state()
store.active_item()
worker.call(store.close)