From 6ae2375194c284924895164ad63be62583510aa1 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 8 Oct 2026 03:12:58 +0000 Subject: [PATCH] M2.8: switch over to the new loop nisse-dev now runs only the conversational loop. The gated state machine it was built alongside is gone: - Removed control.py, planner.py, executor.py, approvals.py, gates.py (command_form moves to loop/reader.py), publish.py and poller.py, the old assembly in agent.py (its shared helpers move to loop/build.py), the store's state-machine view (get_state, transition), their tests, the old end-to-end and live planning/milestone/gate tests, and specs/orchestrator.qnt with its scenarios. - Roles are driver, helper, expert and classifier; driver is required. loop:, roles.planner, roles.executor and tuning.executor are refused with what to use instead; tuning.driver takes the turn cap and failure limit. - The benchmark and the capability matrix measure the driver: the loop's own system prompt and tools, the task's plan as the agreed restatement. Task plans end with finish_milestone, so earlier runs show as (old); stored runs' executor tuning reads as the driver's. - A delegation on the loop records what its helper's alias resolved to (L9), as the old path's did. - The dump shows a work item's revision and what it waits for. - Behaviour the old tests covered and the loop shares stays covered: driver runs that fail on a model call, stop mid-turn, reset the failure count; go-aheads ignoring case and punctuation; a request too big to fit is never sent; the assembly tests, now on build_loop_agent. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE --- docs/Development-Plan.md | 46 +- docs/Tuning-Guide.md | 80 +- nisse-dev.example.yaml | 17 +- nisse-dev/src/nisse_dev/agent.py | 342 ----- nisse-dev/src/nisse_dev/approvals.py | 86 -- nisse-dev/src/nisse_dev/bench/__init__.py | 4 +- nisse-dev/src/nisse_dev/bench/results.py | 11 +- nisse-dev/src/nisse_dev/bench/runner.py | 86 +- nisse-dev/src/nisse_dev/bench/tasks.py | 6 +- .../nisse_dev/bench/tasks/fix-bug/task.yaml | 2 +- .../bench/tasks/long-milestone/task.yaml | 2 +- .../bench/tasks/long-output/task.yaml | 2 +- .../bench/tasks/multi-file/task.yaml | 2 +- .../bench/tasks/refactor-rename/task.yaml | 2 +- .../bench/tasks/spec-parser/task.yaml | 2 +- .../nisse_dev/bench/tasks/trace-bug/task.yaml | 2 +- .../bench/tasks/write-function/task.yaml | 2 +- nisse-dev/src/nisse_dev/config.py | 88 +- nisse-dev/src/nisse_dev/control.py | 1331 ----------------- nisse-dev/src/nisse_dev/delegation.py | 4 +- nisse-dev/src/nisse_dev/dump.py | 9 +- nisse-dev/src/nisse_dev/executor.py | 318 ---- nisse-dev/src/nisse_dev/gates.py | 109 -- nisse-dev/src/nisse_dev/llm.py | 15 +- nisse-dev/src/nisse_dev/loop/__init__.py | 4 +- nisse-dev/src/nisse_dev/loop/agent.py | 25 +- nisse-dev/src/nisse_dev/loop/build.py | 175 ++- nisse-dev/src/nisse_dev/loop/help.py | 3 + nisse-dev/src/nisse_dev/loop/reader.py | 9 + nisse-dev/src/nisse_dev/main.py | 12 +- nisse-dev/src/nisse_dev/planner.py | 193 --- nisse-dev/src/nisse_dev/poller.py | 118 -- nisse-dev/src/nisse_dev/publish.py | 205 --- nisse-dev/src/nisse_dev/state.py | 72 +- nisse-dev/src/nisse_dev/tools.py | 2 +- nisse-dev/src/nisse_dev/views.py | 8 +- nisse-dev/tests/test_abandon.py | 186 --- nisse-dev/tests/test_agent_build.py | 328 ++-- nisse-dev/tests/test_approval_binding.py | 147 -- nisse-dev/tests/test_approvals.py | 210 --- nisse-dev/tests/test_bench.py | 54 +- nisse-dev/tests/test_config.py | 115 +- nisse-dev/tests/test_context.py | 76 +- nisse-dev/tests/test_control.py | 289 ---- nisse-dev/tests/test_dump.py | 27 +- nisse-dev/tests/test_executor.py | 397 ----- nisse-dev/tests/test_expert.py | 3 +- nisse-dev/tests/test_gates.py | 105 -- nisse-dev/tests/test_gitea.py | 18 +- nisse-dev/tests/test_llm.py | 23 +- nisse-dev/tests/test_loop.py | 81 +- nisse-dev/tests/test_loop_help.py | 38 +- nisse-dev/tests/test_loop_properties.py | 2 +- nisse-dev/tests/test_no_stranding.py | 238 --- nisse-dev/tests/test_oplog.py | 48 +- nisse-dev/tests/test_planner.py | 312 ---- nisse-dev/tests/test_poller.py | 132 -- nisse-dev/tests/test_pr_open.py | 202 --- nisse-dev/tests/test_pr_open_handling.py | 250 ---- nisse-dev/tests/test_publish.py | 289 ---- nisse-dev/tests/test_quint_counterexamples.py | 400 ----- nisse-dev/tests/test_review_loop.py | 184 --- nisse-dev/tests/test_state.py | 23 +- nisse-dev/tests/test_stop.py | 365 ----- nisse-dev/tests/test_tools.py | 6 +- nisse-dev/tests/test_views.py | 183 +-- nisse-dev/tests/test_work_items.py | 177 --- nisse-dev/tests/test_worker.py | 4 +- scripts/benchmark.py | 9 +- scripts/e2e.sh | 2 +- scripts/evaluate_model.sh | 2 +- specs/README.md | 32 +- specs/orchestrator.qnt | 488 ------ specs/orchestrator_scenarios.qnt | 125 -- tests/integration/conftest.py | 4 +- tests/integration/e2e_support.py | 19 +- tests/integration/nisse-test.yaml | 3 +- tests/integration/test_benchmark_live.py | 6 +- tests/integration/test_crash_durability.py | 11 +- tests/integration/test_end_to_end.py | 214 --- tests/integration/test_end_to_end_loop.py | 58 +- tests/integration/test_expert_live.py | 2 +- tests/integration/test_gate_restart.py | 65 - tests/integration/test_gitea_live.py | 2 +- tests/integration/test_llm_live.py | 6 +- tests/integration/test_milestone_live.py | 99 -- tests/integration/test_model_capability.py | 37 +- tests/integration/test_passthrough_live.py | 6 +- tests/integration/test_planning_live.py | 47 - 89 files changed, 951 insertions(+), 8592 deletions(-) delete mode 100644 nisse-dev/src/nisse_dev/agent.py delete mode 100644 nisse-dev/src/nisse_dev/approvals.py delete mode 100644 nisse-dev/src/nisse_dev/control.py delete mode 100644 nisse-dev/src/nisse_dev/executor.py delete mode 100644 nisse-dev/src/nisse_dev/gates.py delete mode 100644 nisse-dev/src/nisse_dev/planner.py delete mode 100644 nisse-dev/src/nisse_dev/poller.py delete mode 100644 nisse-dev/src/nisse_dev/publish.py delete mode 100644 nisse-dev/tests/test_abandon.py delete mode 100644 nisse-dev/tests/test_approval_binding.py delete mode 100644 nisse-dev/tests/test_approvals.py delete mode 100644 nisse-dev/tests/test_control.py delete mode 100644 nisse-dev/tests/test_executor.py delete mode 100644 nisse-dev/tests/test_gates.py delete mode 100644 nisse-dev/tests/test_no_stranding.py delete mode 100644 nisse-dev/tests/test_planner.py delete mode 100644 nisse-dev/tests/test_poller.py delete mode 100644 nisse-dev/tests/test_pr_open.py delete mode 100644 nisse-dev/tests/test_pr_open_handling.py delete mode 100644 nisse-dev/tests/test_publish.py delete mode 100644 nisse-dev/tests/test_quint_counterexamples.py delete mode 100644 nisse-dev/tests/test_review_loop.py delete mode 100644 nisse-dev/tests/test_stop.py delete mode 100644 nisse-dev/tests/test_work_items.py delete mode 100644 specs/orchestrator.qnt delete mode 100644 specs/orchestrator_scenarios.qnt delete mode 100644 tests/integration/test_end_to_end.py delete mode 100644 tests/integration/test_gate_restart.py delete mode 100644 tests/integration/test_milestone_live.py delete mode 100644 tests/integration/test_planning_live.py diff --git a/docs/Development-Plan.md b/docs/Development-Plan.md index 456cdcd..5642448 100644 --- a/docs/Development-Plan.md +++ b/docs/Development-Plan.md @@ -24,8 +24,8 @@ Only `nisse-dev` has real code; `nisse-core` and `nisse-ops` are placeholders. | Section | Status | Notes | |---|---|---| | **L1** Work model | mostly done | Work items with an active pointer (M1.1); issue intake (M2.1); queue, parking and blocked items (M2.2) on the new loop; no grace period yet | -| **L2** Agent loop | partial | `control.py`, a fixed state machine with no-stranding, `stop`, `abandon`. Replaced by the new loop (Phase 2) | -| **L3** Authority | partial | Revision binding and arrival stamping (`gates.py`); scope is still plan approval, not the issue and its conversation | +| **L2** Agent loop | mostly done | The loop (Phase 2), the only one since the switch-over (M2.8): driver runs, waits, `stop`, `abandon`, question runs | +| **L3** Authority | partial | Go-aheads bound to the restatement's revision, on the issue and its conversation (M2.1); no action classes yet (M3.1) | | **L4** Classifier | none | | | **L5** Conversation | partial | Matrix transport, mentions, lenient commands; on the new loop, a thread per item, named and ambiguous commands, plain-language go-aheads (M2.3); little explanation | | **L6** Context | partial | Context fitting, per-role views and orchestrator facts (M1.2); no scope or agent memory | @@ -41,7 +41,7 @@ Only `nisse-dev` has real code; `nisse-core` and `nisse-ops` are placeholders. | **D5** Review | partial | PR conversation comments only (#69), watch stall (#70) | | **D7–D8** Deployment, projects | partial | Runs from a venv as a systemd unit on this box; project entries in config (M1.4); egress not enforced | | **O**, **X** | none | No container image yet | -| `specs/` | done for now | `orchestrator.qnt` (current loop), `loop.qnt` (the architecture, #91) | +| `specs/` | done for now | `loop.qnt` (the architecture, #91); `orchestrator.qnt` went with `control.py` (M2.8) | --- @@ -51,8 +51,9 @@ Only `nisse-dev` has real code; `nisse-core` and `nisse-ops` are placeholders. Authelia. The infrastructure repository exists in Gitea. - **Local inference:** llama-swap on an RTX 4090 host (320 W cap). The benchmark (PR #55) chose `qwen3.8-27b` for local work. -- **Models by role today:** executor `qwen3.8-27b`, planner and helper - `claude-opus-5-5`, expert `claude-haiku-4-5`. Phase 2 moves to driver and helpers. +- **Models by role today:** driver and helper `claude-opus-5-5`, expert + `claude-haiku-4-5`. `qwen3.8-27b` has had no role since the executor's went + (M2.8); the driver benchmark measures where it fits. - **Bring-up** runs on this development VM (`scripts/bringup.sh`), against test resources: a test repository, the test Matrix room, a budget-capped LiteLLM key; configuration in `/root/nisse-e2e/`. @@ -85,11 +86,9 @@ Only `nisse-dev` has real code; `nisse-core` and `nisse-ops` are placeholders. (`-k`) or diagnosed (`--lf`); `scripts/e2e.sh` reports what a run spent. Manual live runs are for the new loop's first runs and phase ends. Findings are filed as issues and either fixed before moving on or deliberately deferred. -- **The Quint models move with the loop:** `orchestrator.qnt` while `control.py` - runs, `loop.qnt` for the new loop, whose properties become tests as each piece - lands (Phase 2). -- **`control.py` gets fixes only,** and only for what blocks a live run. New - behaviour goes into the new loop. +- **The Quint model moves with the loop:** `loop.qnt` models the architecture, + and its properties are tests (the property walk), changed in the same PR as a + rule they cover. - **Bring-up runs on this box as root,** against test resources only, with today's gates on until the classifier and trust dial are proven (Phase 3). - **This plan is revised as we go,** in the same PR as the work that changes it. @@ -230,8 +229,8 @@ deletes afterwards. **M2.6 Roles and capability profiles.** *Pre-approved.* **Done** (this PR). `driver`, `helper`, `expert`, `classifier` (L9); escalation triggers; delegation and `ask_expert` on the new loop. -- `roles.driver` (unset: the planner's model) and `roles.classifier` (unset: the - expert's), pinned per item; the driver and the go-ahead reader run on the item's. +- `roles.driver` (required since M2.8; until then, unset meant the planner's + model) and `roles.classifier` (unset: the expert's), pinned per item; the driver and the go-ahead reader run on the item's. - `profiles:` in configuration (tools, strengths, weaknesses), used in briefings: the driver is told what its helper and expert are good at, the helper who will read its handback; never a model's name. @@ -264,9 +263,8 @@ in prose, asking mid-work (L5, #42, #77). defaults, secrets by name with their visibility, egress additions. - Acceptance: the bring-up configuration is one project entry; a second project needs only a second entry. -- `projects: {name: entry}`. The old loop runs exactly one project and refuses - more. On the new loop each project has its own clone, workspace, push credential - and milestones. Issues are named `project#14` when two projects could both have +- `projects: {name: entry}`. Each project has its own clone, workspace, push + credential and milestones. Issues are named `project#14` when two projects could both have `#14`, and a bare `#14` that matches two is asked about. Switching projects parks the one worked on last. - Secrets are by `${VAR}` reference only, `commands` or `harness`. `commands` @@ -295,8 +293,20 @@ the release's own tests over it). - The live bring-up's switch to the image waits for the next live test (it needs its own test repository too); `bringup.sh plan` shows what it will run. -**M2.8 Switch over.** *Pre-approved, after acceptance.* `nisse-dev` runs the new -loop; `control.py`, its tests and `orchestrator.qnt` are removed. +**M2.8 Switch over.** *Pre-approved, after acceptance.* **Done** (this PR, after +the live session's fixes in #119 and #122). `nisse-dev` runs only the loop. +- Removed: `control.py`, `planner.py`, `executor.py`, `approvals.py`, `gates.py`, + `publish.py`, `poller.py`, the old assembly (`agent.py`, folded into + `loop/build.py`), the state store's state-machine view, their tests, the old + end-to-end and live planning tests, and `orchestrator.qnt` with its scenarios. +- Configuration: `loop:`, `roles.planner`, `roles.executor` and + `tuning.executor` are refused with what to use instead; `roles.driver` is + required, and `tuning.driver` takes the driver's turn cap and failure limit. +- The benchmark and the capability matrix measure the driver: the loop's own + system prompt and tools, the task's plan as the agreed restatement. Task plans + end with `finish_milestone`, so earlier runs show as `(old)`. +- A delegation on the loop records what its helper's alias resolved to (L9), as + the old path's did. ### Phase 3 — Authority @@ -392,7 +402,7 @@ if the implementation differs. lines, then drop the oldest events with a note saying so; never the system prompt, the current plan or the most recent turns. View labels: `[orchestrator]`, `[operator, Matrix]`, `[operator, comment on PR #N]`, - `[helper report]`, `[executor]` narration. + `[helper report]`, `[driver]` narration. - **L7 rendering:** content is stored as bytes or local paths and uploaded or inlined at render time, never as provider handles. Token budgeting and prompt-cache breakpoints are render-time concerns, never stored. Compaction diff --git a/docs/Tuning-Guide.md b/docs/Tuning-Guide.md index 3e67937..0914767 100644 --- a/docs/Tuning-Guide.md +++ b/docs/Tuning-Guide.md @@ -127,9 +127,10 @@ a local model can be held to less without affecting the frontier roles. response inside the context window. With a 32k context and a 16k ceiling, only 16k is left for the prompt. Some servers reject any request where prompt plus ceiling exceeds the context. -- **Too low:** truncated plans and tool calls. -- **Deciding:** executor turns are short, so 2–4k is plenty for a local - executor. Frontier planning wants the large default. Leave this high, and +- **Too low:** truncated restatements and tool calls. +- **Deciding:** a working turn is short, so 2–4k is plenty for a local + model doing a milestone's work. A frontier driver restating and designing + wants the large default. Leave this high, and set a local model's smaller ceiling as its `max_output_tokens` in LiteLLM. ### `llm.request_timeout_s` — default 600 @@ -137,8 +138,8 @@ a local model can be held to less without affecting the frontier roles. How long one model request may take before it is abandoned. It applies to every request, on both routes. -- **Too low:** a slow model's long answer is cut off, and the executor - escalates work that was going well. At 120 s, the old fixed value, a dense +- **Too low:** a slow model's long answer is cut off, and the driver's run + fails on work that was going well. At 120 s, the old fixed value, a dense 27B model on the RTX 4090 timed out writing one module in one turn. - **Too high:** a hung connection holds the milestone for that long before anything notices. @@ -147,10 +148,12 @@ every request, on both routes. expect, which is the model's output ceiling divided by its generation speed. -### `tuning.executor.max_turns` — default 40 +### `tuning.driver.max_turns` — default 40 -How many model calls one milestone may make before the executor stops with -`capped` and asks you what to do. +How many model calls one driver run may make before it stops as `capped` and +asks you what to do. A run lasts until the driver waits (for you, a review or +a merge), so a long milestone is usually one run. Before M2.8 this was +`tuning.executor.max_turns`. - **Too low:** milestones that were making progress stop short. You'll see `capped` outcomes on work that was going well. @@ -167,12 +170,14 @@ How many model calls one milestone may make before the executor stops with `long-milestone`, one of which had in fact finished the work. It is a guard against a model trying forever, not a budget, so it was raised to 40. -### `tuning.executor.failure_limit` — default 2 +### `tuning.driver.failure_limit` — default 2 -How many consecutive failed `run_command` calls trigger escalation: the -sub-task is handed to a frontier model as a delegation (architecture L9). A -successful command resets the count. Failed file reads don't count, since -reading is looking around, not doing the work. +How many consecutive failed `run_command` calls mean the driver is stuck: the +first time in a milestone, a helper takes the problem as a delegation +(architecture L9); the second time, the item waits on you. A successful +command resets the count. Failed file reads don't count, since reading is +looking around, not doing the work. Before M2.8 this was +`tuning.executor.failure_limit`. - **Why it's a count and not a judgment:** small models are poorly calibrated about their own competence. Asked "are you stuck?", they say no until far @@ -194,13 +199,13 @@ How many `ask_expert` calls one milestone may make. `ask_expert` is a single stateless question to a stronger model, with no access to the repository (architecture L9), for things like "is this API deprecated?". -- **Too low:** the executor can't get unstuck on a knowledge question it +- **Too low:** the driver can't get unstuck on a knowledge question it can't answer from the code. -- **Too high:** a confused executor asks variations of the same question +- **Too high:** a confused driver asks variations of the same question instead of doing the work or failing. That's the circling L9 warns about. - **Deciding:** count consults per milestone, and check whether the answers - changed what the executor did next. If it regularly exhausts the budget - without progress, the budget is fine and the model or the plan is the + changed what the driver did next. If it regularly exhausts the budget + without progress, the budget is fine and the model or the brief is the problem. ### `tuning.delegation` — hard caps on one delegation @@ -282,10 +287,9 @@ Tokens are estimated from characters (pessimistically, at 3 per token) and the estimate is corrected from the prompt sizes the server reports. The correction only ever makes the estimate larger. -The executor also sends less in the first place: the approved milestone plan -goes in its system prompt, which is never cut, and it sees only what happened -since that plan. The planning conversation and earlier milestones stay in the -log. +Each role also sends less in the first place: it sees its own work item's +transcript (L6), with other roles' work reduced to what it needs (a helper's +handback is one report to the driver). | Knob | Default | | |---|---|---| @@ -331,14 +335,15 @@ Asking a model that can't honour it can be worse than not asking: the Qwen3-14B turn above costs a full output ceiling of generation, where a plain prose reply would have cost a few hundred tokens. -Where it isn't enforced, the executor holds the contract itself: +Where it isn't enforced, the driver's run holds the contract itself: -- **`finish` is the only way to end a milestone.** It is refused once if - nothing has been changed yet. -- **A prose reply is a violation.** The model is told once; a second escalates. -- **A call to a tool that wasn't offered counts as a failure** toward - `failure_limit`, like a failed command, so an inventing model escalates in a - couple of turns instead of using up `max_turns`. +- **A waiting tool is the only way to end a run** (`finish_milestone`, + `ask_human`, and the like); the harness, not the model, then publishes or + asks. +- **A prose reply is a violation.** The model is told once; a second ends + the run, and the item waits on you. +- **A call to a tool that wasn't offered is answered, not run:** the result + names the tools there are, so the model can put it right on its next turn. The capability matrix checks the declaration: a model declared `true` that answers a required call in prose fails @@ -412,14 +417,17 @@ done. The first four were the original set. By September 2026 all seven newer local models passed all of them, so the last four were added to separate models that can hold the tool contract from models that can do a milestone's real work. -Their plans stay explicit, as the planner is told to write them; what makes +Their plans stay explicit, as an agreed restatement is; what makes them harder is the work itself: tracing a cause, searching a codebase, reading a spec closely, and holding a plan over many turns. -It runs the executor exactly as the agent does (the same client wiring, -context budget and `tool_choice` handling, and your config's tuning knobs), -with **delegation and `ask_expert` turned off**. An escalation is a failure of -the model under test, not a frontier model's success. +It runs the loop's own driver exactly as the agent does: the same system +prompt, tools, client wiring, context budget, `tool_choice` handling and your +config's tuning knobs. The task's plan is given as the agreed restatement, and +finishing the milestone ends the run. **Delegation and `ask_expert` are +turned off**: an escalation is a failure of the model under test, not a +frontier model's success. Until M2.8 it ran the old loop's executor, so runs +from before then are against older task versions and show as `(old)`. **Evaluating a new model** is one command, once its LiteLLM entry declares its limits: @@ -456,7 +464,7 @@ served differently (different note) are kept apart. `failure_limit: 2` mid-recovery. That says as much about the knob as about the model. Tuning a knob and re-running is a fair comparison, because the tuning is recorded with every run. -- **Cost** is per run, straight from LiteLLM, so a frontier model as executor +- **Cost** is per run, straight from LiteLLM, so a frontier model as driver can be compared on success rate *and* dollars. --- @@ -511,8 +519,8 @@ Later work that will add knobs, each with its own section here: - **Compaction.** A model-written summary replacing old turns, instead of dropping them. The log already has an event for it; it needs a summarizing model and a policy for what is safe to summarize. -- **Benchmarking the other roles:** planner and delegate, with delegation - turned back on for comparisons across frontier tiers. +- **Benchmarking the other roles:** the helper, with delegation turned back + on for comparisons across frontier tiers. ### `tuning.loop.bug_grace_s` — default 600 diff --git a/nisse-dev.example.yaml b/nisse-dev.example.yaml index 0884089..5f1e8dc 100644 --- a/nisse-dev.example.yaml +++ b/nisse-dev.example.yaml @@ -41,14 +41,11 @@ llm: # L9: LiteLLM is the only LLM destination request_timeout_s: 600 roles: # L9: role -> LiteLLM model alias - executor: qwen3.8-27b # the local tier: best executor in the benchmark (PR #55) - planner: claude-opus-5-5 # high-end judgement: plans a weaker executor follows - helper: claude-opus-5-5 # takes over what the executor got stuck on - expert: claude-haiku-4-5 # one question at a time, no tools: cheap is enough - # The new loop's roles (loop: new): unset, driver is the planner's model and - # classifier the expert's. Each work item pins the mapping when it opens. - # driver: claude-opus-5-5 # drives the loop: talks, restates, works - # classifier: claude-haiku-4-5 # reads plain-language go-aheads (later: actions, L4) + # Each work item pins the mapping when it opens. + driver: claude-opus-5-5 # drives the loop: talks, restates, works + helper: claude-opus-5-5 # takes over what the driver got stuck on + expert: claude-haiku-4-5 # one question at a time, no tools: cheap is enough + # classifier: claude-haiku-4-5 # reads go-aheads (later: actions, L4); unset, expert's # Capability profiles (L9), from benchmarks: what one role is told about # another (never a model's name). Optional; a model without one is briefed as @@ -83,8 +80,8 @@ gitea: # D4, D5 # These are the defaults; the section can be left out entirely. The file is # read at startup, so a change takes effect on the next restart. tuning: - executor: - max_turns: 40 # model calls per milestone before it stops as capped + driver: + max_turns: 40 # model calls per driver run before it stops as capped failure_limit: 2 # consecutive failed commands before escalating (L9) expert: consult_budget: 3 # ask_expert calls per milestone (L9) diff --git a/nisse-dev/src/nisse_dev/agent.py b/nisse-dev/src/nisse_dev/agent.py deleted file mode 100644 index d7550c1..0000000 --- a/nisse-dev/src/nisse_dev/agent.py +++ /dev/null @@ -1,342 +0,0 @@ -"""Composition root: the one place that knows how the whole agent fits together. - -Every component below has been unit-tested in isolation for some time; nothing -had ever assembled them. That gap hid two live bugs — the delegation harness -and the Gitea comment poller were both fully built and reachable from nothing -but their own tests — which is the argument for having a single root rather -than constructing pieces ad hoc. -""" - -import threading -from collections.abc import Callable -from dataclasses import dataclass - -from nisse_dev.approvals import PlanBoard -from nisse_dev.cancel import Cancel -from nisse_dev.catalog import ModelCatalog, select_client -from nisse_dev.config import AgentConfig, Project -from nisse_dev.context import BudgetedClient -from nisse_dev.control import Orchestrator -from nisse_dev.delegation import Caps, Delegation -from nisse_dev.execution import SSHHost -from nisse_dev.executor import LLMExecutor -from nisse_dev.expert import ExpertDesk -from nisse_dev.git import GitRepo, HTTPSToken, PushCredential, SSHKey -from nisse_dev.gitea import Comment, GiteaClient -from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient -from nisse_dev.matrix import MatrixTransport -from nisse_dev.oplog import OpLog -from nisse_dev.planner import LLMPlanningAgent -from nisse_dev.poller import CommentPoller -from nisse_dev.publish import MilestonePublisher -from nisse_dev.state import StateStore -from nisse_dev.tools import Workspace -from nisse_dev.worker import SerialWorker - -# How long shutdown waits for running work before leaving the store to the -# durability layer (L8). A milestone can run far longer than a service -# manager allows for a stop, and a hard exit mid-work is already safe. -SHUTDOWN_WAIT_S = 10.0 - -TOOL_DRIVING_ROLES = ("planner", "executor", "helper") - - -def discover_models(config: AgentConfig, oplog: OpLog) -> ModelCatalog: - """Resolve every configured role against LiteLLM before anything runs. - - Refusing to boot is the whole point: a role pointed at a model this key - cannot see is a misconfiguration, and the cheapest place to learn that is - here rather than ten minutes into a milestone. An unreachable proxy is - reported separately -- it is an outage, not a mistake, and telling them - apart is what lets an operator know whether to edit config or wait. - """ - catalog = ModelCatalog.discover(config.llm_base_url, config.llm_token) - catalog.validate(config.roles.pinned()) - # Every role's context window must be known before a request is built - # against it (L6): a guess that is too large reaches a server that may - # silently cut the prompt. - catalog.require_limits(config.roles.pinned()) - for role in TOOL_DRIVING_ROLES: - facts = catalog.facts(config.roles.resolve(role)) - if facts.supports_tools is False: - # A warning, not a refusal: models LiteLLM declares tool-less are - # observed driving tool loops anyway, and L9 expects weak local - # models to be accommodated rather than excluded. - oplog.emit( - "model_tools_unsupported", - cause="startup", - role=role, - model=facts.alias, - backing=facts.backing, - ) - return catalog - - -@dataclass -class Agent: - config: AgentConfig - store: StateStore - transport: MatrixTransport - orchestrator: Orchestrator - gitea: GiteaClient - account: str - worker: SerialWorker - _stopped: threading.Event - - def start(self) -> None: - """Boot in the order L8 requires: reconcile against external truth - before accepting a single operator command.""" - self.worker.call(self.orchestrator.start) - self.transport.start() - state = self.worker.call(self.store.get_state) - where = state.state if state else "IDLE" - self.transport.send_message( - f"nisse-dev up as {self.account} — project {self.config.project}, " - f"state {where}, planner {self.config.roles.planner}, " - f"executor {self.config.roles.executor}" - ) - - def wait(self) -> None: - self._stopped.wait() - - def stop(self) -> None: - """Shut down outermost-first: stop accepting work, then stop talking, - then close the state store last so nothing writes to a closed file. - The store belongs to the worker, so the worker closes it.""" - self.orchestrator.stop_watching() - self.transport.close() - self.gitea.close() - try: - self.worker.call(self.store.close, timeout=SHUTDOWN_WAIT_S) - except TimeoutError: - pass # still working; exiting mid-work is what L8 are for - self.worker.stop(timeout=SHUTDOWN_WAIT_S) - self._stopped.set() - - -def make_llm_clients( - config: AgentConfig, catalog: ModelCatalog -) -> tuple[BudgetedClient, BudgetedClient]: - """The two LLM routes, as every model call in the agent uses them. - - Shared with the benchmark, so that what it measures is exactly what the - agent would do: the same context budget, the same declared limits, and - tool_choice asked only of models declared to enforce it. - """ - tuning = config.tuning - - def limits_for(model: str) -> tuple[int, int | None]: - facts = catalog.facts(model) - assert facts.max_input_tokens is not None # require_limits checked - return facts.max_input_tokens, facts.max_output_tokens - - def budgeted(client: object) -> BudgetedClient: - # Every request, on either route, is fitted to its model's window. - return BudgetedClient( - client, - limits_for, - max_output_tokens=config.max_output_tokens, - keep_recent_turns=tuning.context.keep_recent_turns, - safety_margin=tuning.context.safety_margin, - enforces_tool_choice=lambda m: ( - catalog.facts(m).supports_tool_choice is True - ), - ) - - normalized = budgeted( - LiteLLMClient( - config.llm_base_url, - config.llm_token, - max_tokens=config.max_output_tokens, - timeout_s=config.request_timeout_s, - ) - ) - # Delegations prefer the vendor-native route; fidelity is cheap - # elsewhere (L9). Which route a given model gets is decided by - # select_client, not by where the caller happens to sit. - passthrough = budgeted( - AnthropicPassthroughClient( - config.llm_base_url, - config.llm_token, - max_tokens=config.max_output_tokens, - timeout_s=config.request_timeout_s, - ) - ) - return normalized, passthrough - - -def build_host(config: AgentConfig, project: Project | None = None) -> SSHHost: - """A project's build VM (L13; the only project's, if none is named). Its - known-hosts file and connection socket live beside the state store.""" - host = (project or config.only_project()).build_host - return SSHHost( - host.address, - key=host.key, - user=host.user, - port=host.port, - state_dir=config.state_db.parent / "ssh", - ) - - -def push_credential( - config: AgentConfig, project: Project | None = None -) -> PushCredential | None: - """What a project's pushes go by: a push-only account's token over - HTTPS, or a deploy key over SSH (D4).""" - project = project or config.only_project() - if project.push_user and project.push_token: - return HTTPSToken(project.push_user, project.push_token) - if project.push_key is None: - return None - try: - return SSHKey(project.push_key.read_text()) - except OSError as exc: - raise RuntimeError( - f"cannot read the push key {project.push_key}: {exc}" - ) from exc - - -def remote_url(gitea: GiteaClient, repo: str, credential: PushCredential | None) -> str: - """The credential-free URL the workspace clones from and pushes to.""" - if isinstance(credential, SSHKey): - return gitea.ssh_url(repo) - return gitea.clone_url(repo) - - -def prepare_workspace( - host: SSHHost, - config: AgentConfig, - account: str, - remote: str, - credential: PushCredential | None, - project: Project | None = None, -) -> GitRepo: - """Ensure the workspace on the build VM is a clone of the project repo. - - The clone uses the repository's SSH URL, so no credential is ever stored - in it (#89); the push credential is handed to the clone and nothing else. - Cloning here rather than when a work item starts is the single-project - shortcut that M1.4 and D3's fresh workspace replace. - """ - repo = GitRepo(host, (project or config.only_project()).workspace) - if not repo.exists(): - repo.clone(remote, credential=credential) - repo.configure_identity(account, f"{account}@users.noreply.local") - repo.install_excludes() # build artifacts are never the work (#80) - return repo - - -def build_agent( - config: AgentConfig, *, notify: Callable[[str], None] | None = None -) -> Agent: - oplog = OpLog("nisse-dev") - worker = SerialWorker( - on_error=lambda exc: oplog.emit( - "worker.job_failed", cause="unhandled exception", error=repr(exc) - ) - ) - worker.start() - # Opened on the worker, which is the only thread allowed to use it. - store = worker.call(lambda: StateStore(config.state_db, oplog=oplog)) - cancel = Cancel() - # Before the clone, before the token checks: a typo'd role name should - # cost a second, not a repository. - catalog = discover_models(config, oplog) - - gitea = GiteaClient(config.gitea_url, config.gitea_token) - # Whoami rather than another config variable: it names the identity every - # Gitea permission is evaluated against, and proves the token works before - # anything depends on it. - account = gitea.whoami() - - host = build_host(config) - credential = push_credential(config) - remote = remote_url(gitea, config.gitea_repo, credential) - git_repo = prepare_workspace(host, config, account, remote, credential) - tuning = config.tuning - workspace = Workspace( - host, - config.workspace, - timeout_s=tuning.tools.command_timeout_s, - max_output_chars=tuning.tools.max_output_chars, - ) - - normalized, passthrough = make_llm_clients(config, catalog) - - transport = MatrixTransport(config.matrix) - announce = notify or transport.send_message - - expert = ExpertDesk(normalized, config.roles, budget=tuning.expert.consult_budget) - delegate_model = config.roles.resolve("helper") - delegation = Delegation( - select_client(delegate_model, normalized, passthrough, catalog), - store, - workspace, - git_repo, - model=delegate_model, - backing_model=catalog.facts(delegate_model).backing, - caps=Caps( - turns=tuning.delegation.max_turns, - tokens=tuning.delegation.max_tokens, - wall_clock_s=tuning.delegation.wall_clock_s, - ), - cancel=cancel, - ) - executor = LLMExecutor( - normalized, - config.roles, - workspace, - expert=expert, - delegate=delegation, - notify=announce, - cancel=cancel, - max_turns=tuning.executor.max_turns, - failure_limit=tuning.executor.failure_limit, - ) - publisher = MilestonePublisher( - store, - gitea, - config.gitea_repo, - git_repo, - remote_url=remote, - push_credential=credential, - base_branch=config.base_branch, - ) - - def poller_factory(on_comment: Callable[[Comment], None]) -> CommentPoller: - return CommentPoller( - gitea, - config.gitea_repo, - on_comment, - oplog=oplog, - # Its own comments are not new input; without this the agent - # answers itself. - ignore_authors={account}, - ) - - orchestrator = Orchestrator( - store, - transport, - LLMPlanningAgent(normalized, config.roles, cancel=cancel), - operators=config.operators, - roles=config.roles, - executor=executor, - publisher=publisher, - gitea_operators=config.gitea_operators, - board=PlanBoard(gitea, config.gitea_repo), - delegate=delegation, - poller_factory=poller_factory, - catalog=catalog, - worker=worker, - cancel=cancel, - ) - return Agent( - config=config, - store=store, - transport=transport, - orchestrator=orchestrator, - gitea=gitea, - account=account, - worker=worker, - _stopped=threading.Event(), - ) diff --git a/nisse-dev/src/nisse_dev/approvals.py b/nisse-dev/src/nisse_dev/approvals.py deleted file mode 100644 index 3952b73..0000000 --- a/nisse-dev/src/nisse_dev/approvals.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Plan externalisation and Gitea-backed approvals (architecture L3, D2). - -Plans are posted as comments on a per-project Gitea issue. The approval -comment — its author, timestamp, and revision tag — is the durable off-box -record, authenticated by Gitea's own auth. The state store keeps only a -pointer to it, so boot reconciliation can re-derive gate state from Gitea -even after the state volume is lost: external truth wins. - -Approvals arriving here bind to a revision exactly as they do over transport -(L3). The first valid approval on any channel wins; the orchestrator echoes -the outcome to the other channel so the records converge. -""" - -from dataclasses import dataclass - -from nisse_dev.gitea import Comment, GiteaClient, GiteaError - -APPROVAL_HINT = ( - "Reply `approve {tag}` to approve this revision, or comment with feedback " - "to reject it." -) - - -@dataclass(frozen=True) -class PlanPost: - issue: int - comment_id: int - - -class PlanBoard: - """The Gitea side of plan gating: one issue per project, one comment per - plan revision.""" - - def __init__(self, client: GiteaClient, repo: str) -> None: - self._client = client - self._repo = repo - - @property - def repo(self) -> str: - return self._repo - - def open_project_issue(self, project: str, task_id: str) -> int: - return self._client.create_issue( - self._repo, - title=f"nisse-dev: {project}", - body=( - f"Plan gate for project `{project}` (task `{task_id}`).\n\n" - "Each plan revision is posted here as a comment. Approving a " - "revision by comment is equivalent to approving it in Matrix; " - "the first valid approval on either channel wins." - ), - ) - - def post_plan(self, issue: int, gate: str, rev: int, plan: str) -> PlanPost: - tag = f"rev-{rev}" - body = "\n".join( - [ - f"## Plan for `{gate}` ({tag})", - "", - plan, - "", - "---", - APPROVAL_HINT.format(tag=tag), - ] - ) - comment = self._client.post_comment(self._repo, issue, body) - return PlanPost(issue=issue, comment_id=comment.id) - - def echo(self, issue: int, text: str) -> None: - """Mirror an outcome decided on another channel, so the Gitea record - does not silently disagree with Matrix.""" - try: - self._client.post_comment(self._repo, issue, text) - except GiteaError: - pass - - def close(self, issue: int, note: str) -> None: - """Close the plan issue with a note saying why (e.g. abandoned). - Unlike echo, failures raise: the caller reports them.""" - self._client.post_comment(self._repo, issue, note) - self._client.close_issue(self._repo, issue) - - def comments_since(self, issue: int, after_id: int) -> list[Comment]: - return [ - c for c in self._client.list_comments(self._repo, issue) if c.id > after_id - ] diff --git a/nisse-dev/src/nisse_dev/bench/__init__.py b/nisse-dev/src/nisse_dev/bench/__init__.py index 1e5b0a7..bcc92fa 100644 --- a/nisse-dev/src/nisse_dev/bench/__init__.py +++ b/nisse-dev/src/nisse_dev/bench/__init__.py @@ -1,6 +1,6 @@ -"""The executor benchmark: canned milestones, run repeatedly per model. +"""The driver benchmark: canned milestones, run repeatedly per model. -Choosing an executor model, and later tuning its knobs, should rest on +Choosing a driver model, and later tuning its knobs, should rest on measured success rates, not on one run or on reputation. Each task is a fixed approved plan over a seed project, with a hidden check the model never sees; success means the check passes, not that the model said it finished. Runs use diff --git a/nisse-dev/src/nisse_dev/bench/results.py b/nisse-dev/src/nisse_dev/bench/results.py index b39b967..306defe 100644 --- a/nisse-dev/src/nisse_dev/bench/results.py +++ b/nisse-dev/src/nisse_dev/bench/results.py @@ -74,12 +74,17 @@ class Row: def tuning_label(tuning: dict[str, Any]) -> str: """The settings in a run's tuning that differ from today's defaults, as - "executor.max_turns=24"; empty when there are none. Relative to today's - defaults so that rows at the current settings stay unlabelled.""" + "driver.max_turns=24"; empty when there are none. Relative to today's + defaults so that rows at the current settings stay unlabelled. Runs from + before M2.8 called the driver's section ``executor``.""" defaults = asdict(Tuning()) + renamed = { + ("driver" if section == "executor" else section): values + for section, values in tuning.items() + } return ", ".join( f"{section}.{key}={value}" - for section, values in sorted(tuning.items()) + for section, values in sorted(renamed.items()) for key, value in sorted((values or {}).items()) if defaults.get(section, {}).get(key) != value ) diff --git a/nisse-dev/src/nisse_dev/bench/runner.py b/nisse-dev/src/nisse_dev/bench/runner.py index 358bb15..fc197a9 100644 --- a/nisse-dev/src/nisse_dev/bench/runner.py +++ b/nisse-dev/src/nisse_dev/bench/runner.py @@ -1,4 +1,4 @@ -"""Running one task once with one executor model, and recording it.""" +"""Running one task once with one driver model, and recording it.""" import io import shlex @@ -14,7 +14,7 @@ from typing import Any from nisse_dev.bench.tasks import Task from nisse_dev.canonical import ( CanonicalEvent, - PlanSubmitted, + Fact, StoredEvent, ToolCall, ToolResult, @@ -23,11 +23,37 @@ from nisse_dev.canonical import ( from nisse_dev.catalog import ModelFacts from nisse_dev.config import Tuning from nisse_dev.execution import SSHHost -from nisse_dev.executor import LLMExecutor -from nisse_dev.llm import RoleMap +from nisse_dev.loop.agent import SYSTEM +from nisse_dev.loop.driver import Driver, RunOutcome +from nisse_dev.loop.tools import ( + ASK_HUMAN, + CLOSE_ITEM, + FINISH_MILESTONE, + START_MILESTONE, +) from nisse_dev.oplog import OpLog from nisse_dev.state import StateStore -from nisse_dev.tools import Workspace +from nisse_dev.tools import EXECUTION_TOOLS, Workspace + +# What the driver is offered once an item is agreed, less what the benchmark +# turns off: delegation and ask_expert (an escalation is the model's failure, +# not a frontier model's success) and review, which has no PR here. +BENCH_TOOLS = [ + ASK_HUMAN, + START_MILESTONE, + FINISH_MILESTONE, + CLOSE_ITEM, + *EXECUTION_TOOLS, +] + + +def outcome_of(run: RunOutcome) -> str: + """A driver run as the benchmark records it: ``completed`` when the + driver says the work is done, ``escalated`` when it asks the operator, + otherwise how the run ended (capped, stuck, failed).""" + if run.kind == "wait": + return "escalated" if run.tool == "ask_human" else "completed" + return run.kind class MeteringClient: @@ -158,11 +184,13 @@ def run_task( commit: str = "", workdir: Path | None = None, ) -> RunRecord: - """Run one task once: seed a workspace, hand the executor the approved - plan, then judge the result with the hidden check. + """Run one task once: seed a workspace, hand the driver an agreed item + whose restatement is the task's plan, then judge the result with the + hidden check. - Delegation and ask_expert are off: an escalation is a failure of the - model under test, not a frontier model's success.""" + It is the loop's own driver, system prompt and tools, so what is measured + is what the agent would do, without the harness around it: the milestone + is started but no branch is cut, and finishing it ends the run.""" with tempfile.TemporaryDirectory(dir=workdir) as tmp: root = Path(tmp) # The workspace is on the build VM, like the agent's own (L13). @@ -174,39 +202,51 @@ def run_task( ) task.seed_workspace(workspace) store = StateStore(root / "state.db", oplog=OpLog("nisse-bench", io.StringIO())) - # One work item per run, as the agent's own tasks are (L1). + # One work item per run, as the agent's own are (L1). store.open_work_item( - "bench", title=task.name, position="EXECUTING", cause="benchmark run" + "bench", title=task.name, position="RUNNING", cause="benchmark run" ) - store.transcript("bench", "planner").append( - PlanSubmitted(plan=task.plan, model="benchmark") + store.transcript("bench", "orchestrator").append( + Fact(text=f"The operator agreed to this restatement:\n\n{task.plan}") ) store.transcript("bench", "operator").append( - UserMessage(text="start milestone 1", source="benchmark") + UserMessage(text="go", source="benchmark") ) meter = MeteringClient(client, facts) trims_before = getattr(client, "trimmed_requests", 0) model = facts.alias - roles = RoleMap(executor=model, planner=model, helper=model, expert=model) - executor = LLMExecutor( + driver = Driver( meter, - roles, - workspace, - max_turns=tuning.executor.max_turns, - failure_limit=tuning.executor.failure_limit, + model, + max_turns=tuning.driver.max_turns, + failure_limit=tuning.driver.failure_limit, + ) + system = SYSTEM.format( + repo=f"bench/{task.name}", + issue=1, + title=task.title, + labels="", + helpers="", + workspace=workspace.describe(), ) started = time.monotonic() try: - outcome = executor.run(store.transcript("bench", "executor"), 1) - status, detail = outcome.status, outcome.note + run = driver.run( + store.transcript("bench", "driver"), + system=system, + tools=BENCH_TOOLS, + workspace=workspace, + handlers={"start_milestone": lambda args: ("started", True)}, + ) + status, detail = outcome_of(run), run.note except Exception as exc: # noqa: BLE001 -- recorded, the benchmark goes on status, detail = "error", repr(exc) wall = time.monotonic() - started success, check_output = task.check_workspace(workspace) host.execute(f"rm -rf -- {shlex.quote(str(workspace.root))}", timeout_s=60) - failed, invented = _count(store.transcript("bench", "executor").read()) + failed, invented = _count(store.transcript("bench", "driver").read()) store.close() return RunRecord( diff --git a/nisse-dev/src/nisse_dev/bench/tasks.py b/nisse-dev/src/nisse_dev/bench/tasks.py index 2985189..71afa12 100644 --- a/nisse-dev/src/nisse_dev/bench/tasks.py +++ b/nisse-dev/src/nisse_dev/bench/tasks.py @@ -3,11 +3,11 @@ Each task is a directory under ``tasks/``: - ``task.yaml``: ``title``, ``plan`` (the approved milestone plan, written the - way the planner is asked to write them), and ``check`` (a shell command run + way the driver restates them), and ``check`` (a shell command run in the workspace afterwards). -- ``seed/``: the project the executor starts from. +- ``seed/``: the project the driver starts from. - ``hidden/``: files the check needs, copied in only after the run, into - ``.bench_check/``. The executor never sees them. + ``.bench_check/``. The driver never sees them. - ``solution/``: a reference solution, overlaid on the seed by the tests to prove each task can be solved and that its check tells the difference. """ diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml index 7a676d1..2dedf7d 100644 --- a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml @@ -9,4 +9,4 @@ plan: | 3. Fix `total_value` so that each item contributes its unit price times its quantity. 4. Run `python -m pytest -q` until every test passes. - 5. Call finish. + 5. Call finish_milestone, with final yes. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/long-milestone/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/long-milestone/task.yaml index 2b6cfee..bf90afb 100644 --- a/nisse-dev/src/nisse_dev/bench/tasks/long-milestone/task.yaml +++ b/nisse-dev/src/nisse_dev/bench/tasks/long-milestone/task.yaml @@ -37,4 +37,4 @@ plan: | then the same with `list --overdue`, which must print `[!] 1 Pay rent (due 2026-10-01)`. Delete /tmp/tracker-check.json before and after. - 9. Call finish. + 9. Call finish_milestone, with final yes. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml index 3ac8ae3..5529f54 100644 --- a/nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml +++ b/nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml @@ -9,4 +9,4 @@ plan: | (NNN mod 25) / 100. Correct only the failing region's entry; leave every other line as it is. 3. Run `python -m pytest -q` until every test passes. - 4. Call finish. + 4. Call finish_milestone, with final yes. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml index 44f6902..8f0c24f 100644 --- a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml @@ -13,4 +13,4 @@ plan: | using pytest's capsys fixture. 4. Run `python -m pytest -q` until every test passes, and check that `python -m greeter --shout Bob` prints `HELLO, BOB!`. - 5. Call finish. + 5. Call finish_milestone, with final yes. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/refactor-rename/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/refactor-rename/task.yaml index eaa6d53..03453f5 100644 --- a/nisse-dev/src/nisse_dev/bench/tasks/refactor-rename/task.yaml +++ b/nisse-dev/src/nisse_dev/bench/tasks/refactor-rename/task.yaml @@ -15,4 +15,4 @@ plan: | are different functions and keep their names. Change no behaviour. 4. Update the tests under `tests/` to use the new name, and run `python -m pytest -q` until every test passes. - 5. Call finish. + 5. Call finish_milestone, with final yes. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/spec-parser/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/spec-parser/task.yaml index 8a6cd97..abaa95d 100644 --- a/nisse-dev/src/nisse_dev/bench/tasks/spec-parser/task.yaml +++ b/nisse-dev/src/nisse_dev/bench/tasks/spec-parser/task.yaml @@ -12,4 +12,4 @@ plan: | invalid input for each of the eight rules. 4. Run `python -m pytest -q test_durations.py`, and fix `durations.py` until every test passes. - 5. Call finish. + 5. Call finish_milestone, with final yes. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/trace-bug/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/trace-bug/task.yaml index 71af44a..3d33a85 100644 --- a/nisse-dev/src/nisse_dev/bench/tasks/trace-bug/task.yaml +++ b/nisse-dev/src/nisse_dev/bench/tasks/trace-bug/task.yaml @@ -18,4 +18,4 @@ plan: | `invoicing/currency.py` have up to six decimal places (JPY is 0.006712), and no step may round a rate or fraction to fewer places than that. 4. Run `python -m pytest -q` until every test passes. - 5. Call finish. + 5. Call finish_milestone, with final yes. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml index 9955265..ab4c6b2 100644 --- a/nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml +++ b/nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml @@ -10,4 +10,4 @@ plan: | "a--b" -> "a-b"; "" -> "". 3. Create `test_slugify.py` with pytest tests covering the four examples. 4. Run `python -m pytest -q test_slugify.py`, and fix anything until it passes. - 5. Call finish. + 5. Call finish_milestone, with final yes. diff --git a/nisse-dev/src/nisse_dev/config.py b/nisse-dev/src/nisse_dev/config.py index 872db62..8e184fd 100644 --- a/nisse-dev/src/nisse_dev/config.py +++ b/nisse-dev/src/nisse_dev/config.py @@ -17,7 +17,7 @@ silently ignored would look exactly like a knob that has no effect. import os import re from collections.abc import Mapping -from dataclasses import dataclass, field, fields, replace +from dataclasses import dataclass, field, fields from pathlib import Path from typing import Any @@ -32,8 +32,7 @@ REQUIRED_KEYS = ( "state_db", "llm.base_url", "llm.token", - "roles.executor", - "roles.planner", + "roles.driver", "roles.helper", "roles.expert", "matrix.homeserver", @@ -46,12 +45,16 @@ REQUIRED_KEYS = ( "gitea.operators", ) OPTIONAL_KEYS = ( - "roles.driver", "roles.classifier", "llm.max_output_tokens", "llm.request_timeout_s", - "loop", ) +# Settings the old loop had (until M2.8), and what to do instead. +RETIRED_KEYS = { + "loop": "the loop is the only one now; remove it", + "roles.planner": "use roles.driver", + "roles.executor": "the driver does the work now; remove it", +} # A project entry (D8), under `projects: {name: {...}}`. PROJECT_REQUIRED = ("repo", "workspace", "build_host.address", "build_host.key") PROJECT_OPTIONAL = ( @@ -85,7 +88,7 @@ class ConfigError(RuntimeError): @dataclass(frozen=True) -class ExecutorTuning: +class DriverTuning: max_turns: int = 40 failure_limit: int = 2 @@ -125,7 +128,7 @@ class LoopTuning: @dataclass(frozen=True) class Tuning: - executor: ExecutorTuning = field(default_factory=ExecutorTuning) + driver: DriverTuning = field(default_factory=DriverTuning) expert: ExpertTuning = field(default_factory=ExpertTuning) delegation: DelegationTuning = field(default_factory=DelegationTuning) tools: ToolsTuning = field(default_factory=ToolsTuning) @@ -148,6 +151,9 @@ def _parse_tuning(raw: Any, errors: list[str]) -> Tuning: sections: dict[str, Any] = {} for name, section in raw.items(): cls = _TUNING_SECTIONS.get(name) + if name == "executor": # until M2.8 + errors.append("tuning.executor: retired — it is tuning.driver now") + continue if cls is None: errors.append(f"tuning.{name}: unknown section") continue @@ -474,60 +480,8 @@ class AgentConfig: max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS request_timeout_s: float = DEFAULT_REQUEST_TIMEOUT_S tuning: Tuning = field(default_factory=Tuning) - # Which loop runs: "old" (control.py) until the switch-over (M2.8), or - # "new" (nisse_dev.loop). - loop: str = "old" config_file: Path | None = None - # -- one project, for the old loop ---------------------------------------- - # control.py (until M2.8) knows one project; these read it, and refuse a - # configuration with more. - - def only_project(self) -> Project: - if len(self.projects) != 1: - names = ", ".join(p.name for p in self.projects) - raise ConfigError( - f"cannot start: the old loop runs exactly one project, but " - f"{len(self.projects)} are configured ({names}); use `loop: new`" - ) - return self.projects[0] - - def with_project(self, **changes: Any) -> "AgentConfig": - """This configuration with its one project changed (for tests).""" - return replace(self, projects=(replace(self.only_project(), **changes),)) - - @property - def project(self) -> str: - return self.only_project().name - - @property - def gitea_repo(self) -> str: - return self.only_project().repo - - @property - def workspace(self) -> Path: - return self.only_project().workspace - - @property - def build_host(self) -> BuildHost: - return self.only_project().build_host - - @property - def base_branch(self) -> str: - return self.only_project().base_branch - - @property - def push_key(self) -> Path | None: - return self.only_project().push_key - - @property - def push_user(self) -> str | None: - return self.only_project().push_user - - @property - def push_token(self) -> str | None: - return self.only_project().push_token - @classmethod def load(cls, env: Mapping[str, str] = os.environ) -> "AgentConfig": """The file named by NISSE_CONFIG, or ./nisse-dev.yaml.""" @@ -573,7 +527,10 @@ class AgentConfig: known = set(REQUIRED_KEYS) | set(OPTIONAL_KEYS) for key in sorted(set(flat) - known): - errors.append(f"{key}: unknown setting") + if key in RETIRED_KEYS: + errors.append(f"{key}: retired — {RETIRED_KEYS[key]}") + else: + errors.append(f"{key}: unknown setting") values = { key: _resolve(value, key, env, errors) for key, value in flat.items() @@ -607,9 +564,6 @@ class AgentConfig: raise ValueError except (TypeError, ValueError): errors.append("llm.request_timeout_s: must be a positive number of seconds") - loop = str(values.get("loop") or "old") - if loop not in ("old", "new"): - errors.append("loop: must be old or new") if errors: raise ConfigError("cannot start: " + "; ".join(errors)) @@ -618,11 +572,9 @@ class AgentConfig: llm_base_url=str(values["llm.base_url"]), llm_token=str(values["llm.token"]), roles=RoleMap( - executor=str(values["roles.executor"]), - planner=str(values["roles.planner"]), + driver=str(values["roles.driver"]), helper=str(values["roles.helper"]), expert=str(values["roles.expert"]), - driver=str(values.get("roles.driver") or ""), classifier=str(values.get("roles.classifier") or ""), ), matrix=MatrixConfig( @@ -640,7 +592,6 @@ class AgentConfig: max_output_tokens=max_output, request_timeout_s=request_timeout, tuning=tuning, - loop=loop, config_file=config_file, ) config._refuse_secrets_inside_the_clone(env) @@ -685,7 +636,6 @@ class AgentConfig: f"{p.name} ({p.repo}, {p.build_host.address}:{p.workspace})" for p in self.projects ), - "planner": self.roles.planner, - "executor": self.roles.executor, + "driver": self.roles.driver, "helper": self.roles.helper, } diff --git a/nisse-dev/src/nisse_dev/control.py b/nisse-dev/src/nisse_dev/control.py deleted file mode 100644 index 4c8c442..0000000 --- a/nisse-dev/src/nisse_dev/control.py +++ /dev/null @@ -1,1331 +0,0 @@ -"""Control loop and state machine (architecture L2, L3; retired by M2.6). - -Gating is structural, not advisory: the agent's only actions are ``SubmitPlan`` -(posts the plan for approval and ends the agent's turn) and ``AskHuman`` -(pause, ask, resume on answer). There is deliberately no proceed/start action — -advancement past a gate happens only here, when an operator approval or command -arrives over transport. Every position change is checkpointed through the state -store before the loop continues, so a restart lands mid-gate, not mid-air. - -All of it runs on one worker (nisse_dev.worker), never on the thread that -delivered a message. The one exception is `stop` (L2), which has to reach -work that is already running: the transport's thread sets the cancel flag -directly, and the worker records the stop once the work has returned. -""" - -import sys -import traceback -import uuid -from collections.abc import Callable, Iterable, Iterator, Mapping -from contextlib import contextmanager -from dataclasses import dataclass -from typing import Any, Protocol - -from nisse_dev.cancel import Cancel -from nisse_dev.canonical import Fact, GateOpened, PlanSubmitted, UserMessage -from nisse_dev.gates import ( - Approved, - Gate, - Ignored, - PlanRevision, - Rejected, - command_form, -) -from nisse_dev.git import GitError -from nisse_dev.gitea import GiteaError -from nisse_dev.llm import RoleMap -from nisse_dev.reconcile import Reconciler, reconcile_on_boot -from nisse_dev.state import StateStore, Transcript -from nisse_dev.transport import InboundMessage, Transport -from nisse_dev.worker import InlineWorker, Worker - -IDLE = "IDLE" -PLANNING_ARCH = "PLANNING_ARCH" -AWAIT_ARCH_SIGNOFF = "AWAIT_ARCH_SIGNOFF" -MILESTONE_PLANNING = "MILESTONE_PLANNING" -AWAIT_MILESTONE_SIGNOFF = "AWAIT_MILESTONE_SIGNOFF" -EXECUTING = "EXECUTING" -PR_OPEN = "PR_OPEN" -AWAIT_INPUT = "AWAIT_INPUT" - -TRANSITIONS = { - (IDLE, PLANNING_ARCH), - (PLANNING_ARCH, AWAIT_ARCH_SIGNOFF), - (AWAIT_ARCH_SIGNOFF, PLANNING_ARCH), # rejected: replan with feedback - (AWAIT_ARCH_SIGNOFF, MILESTONE_PLANNING), - (MILESTONE_PLANNING, AWAIT_MILESTONE_SIGNOFF), - (AWAIT_MILESTONE_SIGNOFF, MILESTONE_PLANNING), - (AWAIT_MILESTONE_SIGNOFF, EXECUTING), - (EXECUTING, PR_OPEN), - # PR acceptance is the milestone's final gate (D5): a merged PR is the - # durable record, so completion returns to planning or to idle rather - # than passing through a separate FINISHED state. - (PR_OPEN, MILESTONE_PLANNING), - (PR_OPEN, IDLE), - # abandon (L2): the operator may end the task from any waiting state. - # (AWAIT_INPUT may go anywhere already.) - (AWAIT_ARCH_SIGNOFF, IDLE), - (AWAIT_MILESTONE_SIGNOFF, IDLE), -} - - -# States that are only right while work is running. Once a job ends the agent -# must be somewhere it can wait for the operator, or every message would be -# answered "busy" until a restart. -_WORKING = (PLANNING_ARCH, MILESTONE_PLANNING, EXECUTING) - -_BOOT_CHOICES = ( - "reply 'continue' to pick up where the restart cut in, " - "'re-plan' to plan the current step again, or 'abandon' to drop the task" -) - - -class TransitionError(RuntimeError): - pass - - -class PlanningError(RuntimeError): - """A planning turn produced nothing usable. Part of the PlanningAgent - contract: the orchestrator gates to AWAIT_INPUT rather than inventing a - plan or retrying forever.""" - - -class PlanningStopped(RuntimeError): - """The operator said `stop` (L2) before a planning turn could run.""" - - -@dataclass(frozen=True) -class SubmitPlan: - content: str - - -@dataclass(frozen=True) -class AskHuman: - question: str - - -class MilestoneExecutor(Protocol): - """What the orchestrator needs from an executor loop; the concrete one - lives in nisse_dev.executor (imported there, not here, to keep the - dependency pointing one way).""" - - def run(self, transcript: Transcript, milestone: int) -> Any: ... - - -class MilestonePublisher(Protocol): - """What the orchestrator needs to publish a milestone; the concrete one - lives in nisse_dev.publish.""" - - def reconcilers(self) -> Mapping[str, Reconciler]: ... - - def publish( - self, *, task_id: str, milestone: int, project: str, plan: str - ) -> Any: ... - - def push_followup( - self, *, task_id: str, milestone: int, message: str - ) -> str | None: ... - - def comment(self, index: int, body: str) -> None: ... - - def close_pull_request(self, index: int, note: str) -> None: ... - - def published_pull_request(self, task_id: str, milestone: int) -> Any: ... - - -class PlanBoard(Protocol): - """The Gitea side of plan gating; the concrete one is in - nisse_dev.approvals.""" - - def open_project_issue(self, project: str, task_id: str) -> int: ... - - def post_plan(self, issue: int, gate: str, rev: int, plan: str) -> Any: ... - - def echo(self, issue: int, text: str) -> None: ... - - def close(self, issue: int, note: str) -> None: ... - - -class PlanningAgent(Protocol): - def plan( - self, - transcript: Transcript, - gate: str, - project: str, - feedback: str | None = None, - answer: str | None = None, - ) -> SubmitPlan | AskHuman: ... - - -class StubAgent: - """Echoes canned plans so the approval loop is exercisable with no LLM.""" - - def plan( - self, - transcript: Transcript, - gate: str, - project: str, - feedback: str | None = None, - answer: str | None = None, - ) -> SubmitPlan | AskHuman: - notes = "" - if feedback: - notes += f" (revised after feedback: {feedback})" - if answer: - notes += f" (operator answered: {answer})" - return SubmitPlan(f"[stub] {gate} plan for {project}{notes}") - - -def _rev_before(board_state: dict[str, Any] | None, comment_id: int) -> int | None: - """The revision a Gitea comment was about: the latest plan posted before - it. None for a record kept before plan posts were.""" - posts = (board_state or {}).get("posts") - if posts is None: - return None - earlier = [int(rev) for cid, rev in posts if int(cid) < comment_id] - return max(earlier) if earlier else 0 - - -def _project_name(text: str) -> str: - """The name in 'use ', as written: only the command word is - case-insensitive.""" - return text.strip().rstrip(".!")[len("use") :].strip() - - -def _fresh_data() -> dict[str, Any]: - return { - # Latest revision per gate; revision numbers themselves run across - # the whole work item (last_rev), so rev-N names one plan (L3). - "revs": {}, - "last_rev": 0, - # The revision last put up for sign-off: a reply that names none - # binds to the one up when it arrived. - "shown_rev": 0, - "gate": None, - # What an operator reply in AWAIT_INPUT should resume. Every path into - # AWAIT_INPUT must set this, or the reply has nowhere to go. - "pending": None, - "milestone": 1, - "task_id": None, - "roles": None, # L9 mapping, pinned at task start - # What each pinned alias resolved to, discovered from LiteLLM. An - # alias is free to point somewhere else tomorrow, so the transcript - # has to say what actually answered (L9). - "models": None, - "pr": None, # the open PR for the current milestone, once published - # Pointer to the Gitea plan record (D2): the state store holds - # only this, so gate state can be re-derived from Gitea after a - # state-volume loss. - "board": None, - } - - -class Orchestrator: - def __init__( - self, - store: StateStore, - transport: Transport, - agent: PlanningAgent, - operators: Iterable[str], - roles: RoleMap | None = None, - executor: "MilestoneExecutor | None" = None, - publisher: "MilestonePublisher | None" = None, - gitea_operators: Iterable[str] = (), - board: "PlanBoard | None" = None, - delegate: Any = None, - poller_factory: Any = None, - catalog: Any = None, - worker: Worker | None = None, - cancel: Cancel | None = None, - ) -> None: - self._store = store - self._transport = transport - self._agent = agent - self._roles = roles - self._catalog = catalog - self._executor = executor - self._publisher = publisher - self._operators = set(operators) - # Per-channel allowlists (L3): a Matrix MXID and a Gitea login - # are different names for the same person, and neither implies - # the other. - self._gitea_operators = set(gitea_operators) - self._board = board - self._delegate = delegate - # Builds a poller bound to a comment handler; the orchestrator - # owns *when* to watch, which is a state-machine concern, while - # the factory owns how to build one. - self._poller_factory = poller_factory - self._pollers: dict[str, Any] = {} - # Transport and the comment poller deliver on their own threads, and - # every handler runs on the worker, one at a time, so state - # transitions never interleave. - self._worker = worker or InlineWorker() - # Shared with the executor, delegation and planner, which check it - # at each tool boundary. - self._cancel = cancel or Cancel() - # Read from the transport's thread, so plain values only: whether - # long work is running, and the state name for a "busy" reply. - self._working = False - self._state_name = IDLE - # The revision up for sign-off, stamped on each message as it arrives. - self._shown_rev = 0 - # Set by whichever path halted because of a stop, so the stop's own - # job knows whether it stopped anything. - self._halted_by_stop = False - self._data = _fresh_data() - self._project: str | None = None - transport.on_message(self._handle) - - def start(self, reconcilers: Mapping[str, Reconciler] | None = None) -> None: - # L8 boot step 5 runs before the generic sweep: an interrupted delegation is - # resumed or discarded by inspecting the branch and the log, and only a - # genuinely unclean cut is left for the sweep below to gate on. - if self._delegate is not None: - with self._long_work(): - self._delegate.recover(self._transport.send_message) - # The publisher owns the external-truth query for its own intents, so - # a configured publisher always contributes its reconciler (L8). - registered: dict[str, Reconciler] = dict(reconcilers or {}) - if self._publisher is not None: - registered = {**self._publisher.reconcilers(), **registered} - report = reconcile_on_boot(self._store, registered) - state = self._store.get_state() - if state is not None: - self._state_name = state.state - if state is None: - self._transition(IDLE, cause="boot: fresh state") - self._transport.send_message("nisse-dev ready (idle)") - return - self._data = {**_fresh_data(), **state.data} - self._project = state.project - self._store.oplog.task_id = self._data.get("task_id") - gate = self._data.get("gate") - # An open gate is announced again below, so its plan counts as up. - self._shown_rev = max( - int(self._data.get("shown_rev") or 0), - int(gate["rev"]) - if gate and state.state in (AWAIT_ARCH_SIGNOFF, AWAIT_MILESTONE_SIGNOFF) - else 0, - ) - if report.action == "await_input": - self._await_input( - cause="boot: unresolved intents", - prompt=( - "restart found unresolved work: " - + "; ".join(report.notes) - + f" — {_BOOT_CHOICES}" - ), - pending={"kind": "boot", "interrupted": state.state}, - ) - elif state.state == AWAIT_INPUT and self._data.get("pending"): - # Waiting on the operator is a committed state (L8 boot step 4): - # nothing was in flight, and the question and what answering it - # resumes are durable. Ask it again rather than replacing it. - pending = self._data["pending"] - self._transport.ask_question( - "restarted while waiting on you: " - + pending.get("prompt", "my last question still stands") - ) - elif state.state == EXECUTING and (published := self._published_pr()): - # L8 boot step 3: Gitea shows this milestone's PR open, so it was - # published and only PR_OPEN went unrecorded. Adopt the PR rather - # than ask, and never publish it again. Nothing watched it yet, - # so every comment on it is new. - milestone = int(self._data.get("milestone", 1)) - self._data["pr"] = { - "number": published.number, - "url": published.url, - "milestone": milestone, - "last_comment": self._watch( - "review", published.number, self.handle_review_comment, after=0 - ), - } - self._unwatch("plan") - self._transition(PR_OPEN, cause=f"boot: milestone {milestone} published") - self._transport.send_message( - f"restarted after milestone {milestone} was published: PR " - f"#{published.number} is open ({published.url}); watching it for " - "review comments" - ) - elif state.state == PR_OPEN: - # Waiting on review is a committed state like a gate (L8 boot step - # 8): resume, and watch the PR again from its cursor. - pr = self._data.get("pr") or {} - self._watch_pr(pr) - if pr.get("reviewing"): - self._transport.send_message( - f"restarted while answering review comment {pr['reviewing']} " - f"on PR #{pr.get('number')}; answering it again" - ) - else: - self._transport.send_message( - f"restarted with PR #{pr.get('number')} open; still watching " - "it for review comments" - ) - elif state.state == IDLE or ( - state.state in (AWAIT_ARCH_SIGNOFF, AWAIT_MILESTONE_SIGNOFF) and gate - ): - # A gate is a safe place to have been interrupted: nothing was in - # flight, the plan and revision are durable. Announce and resume, - # watching the plan issue again from its cursor. - if gate: - board_state = self._data.get("board") - if board_state and board_state.get("comment_id"): - self._watch( - "plan", - board_state["issue"], - self.handle_plan_comment, - # No cursor means data from before cursors were - # kept: skip what is there, as a fresh watch does, - # so an old approval cannot approve this revision. - after=board_state.get("last_comment") or None, - ) - self._transport.send_message( - f"restarted while awaiting sign-off on {gate['name']} " - f"rev-{gate['rev']}; the gate is still open" - ) - else: - # Interrupted mid-turn (planning, executing, unanswered question): - # never blind-resume (L8 boot step 5). - self._await_input( - cause=f"boot: restarted in {state.state}", - prompt=f"restarted mid-{state.state}; {_BOOT_CHOICES}", - pending={"kind": "boot", "interrupted": state.state}, - ) - - def _handle(self, msg: InboundMessage) -> None: - """Runs on the transport's thread, so it touches nothing durable and - hands everything to the worker.""" - if msg.sender not in self._operators: - return - text = msg.text.strip() - if command_form(text) == "stop": - # Straight to the flag, not behind the work in the queue: the - # work is what it has to reach (L2). - if self._worker.busy: - self._cancel.set() - self._worker.submit(lambda: self._finish_stop(msg)) - return - if self._working: - # Queued, it would run after the work, in whatever state the work - # left behind, which is not the state the operator was answering. - self._transport.send_message( - f"busy ({self._state_name}); say 'stop' to halt" - ) - self._worker.submit(lambda: self._record(msg)) - return - # Stamped here, on arrival: by the time the worker reaches it, the - # plan it was about may have been replaced. - about = self._shown_rev - self._worker.submit( - lambda: self._settled("your message", lambda: self._handle_now(msg, about)) - ) - - def _settled(self, what: str, job: Callable[[], None]) -> None: - """Run one job and make sure it ends in a state that can wait for the - operator, however it ended. An exception is reported rather than left - to the worker, which would only print it; a job left in a working - state is moved to AWAIT_INPUT with the choice a restart offers.""" - error: Exception | None = None - try: - job() - except Exception as exc: # noqa: BLE001 -- reported below - error = exc - traceback.print_exception(exc, file=sys.stderr) - state = self._store.get_state() - current = state.state if state else IDLE - reason = f"{type(error).__name__}: {error}" if error else "no outcome" - try: - if current in _WORKING: - self._await_input( - cause=f"stranded in {current}: {reason}", - prompt=f"{what} ended mid-{current} ({reason}); {_BOOT_CHOICES}", - # The same choice as after a restart, answered the same way. - pending={"kind": "boot", "interrupted": current}, - ) - elif error is not None: - self._transport.send_message( - f"could not handle {what} ({reason}); still {current}" - ) - except Exception as exc: # noqa: BLE001 -- nothing left to tell - traceback.print_exception(exc, file=sys.stderr) - - def _log(self, actor: str) -> Transcript: - """The active work item's transcript as one actor writes it; the - agent-level one when no work item is active (L6, L7).""" - item = self._store.active_item() - return self._store.transcript(item.id if item else None, actor) - - def _fact(self, text: str) -> None: - """Record what the models must know and cannot see for themselves: - every role is shown it, labelled as the orchestrator's (L6).""" - self._log("orchestrator").append(Fact(text=text)) - - def _close_item(self, outcome: str, cause: str) -> None: - item = self._store.active_item() - if item is not None and item.status == "open": - self._store.close_work_item(item.id, outcome=outcome, cause=cause) - - def _record(self, msg: InboundMessage) -> None: - self._log("operator").append( - UserMessage( - text=(msg.original or msg.text).strip(), - source=f"transport:{msg.sender}", - ) - ) - - def _finish_stop(self, msg: InboundMessage) -> None: - """The worker's half of `stop`. Whatever was running has returned by - now, and has already moved to AWAIT_INPUT if the stop reached it.""" - self._record(msg) - halted = self._halted_by_stop - self._halted_by_stop = False - self._cancel.clear() - if not halted: - self._transport.send_message(f"nothing to stop (state={self._state_name})") - - @contextmanager - def _long_work(self) -> Iterator[None]: - self._working = True - try: - yield - finally: - self._working = False - - def _halt_for_stop( - self, *, what: str, note: str, next_step: str, pending: dict[str, Any] - ) -> None: - """L2: stopped work moves to AWAIT_INPUT, and what happens next is the - operator's call. The prompt asks why, because the model should hear - the reason before it tries again.""" - self._halted_by_stop = True - self._await_input( - cause=f"operator: stop ({what})", - prompt=( - f"stopped {what} ({note}). Why did you stop it? Your reply is " - f"recorded for the model. {next_step}" - ), - pending=pending, - ) - - def _handle_now(self, msg: InboundMessage, about: int | None = None) -> None: - # Every operator message enters the canonical transcript, so planning - # turns see feedback and answers as conversation rather than as - # out-of-band parameters (L7). - self._log("operator").append( - UserMessage( - text=(msg.original or msg.text).strip(), - source=f"transport:{msg.sender}", - ) - ) - state = self._store.get_state() - current = state.state if state else IDLE - text = msg.text.strip() - command = command_form(text) - if command == "status": - self._send_status(current) - elif command == "abandon": - # Like status, a command in every waiting state: it takes - # precedence over reading the word as an answer or feedback. - if current == IDLE: - self._transport.send_message("no active task; nothing to abandon") - else: - self._abandon(msg.sender) - elif current == IDLE: - if command.startswith("use "): - self._begin_project(_project_name(text)) - else: - self._transport.send_message("no active project; say 'use '") - elif current in (AWAIT_ARCH_SIGNOFF, AWAIT_MILESTONE_SIGNOFF): - if not self._data.get("gate"): - # A sign-off state with no plan to sign off: replies could - # never be applied, so plan again from the operator's choice. - self._await_input( - cause=f"{current} without an open gate", - prompt=f"the plan awaiting sign-off is missing; {_BOOT_CHOICES}", - pending={ - "kind": "boot", - "interrupted": ( - PLANNING_ARCH - if current == AWAIT_ARCH_SIGNOFF - else MILESTONE_PLANNING - ), - }, - ) - return - self._handle_gate_reply(msg, text, about) - elif current == PR_OPEN: - self._handle_pr_open(text, msg.sender) - elif current == AWAIT_INPUT: - self._handle_answer(text) - else: - self._transport.send_message(f"busy ({current}); 'status' for details") - - def _send_status(self, current: str) -> None: - gate = self._data.get("gate") - gate_note = ( - f", awaiting sign-off on {gate['name']} rev-{gate['rev']}" if gate else "" - ) - self._transport.send_message( - f"state={current} project={self._project}{gate_note}" - ) - - def _begin_project(self, project: str) -> None: - task_id = f"{project}-{uuid.uuid4().hex[:8]}" - # Pin the role mapping now (L9): a mid-task config change must not - # split one task across two planners. - pinned = self._roles.pinned() if self._roles is not None else None - resolved = ( - self._catalog.pinned(pinned) - if self._catalog is not None and pinned is not None - else None - ) - self._data = { - **_fresh_data(), - "task_id": task_id, - "roles": pinned, - "models": resolved, - } - self._project = project - self._store.oplog.task_id = task_id - # A work item of its own (L1). One left active by an unfinished task - # (reachable only through a lost pending item) is closed first. - replaced = self._store.active_item() - if replaced is not None: - self._close_item("abandoned", f"replaced by use {project}") - self._store.set_active(None, cause=f"replaced by use {project}") - self._store.open_work_item( - task_id, - title=project, - scope=project, - position=IDLE, - data=self._data, - cause=f"operator command: use {project}", - ) - self._store.set_active(task_id, cause=f"operator command: use {project}") - # What the operator said before `use` is part of this task (D1). - self._store.carry_idle_messages(task_id) - self._fact(f"work item opened: project {project}, task {task_id}") - if self._board is not None: - try: - self._data["board"] = { - "issue": self._board.open_project_issue(project, task_id), - "last_comment": 0, - } - except GiteaError as exc: - # Gitea being down must not block planning; Matrix remains a - # complete approval channel on its own. - self._transport.send_message( - f"note: could not open the Gitea plan issue ({exc}); " - "approvals will be Matrix-only for this task" - ) - self._transition(PLANNING_ARCH, cause=f"operator command: use {project}") - self._run_planning_turn("arch") - - def _run_planning_turn( - self, gate_name: str, feedback: str | None = None, answer: str | None = None - ) -> None: - assert self._project is not None - try: - with self._long_work(): - action = self._agent.plan( - self._log("planner"), - gate_name, - self._project, - feedback=feedback, - answer=answer, - ) - except PlanningStopped as exc: - # Answering resumes planning with the answer, so the reason for - # the stop is the first thing the planner hears. - self._halt_for_stop( - what=f"planning {gate_name}", - note=str(exc), - next_step="Planning resumes with your reply.", - pending={"kind": "question", "gate": gate_name}, - ) - return - except PlanningError as exc: - self._await_input( - cause=f"planning failed: {exc}", - prompt=f"planning for {gate_name} failed ({exc}); how should I proceed?", - pending={"kind": "planning_failed", "gate": gate_name}, - ) - return - if isinstance(action, AskHuman): - self._await_input( - cause=f"agent question during {gate_name} planning", - prompt=action.question, - pending={"kind": "question", "gate": gate_name}, - ) - return - rev = int(self._data.get("last_rev") or 0) + 1 - self._data["last_rev"] = rev - self._data["revs"][gate_name] = rev - self._data["gate"] = { - "name": gate_name, - "rev": rev, - "plan": action.content, - "approved": False, - } - await_state = ( - AWAIT_ARCH_SIGNOFF if gate_name == "arch" else AWAIT_MILESTONE_SIGNOFF - ) - roles = self._data.get("roles") or {} - self._log("planner").append( - PlanSubmitted(plan=action.content, model=roles.get("planner", "stub")) - ) - self._log("orchestrator").append( - GateOpened(gate=gate_name, reason=f"plan submitted (rev-{rev})") - ) - self._transition(await_state, cause=f"plan submitted ({gate_name} rev-{rev})") - self._transport.send_approval_request( - f"{action.content}\n\nreply 'approve' to approve {gate_name} rev-{rev}; " - "any other reply is rejection feedback" - ) - # Up only once sent: a reply that arrived before was about another. - self._data["shown_rev"] = rev - self._shown_rev = rev - self._post_plan_to_board(gate_name, rev, action.content) - self._store.oplog.emit( - "gate.open", - cause=f"plan submitted (rev-{rev})", - gate=gate_name, - rev=f"rev-{rev}", - channel="transport", - ) - - def _handle_gate_reply( - self, msg: InboundMessage, text: str, about: int | None - ) -> None: - self._apply_gate_reply( - sender=msg.sender, text=text, channel="transport", about=about - ) - - def _apply_gate_reply( - self, *, sender: str, text: str, channel: str, about: int | None = None - ) -> None: - """Shared by every approval source (L3). The allowlist is per channel: - a Matrix MXID and a Gitea login are different names, and holding one - says nothing about holding the other.""" - gate_data = self._data["gate"] - name, rev = gate_data["name"], gate_data["rev"] - milestone = self._data["milestone"] - if gate_data["approved"]: - if command_form(text) == f"start milestone {milestone}": - self._transition( - EXECUTING, cause=f"operator command: start milestone {milestone}" - ) - self._execute_milestone(milestone) - else: - self._transport.send_message( - f"plan approved; say 'start milestone {milestone}' to begin" - ) - return - revision = PlanRevision(gate=name, rev=rev, content=gate_data["plan"]) - allowlist = self._operators if channel == "transport" else self._gitea_operators - result = Gate(revision, allowlist).evaluate( - sender, text, channel=channel, about=about - ) - if isinstance(result, Ignored): - if result.notice: - self._transport.send_message(result.notice) - return - if isinstance(result, Rejected): - self._store.oplog.emit( - "gate.reject", - cause=f"rejected by {result.approver}", - gate=name, - rev=f"rev-{rev}", - channel=result.channel, - ) - self._echo_outcome( - channel, - f"{name} rev-{rev} rejected by {result.approver}: {result.feedback}", - ) - self._fact( - f"{name} plan rev-{rev} rejected by {result.approver}; " - "their feedback is the operator's message above" - ) - self._data["gate"] = None - back = PLANNING_ARCH if name == "arch" else MILESTONE_PLANNING - self._transition(back, cause=f"plan rejected ({name} rev-{rev})") - self._run_planning_turn(name, feedback=result.feedback) - return - assert isinstance(result, Approved) - self._echo_outcome( - channel, f"{name} rev-{rev} approved by {result.approver} via {channel}" - ) - self._store.oplog.emit( - "gate.approve", - cause=f"approved by {result.approver}", - gate=name, - rev=f"rev-{rev}", - channel=result.channel, - ) - self._fact(f"{name} plan rev-{rev} approved by {result.approver}") - if name == "arch": - self._data["gate"] = None - self._transition( - MILESTONE_PLANNING, cause=f"arch plan approved (rev-{rev})" - ) - self._transport.send_message( - f"architecture approved; planning milestone {milestone}" - ) - self._run_planning_turn(f"milestone-{milestone}") - else: - gate_data["approved"] = True - self._transition( - AWAIT_MILESTONE_SIGNOFF, - cause=f"milestone plan approved (rev-{rev})", - ) - self._transport.send_message( - f"milestone plan approved; say 'start milestone {milestone}' to begin" - ) - - def _execute_milestone(self, milestone: int) -> None: - self._fact(f"milestone {milestone} started") - if self._executor is None: - # The stub completes at once, so it still leaves EXECUTING. - self._transport.send_message(f"[stub] executing milestone {milestone}") - self._publish_milestone(milestone) - return - self._transport.send_message(f"executing milestone {milestone}") - with self._long_work(): - outcome = self._executor.run(self._log("executor"), milestone) - if outcome.status == "stopped": - self._fact(f"milestone {milestone} stopped by the operator") - self._halt_for_stop( - what=f"milestone {milestone}", - note=outcome.note, - next_step=( - f"The plan is still approved; say 'start milestone " - f"{milestone}' to run it again." - ), - pending={"kind": "milestone_stopped", "milestone": milestone}, - ) - return - if outcome.status == "completed": - self._fact(f"milestone {milestone} finished: {outcome.note}") - self._transport.send_message(f"milestone {milestone} done: {outcome.note}") - self._publish_milestone(milestone) - return - self._fact(f"milestone {milestone} {outcome.status}: {outcome.note}") - self._await_input( - cause=f"milestone {milestone} {outcome.status}", - prompt=f"milestone {milestone} stopped ({outcome.status}): {outcome.note}", - pending={"kind": "milestone_stopped", "milestone": milestone}, - ) - - def _echo_outcome(self, deciding_channel: str, text: str) -> None: - """A gate satisfied on one channel is echoed to the other, so neither - record silently disagrees about what was approved (L3).""" - board_state = self._data.get("board") - if deciding_channel != "gitea" and self._board is not None and board_state: - self._board.echo(board_state["issue"], text) - elif deciding_channel == "gitea": - self._transport.send_message(text) - - def _post_plan_to_board(self, gate_name: str, rev: int, plan: str) -> None: - board_state = self._data.get("board") - if self._board is None or not board_state: - return - try: - post = self._board.post_plan(board_state["issue"], gate_name, rev, plan) - except GiteaError: - # Matrix already carries the plan; a Gitea outage degrades the - # durability of the record, not the ability to approve. - return - board_state["comment_id"] = post.comment_id - # Which comment put which revision up: a Gitea reply binds to the - # latest plan posted before it, since comment IDs only grow. - board_state.setdefault("posts", []).append([post.comment_id, rev]) - # Approvals can arrive on the issue as well as over transport (L3), - # so watch it for as long as the gate stands. A new revision skips - # what came before it: an early "approve" must not approve it. - board_state["last_comment"] = self._watch( - "plan", board_state["issue"], self.handle_plan_comment - ) - self._save(f"{gate_name} rev-{rev} posted to Gitea") - - def handle_plan_comment(self, comment: Any) -> None: - """A comment on the project issue may satisfy an open plan gate (L3). - - Approvals bind to a revision here exactly as over transport, and the - first valid approval on either channel wins. Called from the poller - thread, which waits for the worker so its cursor never moves past a - comment that has not been handled. - """ - self._worker.call( - lambda: self._settled( - "a plan comment", lambda: self._handle_plan_comment_now(comment) - ) - ) - - def _handle_plan_comment_now(self, comment: Any) -> None: - board_state = self._data.get("board") - if comment.author not in self._gitea_operators: - self._take_comment(board_state, comment) # never actionable - return - state = self._store.get_state() - if state is None or state.state not in ( - AWAIT_ARCH_SIGNOFF, - AWAIT_MILESTONE_SIGNOFF, - ): - return # not taken: a watch attached later still sees it - self._take_comment(board_state, comment) - issue = (board_state or {}).get("issue") - self._log("operator").append( - UserMessage( - text=comment.body, source=f"gitea:issue:{issue}:comment:{comment.id}" - ) - ) - self._apply_gate_reply( - sender=comment.author, - text=comment.body.strip(), - channel="gitea", - about=_rev_before(board_state, comment.id), - ) - - def handle_review_comment(self, comment: Any) -> None: - """React to a PR review comment (D5). Called from the poller thread. - - Comment authors are checked against the Gitea operator allowlist: - a PR comment is an instruction channel into an agent that writes - code, so it is operator-scoped like any other (L11, and the same - reasoning O5 applies to tickets). - """ - self._worker.call( - lambda: self._settled( - "a review comment", lambda: self._handle_review_comment_now(comment) - ) - ) - - def _handle_review_comment_now(self, comment: Any) -> None: - if comment.author not in self._gitea_operators: - self._take_comment(self._data.get("pr"), comment) # never actionable - self._store.oplog.emit( - "gitea.comment_ignored", - cause=f"author {comment.author!r} not an operator", - comment_id=comment.id, - ) - return - state = self._store.get_state() - if state is None or state.state != PR_OPEN: - # Not taken: back in PR_OPEN, the watch resumes from the cursor - # and delivers it again. - self._store.oplog.emit( - "gitea.comment_ignored", - cause=f"not in PR_OPEN (state={state.state if state else None})", - comment_id=comment.id, - ) - return - self._take_comment(self._data.get("pr"), comment) - if self._executor is None or self._publisher is None: - return - - pr = self._data.get("pr") or {} - milestone = int(pr.get("milestone", self._data.get("milestone", 1))) - # Taken at most once, but answered at least once: until the executor - # returns, the comment is marked, and a restart delivers it again. - pr["reviewing"] = comment.id - self._save(f"answering review comment {comment.id}") - # The comment enters the transcript, so the executor sees review - # feedback as conversation rather than as an out-of-band instruction. - self._log("operator").append( - UserMessage( - text=comment.body, - source=f"gitea:pr:{pr.get('number')}:comment:{comment.id}", - ) - ) - try: - with self._long_work(): - outcome = self._executor.run(self._log("executor"), milestone) - except Exception: - # Reported to the operator by _settled: answered, if not well. - self._done_reviewing(pr) - raise - self._done_reviewing(pr) - if outcome.status == "stopped": - self._halt_for_stop( - what=f"the response to review comment {comment.id}", - note=outcome.note, - next_step="Your reply returns to the open PR.", - pending={"kind": "review_stopped", "milestone": milestone}, - ) - return - if outcome.status != "completed": - self._await_input( - cause=f"review response {outcome.status}", - prompt=( - f"could not address the review comment ({outcome.status}): " - f"{outcome.note}. Reply to return to the open PR; comment " - "again there to retry." - ), - pending={"kind": "review_stopped", "milestone": milestone}, - ) - return - try: - sha = self._publisher.push_followup( - task_id=str(self._data.get("task_id")), - milestone=milestone, - message=f"Address review comment {comment.id}", - ) - except (GiteaError, GitError) as exc: - self._transport.send_message(f"could not push the fix: {exc}") - return - note = ( - f"pushed {sha[:8]} addressing the review comment" - if sha - else "no code change was needed for that comment" - ) - self._fact(f"review comment {comment.id}: {note}") - self._transport.send_message(note) - if pr.get("number"): - self._publisher.comment(int(pr["number"]), note) - - def _done_reviewing(self, pr: dict[str, Any]) -> None: - """The executor returned: the comment is being answered one way or - another. Deliberately not in a `finally`: if the process dies - mid-response, the mark must survive so the comment is redelivered.""" - if pr.pop("reviewing", None) is not None: - self._save("review comment answered") - - def _publish_milestone(self, milestone: int) -> None: - if self._publisher is None: - # No PR to wait on, but PR_OPEN's commands still lead on. - self._fact(f"milestone {milestone} done; no publisher, so no PR") - self._transition(PR_OPEN, cause=f"[stub] milestone {milestone} done") - self._transport.send_message( - f"[stub] milestone {milestone} done; no publisher, so no PR — " - "say 'next milestone', 'project complete' or 'abandon'" - ) - return - gate = self._data.get("gate") or {} - try: - result = self._publisher.publish( - task_id=str(self._data.get("task_id")), - milestone=milestone, - project=str(self._project), - plan=str(gate.get("plan", "")), - ) - except (GiteaError, GitError) as exc: - # The intent log already records what was in flight, so boot - # reconciliation can settle it against Gitea. Gate rather than - # retry blindly into a possible duplicate. - self._fact(f"publishing milestone {milestone} failed: {exc}") - self._await_input( - cause=f"publishing milestone {milestone} failed: {exc}", - prompt=f"could not publish milestone {milestone}: {exc}", - pending={"kind": "milestone_stopped", "milestone": milestone}, - ) - return - if result.pull_request is None: - # Nothing left EXECUTING behind: the operator decides what next, - # and answering returns to the still-approved plan. - self._fact(f"milestone {milestone} not published: {result.note}") - self._await_input( - cause=f"milestone {milestone}: {result.note}", - prompt=( - f"milestone {milestone}: {result.note}. Reply to return to " - "the approved plan." - ), - pending={"kind": "milestone_stopped", "milestone": milestone}, - ) - return - self._fact( - f"milestone {milestone} published: {result.note} " - f"(PR #{result.pull_request.number}, {result.pull_request.url})" - ) - self._unwatch("plan") # the gate is behind us; the PR is the thread now - self._data["pr"] = { - "number": result.pull_request.number, - "url": result.pull_request.url, - "milestone": milestone, - "last_comment": self._watch( - "review", result.pull_request.number, self.handle_review_comment - ), - } - self._transition(PR_OPEN, cause=f"milestone {milestone} published") - self._transport.send_message( - f"{result.note} — review it, then say 'next milestone' or " - "'project complete'" - ) - - def _abandon(self, sender: str) -> None: - """End the task on the operator's word (L2: continue, re-plan, or - abandon), closing its Gitea records. Only reached from a waiting - state: while work runs the reply is "busy; say 'stop'" instead. - - The move to IDLE is made durable first, so a failure while tidying - Gitea can never leave a live task behind a closed plan issue.""" - note = f"abandoned by {sender}" - board_state = self._data.get("board") or {} - pr = self._data.get("pr") or {} - project = self._project - self.stop_watching() - self._fact(f"work item abandoned by {sender}") - self._close_item("abandoned", f"operator: abandon ({sender})") - self._data = {**_fresh_data()} - self._project = None - self._store.oplog.task_id = None - self._transition(IDLE, cause=f"operator: abandon ({sender})") - - # Dropped on purpose: boot reconciliation must not gate on it. - for intent in self._store.pending_intents(): - self._store.abort_intent(intent.id, cause=note) - problems: list[str] = [] - if problem := self._close_plan_issue(board_state, note): - problems.append(problem) - if self._publisher is not None and pr.get("number"): - try: - self._publisher.close_pull_request(int(pr["number"]), note) - except GiteaError as exc: - problems.append(f"PR #{pr['number']}: {exc}") - closed = "; could not close " + ", ".join(problems) if problems else "" - self._transport.send_message( - f"task on {project} abandoned{closed}. Say 'use ' to start another" - ) - - def _published_pr(self) -> Any: - """The open PR for the current milestone, if Gitea shows one; None - when there is none or Gitea cannot say (then the restart question - is asked as before).""" - task_id = self._data.get("task_id") - if self._publisher is None or not task_id: - return None - try: - return self._publisher.published_pull_request( - str(task_id), int(self._data.get("milestone", 1)) - ) - except GiteaError: - return None - - def _close_plan_issue(self, board_state: dict[str, Any], note: str) -> str | None: - """Close a finished task's plan issue with a note; what went wrong, - if anything, for the operator's message.""" - if self._board is None or not board_state.get("issue"): - return None - try: - self._board.close(int(board_state["issue"]), note) - except GiteaError as exc: - return f"plan issue #{board_state['issue']}: {exc}" - return None - - def _return_to_pr_open(self, cause: str) -> None: - self._transition(PR_OPEN, cause=cause) - pr = self._data.get("pr") or {} - self._watch_pr(pr) - self._transport.send_message( - f"PR #{pr.get('number')} is open and watched for review comments; " - "say 'next milestone', 'project complete' or 'abandon'" - ) - - def _watch_pr(self, pr: dict[str, Any]) -> None: - """Watch the open PR from its recorded cursor, catching up on what - arrived while it was not watched.""" - if pr.get("number"): - after = pr.get("last_comment") - reviewing = pr.get("reviewing") - if reviewing: - # Its answer never finished: resume just before it. - after = min( - int(after if after is not None else reviewing), reviewing - 1 - ) - self._watch( - "review", int(pr["number"]), self.handle_review_comment, after=after - ) - - def _handle_pr_open(self, text: str, sender: str = "the operator") -> None: - command = command_form(text) - if command == "next milestone": - self._unwatch("review") - pr = self._data.get("pr") or {} - self._data["milestone"] = int(self._data.get("milestone", 1)) + 1 - self._fact( - f"PR #{pr.get('number')} left for review; planning milestone " - f"{self._data['milestone']}" - if pr.get("number") - else f"planning milestone {self._data['milestone']}" - ) - self._data["gate"] = None - self._data["pr"] = None - self._transition(MILESTONE_PLANNING, cause="operator: next milestone") - self._run_planning_turn(f"milestone-{self._data['milestone']}") - elif command == "project complete": - board_state = self._data.get("board") or {} - self.stop_watching() - self._fact(f"work item completed: {sender} said project complete") - self._close_item("completed", "operator: project complete") - self._data = {**_fresh_data()} - self._project = None - self._store.oplog.task_id = None - self._transition(IDLE, cause="operator: project complete") - # IDLE first, as for abandon; the PR is the operator's to merge. - problem = self._close_plan_issue(board_state, f"completed by {sender}") - self._transport.send_message( - "project closed" - + (f"; could not close {problem}" if problem else "") - + ". Say 'use ' to start another" - ) - else: - pr = self._data.get("pr") or {} - self._transport.send_message( - f"PR #{pr.get('number')} is open; say 'next milestone', " - "'project complete' or 'abandon'" - ) - - def _watch( - self, kind: str, index: int, handler: Any, *, after: int | None = None - ) -> int: - """Start watching one Gitea thread, replacing any previous watch of - the same kind, and return the comment cursor to record. - - A fresh watch (after=None) skips everything already there, so it - does not replay comments already addressed. Watching again from a - recorded cursor delivers what came in while nothing was watching.""" - if self._poller_factory is None: - return after or 0 - self._unwatch(kind) - poller = self._poller_factory(handler) - if after is None: - cursor = poller.prime(index) - else: - poller.resume_after(after) - cursor = after - poller.start(index) - self._pollers[kind] = poller - return cursor - - def _save(self, cause: str) -> None: - """Make changes to the task's data durable without a state change.""" - state = self._store.get_state() - self._transition(state.state if state else IDLE, cause=cause) - - def _take_comment(self, record: dict[str, Any] | None, comment: Any) -> None: - """Advance a thread's cursor past a comment being acted on, durably, - before acting: a comment is handled at most once, even across a - crash mid-way.""" - if record is not None: - record["last_comment"] = max( - int(record.get("last_comment") or 0), comment.id - ) - self._save(f"gitea comment {comment.id} taken") - - def _unwatch(self, kind: str) -> None: - poller = self._pollers.pop(kind, None) - if poller is not None: - poller.stop() - - def stop_watching(self) -> None: - for kind in list(self._pollers): - self._unwatch(kind) - - def _await_input(self, *, cause: str, prompt: str, pending: dict[str, Any]) -> None: - # The prompt is kept with what it resumes, so a restart can ask the - # same question again. - self._data["pending"] = {**pending, "prompt": prompt} - self._transition(AWAIT_INPUT, cause=cause) - self._transport.ask_question(prompt) - - def _handle_answer(self, text: str) -> None: - pending = self._data.get("pending") - if pending is None: - # Reached only if state was forced externally or a pending item - # was lost; starting over must still work from here. - if command_form(text).startswith("use "): - self.stop_watching() - self._begin_project(_project_name(text)) - return - self._transport.send_message( - "waiting for direction, but I have no pending item to resume; " - "try 'status', or 'use ' to start over" - ) - return - self._data["pending"] = None - match pending.get("kind"): - case "question" | "planning_failed": - gate_name = pending["gate"] - back = PLANNING_ARCH if gate_name == "arch" else MILESTONE_PLANNING - self._transition(back, cause="operator answered; resuming planning") - self._run_planning_turn(gate_name, answer=text) - case "review_stopped": - self._return_to_pr_open("operator answered; back to the open PR") - case "boot": - self._resume_after_restart(pending, text) - case "milestone_stopped": - milestone = pending["milestone"] - self._transition( - AWAIT_MILESTONE_SIGNOFF, - cause="operator answered; plan still approved", - ) - self._transport.send_message( - f"noted. the milestone {milestone} plan is still approved — " - f"say 'start milestone {milestone}' to run it again" - ) - case _: - self._transport.send_message( - "noted. say 'status' for where things stand" - ) - - def _resume_after_restart(self, pending: dict[str, Any], text: str) -> None: - """Answer to the question a restart asks (L8 boot step 5). Never a blind - resume: 'continue' re-runs the interrupted planning turn, or returns - an interrupted milestone to its approved plan to be started again.""" - choice = command_form(text) - if choice not in ("continue", "re-plan", "replan"): - self._data["pending"] = pending - self._transport.ask_question(_BOOT_CHOICES) - return - milestone = int(self._data.get("milestone", 1)) - gate = self._data.get("gate") or {} - interrupted = pending.get("interrupted") - if interrupted == PR_OPEN: - # Recorded before a restart in PR_OPEN resumed quietly. Its PR is - # still open, so either reply goes back to it; re-planning the - # milestone would open a second PR. - self._return_to_pr_open(f"operator: {choice} after restart") - return - if choice == "continue" and interrupted == EXECUTING and gate.get("approved"): - self._transition( - AWAIT_MILESTONE_SIGNOFF, cause="operator: continue after restart" - ) - self._transport.send_message( - f"the milestone {milestone} plan is still approved; say " - f"'start milestone {milestone}' to run it" - ) - return - if interrupted == PLANNING_ARCH: - gate_name = "arch" - elif interrupted in (MILESTONE_PLANNING, EXECUTING): - gate_name = f"milestone-{milestone}" - else: - # Not recorded (a question asked before the interrupted state - # was): judge by how far planning got. - planned = self._data.get("revs", {}) - gate_name = gate.get("name") or ( - "arch" if "arch" not in planned else f"milestone-{milestone}" - ) - self._data["gate"] = None - back = PLANNING_ARCH if gate_name == "arch" else MILESTONE_PLANNING - self._transition(back, cause=f"operator: {choice} after restart") - self._run_planning_turn(gate_name) - - def _transition(self, new_state: str, *, cause: str) -> None: - state = self._store.get_state() - current = state.state if state else None - allowed = ( - current is None - or current == new_state - or new_state == AWAIT_INPUT - or current == AWAIT_INPUT - or (current, new_state) in TRANSITIONS - ) - if not allowed: - raise TransitionError(f"illegal transition {current} -> {new_state}") - self._store.transition( - new_state, cause=cause, project=self._project, data=self._data - ) - self._state_name = new_state - self._shown_rev = int(self._data.get("shown_rev") or 0) diff --git a/nisse-dev/src/nisse_dev/delegation.py b/nisse-dev/src/nisse_dev/delegation.py index 9825493..c0d44af 100644 --- a/nisse-dev/src/nisse_dev/delegation.py +++ b/nisse-dev/src/nisse_dev/delegation.py @@ -43,7 +43,7 @@ from nisse_dev.git import GitError, GitRepo from nisse_dev.llm import LLMClient, LLMError, UpstreamUnavailable from nisse_dev.reconcile import Reconciler from nisse_dev.state import Intent, StateStore, Transcript -from nisse_dev.tools import EXECUTOR_TOOLS, MUTATING_TOOLS, Workspace +from nisse_dev.tools import EXECUTION_TOOLS, MUTATING_TOOLS, Workspace DELEGATION_INTENT = "delegation" @@ -97,7 +97,7 @@ HAND_BACK_TOOL: dict[str, Any] = { }, } -DELEGATE_TOOLS = [*EXECUTOR_TOOLS, HAND_BACK_TOOL] +DELEGATE_TOOLS = [*EXECUTION_TOOLS, HAND_BACK_TOOL] @dataclass(frozen=True) diff --git a/nisse-dev/src/nisse_dev/dump.py b/nisse-dev/src/nisse_dev/dump.py index 6a91341..e09e4ae 100644 --- a/nisse-dev/src/nisse_dev/dump.py +++ b/nisse-dev/src/nisse_dev/dump.py @@ -62,13 +62,14 @@ def render(path: Path, *, full: bool = False) -> str: active = agent["active_item"] if agent else None lines.append(f"active work item: {active or '(none: idle)'}") for item in conn.execute("SELECT * FROM work_items ORDER BY opened_at"): - data = json.loads(item["data"]) - gate = data.get("gate") or {} + loop = json.loads(item["data"]).get("loop") or {} status = item["status"] + (f" ({item['outcome']})" if item["outcome"] else "") + waiting = ( + f" waits for: {loop['wait']}" if item["position"] == "WAITING" else "" + ) lines.append( f" {item['id']} {status} position: {item['position']} " - f"scope: {item['scope']} gate: " - + (f"{gate.get('name')} rev-{gate.get('rev')}" if gate else "none") + f"scope: {item['scope']} rev-{loop.get('rev', 0)}{waiting}" ) pending = conn.execute( "SELECT id, kind, payload, work_item FROM intent_log WHERE status = 'pending'" diff --git a/nisse-dev/src/nisse_dev/executor.py b/nisse-dev/src/nisse_dev/executor.py deleted file mode 100644 index 2ee0340..0000000 --- a/nisse-dev/src/nisse_dev/executor.py +++ /dev/null @@ -1,318 +0,0 @@ -"""Local executor loop (architecture L9 escalation triggers, L9 accommodations). - -Runs the approved plan with the executor role — the cheap local model doing -mechanical work. Every tool call and result is checkpointed to the canonical -log as it happens (L9), so a crash mid-milestone leaves a resumable record. - -Escalation is decided from outside the model (L9): consecutive command -failures and hard caps on turns, never the model's own claim that it is stuck -or finished-enough. Until delegation exists (step 7), an escalation gates to -the human with the failures attached rather than engaging a frontier model. -""" - -from collections.abc import Callable -from dataclasses import dataclass -from typing import Any, Literal - -from nisse_dev.cancel import Cancel, skip_calls -from nisse_dev.canonical import PlanSubmitted, StoredEvent, ToolCall, ToolResult -from nisse_dev.delegation import Delegation -from nisse_dev.expert import ASK_EXPERT_TOOL, BudgetExhausted, ExpertDesk -from nisse_dev.llm import LLMClient, LLMError, RoleMap -from nisse_dev.state import Transcript -from nisse_dev.tools import EXECUTOR_TOOLS, MUTATING_TOOLS, ToolOutcome, Workspace - -EXECUTOR_SYSTEM = """You are nisse-dev's executor, doing mechanical work. - -Follow the approved plan below exactly. It was written to be executed -verbatim; do not redesign it, expand its scope, or skip steps. - -Every reply must be a tool call. Verify your work by running the project's -tests. When every step of the plan is done and the tests pass, call finish -with a short summary — that, and nothing else, ends the milestone.""" - -DEFAULT_MAX_TURNS = 24 -DEFAULT_FAILURE_LIMIT = 2 - -FINISH_TOOL: dict[str, Any] = { - "type": "function", - "function": { - "name": "finish", - "description": ( - "End the milestone: every step of the plan is done and the tests pass." - ), - "parameters": { - "type": "object", - "properties": { - "summary": {"type": "string", "description": "What was done."} - }, - "required": ["summary"], - }, - }, -} - -# Finishing before any real work has happened would record an empty -# milestone as a success, so the claim is refused once first. -_NOT_YET = ( - "You have not made any change yet, so the plan cannot be complete. Carry " - "out the next step of the plan, or run the tests to show the work is " - "already done, before calling finish." -) -# A reply in prose breaks the tool contract. A server that enforces -# tool_choice never allows one; on one that doesn't, the model is told once. -_PROSE_NUDGE = ( - "\n\nYour last reply was prose. Every reply must be a tool call: carry " - "out the next step, or call finish if the plan is complete." -) - - -@dataclass(frozen=True) -class ExecutorOutcome: - status: Literal["completed", "capped", "escalate", "stopped"] - note: str - - -class LLMExecutor: - def __init__( - self, - client: LLMClient, - roles: RoleMap, - workspace: Workspace, - *, - max_turns: int = DEFAULT_MAX_TURNS, - failure_limit: int = DEFAULT_FAILURE_LIMIT, - expert: ExpertDesk | None = None, - delegate: Delegation | None = None, - notify: Callable[[str], None] | None = None, - cancel: Cancel | None = None, - ) -> None: - self._client = client - self._roles = roles - self._workspace = workspace - self._max_turns = max_turns - self._failure_limit = failure_limit - self._expert = expert - self._delegate = delegate - # The loop runs synchronously inside the transport callback, so - # without this the agent is silent for the whole milestone and a - # working run looks exactly like a hung one. - self._notify = notify - self._cancel = cancel or Cancel() - - def _report(self, turn: int, calls: list[ToolCall]) -> None: - if self._notify is None: - return - summary = ", ".join( - f"{c.tool}({c.arguments.get('path') or c.arguments.get('command', '')[:40]})" - if c.arguments - else c.tool - for c in calls - ) - self._notify(f"turn {turn + 1}: {summary}") - - def _escalate( - self, transcript: Transcript, milestone: int, failures: int, last_failure: str - ) -> ExecutorOutcome | None: - """Hand the sub-task to a stronger model, per L9. - - Returns None when the delegation handed back usable work and the local - loop should carry on; an ExecutorOutcome when the caller must stop. - """ - note = f"{failures} consecutive command failures; last: {last_failure[:500]}" - if self._delegate is None: - return ExecutorOutcome("escalate", note) - - # L9 says the failures go in the brief: the delegate must not have to - # rediscover what the local model already learned the hard way. - brief = ( - f"Milestone {milestone} is stuck. The local executor tried and its " - f"commands failed {failures} times in a row.\n\n" - f"Most recent failure:\n{last_failure[:2000]}\n\n" - "The approved plan and the milestone so far are in the conversation " - "above. Finish this sub-task, staying inside the approved plan." - ) - result = self._delegate.run( - transcript.as_actor("helper"), - brief=brief, - branch=f"delegation-milestone-{milestone}-{failures}", - ) - if result.usable: - return None - if result.status == "stopped": - return ExecutorOutcome("stopped", f"during a delegation: {result.note}") - # L9: a cap breach or failure leaves a half-edited tree, so the local - # loop must not resume on top of it. - return ExecutorOutcome( - "escalate", f"{note}; delegation {result.status}: {result.note}" - ) - - def _tools(self) -> list[dict[str, Any]]: - # L9: ask_expert exists in addition to the observable escalation - # triggers, never as a replacement for them. - if self._expert is None: - return [*EXECUTOR_TOOLS, FINISH_TOOL] - return [*EXECUTOR_TOOLS, ASK_EXPERT_TOOL, FINISH_TOOL] - - def _consult( - self, transcript: Transcript, arguments: dict[str, Any] - ) -> ToolOutcome: - assert self._expert is not None - question = str(arguments.get("question", "")).strip() - if not question: - return ToolOutcome("ask_expert needs a question", success=False) - try: - answer = self._expert.ask(transcript.as_actor("expert"), question) - return ToolOutcome(answer.render(), success=True) - except BudgetExhausted as exc: - # Not an error the model should route around: it is the signal - # that it is circling and should get on with the work or fail. - return ToolOutcome(str(exc), success=False) - except LLMError as exc: - return ToolOutcome(f"the expert did not answer: {exc}", success=False) - - def _scope(self, transcript: Transcript) -> tuple[str | None, int]: - """The approved plan, and where this milestone's events begin. - - The most recent plan in the log is the approved milestone plan: no - plan is submitted between approval and execution, or during review - responses. The executor sees that plan and what came after it, and - not the planning conversation or earlier milestones, which a small - context window cannot afford and the plan already distils. - """ - latest: StoredEvent | None = None - for stored in transcript.read(): - if isinstance(stored.event, PlanSubmitted): - latest = stored - if latest is None: - return None, 0 - assert isinstance(latest.event, PlanSubmitted) - return latest.event.plan, latest.id - - def run(self, transcript: Transcript, milestone: int) -> ExecutorOutcome: - """Run a milestone of the work item whose transcript this is, writing - as the executor (L6, L7).""" - model = self._roles.resolve("executor") - plan, after = self._scope(transcript) - system = ( - f"{EXECUTOR_SYSTEM}\n\n{self._workspace.describe()}" - f"\n\nYou are executing milestone {milestone}." - ) - if plan is not None: - # In the system prompt, so the context budget never drops it. - system += f"\n\nThe approved plan:\n{plan}" - tools = self._tools() - offered = {tool["function"]["name"] for tool in tools} - consecutive_failures = 0 - did_work = False - finish_refused = False - prose_nudged = False - - for turn in range(self._max_turns): - # L2: `stop` takes effect at the next tool boundary. The start of - # a turn is one, and so is the gap between two tool calls below. - if self._cancel.requested: - return ExecutorOutcome("stopped", f"before turn {turn + 1}") - try: - produced = self._client.complete( - transcript.view(after=after), - tools=tools, - model=model, - system=system, - ) - except LLMError as exc: - return ExecutorOutcome("escalate", f"executor call failed: {exc}") - - calls: list[ToolCall] = [] - for event in produced: - transcript.append(event) - if isinstance(event, ToolCall): - calls.append(event) - - if not calls: - if not prose_nudged: - prose_nudged = True - system += _PROSE_NUDGE - continue - return ExecutorOutcome( - "escalate", "executor kept replying in prose instead of tools" - ) - - self._report(turn, calls) - for index, call in enumerate(calls): - if self._cancel.requested: - skip_calls(transcript, calls[index:]) - return ExecutorOutcome("stopped", f"during turn {turn + 1}") - if call.tool == "finish": - if not did_work and not finish_refused: - finish_refused = True - outcome = ToolOutcome(_NOT_YET, success=False) - elif not did_work: - return ExecutorOutcome( - "escalate", - "executor claimed the milestone was done without " - "making any change", - ) - else: - transcript.append( - ToolResult( - call_id=call.call_id, - result="milestone finished", - success=True, - ) - ) - skip_calls( - transcript, - calls[index + 1 :], - "not run: finish had already ended the milestone", - ) - summary = str(call.arguments.get("summary", "")).strip() - return ExecutorOutcome( - "completed", summary or "executor reported the plan done" - ) - elif call.tool not in offered: - # A model inventing tools is not doing the work, and with - # no server enforcement nothing else would stop it, so it - # counts toward escalation like a failed command (L9). - outcome = ToolOutcome( - f"no such tool: {call.tool}; the tools are " - + ", ".join(sorted(offered)), - success=False, - ) - elif call.tool == "ask_expert": - outcome = self._consult(transcript, call.arguments) - else: - if call.tool in MUTATING_TOOLS: - did_work = True - outcome = self._workspace.dispatch(call.tool, call.arguments) - transcript.append( - ToolResult( - call_id=call.call_id, - result=outcome.result, - success=outcome.success, - ) - ) - # Commands and invented tools count toward escalation: a - # failed read is the model looking around, a failed command - # is work not working. - if call.tool == "run_command" or call.tool not in offered: - consecutive_failures = ( - 0 if outcome.success else consecutive_failures + 1 - ) - if consecutive_failures >= self._failure_limit: - # L9: this is the trigger delegation exists for. - # Escalating to the human is the fallback when there - # is no delegate, not the intended path. - escalation = self._escalate( - transcript, - milestone, - consecutive_failures, - outcome.result, - ) - if escalation is not None: - return escalation - consecutive_failures = 0 - did_work = True - - return ExecutorOutcome( - "capped", f"hit the {self._max_turns}-turn cap for milestone {milestone}" - ) diff --git a/nisse-dev/src/nisse_dev/gates.py b/nisse-dev/src/nisse_dev/gates.py deleted file mode 100644 index 7b2fbf2..0000000 --- a/nisse-dev/src/nisse_dev/gates.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Approval gating primitives (architecture L3). - -Approvals bind to a plan revision, never to "the current gate": every revision -carries a monotonic ``rev-N`` tag, an approval naming a stale revision is -answered with a notice and never applied to a newer one, and the first valid -approval wins. Approval is always the typed word ``approve`` (optionally -``approve rev-N``); any other reply is a rejection with feedback. The approver -is authenticated against a per-channel operator allowlist. - -Revisions are numbered across a whole work item, not per gate, so ``rev-N`` -names one plan. A reply that does not name a revision binds to the one that -was up when it arrived (its ``about``): if that is not the current revision, -it is ambiguous and not applied, neither as an approval nor as feedback. That -is how a second "approve", or an afterthought, sent while one plan was up can -never land on the next. - -The channel seam: any approval source (Matrix transport, Gitea comment poller, -Vikunja comments) feeds operator replies to ``Gate.evaluate`` with its channel -name. -""" - -import re -from collections.abc import Set as AbstractSet -from dataclasses import dataclass - - -@dataclass(frozen=True) -class PlanRevision: - gate: str - rev: int - content: str - - @property - def tag(self) -> str: - return f"rev-{self.rev}" - - -@dataclass(frozen=True) -class Approved: - revision: PlanRevision - approver: str - channel: str - - -@dataclass(frozen=True) -class Rejected: - feedback: str - approver: str - channel: str - - -@dataclass(frozen=True) -class Ignored: - notice: str | None # posted back to the channel when not None - - -_APPROVE_RE = re.compile(r"^approve(?:\s+(rev-\d+))?$") - - -def command_form(text: str) -> str: - """How a message reads as a command: case, surrounding whitespace, runs - of inner whitespace and trailing '.' or '!' do not matter, so a phone's - "Approve." is still an approval. Only for matching; what the operator - wrote is what gets recorded and passed on.""" - return " ".join(text.strip().rstrip(".!").split()).lower() - - -class Gate: - def __init__(self, revision: PlanRevision, operators: AbstractSet[str]) -> None: - self._revision = revision - self._operators = operators - - def evaluate( - self, sender: str, text: str, channel: str, about: int | None = None - ) -> Approved | Rejected | Ignored: - """`about` is the revision that was up when the reply arrived; None - only for records kept before replies were stamped.""" - if sender not in self._operators: - return Ignored(notice=None) - text = text.strip() - match = _APPROVE_RE.match(command_form(text)) - named = match.group(1) if match else None - current = self._revision - if named is None and about is not None and about != current.rev: - was = f"rev-{about}" if about else "no plan" - if match is None: - return Ignored( - notice=( - f"noted, but it arrived while {was} was up, so it is not " - f"taken as feedback on {current.gate} {current.tag}" - ) - ) - return Ignored( - notice=( - f"that approval arrived while {was} was up; the current plan is " - f"{current.gate} {current.tag}, not applied. Reply again to " - "approve it" - ) - ) - if match is None: - return Rejected(feedback=text, approver=sender, channel=channel) - if named is not None and named != self._revision.tag: - return Ignored( - notice=( - f"approval names {named}, but the current plan is " - f"{self._revision.tag}; not applied" - ) - ) - return Approved(revision=self._revision, approver=sender, channel=channel) diff --git a/nisse-dev/src/nisse_dev/llm.py b/nisse-dev/src/nisse_dev/llm.py index 432f4f7..2e2dcb8 100644 --- a/nisse-dev/src/nisse_dev/llm.py +++ b/nisse-dev/src/nisse_dev/llm.py @@ -114,22 +114,13 @@ class LLMClient(Protocol): @dataclass(frozen=True) class RoleMap: - executor: str # the old loop's (control.py), until M2.8 - planner: str # the old loop's, until M2.8 + driver: str # drives the loop (L2) helper: str # delegations (L9) expert: str # ask_expert (L9) - driver: str = "" # drives the new loop; unset, the planner's model classifier: str = "" # reads go-aheads now, actions later (L4); unset, expert's - ROLES: ClassVar[tuple[str, ...]] = ( - "executor", - "planner", - "helper", - "expert", - "driver", - "classifier", - ) - _FALLBACK: ClassVar[dict[str, str]] = {"driver": "planner", "classifier": "expert"} + ROLES: ClassVar[tuple[str, ...]] = ("driver", "helper", "expert", "classifier") + _FALLBACK: ClassVar[dict[str, str]] = {"classifier": "expert"} def resolve(self, role: str) -> str: if role not in self.ROLES: diff --git a/nisse-dev/src/nisse_dev/loop/__init__.py b/nisse-dev/src/nisse_dev/loop/__init__.py index 3a399bb..3cd7d40 100644 --- a/nisse-dev/src/nisse_dev/loop/__init__.py +++ b/nisse-dev/src/nisse_dev/loop/__init__.py @@ -1,5 +1,5 @@ """The conversational loop (architecture L1, L2, L5; Development Plan M2.1). -Built alongside control.py, which keeps running until the switch-over -(M2.8); the `loop: new` setting chooses this one. +The agent's only loop since the switch-over (M2.8), which retired the gated +state machine it was built alongside. """ diff --git a/nisse-dev/src/nisse_dev/loop/agent.py b/nisse-dev/src/nisse_dev/loop/agent.py index e4c55bb..6ac78ae 100644 --- a/nisse-dev/src/nisse_dev/loop/agent.py +++ b/nisse-dev/src/nisse_dev/loop/agent.py @@ -37,7 +37,6 @@ from nisse_dev.canonical import ( from nisse_dev.config import GENERAL from nisse_dev.delegation import DELEGATION_INTENT, Delegation from nisse_dev.expert import BudgetExhausted, ExpertDesk -from nisse_dev.gates import command_form from nisse_dev.git import GitError from nisse_dev.gitea import GiteaClient, GiteaError, Issue from nisse_dev.llm import LLMError, RoleMap @@ -61,7 +60,7 @@ from nisse_dev.loop.items import ( save, ) from nisse_dev.loop.milestones import OPEN_PR, Milestones -from nisse_dev.loop.reader import GoAheadReader, is_question +from nisse_dev.loop.reader import GoAheadReader, command_form, is_question from nisse_dev.loop.tools import ( ASK_HUMAN, CLOSE_ITEM, @@ -75,7 +74,7 @@ from nisse_dev.loop.tools import ( ) from nisse_dev.oplog import OpLog from nisse_dev.state import StateStore, WorkItem -from nisse_dev.tools import EXECUTOR_TOOLS, Workspace +from nisse_dev.tools import EXECUTION_TOOLS, Workspace from nisse_dev.transport import InboundMessage, Transport from nisse_dev.worker import Worker @@ -113,7 +112,7 @@ _MARKER = re.compile(r"") # Before the go-ahead the driver may look around (read, list, run a command # to reproduce a bug) but not change anything (D1); after it, everything. -_LOOK = [t for t in EXECUTOR_TOOLS if t["function"]["name"] != "write_file"] +_LOOK = [t for t in EXECUTION_TOOLS if t["function"]["name"] != "write_file"] SYSTEM = """You are nisse-dev, a software agent working with the operator on the repository {repo}. You are on one work item: issue #{issue}, "{title}". @@ -1411,12 +1410,7 @@ class LoopAgent: default = ( self._helpers.roles if self._helpers - else RoleMap( - executor="", - planner=self._roles.get("planner", ""), - helper="", - expert="", - ) + else RoleMap(driver=self._roles.get("driver", ""), helper="", expert="") ) pinned = item.data.get("roles") or self._roles return RoleMap.from_pinned(dict(pinned), default) @@ -1454,6 +1448,9 @@ class LoopAgent: workspace, milestones.git, model=helper, + backing_model=( + self._helpers.backing(helper) if self._helpers.backing else "" + ), caps=self._helpers.caps, cancel=self._cancel, reader=self._helpers.reader(roles), @@ -1737,7 +1734,7 @@ class LoopAgent: *([WAIT_FOR_REVIEW] if loop.pr else []), *([DELEGATE, ASK_EXPERT] if self._helpers else []), *([REPLY] if loop.back_to else []), - *EXECUTOR_TOOLS, + *EXECUTION_TOOLS, ] ), ] @@ -1826,7 +1823,7 @@ class LoopAgent: text = str(args.get("text", "")).strip() rev = loop.rev + 1 self._store.transcript(item.id, "driver").append( - PlanSubmitted(plan=text, model=self._roles.get("planner", "")) + PlanSubmitted(plan=text, model=self._roles.get("driver", "")) ) self._fact(item, f"restatement rev-{rev} put up for a go-ahead") self._shown[loop.thread] = rev @@ -1869,7 +1866,7 @@ class LoopAgent: summary = str(args.get("summary", "")).strip() rev = loop.rev + 1 self._store.transcript(item.id, "driver").append( - PlanSubmitted(plan=summary, model=self._roles.get("planner", "")) + PlanSubmitted(plan=summary, model=self._roles.get("driver", "")) ) self._fact(item, f"work proposed as rev-{rev}: {project.repo}, {title}") self._shown[loop.thread] = rev @@ -1899,7 +1896,7 @@ class LoopAgent: brief = f"{purpose}\n\nDecisions:\n{decisions}" if decisions else purpose rev = loop.rev + 1 self._store.transcript(item.id, "driver").append( - PlanSubmitted(plan=brief, model=self._roles.get("planner", "")) + PlanSubmitted(plan=brief, model=self._roles.get("driver", "")) ) self._fact(item, f"new project proposed as rev-{rev}: {name}") self._shown[loop.thread] = rev diff --git a/nisse-dev/src/nisse_dev/loop/build.py b/nisse-dev/src/nisse_dev/loop/build.py index 25b6d73..4e149d8 100644 --- a/nisse-dev/src/nisse_dev/loop/build.py +++ b/nisse-dev/src/nisse_dev/loop/build.py @@ -1,25 +1,22 @@ -"""Assembling the new loop (M2.1): the same services as the old agent, with -LoopAgent in place of the Orchestrator. Chosen with `loop: new` in config.""" +"""Composition root: the one place that knows how the whole agent fits +together. Every component is unit-tested in isolation; this is where they meet, +which is what keeps a built piece from being reachable only from its own tests +(the delegation harness and the comment poller once were).""" import threading from collections.abc import Callable from dataclasses import dataclass -from nisse_dev.agent import ( - SHUTDOWN_WAIT_S, - build_host, - discover_models, - make_llm_clients, - prepare_workspace, - push_credential, - remote_url, -) from nisse_dev.cancel import Cancel -from nisse_dev.catalog import select_client +from nisse_dev.catalog import ModelCatalog, select_client from nisse_dev.config import AgentConfig, ConfigError from nisse_dev.config import Project as ProjectEntry +from nisse_dev.context import BudgetedClient from nisse_dev.delegation import Caps +from nisse_dev.execution import SSHHost +from nisse_dev.git import GitRepo, HTTPSToken, PushCredential, SSHKey from nisse_dev.gitea import GiteaClient +from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient from nisse_dev.loop.agent import LoopAgent, Project from nisse_dev.loop.driver import Driver from nisse_dev.loop.help import Helpers @@ -31,6 +28,44 @@ from nisse_dev.state import StateStore from nisse_dev.tools import Workspace from nisse_dev.worker import SerialWorker +# How long shutdown waits for running work before leaving the store to the +# durability layer (L8). A milestone can run far longer than a service +# manager allows for a stop, and a hard exit mid-work is already safe. +SHUTDOWN_WAIT_S = 10.0 + +TOOL_DRIVING_ROLES = ("driver", "helper") + + +def discover_models(config: AgentConfig, oplog: OpLog) -> ModelCatalog: + """Resolve every configured role against LiteLLM before anything runs. + + Refusing to boot is the whole point: a role pointed at a model this key + cannot see is a misconfiguration, and the cheapest place to learn that is + here rather than ten minutes into a milestone. An unreachable proxy is + reported separately -- it is an outage, not a mistake, and telling them + apart is what lets an operator know whether to edit config or wait. + """ + catalog = ModelCatalog.discover(config.llm_base_url, config.llm_token) + catalog.validate(config.roles.pinned()) + # Every role's context window must be known before a request is built + # against it (L6): a guess that is too large reaches a server that may + # silently cut the prompt. + catalog.require_limits(config.roles.pinned()) + for role in TOOL_DRIVING_ROLES: + facts = catalog.facts(config.roles.resolve(role)) + if facts.supports_tools is False: + # A warning, not a refusal: models LiteLLM declares tool-less are + # observed driving tool loops anyway, and L9 expects weak local + # models to be accommodated rather than excluded. + oplog.emit( + "model_tools_unsupported", + cause="startup", + role=role, + model=facts.alias, + backing=facts.backing, + ) + return catalog + @dataclass class RunningLoop: @@ -46,7 +81,7 @@ class RunningLoop: def start(self) -> None: self.loop.start() self.transport.post( - f"nisse-dev up as {self.account} (new loop) on " + f"nisse-dev up as {self.account} on " f"{', '.join(p.repo for p in self.config.projects)}; " f"{self.loop_status()}" ) @@ -69,6 +104,113 @@ class RunningLoop: self._stopped.wait() +def make_llm_clients( + config: AgentConfig, catalog: ModelCatalog +) -> tuple[BudgetedClient, BudgetedClient]: + """The two LLM routes, as every model call in the agent uses them. + + Shared with the benchmark, so that what it measures is exactly what the + agent would do: the same context budget, the same declared limits, and + tool_choice asked only of models declared to enforce it. + """ + tuning = config.tuning + + def limits_for(model: str) -> tuple[int, int | None]: + facts = catalog.facts(model) + assert facts.max_input_tokens is not None # require_limits checked + return facts.max_input_tokens, facts.max_output_tokens + + def budgeted(client: object) -> BudgetedClient: + # Every request, on either route, is fitted to its model's window. + return BudgetedClient( + client, + limits_for, + max_output_tokens=config.max_output_tokens, + keep_recent_turns=tuning.context.keep_recent_turns, + safety_margin=tuning.context.safety_margin, + enforces_tool_choice=lambda m: ( + catalog.facts(m).supports_tool_choice is True + ), + ) + + normalized = budgeted( + LiteLLMClient( + config.llm_base_url, + config.llm_token, + max_tokens=config.max_output_tokens, + timeout_s=config.request_timeout_s, + ) + ) + # Delegations prefer the vendor-native route; fidelity is cheap + # elsewhere (L9). Which route a given model gets is decided by + # select_client, not by where the caller happens to sit. + passthrough = budgeted( + AnthropicPassthroughClient( + config.llm_base_url, + config.llm_token, + max_tokens=config.max_output_tokens, + timeout_s=config.request_timeout_s, + ) + ) + return normalized, passthrough + + +def build_host(config: AgentConfig, project: ProjectEntry) -> SSHHost: + """A project's build VM (L13). Its known-hosts file and connection socket + live beside the state store.""" + host = project.build_host + return SSHHost( + host.address, + key=host.key, + user=host.user, + port=host.port, + state_dir=config.state_db.parent / "ssh", + ) + + +def push_credential(project: ProjectEntry) -> PushCredential | None: + """What a project's pushes go by: a push-only account's token over + HTTPS, or a deploy key over SSH (D4).""" + if project.push_user and project.push_token: + return HTTPSToken(project.push_user, project.push_token) + if project.push_key is None: + return None + try: + return SSHKey(project.push_key.read_text()) + except OSError as exc: + raise RuntimeError( + f"cannot read the push key {project.push_key}: {exc}" + ) from exc + + +def remote_url(gitea: GiteaClient, repo: str, credential: PushCredential | None) -> str: + """The credential-free URL the workspace clones from and pushes to.""" + if isinstance(credential, SSHKey): + return gitea.ssh_url(repo) + return gitea.clone_url(repo) + + +def prepare_workspace( + host: SSHHost, + config: AgentConfig, + account: str, + remote: str, + credential: PushCredential | None, + project: ProjectEntry, +) -> GitRepo: + """Ensure the workspace on the build VM is a clone of the project repo. + + The clone uses the repository's SSH URL, so no credential is ever stored + in it (#89); the push credential is handed to the clone and nothing else. + """ + repo = GitRepo(host, project.workspace) + if not repo.exists(): + repo.clone(remote, credential=credential) + repo.configure_identity(account, f"{account}@users.noreply.local") + repo.install_excludes() # build artifacts are never the work (#80) + return repo + + def project_runtime( config: AgentConfig, entry: ProjectEntry, @@ -80,7 +222,7 @@ def project_runtime( (with the secrets its commands may see, redacted on return), push credential and milestones.""" host = build_host(config, entry) - credential = push_credential(config, entry) + credential = push_credential(entry) remote = remote_url(gitea, entry.repo, credential) git = prepare_workspace(host, config, account, remote, credential, entry) return Project( @@ -133,15 +275,16 @@ def build_loop_agent( driver = Driver( normalized, config.roles.resolve("driver"), - max_turns=config.tuning.executor.max_turns, + max_turns=config.tuning.driver.max_turns, cancel=cancel, - failure_limit=config.tuning.executor.failure_limit, + failure_limit=config.tuning.driver.failure_limit, ) tuning = config.tuning helpers = Helpers( client=normalized, roles=config.roles, client_for=lambda model: select_client(model, normalized, passthrough, catalog), + backing=lambda model: catalog.facts(model).backing, caps=Caps( turns=tuning.delegation.max_turns, tokens=tuning.delegation.max_tokens, diff --git a/nisse-dev/src/nisse_dev/loop/help.py b/nisse-dev/src/nisse_dev/loop/help.py index 63fab77..bc49b36 100644 --- a/nisse-dev/src/nisse_dev/loop/help.py +++ b/nisse-dev/src/nisse_dev/loop/help.py @@ -42,6 +42,9 @@ class Helpers: # The helper's client by model (vendor passthrough where it fits, L9); # unset, ``client``. client_for: Callable[[str], LLMClient] | None = None + # What an alias resolves to, recorded with each delegation: eval history + # keyed on an alias alone averages over whatever it pointed at (L9). + backing: Callable[[str], str] | None = None caps: Caps = field(default_factory=Caps) expert_budget: int = DEFAULT_BUDGET profiles: Mapping[str, Profile] = field(default_factory=dict) diff --git a/nisse-dev/src/nisse_dev/loop/reader.py b/nisse-dev/src/nisse_dev/loop/reader.py index eec7f68..0aaea2b 100644 --- a/nisse-dev/src/nisse_dev/loop/reader.py +++ b/nisse-dev/src/nisse_dev/loop/reader.py @@ -13,6 +13,15 @@ from typing import Any from nisse_dev.canonical import StoredEvent, ToolCall, UserMessage from nisse_dev.llm import LLMClient, LLMError + +def command_form(text: str) -> str: + """How a message reads as a command: case, surrounding whitespace, runs + of inner whitespace and trailing '.' or '!' do not matter, so a phone's + "Go." is still a go-ahead (#64). Only for matching; what the operator + wrote is what gets recorded and passed on.""" + return " ".join(text.strip().rstrip(".!").split()).lower() + + VERDICT = { "type": "function", "function": { diff --git a/nisse-dev/src/nisse_dev/main.py b/nisse-dev/src/nisse_dev/main.py index ec1a817..397bec6 100644 --- a/nisse-dev/src/nisse_dev/main.py +++ b/nisse-dev/src/nisse_dev/main.py @@ -12,9 +12,9 @@ import sys from pathlib import Path from types import FrameType -from nisse_dev.agent import build_agent from nisse_dev.config import AgentConfig, ConfigError from nisse_dev.envfile import load_env_file +from nisse_dev.loop.build import build_loop_agent def main() -> int: @@ -27,12 +27,7 @@ def main() -> int: print(f"nisse-dev: {exc}", file=sys.stderr) return 2 - if config.loop == "new": - from nisse_dev.loop.build import build_loop_agent - - agent = build_loop_agent(config) - else: - agent = build_agent(config) + agent = build_loop_agent(config) def shutdown(_signum: int, _frame: FrameType | None) -> None: print("nisse-dev: stopping", file=sys.stderr) @@ -42,7 +37,8 @@ def main() -> int: signal.signal(signal.SIGTERM, shutdown) agent.start() - print(f"nisse-dev: ready as {agent.account} on {config.project}", file=sys.stderr) + names = ", ".join(p.name for p in config.projects) + print(f"nisse-dev: ready as {agent.account} on {names}", file=sys.stderr) agent.wait() return 0 diff --git a/nisse-dev/src/nisse_dev/planner.py b/nisse-dev/src/nisse_dev/planner.py deleted file mode 100644 index b291cde..0000000 --- a/nisse-dev/src/nisse_dev/planner.py +++ /dev/null @@ -1,193 +0,0 @@ -"""LLM-backed planning agent (architecture L3 approvals, L9 roles, D2 plans). - -Gating is structural at the tool layer: the planner is offered exactly two -tools — ``submit_plan``, which posts a plan for approval and ends the turn, -and ``ask_human``, which pauses for an answer. No "proceed" tool exists, so -the model has no path to advance itself past a gate. - -Model output is appended to the canonical log as it arrives (L8/L7), so a -crash mid-planning leaves a truthful transcript. -""" - -from typing import Any - -from nisse_dev.cancel import Cancel -from nisse_dev.canonical import AssistantText, ToolCall, ToolResult -from nisse_dev.control import AskHuman, PlanningError, PlanningStopped, SubmitPlan -from nisse_dev.llm import BudgetExhausted, LLMClient, LLMError, RoleMap -from nisse_dev.state import Transcript - -# Plans may be executed by a materially weaker model, which cannot -# reconstruct intent. This predates the architecture's capability profiles -# and plans that specify rather than implement (L9, D2); Development Plan -# M2.3 replaces it. -SYSTEM_PROMPT = """You are nisse-dev, a software engineering agent. - -You are planning, not implementing. Produce plans that are explicit and -low-inference: a materially weaker model will execute them and cannot -reconstruct your intent. Name files, commands, and acceptance criteria -concretely; never write "handle the edge cases" or "wire it up". - -You must answer by calling exactly one tool: -- submit_plan: your plan is posted for human approval and your turn ends. -- ask_human: ask one question when a decision is genuinely the operator's. - -You cannot approve your own plan or start work; a human gate follows every -plan you submit.""" - -PLANNER_TOOLS: list[dict[str, Any]] = [ - { - "type": "function", - "function": { - "name": "submit_plan", - "description": "Post a plan for human approval. Ends your turn.", - "parameters": { - "type": "object", - "properties": { - "type": { - "type": "string", - "description": "Which plan this is, e.g. 'arch' or 'milestone-1'.", - }, - "content": { - "type": "string", - "description": "The plan itself, explicit enough to execute verbatim.", - }, - }, - "required": ["type", "content"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "ask_human", - "description": "Ask the operator one question, then wait.", - "parameters": { - "type": "object", - "properties": { - "question": {"type": "string"}, - }, - "required": ["question"], - }, - }, - }, -] - -_NUDGE = ( - "You did not call a tool. Call submit_plan or ask_human now; " - "plain prose is not accepted." -) - - -class LLMPlanningAgent: - def __init__( - self, - client: LLMClient, - roles: RoleMap, - *, - system_prompt: str = SYSTEM_PROMPT, - cancel: Cancel | None = None, - ) -> None: - self._client = client - self._roles = roles - self._system_prompt = system_prompt - # Checked before each turn only. A turn is one model call, and a turn - # that completes ends at a gate or a question, which is a halt anyway. - self._cancel = cancel or Cancel() - - def plan( - self, - transcript: Transcript, - gate: str, - project: str, - feedback: str | None = None, - answer: str | None = None, - ) -> SubmitPlan | AskHuman: - """Operator feedback and answers are already durable in the canonical - log (the orchestrator appends every inbound message), so they are not - re-injected here; the log is the single source of context. The - transcript is the work item's, written as the planner (L6, L7).""" - model = self._roles.resolve("planner") - instruction = ( - f"{self._system_prompt}\n\n" - f"Current task: draft the {gate} plan for project {project!r}." - ) - self._check_stop() - action, first_error = self._try_turn(transcript, model, instruction) - if action is not None: - return action - self._check_stop() - action, second_error = self._try_turn( - transcript, model, f"{instruction}\n\n{_NUDGE}" - ) - if action is None: - detail = second_error or first_error - raise PlanningError( - f"planner ({model}) produced no submit_plan or ask_human call " - f"after a retry{f': {detail}' if detail else ''}" - ) - return action - - def _check_stop(self) -> None: - if self._cancel.requested: - raise PlanningStopped("stopped by the operator") - - def _try_turn( - self, transcript: Transcript, model: str, system: str - ) -> tuple[SubmitPlan | AskHuman | None, str | None]: - """A malformed or failed call is a bad turn, not a crash: the caller - retries once and then gates to the human (L9 — judgment about model - output is observable, never the model's own claim).""" - try: - return self._turn(transcript, model, system), None - except BudgetExhausted: - # Retrying cannot help: the budget will not reappear within a - # turn. Surface it now so the operator is told rather than the - # loop spending a second call to fail identically. - raise - except LLMError as exc: - return None, str(exc) - - def _turn( - self, transcript: Transcript, model: str, system: str - ) -> SubmitPlan | AskHuman | None: - events = transcript.view() - produced = self._client.complete( - events, tools=PLANNER_TOOLS, model=model, system=system - ) - action: SubmitPlan | AskHuman | None = None - for event in produced: - # Append before interpreting: the transcript must be truthful even - # if the call below rejects the arguments. - transcript.append(event) - if isinstance(event, AssistantText) or action is not None: - continue - if isinstance(event, ToolCall): - action = self._interpret(event) - if action is not None: - # The gate is this call's outcome. Recording it keeps the - # transcript renderable: providers reject a tool call with - # no result, and the next planning turn replays this log. - transcript.append( - ToolResult( - call_id=event.call_id, - result=( - "plan posted for approval; turn ended" - if isinstance(action, SubmitPlan) - else "question posted; awaiting the operator" - ), - success=True, - ) - ) - return action - - def _interpret(self, call: ToolCall) -> SubmitPlan | AskHuman | None: - if call.tool == "submit_plan": - content = call.arguments.get("content") - if isinstance(content, str) and content.strip(): - return SubmitPlan(content.strip()) - elif call.tool == "ask_human": - question = call.arguments.get("question") - if isinstance(question, str) and question.strip(): - return AskHuman(question.strip()) - return None diff --git a/nisse-dev/src/nisse_dev/poller.py b/nisse-dev/src/nisse_dev/poller.py deleted file mode 100644 index c0bd840..0000000 --- a/nisse-dev/src/nisse_dev/poller.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Gitea comment intake (architecture D5). - -Polling, not webhooks: the agent VM stays outbound-only (D7), so nothing -needs to reach in. Intake is structured JSON straight from the API and never -routed through a chat bridge. - -Outage posture, per L5: back off exponentially, cap at a few minutes, retry -indefinitely, and log. Polling is pure intake — an outage delays a reaction -but must never fail a task or force a state transition. -""" - -import threading -from collections.abc import Callable, Iterable - -from nisse_dev.gitea import Comment, GiteaClient, GiteaError -from nisse_dev.oplog import OpLog - -DEFAULT_INTERVAL_S = 20.0 -INITIAL_BACKOFF_S = 5.0 -MAX_BACKOFF_S = 300.0 - - -class CommentPoller: - def __init__( - self, - client: GiteaClient, - repo: str, - on_comment: Callable[[Comment], None], - *, - oplog: OpLog | None = None, - ignore_authors: Iterable[str] = (), - interval_s: float = DEFAULT_INTERVAL_S, - initial_backoff_s: float = INITIAL_BACKOFF_S, - max_backoff_s: float = MAX_BACKOFF_S, - ) -> None: - self._client = client - self._repo = repo - self._on_comment = on_comment - self._oplog = oplog - self._ignore = set(ignore_authors) - self._interval_s = interval_s - self._initial_backoff_s = initial_backoff_s - self._max_backoff_s = max_backoff_s - self._seen: set[int] = set() - # Comments at or below this ID were handled before this poller - # existed (a restart, or a watch attached again); Gitea's comment IDs - # only grow, so one number says it all. - self._after = 0 - self._stop = threading.Event() - self._thread: threading.Thread | None = None - - def prime(self, index: int) -> int: - """Mark existing comments as seen without delivering them, and - return the highest ID marked (0 if none). - - A poller that starts by replaying the whole thread would re-act to - feedback already addressed — including its own earlier replies. - """ - highest = 0 - try: - for comment in self._client.list_comments(self._repo, index): - self._seen.add(comment.id) - highest = max(highest, comment.id) - except GiteaError: - # Nothing seen means the first successful poll delivers the - # backlog once. Better than refusing to start. - pass - return max(highest, self._after) - - def resume_after(self, comment_id: int) -> None: - """Deliver only comments newer than comment_id: the ones posted - while nothing was watching are caught up, the handled ones are not - replayed. Needs no request, so an outage cannot cause a replay.""" - self._after = comment_id - - def poll_once(self, index: int) -> list[Comment]: - """One pass; returns what was newly delivered. Separated from the - loop so behaviour is testable without threads or sleeps.""" - delivered: list[Comment] = [] - for comment in self._client.list_comments(self._repo, index): - if comment.id in self._seen or comment.id <= self._after: - continue - self._seen.add(comment.id) - if comment.author in self._ignore: - continue # the agent's own replies are not new input - delivered.append(comment) - self._on_comment(comment) - return delivered - - def start(self, index: int) -> None: - self._stop.clear() - self._thread = threading.Thread(target=self._loop, args=(index,), daemon=True) - self._thread.start() - - def stop(self) -> None: - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=self._interval_s + 5) - self._thread = None - - def _loop(self, index: int) -> None: - backoff = self._initial_backoff_s - while not self._stop.is_set(): - try: - self.poll_once(index) - except GiteaError as exc: - if self._oplog is not None: - self._oplog.emit( - "gitea.poll_failed", - cause="gitea unreachable", - error=str(exc)[:300], - retry_in_s=backoff, - ) - self._stop.wait(backoff) - backoff = min(backoff * 2, self._max_backoff_s) - continue - backoff = self._initial_backoff_s - self._stop.wait(self._interval_s) diff --git a/nisse-dev/src/nisse_dev/publish.py b/nisse-dev/src/nisse_dev/publish.py deleted file mode 100644 index 219eb50..0000000 --- a/nisse-dev/src/nisse_dev/publish.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Publishing a finished milestone as a Gitea PR (architecture D4, L8). - -The ordering here is the whole point. Intent is recorded *before* the first -externally visible act, so a crash anywhere in the sequence leaves a durable -record of what was in flight. Boot reconciliation then asks Gitea whether the -PR exists rather than repeating the push-and-open — the spec's own example of -an idempotent transition reconciled against external truth. -""" - -from collections.abc import Sequence -from dataclasses import dataclass - -from nisse_dev.git import GitRepo, PushCredential -from nisse_dev.gitea import ( - GiteaClient, - GiteaError, - PullRequest, - pull_request_reconciler, -) -from nisse_dev.reconcile import Reconciler -from nisse_dev.state import StateStore - -OPEN_PR_INTENT = "open_pr" - - -@dataclass(frozen=True) -class PublishResult: - pull_request: PullRequest | None - note: str - - -class MilestonePublisher: - def __init__( - self, - store: StateStore, - client: GiteaClient, - repo: str, - git_repo: GitRepo, - *, - remote_url: str, - push_credential: PushCredential | None = None, - base_branch: str = "main", - ) -> None: - self._store = store - self._client = client - self._repo = repo - self._git = git_repo - self._base_branch = base_branch - # A URL with no credential in it; the push credential is handed to - # each push on the build VM and nothing else (D4, L12). - self._remote = remote_url - self._credential = push_credential - - def reconcilers(self) -> dict[str, Reconciler]: - return {OPEN_PR_INTENT: pull_request_reconciler(self._client, self._repo)} - - def branch_for(self, task_id: str, milestone: int) -> str: - # Namespaced by task so re-running a project against the same repo - # cannot collide with an earlier run's branch. Flat, with no "/": - # git stores branch names as paths, so a branch named after the task - # (which a model did make, live) would block "/milestone-1". - return f"{task_id}-milestone-{milestone}" - - def publish( - self, - *, - task_id: str, - milestone: int, - project: str, - plan: str, - service_url: str | None = None, - ) -> PublishResult: - branch = self.branch_for(task_id, milestone) - if self._git.current_branch() != branch: - self._git.checkout_new_branch(branch) - sha = self._git.commit_all(f"Milestone {milestone}: {project}") - if sha is None: - if self._git.commits_since(f"origin/{self._base_branch}") == 0: - # Not an error: the executor may have concluded there was - # nothing to change. Report it and let the operator decide. - return PublishResult(None, "no changes to publish") - # The executor committed its work itself; publish that. - sha = self._git.head_sha() - # D4: what was left out (build artifacts) is reported, not dropped. - left_out = self._git.left_out() - - intent_id = self._store.begin_intent( - OPEN_PR_INTENT, - {"branch": branch, "milestone": milestone, "sha": sha}, - cause=f"publishing milestone {milestone}", - work_item=task_id, # owned by the work item it publishes (L1) - ) - self._git.push(self._remote, branch, credential=self._credential) - # Idempotent: if a PR for this branch is already open (a crash after - # opening it, or a response lost on the way back), the push above - # has updated it; adopt it rather than fail on a second. - existing = self.published_pull_request(task_id, milestone) - if existing is not None: - self._store.commit_intent( - intent_id, cause=f"adopted open PR #{existing.number}" - ) - return PublishResult( - existing, - f"PR #{existing.number} was already open for this milestone; " - f"pushed to it: {existing.url}" + _left_out_note(left_out), - ) - pr = self._client.open_pull_request( - self._repo, - head=branch, - base=self._base_branch, - title=f"Milestone {milestone}: {project}", - body=_pr_body(milestone, plan, service_url, left_out), - ) - self._store.commit_intent(intent_id, cause=f"opened PR #{pr.number}") - return PublishResult( - pr, f"opened PR #{pr.number}: {pr.url}" + _left_out_note(left_out) - ) - - def published_pull_request( - self, task_id: str, milestone: int - ) -> PullRequest | None: - """The open PR for a milestone's branch, if there is one. A closed one - does not count: the operator closed it, so a new one may be opened. - Raises GiteaError when Gitea cannot say.""" - pr = self._client.find_pull_request( - self._repo, self.branch_for(task_id, milestone) - ) - return pr if pr is not None and pr.state == "open" else None - - def comment(self, index: int, body: str) -> None: - """Reply on the PR. Courtesy only — the pushed commit is the real - answer — so a failure here must not derail the review loop.""" - try: - self._client.post_comment(self._repo, index, body) - except GiteaError: - pass - - def close_pull_request(self, index: int, note: str) -> None: - """Close a milestone PR with a note saying why. The branch is kept, - so the work can be reopened in Gitea.""" - self._client.post_comment(self._repo, index, note) - self._client.close_pull_request(self._repo, index) - - def push_followup( - self, *, task_id: str, milestone: int, message: str - ) -> str | None: - """Push a fix commit onto an open milestone's branch. - - No intent is recorded: the PR already exists, so a crash mid-push - costs a repeated push of the same commit, which is harmless. That is - the difference from publish(), where the side effect is a new PR. - """ - branch = self.branch_for(task_id, milestone) - if self._git.current_branch() != branch: - self._git.checkout_new_branch(branch) - sha = self._git.commit_all(message) - if sha is None: - # The executor may have committed the fix itself. - sha = self._git.head_sha() - pushed = self._git.remote_sha( - self._remote, branch, credential=self._credential - ) - if sha == pushed: - return None - self._git.push(self._remote, branch, credential=self._credential) - return sha - - -def _left_out_note(paths: list[str]) -> str: - if not paths: - return "" - shown = ", ".join(paths[:5]) + ( - f" and {len(paths) - 5} more" if len(paths) > 5 else "" - ) - return f" (left out {len(paths)} build artifact(s): {shown})" - - -def _pr_body( - milestone: int, - plan: str, - service_url: str | None, - left_out: Sequence[str] = (), -) -> str: - parts = [ - f"Milestone {milestone}, implemented by nisse-dev against the approved plan.", - "", - "## Approved plan", - "", - plan, - ] - if service_url: - # D4: a runnable test service's address belongs in the PR description - # so a reviewer can exercise it. Nothing reports one yet; the browser - # tool (D7) is what will. - parts += ["", "## Running test service", "", service_url] - if left_out: - parts += [ - "", - "## Left out", - "", - "Build artifacts in the workspace, never part of the work:", - "", - *(f"- `{path}`" for path in left_out), - ] - return "\n".join(parts) diff --git a/nisse-dev/src/nisse_dev/state.py b/nisse-dev/src/nisse_dev/state.py index a59bcd7..382f057 100644 --- a/nisse-dev/src/nisse_dev/state.py +++ b/nisse-dev/src/nisse_dev/state.py @@ -12,7 +12,6 @@ resolved against external truth on boot). import json import sqlite3 -import uuid from collections.abc import Callable, Mapping from dataclasses import dataclass from datetime import UTC, datetime @@ -42,10 +41,9 @@ Migration = Callable[[sqlite3.Connection], None] # Maps version N to the migration that takes the schema from N to N+1. MIGRATIONS: Mapping[int, Migration] = {} -IDLE = "IDLE" - # Who produced an event (L7): the operator, the orchestrator's own code, or a -# model role (L9). planner and executor become driver in the new loop (M2.3). +# model role (L9). planner and executor are the old loop's (until M2.8), kept +# so its records still read. ACTORS = frozenset( { "operator", @@ -64,17 +62,6 @@ class StateSchemaError(RuntimeError): pass -@dataclass(frozen=True) -class AgentState: - """The state machine's view of the active work item (control.py, until - Development Plan M2.6 retires it).""" - - state: str - project: str | None - data: dict[str, Any] - updated_at: str - - @dataclass(frozen=True) class WorkItem: id: str @@ -406,61 +393,6 @@ class StateStore: ) return len(rows) - # --- the state machine's view (control.py, until M2.6) ----------------- - - def get_state(self) -> AgentState | None: - """The active work item as the state machine sees it: its position - is the state, IDLE when there is none. None only for a store the - agent has never started on.""" - agent = self._conn.execute("SELECT * FROM agent WHERE id = 1").fetchone() - if agent is None: - return None - item = self.active_item() - if item is None: - return AgentState(IDLE, None, {}, agent["updated_at"]) - return AgentState(item.position, item.scope, item.data, item.updated_at) - - def transition( - self, - state: str, - *, - cause: str, - project: str | None = None, - data: dict[str, Any] | None = None, - ) -> None: - """The state machine's transition, over work items: IDLE closes and - deactivates the active item (its outcome should already be recorded - with close_work_item); any other state updates the active item, - opening one if there is none.""" - self._ensure_agent_row() - active = self.active_item() - if state == IDLE: - if active is not None: - if active.status == "open": - self.close_work_item(active.id, outcome="closed", cause=cause) - self.set_active(None, cause=cause) - self.oplog.emit( - "state.transition", - cause=cause, - **{ - "from": active.position if active else None, - "to": IDLE, - "project": project, - }, - ) - return - if active is None: - item_id = str((data or {}).get("task_id") or f"task-{uuid.uuid4().hex[:8]}") - active = self.open_work_item( - item_id, - title=project or item_id, - scope=project, - position=IDLE, - cause=cause, - ) - self.set_active(item_id, cause=cause) - self.update_work_item(active.id, position=state, data=data or {}, cause=cause) - # --- intents (L8) ------------------------------------------------------ def begin_intent( diff --git a/nisse-dev/src/nisse_dev/tools.py b/nisse-dev/src/nisse_dev/tools.py index 1ae4686..ebc186e 100644 --- a/nisse-dev/src/nisse_dev/tools.py +++ b/nisse-dev/src/nisse_dev/tools.py @@ -31,7 +31,7 @@ MUTATING_TOOLS = frozenset({"write_file", "run_command"}) DEFAULT_TIMEOUT_S = 120 MAX_OUTPUT_CHARS = 4000 -EXECUTOR_TOOLS: list[dict[str, Any]] = [ +EXECUTION_TOOLS: list[dict[str, Any]] = [ { "type": "function", "function": { diff --git a/nisse-dev/src/nisse_dev/views.py b/nisse-dev/src/nisse_dev/views.py index f677302..3fea588 100644 --- a/nisse-dev/src/nisse_dev/views.py +++ b/nisse-dev/src/nisse_dev/views.py @@ -38,10 +38,8 @@ NARRATION_RESULT_CHARS = 300 # The roles whose work the viewer is shown, and how: "report" keeps only the # handback, "narrate" keeps a line per step. Anything not listed is not shown. _SHOWN: dict[str, dict[str, str]] = { - "planner": {}, "driver": {"helper": "report"}, - "executor": {"helper": "report"}, - "helper": {"executor": "narrate", "driver": "narrate"}, + "helper": {"driver": "narrate"}, } @@ -101,8 +99,8 @@ def project(events: list[StoredEvent], viewer: str) -> list[StoredEvent]: for stored in events: event, actor = stored.event, stored.actor if isinstance(event, PlanSubmitted): - # A record for the executor's scope and the dump, not a turn: - # the planner's own submit_plan call already carries the plan. + # A record for a helper's scope and the dump, not a turn: the + # driver's own call already carries what was agreed. if viewer == "helper": out.append(_said(stored, "approved plan", event.plan)) continue diff --git a/nisse-dev/tests/test_abandon.py b/nisse-dev/tests/test_abandon.py deleted file mode 100644 index c0dc7aa..0000000 --- a/nisse-dev/tests/test_abandon.py +++ /dev/null @@ -1,186 +0,0 @@ -"""The `abandon` command: ending a task from any waiting state (L2).""" - -import io -from pathlib import Path -from typing import Any - -import pytest - -from nisse_dev.control import ( - AWAIT_ARCH_SIGNOFF, - AWAIT_INPUT, - AWAIT_MILESTONE_SIGNOFF, - IDLE, - PR_OPEN, - AskHuman, - Orchestrator, - SubmitPlan, -) -from nisse_dev.executor import ExecutorOutcome -from nisse_dev.gitea import GiteaError, PullRequest -from nisse_dev.oplog import OpLog -from nisse_dev.publish import PublishResult -from nisse_dev.state import StateStore -from nisse_dev.transport import InMemoryTransport - - -class Planner: - def __init__(self, ask_first: bool = False) -> None: - self._ask = ask_first - - def plan( - self, - transcript: object, - gate: str, - project: str, - feedback: str | None = None, - answer: str | None = None, - ) -> SubmitPlan | AskHuman: - if self._ask and answer is None: - return AskHuman("which language?") - return SubmitPlan(f"{gate} plan") - - -class Board: - def __init__(self, *, fail_close: bool = False) -> None: - self.closed: list[tuple[int, str]] = [] - self._fail_close = fail_close - - def open_project_issue(self, project: str, task_id: str) -> int: - return 10 - - def post_plan(self, issue: int, gate: str, rev: int, plan: str) -> Any: - class Posted: - comment_id = 1 - - return Posted() - - def echo(self, issue: int, text: str) -> None: - pass - - def close(self, issue: int, note: str) -> None: - if self._fail_close: - raise GiteaError("gitea is down") - self.closed.append((issue, note)) - - -class Executor: - def run(self, transcript: object, milestone: int) -> ExecutorOutcome: - return ExecutorOutcome("completed", "done") - - -class Publisher: - def __init__(self) -> None: - self.closed: list[tuple[int, str]] = [] - - def reconcilers(self) -> dict[str, Any]: - return {} - - def publish(self, **kwargs: Any) -> PublishResult: - pr = PullRequest(number=20, state="open", head="h", url="u") - return PublishResult(pr, "opened PR #20") - - def close_pull_request(self, index: int, note: str) -> None: - self.closed.append((index, note)) - - -class Harness: - def __init__( - self, tmp_path: Path, *, ask_first: bool = False, board: Board | None = None - ) -> None: - self.store = StateStore( - tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()) - ) - self.transport = InMemoryTransport() - self.board = board or Board() - self.publisher = Publisher() - Orchestrator( - self.store, - self.transport, - Planner(ask_first), - operators={"don"}, - board=self.board, - executor=Executor(), - publisher=self.publisher, - ).start() - - @property - def state(self) -> str: - state = self.store.get_state() - assert state is not None - return state.state - - def say(self, *replies: str) -> None: - for reply in replies: - self.transport.receive("don", reply) - - -WAITING = [ - ("arch sign-off", ["use demo"], AWAIT_ARCH_SIGNOFF), - ("milestone sign-off", ["use demo", "approve"], AWAIT_MILESTONE_SIGNOFF), - ( - "approved, not started", - ["use demo", "approve", "approve"], - AWAIT_MILESTONE_SIGNOFF, - ), - ("PR open", ["use demo", "approve", "approve", "start milestone 1"], PR_OPEN), -] - - -@pytest.mark.parametrize( - ("replies", "waiting"), - [(r, w) for _, r, w in WAITING], - ids=[n for n, _, _ in WAITING], -) -def test_abandon_ends_the_task_from_any_waiting_state( - tmp_path: Path, replies: list[str], waiting: str -) -> None: - h = Harness(tmp_path) - h.say(*replies) - assert h.state == waiting - h.say("abandon") - state = h.store.get_state() - assert state.state == IDLE - assert state.project is None - assert "abandoned" in h.transport.outbound[-1].text - assert h.board.closed == [(10, "abandoned by don")] - # A new task starts cleanly afterwards. - h.say("use other") - assert h.state == AWAIT_ARCH_SIGNOFF - - -def test_abandon_answers_a_question_by_ending_the_task(tmp_path: Path) -> None: - h = Harness(tmp_path, ask_first=True) - h.say("use demo") - assert h.state == AWAIT_INPUT - h.say("abandon") - assert h.state == IDLE - - -def test_abandon_in_pr_open_closes_the_pr(tmp_path: Path) -> None: - h = Harness(tmp_path) - h.say("use demo", "approve", "approve", "start milestone 1") - h.say("abandon") - assert h.publisher.closed == [(20, "abandoned by don")] - - -def test_abandon_aborts_a_pending_publish_intent(tmp_path: Path) -> None: - h = Harness(tmp_path) - h.say("use demo") - h.store.begin_intent("open_pr", {"branch": "b"}, cause="publish cut short") - h.say("abandon") - assert h.store.pending_intents() == [] - - -def test_abandon_still_ends_the_task_when_gitea_is_down(tmp_path: Path) -> None: - h = Harness(tmp_path, board=Board(fail_close=True)) - h.say("use demo", "abandon") - assert h.state == IDLE - assert "could not close plan issue #10" in h.transport.outbound[-1].text - - -def test_abandon_with_no_task_says_so(tmp_path: Path) -> None: - h = Harness(tmp_path) - h.say("abandon") - assert h.state == IDLE - assert "nothing to abandon" in h.transport.outbound[-1].text diff --git a/nisse-dev/tests/test_agent_build.py b/nisse-dev/tests/test_agent_build.py index 831f185..bef461c 100644 --- a/nisse-dev/tests/test_agent_build.py +++ b/nisse-dev/tests/test_agent_build.py @@ -11,10 +11,12 @@ import httpx import pytest from test_config import SECRETS, P, tree -from nisse_dev.agent import build_agent, discover_models, prepare_workspace from nisse_dev.catalog import ModelCatalog, UnknownModel from nisse_dev.config import AgentConfig from nisse_dev.execution import SSHHost +from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient +from nisse_dev.loop.build import RunningLoop, build_loop_agent, discover_models +from nisse_dev.loop.build import prepare_workspace as prepare from nisse_dev.oplog import OpLog # What the test roles resolve to, as LiteLLM would report them. @@ -92,124 +94,115 @@ def origin(tmp_path: Path) -> Path: return remote +def prepare_workspace(config: AgentConfig, origin: Path) -> Any: + [project] = config.projects + return prepare(HOST, config, "nisse-dev-testing", str(origin), None, project) + + def test_workspace_is_cloned_on_the_build_host_when_absent( tmp_path: Path, origin: Path ) -> None: config = config_for(tmp_path) - repo = prepare_workspace(HOST, config, "nisse-dev-testing", str(origin), None) - - assert (config.workspace / "README.md").exists() + repo = prepare_workspace(config, origin) + workspace = config.projects[0].workspace + assert (workspace / "README.md").exists() assert repo.current_branch() == "main" # The clone's configuration holds the plain URL it was given (#89). - assert str(origin) in (config.workspace / ".git" / "config").read_text() + assert str(origin) in (workspace / ".git" / "config").read_text() def test_workspace_is_reused_when_already_cloned(tmp_path: Path, origin: Path) -> None: config = config_for(tmp_path) - prepare_workspace(HOST, config, "nisse-dev-testing", str(origin), None) - (config.workspace / "local-work.txt").write_text("in progress") + workspace = config.projects[0].workspace + prepare_workspace(config, origin) + (workspace / "local-work.txt").write_text("in progress") - prepare_workspace(HOST, config, "nisse-dev-testing", str(origin), None) + prepare_workspace(config, origin) # Re-cloning would silently destroy uncommitted work. - assert (config.workspace / "local-work.txt").exists() + assert (workspace / "local-work.txt").exists() class FakeTransport: def __init__(self) -> None: self.sent: list[str] = [] - self.started = False self.closed = False - def send_message(self, text: str) -> None: - self.sent.append(text) - - def send_approval_request(self, text: str) -> None: - self.sent.append(text) - - def ask_question(self, text: str) -> None: + def post(self, text: str, thread: str | None = None) -> str: self.sent.append(text) + return f"$event{len(self.sent)}" def on_message(self, callback: Any) -> None: self.callback = callback - def start(self) -> None: - self.started = True - def close(self) -> None: self.closed = True -def build_offline(tmp_path: Path, origin: Path, monkeypatch: Any) -> Any: - """Assemble the real graph with only the network boundaries faked.""" - config = config_for(tmp_path) +def offline(monkeypatch: Any, origins: dict[str, str] | str, catalog=None) -> None: + """Only the network boundaries faked: Gitea, LiteLLM's model list, Matrix.""" # The proxy here doesn't exist: a call to it fails at once rather than # waiting out an outage's backoff (L5). monkeypatch.setattr("nisse_dev.llm.OUTAGE_BACKOFF_S", ()) + monkeypatch.setattr( + "nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing" + ) + monkeypatch.setattr( + "nisse_dev.loop.build.GiteaClient.clone_url", + lambda self, repo: origins if isinstance(origins, str) else origins[repo], + ) + monkeypatch.setattr( + "nisse_dev.loop.build.ModelCatalog.discover", + classmethod(lambda cls, *a, **kw: catalog or TEST_CATALOG), + ) + monkeypatch.setattr( + "nisse_dev.loop.build.MatrixTransport", lambda cfg: FakeTransport() + ) - monkeypatch.setattr( - "nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing" - ) - monkeypatch.setattr( - "nisse_dev.agent.ModelCatalog.discover", - classmethod(lambda cls, *a, **kw: TEST_CATALOG), - ) - monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport()) - monkeypatch.setattr( - "nisse_dev.agent.GiteaClient.clone_url", lambda self, repo: str(origin) - ) - return build_agent(config) + +def build_offline( + tmp_path: Path, origin: Path, monkeypatch: Any, config: AgentConfig | None = None +) -> RunningLoop: + offline(monkeypatch, str(origin)) + return build_loop_agent(config or config_for(tmp_path)) def test_the_whole_graph_assembles(tmp_path: Path, origin: Path, monkeypatch) -> None: - agent = build_offline(tmp_path, origin, monkeypatch) - orchestrator = agent.orchestrator - - # Every component that was previously unreachable is now connected. - assert orchestrator._executor is not None - assert orchestrator._publisher is not None - assert orchestrator._board is not None - assert orchestrator._delegate is not None - assert orchestrator._poller_factory is not None - assert orchestrator._executor._delegate is not None # L9 escalation path - assert orchestrator._executor._expert is not None # L9 consult path - agent.stop() + running = build_offline(tmp_path, origin, monkeypatch) + loop = running.loop + try: + assert running.account == "nisse-dev-testing" + [project] = loop._projects.values() + assert project.repo == "don/demo" + assert (tmp_path / "work" / "README.md").exists() # cloned on the build host + assert project.milestones is not None + assert loop._driver is not None + assert loop._reader is not None # plain-language go-aheads (L5) + assert loop._helpers is not None # delegation and ask_expert (L9) + finally: + running.stop() -def test_start_announces_where_it_resumed( +def test_start_announces_the_account_and_projects( tmp_path: Path, origin: Path, monkeypatch ) -> None: - agent = build_offline(tmp_path, origin, monkeypatch) - agent.start() - - assert agent.transport.started - readiness = agent.transport.sent[-1] - assert "nisse-dev up as nisse-dev-testing" in readiness - assert "greet" in readiness # the project - assert "claude-opus-5" in readiness # the pinned planner - agent.stop() + running = build_offline(tmp_path, origin, monkeypatch) + running.loop.start = lambda: None # type: ignore[method-assign] # no polling + running.start() + readiness = running.transport.sent[-1] # type: ignore[attr-defined] + assert "nisse-dev up as nisse-dev-testing on don/demo" in readiness + running.stop() def test_stop_closes_everything_and_releases_the_wait( tmp_path: Path, origin: Path, monkeypatch ) -> None: - agent = build_offline(tmp_path, origin, monkeypatch) - agent.start() - agent.stop() + running = build_offline(tmp_path, origin, monkeypatch) + running.stop() - assert agent.transport.closed - agent.wait() # returns immediately rather than hanging - assert agent.store.closed - - -def test_poller_factory_ignores_the_agents_own_comments( - tmp_path: Path, origin: Path, monkeypatch -) -> None: - # Without this the agent answers its own PR replies in a loop. - agent = build_offline(tmp_path, origin, monkeypatch) - poller = agent.orchestrator._poller_factory(lambda c: None) - assert "nisse-dev-testing" in poller._ignore - agent.stop() + assert running.transport.closed # type: ignore[attr-defined] + running.wait() # returns immediately rather than hanging + assert running.store.closed def test_delegation_uses_the_passthrough_route( @@ -217,24 +210,46 @@ def test_delegation_uses_the_passthrough_route( ) -> None: # L9: delegations run vendor-native; the normalized route is for the # places where fidelity is cheap. - from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient - - agent = build_offline(tmp_path, origin, monkeypatch) - delegation = agent.orchestrator._delegate - assert isinstance(delegation._client._inner, AnthropicPassthroughClient) - assert isinstance(agent.orchestrator._executor._client._inner, LiteLLMClient) - agent.stop() + running = build_offline(tmp_path, origin, monkeypatch) + helpers = running.loop._helpers + assert helpers is not None + helper = helpers.delegate_client("claude-haiku-4-5") + assert isinstance(helper._inner, AnthropicPassthroughClient) # type: ignore[attr-defined] + assert isinstance(running.loop._driver._client._inner, LiteLLMClient) # type: ignore[attr-defined] + running.stop() def test_httpx_clients_are_not_shared_between_routes( tmp_path: Path, origin: Path, monkeypatch ) -> None: - agent = build_offline(tmp_path, origin, monkeypatch) - normalized = agent.orchestrator._executor._client._inner._http - passthrough = agent.orchestrator._delegate._client._inner._http + running = build_offline(tmp_path, origin, monkeypatch) + helpers = running.loop._helpers + assert helpers is not None + normalized = running.loop._driver._client._inner._http # type: ignore[attr-defined] + passthrough = helpers.delegate_client("claude-haiku-4-5")._inner._http # type: ignore[attr-defined] assert isinstance(normalized, httpx.Client) assert normalized is not passthrough # different base URLs and headers - agent.stop() + running.stop() + + +def test_delegations_record_what_the_alias_resolved_to( + tmp_path: Path, origin: Path, monkeypatch +) -> None: + running = build_offline(tmp_path, origin, monkeypatch) + helpers = running.loop._helpers + assert helpers is not None and helpers.backing is not None + assert helpers.backing("claude-haiku-4-5") == "anthropic/claude-haiku-4-5" + running.stop() + + +def test_the_configured_roles_are_what_items_pin( + tmp_path: Path, origin: Path, monkeypatch +) -> None: + running = build_offline(tmp_path, origin, monkeypatch) + pinned = running.loop._roles + assert pinned == running.config.roles.pinned() + assert set(pinned) == {"driver", "helper", "expert", "classifier"} + running.stop() # --- startup model discovery (L9) ------------------------------------ @@ -243,12 +258,12 @@ def test_httpx_clients_are_not_shared_between_routes( def test_a_typod_role_refuses_to_boot(tmp_path: Path, monkeypatch) -> None: # Misconfiguration should cost a second at startup, not be discovered ten # minutes into a milestone. - config = config_for(tmp_path, roles__planner="clause-opus-5") + config = config_for(tmp_path, roles__driver="clause-opus-5") monkeypatch.setattr( - "nisse_dev.agent.ModelCatalog.discover", + "nisse_dev.loop.build.ModelCatalog.discover", classmethod(lambda cls, *a, **kw: TEST_CATALOG), ) - with pytest.raises(UnknownModel, match="planner='clause-opus-5'"): + with pytest.raises(UnknownModel, match="driver='clause-opus-5'"): discover_models(config, OpLog("test")) @@ -257,12 +272,12 @@ def test_a_role_this_key_cannot_reach_refuses_to_boot( ) -> None: # /model/info is scoped per key, so the same check catches a role pointed # at a real model that this token has no access to. - config = config_for(tmp_path, roles__executor="local-fast") + config = config_for(tmp_path, roles__helper="local-fast") monkeypatch.setattr( - "nisse_dev.agent.ModelCatalog.discover", + "nisse_dev.loop.build.ModelCatalog.discover", classmethod(lambda cls, *a, **kw: TEST_CATALOG), ) - with pytest.raises(UnknownModel, match="executor='local-fast'"): + with pytest.raises(UnknownModel, match="helper='local-fast'"): discover_models(config, OpLog("test")) @@ -288,7 +303,7 @@ def test_a_tool_less_model_warns_but_still_boots(tmp_path: Path, monkeypatch) -> } ) monkeypatch.setattr( - "nisse_dev.agent.ModelCatalog.discover", + "nisse_dev.loop.build.ModelCatalog.discover", classmethod(lambda cls, *a, **kw: tool_less), ) stream = io.StringIO() @@ -296,36 +311,10 @@ def test_a_tool_less_model_warns_but_still_boots(tmp_path: Path, monkeypatch) -> logged = stream.getvalue() assert "model_tools_unsupported" in logged - assert '"role": "executor"' in logged + assert '"role": "driver"' in logged assert '"role": "expert"' not in logged # ask_expert drives no tools -def test_the_resolved_model_is_pinned_at_task_start( - tmp_path: Path, origin: Path, monkeypatch -) -> None: - agent = build_offline(tmp_path, origin, monkeypatch) - # The store belongs to the worker thread, so the work has to run there. - agent.worker.call(lambda: agent.orchestrator._begin_project("greet")) - - pinned = agent.orchestrator._data["models"] - assert pinned["planner"] == { - "alias": "claude-opus-5", - "backing": "anthropic/claude-opus-5", - "provider": "anthropic", - } - agent.stop() - - -def test_delegation_records_what_the_alias_resolved_to( - tmp_path: Path, origin: Path, monkeypatch -) -> None: - # L9 eval history keyed on an alias alone would average across whatever - # that alias pointed at over time. - agent = build_offline(tmp_path, origin, monkeypatch) - assert agent.orchestrator._delegate._backing_model == "anthropic/claude-opus-5" - agent.stop() - - def test_tuning_reaches_every_component( tmp_path: Path, origin: Path, monkeypatch ) -> None: @@ -333,8 +322,9 @@ def test_tuning_reaches_every_component( from nisse_dev.config import ( DelegationTuning, - ExecutorTuning, + DriverTuning, ExpertTuning, + LoopTuning, ToolsTuning, Tuning, ) @@ -343,83 +333,47 @@ def test_tuning_reaches_every_component( config_for(tmp_path), max_output_tokens=8192, tuning=Tuning( - executor=ExecutorTuning(max_turns=40, failure_limit=3), + driver=DriverTuning(max_turns=40, failure_limit=3), expert=ExpertTuning(consult_budget=5), delegation=DelegationTuning( max_turns=12, max_tokens=50_000, wall_clock_s=60 ), tools=ToolsTuning(command_timeout_s=300, max_output_chars=2000), + loop=LoopTuning(bug_grace_s=120, general_idle_s=900), ), ) - monkeypatch.setattr( - "nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing" - ) - monkeypatch.setattr( - "nisse_dev.agent.ModelCatalog.discover", - classmethod(lambda cls, *a, **kw: TEST_CATALOG), - ) - monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport()) - monkeypatch.setattr( - "nisse_dev.agent.GiteaClient.clone_url", lambda self, repo: str(origin) - ) - agent = build_agent(config) - executor = agent.orchestrator._executor - delegation = agent.orchestrator._delegate + running = build_offline(tmp_path, origin, monkeypatch, config) + loop = running.loop + driver = loop._driver + helpers = loop._helpers + assert helpers is not None + [project] = loop._projects.values() - assert (executor._max_turns, executor._failure_limit) == (40, 3) - assert executor._expert._budget == 5 - assert delegation._caps.turns == 12 - assert delegation._caps.tokens == 50_000 - assert delegation._caps.wall_clock_s == 60 - assert executor._workspace._timeout_s == 300 - assert executor._workspace._max_output_chars == 2000 - assert executor._client._inner._max_tokens == 8192 - assert delegation._client._inner._max_tokens == 8192 - assert executor._client._max_output_tokens == 8192 + assert (driver._max_turns, driver._failure_limit) == (40, 3) + assert helpers.expert_budget == 5 + assert (helpers.caps.turns, helpers.caps.tokens) == (12, 50_000) + assert helpers.caps.wall_clock_s == 60 + assert project.workspace is not None + assert project.workspace._timeout_s == 300 + assert project.workspace._max_output_chars == 2000 + assert (loop._grace, loop._idle) == (120, 900) + client = driver._client + assert client._inner._max_tokens == 8192 # type: ignore[attr-defined] + assert client._max_output_tokens == 8192 # type: ignore[attr-defined] + helper = helpers.delegate_client("claude-haiku-4-5") + assert helper._inner._max_tokens == 8192 # type: ignore[attr-defined] # tool_choice is asked of exactly the models LiteLLM declares enforce it. - assert executor._client._enforces_tool_choice("claude-opus-5") is False - agent.stop() - - -def test_the_new_loop_assembles(tmp_path: Path, origin: Path, monkeypatch) -> None: - """`loop: new` builds the LoopAgent over the same services.""" - from nisse_dev.loop.build import build_loop_agent - - config = config_for(tmp_path, loop="new") - monkeypatch.setattr( - "nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing" - ) - monkeypatch.setattr( - "nisse_dev.loop.build.GiteaClient.clone_url", lambda self, repo: str(origin) - ) - monkeypatch.setattr( - "nisse_dev.agent.ModelCatalog.discover", - classmethod(lambda cls, *a, **kw: TEST_CATALOG), - ) - monkeypatch.setattr( - "nisse_dev.loop.build.MatrixTransport", lambda cfg: FakeTransport() - ) - running = build_loop_agent(config) - try: - assert running.account == "nisse-dev-testing" - [project] = running.loop._projects.values() - assert project.repo == config.gitea_repo - assert (config.workspace / "README.md").exists() # cloned on the build host - finally: - running.worker.call(running.store.close) - running.worker.stop(timeout=5) + assert client._enforces_tool_choice("claude-opus-5") is False # type: ignore[attr-defined] + running.stop() def test_each_project_gets_its_own_clone_and_secrets( tmp_path: Path, origin: Path, monkeypatch ) -> None: """D8: a second project is only a second entry.""" - from nisse_dev.loop.build import build_loop_agent - second = tmp_path / "second" config = config_for( tmp_path, - loop="new", projects__other={ "repo": "don/other", "workspace": str(tmp_path / "other-work"), @@ -436,20 +390,7 @@ def test_each_project_gets_its_own_clone_and_secrets( }, ) git("clone", "-q", "--bare", str(origin), str(second)) - origins = {"don/demo": str(origin), "don/other": str(second)} - monkeypatch.setattr( - "nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing" - ) - monkeypatch.setattr( - "nisse_dev.loop.build.GiteaClient.clone_url", lambda self, repo: origins[repo] - ) - monkeypatch.setattr( - "nisse_dev.agent.ModelCatalog.discover", - classmethod(lambda cls, *a, **kw: TEST_CATALOG), - ) - monkeypatch.setattr( - "nisse_dev.loop.build.MatrixTransport", lambda cfg: FakeTransport() - ) + offline(monkeypatch, {"don/demo": str(origin), "don/other": str(second)}) running = build_loop_agent(config) try: projects = running.loop._projects @@ -460,5 +401,4 @@ def test_each_project_gets_its_own_clone_and_secrets( assert projects["other"].workspace._secrets == {"API_KEY": "sk-x"} assert projects["greet"].workspace._secrets == {} finally: - running.worker.call(running.store.close) - running.worker.stop(timeout=5) + running.stop() diff --git a/nisse-dev/tests/test_approval_binding.py b/nisse-dev/tests/test_approval_binding.py deleted file mode 100644 index 2605c85..0000000 --- a/nisse-dev/tests/test_approval_binding.py +++ /dev/null @@ -1,147 +0,0 @@ -"""Approvals and feedback bind to the plan revision they were about (L3).""" - -import io -from pathlib import Path - -from nisse_dev.canonical import UserMessage -from nisse_dev.control import ( - AWAIT_MILESTONE_SIGNOFF, - Orchestrator, - StubAgent, -) -from nisse_dev.gitea import Comment -from nisse_dev.oplog import OpLog -from nisse_dev.state import StateStore -from nisse_dev.transport import InMemoryTransport - - -class Held: - """A worker that queues jobs until released, like one under load.""" - - def __init__(self) -> None: - self.jobs: list = [] - self.busy = False - - def submit(self, job) -> None: - self.jobs.append(job) - - def call(self, job, timeout=None): - return job() - - def run(self) -> None: - while self.jobs: - self.jobs.pop(0)() - - -class Board: - def __init__(self) -> None: - self.next_id = 100 - - def open_project_issue(self, project: str, task_id: str) -> int: - return 10 - - def post_plan(self, issue: int, gate: str, rev: int, plan: str): - self.next_id += 1 - - class Posted: - comment_id = self.next_id - - return Posted() - - def echo(self, issue: int, text: str) -> None: - pass - - -def build(tmp_path: Path, worker: Held | None = None, board: Board | None = None): - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - transport = InMemoryTransport() - orch = Orchestrator( - store, - transport, - StubAgent(), - operators={"don"}, - gitea_operators={"don"}, - board=board, - **({"worker": worker} if worker else {}), - ) - orch.start() - return store, transport, orch - - -def gate(store: StateStore) -> dict: - return store.get_state().data.get("gate") or {} - - -def test_revisions_run_across_the_task(tmp_path: Path) -> None: - store, transport, _ = build(tmp_path) - transport.receive("don", "use demo") - assert gate(store)["rev"] == 1 - transport.receive("don", "approve") - assert gate(store) == {**gate(store), "name": "milestone-1", "rev": 2} - assert "milestone-1 rev-2" in transport.outbound[-1].text - - -def test_a_stale_approval_is_answered_and_a_fresh_one_works(tmp_path: Path) -> None: - worker = Held() - store, transport, _ = build(tmp_path, worker) - transport.receive("don", "use demo") - worker.run() - transport.receive("don", "approve") - transport.receive("don", "approve") # both while arch rev-1 was up - worker.run() - assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF - assert not gate(store)["approved"] - assert "arrived while rev-1 was up" in transport.outbound[-1].text - transport.receive("don", "approve") # now about milestone-1 rev-2 - worker.run() - assert gate(store)["approved"] - - -def test_stale_feedback_is_recorded_but_rejects_nothing(tmp_path: Path) -> None: - worker = Held() - store, transport, _ = build(tmp_path, worker) - transport.receive("don", "use demo") - worker.run() - transport.receive("don", "approve") - transport.receive("don", "also, keep it in one file") - worker.run() - assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF - assert gate(store)["rev"] == 2 - assert "not taken as feedback" in transport.outbound[-1].text - said = [ - e.event.text for e in store.all_events() if isinstance(e.event, UserMessage) - ] - assert "also, keep it in one file" in said # the planner still hears it - - -def test_a_gitea_reply_binds_to_the_latest_plan_posted_before_it( - tmp_path: Path, -) -> None: - board = Board() - store, transport, orch = build(tmp_path, board=board) - transport.receive("don", "use demo") # arch rev-1 posted as comment 101 - orch._handle_plan_comment_now( - Comment(id=102, author="don", body="approve", created_at="t") - ) - # Approved arch rev-1; milestone-1 rev-2 is now posted as comment 103. - assert gate(store) == {**gate(store), "name": "milestone-1", "approved": False} - # A reply older than that post was about arch rev-1: not applied. - orch._handle_plan_comment_now( - Comment(id=102, author="don", body="approve", created_at="t") - ) - assert not gate(store)["approved"] - # One after it is about milestone-1 rev-2. - orch._handle_plan_comment_now( - Comment(id=104, author="don", body="approve", created_at="t") - ) - assert gate(store)["approved"] - - -def test_the_plan_up_survives_a_restart(tmp_path: Path) -> None: - _, transport, _ = build(tmp_path) - transport.receive("don", "use demo") # arch rev-1 up; no board, so unsaved - store, transport, _ = build(tmp_path) # restart at the open gate - transport.receive("don", "approve") # stamped with the restored rev-1 - assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF - assert gate(store)["name"] == "milestone-1" - assert gate(store)["rev"] == 2 diff --git a/nisse-dev/tests/test_approvals.py b/nisse-dev/tests/test_approvals.py deleted file mode 100644 index 6734999..0000000 --- a/nisse-dev/tests/test_approvals.py +++ /dev/null @@ -1,210 +0,0 @@ -"""Plan externalisation to Gitea and approvals arriving on either channel -(architecture L3, D2).""" - -import io -from pathlib import Path -from typing import Any - -from nisse_dev.approvals import PlanBoard -from nisse_dev.canonical import CanonicalEvent, ToolCall -from nisse_dev.control import ( - AWAIT_ARCH_SIGNOFF, - AWAIT_MILESTONE_SIGNOFF, - MILESTONE_PLANNING, - Orchestrator, -) -from nisse_dev.gitea import Comment, GiteaError -from nisse_dev.llm import RoleMap -from nisse_dev.oplog import OpLog -from nisse_dev.planner import LLMPlanningAgent -from nisse_dev.state import StateStore -from nisse_dev.transport import InMemoryTransport - -ROLES = RoleMap(executor="x", planner="y", helper="y", expert="y") - - -class FakeGitea: - def __init__(self, *, fail: bool = False) -> None: - self.issues: list[dict[str, str]] = [] - self.comments: list[tuple[int, str]] = [] - self._fail = fail - self._next_id = 100 - - def create_issue(self, repo: str, title: str, body: str) -> int: - if self._fail: - raise GiteaError("gitea is down") - self.issues.append({"title": title, "body": body}) - return len(self.issues) - - def post_comment(self, repo: str, index: int, body: str) -> Comment: - if self._fail: - raise GiteaError("gitea is down") - self.comments.append((index, body)) - self._next_id += 1 - return Comment( - id=self._next_id, author="nisse-dev-testing", body=body, created_at="t" - ) - - def list_comments(self, repo: str, index: int, since: str | None = None): - return [] - - -class ScriptedClient: - def __init__(self) -> None: - self.calls = 0 - - def complete(self, events, tools, model, system=None) -> list[CanonicalEvent]: - self.calls += 1 - return [ - ToolCall( - call_id=f"c{self.calls}", - tool="submit_plan", - arguments={"type": "x", "content": f"plan {self.calls}"}, - ) - ] - - -def build( - tmp_path: Path, gitea: Any -) -> tuple[Orchestrator, StateStore, InMemoryTransport, PlanBoard]: - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - transport = InMemoryTransport() - board = PlanBoard(gitea, "o/sandbox") - orch = Orchestrator( - store, - transport, - LLMPlanningAgent(ScriptedClient(), ROLES), - operators={"don"}, - roles=ROLES, - gitea_operators={"dongar"}, - board=board, - ) - orch.start() - return orch, store, transport, board - - -def comment(body: str, author: str = "dongar", cid: int = 900) -> Comment: - return Comment(id=cid, author=author, body=body, created_at="t") - - -def test_project_start_opens_an_issue_and_posts_the_plan(tmp_path: Path) -> None: - gitea = FakeGitea() - _, store, transport, _ = build(tmp_path, gitea) - transport.receive("don", "use demo") - - assert gitea.issues[0]["title"] == "nisse-dev: demo" - issue, body = gitea.comments[0] - assert issue == 1 - assert "plan 1" in body - assert "approve rev-1" in body # the tag is printed for copy-paste (L3) - # The store keeps a pointer, not the plan record itself (D2). - assert store.get_state().data["board"]["issue"] == 1 - - -def test_approval_by_gitea_comment_satisfies_the_gate(tmp_path: Path) -> None: - gitea = FakeGitea() - orch, store, _, _ = build(tmp_path, gitea) - orch._transport.receive("don", "use demo") - assert store.get_state().state == AWAIT_ARCH_SIGNOFF - - orch.handle_plan_comment(comment("approve rev-1")) - - assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF - - -def test_stale_revision_from_gitea_is_refused(tmp_path: Path) -> None: - gitea = FakeGitea() - orch, store, transport, _ = build(tmp_path, gitea) - transport.receive("don", "use demo") - transport.receive("don", "too vague") # now at rev-2 - - orch.handle_plan_comment(comment("approve rev-1")) - assert store.get_state().state == AWAIT_ARCH_SIGNOFF # not advanced - - orch.handle_plan_comment(comment("approve rev-2", cid=901)) - assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF - - -def test_non_operator_gitea_comment_cannot_approve(tmp_path: Path) -> None: - gitea = FakeGitea() - orch, store, transport, _ = build(tmp_path, gitea) - transport.receive("don", "use demo") - - orch.handle_plan_comment(comment("approve rev-1", author="drive-by")) - assert store.get_state().state == AWAIT_ARCH_SIGNOFF - - -def test_agents_own_comment_cannot_approve_its_plan(tmp_path: Path) -> None: - # The agent posts plans as nisse-dev-testing; that login is not on the - # operator allowlist, so its own comments can never satisfy its gate (L3). - gitea = FakeGitea() - orch, store, transport, _ = build(tmp_path, gitea) - transport.receive("don", "use demo") - - orch.handle_plan_comment(comment("approve rev-1", author="nisse-dev-testing")) - assert store.get_state().state == AWAIT_ARCH_SIGNOFF - - -def test_rejection_by_gitea_comment_replans(tmp_path: Path) -> None: - gitea = FakeGitea() - orch, store, transport, _ = build(tmp_path, gitea) - transport.receive("don", "use demo") - - orch.handle_plan_comment(comment("needs more detail on the schema")) - - assert store.get_state().state == AWAIT_ARCH_SIGNOFF - assert "plan 2" in transport.outbound[-1].text # replanned - - -def test_matrix_approval_is_echoed_to_gitea(tmp_path: Path) -> None: - gitea = FakeGitea() - _, _, transport, _ = build(tmp_path, gitea) - transport.receive("don", "use demo") - transport.receive("don", "approve") - - echoes = [body for _, body in gitea.comments if "approved by don" in body] - assert echoes, "the Gitea record must not silently disagree with Matrix" - - -def test_gitea_approval_is_echoed_to_matrix(tmp_path: Path) -> None: - gitea = FakeGitea() - orch, _, transport, _ = build(tmp_path, gitea) - transport.receive("don", "use demo") - orch.handle_plan_comment(comment("approve rev-1")) - - assert any("approved by dongar via gitea" in m.text for m in transport.outbound) - - -def test_gitea_outage_leaves_matrix_a_complete_channel(tmp_path: Path) -> None: - gitea = FakeGitea(fail=True) - _, store, transport, _ = build(tmp_path, gitea) - transport.receive("don", "use demo") - - # Planning proceeded and the gate is usable despite Gitea being down. - assert store.get_state().state == AWAIT_ARCH_SIGNOFF - assert any("Matrix-only" in m.text for m in transport.outbound) - transport.receive("don", "approve") - assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF - - -def test_comments_are_ignored_outside_a_gate(tmp_path: Path) -> None: - gitea = FakeGitea() - orch, store, transport, _ = build(tmp_path, gitea) - transport.receive("don", "use demo") - transport.receive("don", "approve") # arch approved; now at milestone gate - transport.receive("don", "approve") # milestone approved, awaiting start - before = store.get_state().state - - orch.handle_plan_comment(comment("approve rev-1", cid=902)) - assert store.get_state().state == before - - -def test_board_records_the_milestone_plan_too(tmp_path: Path) -> None: - gitea = FakeGitea() - _, store, transport, _ = build(tmp_path, gitea) - transport.receive("don", "use demo") - transport.receive("don", "approve") - - assert store.get_state().state in (MILESTONE_PLANNING, AWAIT_MILESTONE_SIGNOFF) - posted = [body for _, body in gitea.comments if "milestone-1" in body] - assert posted, "the milestone plan must be externalised as well" diff --git a/nisse-dev/tests/test_bench.py b/nisse-dev/tests/test_bench.py index 5efbe53..cfe82bc 100644 --- a/nisse-dev/tests/test_bench.py +++ b/nisse-dev/tests/test_bench.py @@ -1,4 +1,4 @@ -"""The executor benchmark.""" +"""The driver benchmark.""" import json import shutil @@ -19,7 +19,7 @@ from nisse_dev.bench.runner import MeteringClient, run_task from nisse_dev.bench.tasks import CHECK_DIR, load_tasks from nisse_dev.canonical import AssistantText, ToolCall from nisse_dev.catalog import ModelFacts -from nisse_dev.config import ExecutorTuning, Tuning +from nisse_dev.config import DriverTuning, Tuning from nisse_dev.execution import SSHHost # The build VM for every test here: the session's throwaway sshd. @@ -79,7 +79,7 @@ def test_the_seed_fails_its_check_and_the_solution_passes(task, tmp_path: Path) def test_the_hidden_check_is_not_in_the_seed(task, tmp_path: Path) -> None: task.seed_into(tmp_path) assert not (tmp_path / CHECK_DIR).exists() - assert "Call finish" in task.plan # plans end the way the executor must + assert "Call finish_milestone" in task.plan # as the driver must end def test_long_output_really_is_long(tmp_path: Path) -> None: @@ -192,7 +192,7 @@ SOLVE_SLUGIFY = [ ) ], [call("run_command", "r", command="python -c 'import slugify'")], - [call("finish", "f", summary="added slugify")], + [call("finish_milestone", "f", summary="added slugify", final="yes")], ] @@ -208,7 +208,7 @@ def test_a_solved_task_is_a_success(tmp_path: Path) -> None: assert record.tokens_in == 3000 assert record.task_hash == task.content_hash assert record.model_info["supports_tool_choice"] is True - assert record.tuning["executor"]["max_turns"] == 40 + assert record.tuning["driver"]["max_turns"] == 40 def test_saying_finished_is_not_success(tmp_path: Path) -> None: @@ -223,7 +223,7 @@ def test_saying_finished_is_not_success(tmp_path: Path) -> None: content="def slugify(t):\n return t\n", ) ], - [call("finish", "f", summary="done")], + [call("finish_milestone", "f", summary="done", final="yes")], ] record = run_task(task, FACTS, Scripted(wrong), Tuning(), host=HOST) assert record.outcome == "completed" @@ -239,7 +239,7 @@ def test_misbehaviour_is_counted(tmp_path: Path) -> None: ] # A higher failure limit, so the invented tool and the failed command # don't escalate before the prose turn is reached. - tuning = Tuning(executor=ExecutorTuning(failure_limit=5)) + tuning = Tuning(driver=DriverTuning(failure_limit=5)) record = run_task(task, FACTS, Scripted(turns), tuning, host=HOST) assert record.invented_tools == 1 assert record.failed_commands == 1 @@ -247,6 +247,31 @@ def test_misbehaviour_is_counted(tmp_path: Path) -> None: assert not record.success +def test_asking_the_operator_is_an_escalation(tmp_path: Path) -> None: + [task] = load_tasks(["write-function"]) + asks = [[call("ask_human", "q", question="which file?")]] + record = run_task(task, FACTS, Scripted(asks), Tuning(), host=HOST) + assert (record.outcome, record.success) == ("escalated", False) + + +def test_the_driver_gets_the_loops_own_prompt_and_tools(tmp_path: Path) -> None: + from nisse_dev.loop.agent import SYSTEM + + seen: list[tuple[str, set[str]]] = [] + + class Spy(Scripted): + def complete(self, events, tools, model, system=None): # type: ignore[no-untyped-def] + seen.append((system or "", {t["function"]["name"] for t in tools})) + return super().complete(events, tools, model, system) + + [task] = load_tasks(["write-function"]) + run_task(task, FACTS, Spy(list(SOLVE_SLUGIFY)), Tuning(), host=HOST) + system, tools = seen[0] + assert system.startswith(SYSTEM.split("{")[0]) + assert {"start_milestone", "finish_milestone", "run_command"} <= tools + assert not {"delegate", "ask_expert", "restate"} & tools + + def test_a_fallback_answer_can_never_be_a_success(tmp_path: Path) -> None: [task] = load_tasks(["write-function"]) client = Scripted(list(SOLVE_SLUGIFY), answered="claude-sonnet-5") @@ -383,11 +408,11 @@ def test_results_stored_before_waits_were_recorded_still_load() -> None: def test_runs_with_different_tuning_are_kept_apart() -> None: current = {t.name: t.content_hash for t in TASKS} capped = record_json() - capped["tuning"]["executor"]["max_turns"] = 7 + capped["tuning"]["driver"]["max_turns"] = 7 rows = summarise([record_json(), capped], current) - assert sorted(row.tuning for row in rows) == ["", "executor.max_turns=7"] + assert sorted(row.tuning for row in rows) == ["", "driver.max_turns=7"] table = render(rows) - assert "executor.max_turns=7" in table + assert "driver.max_turns=7" in table assert "tuning = settings that differ" in table @@ -395,3 +420,12 @@ def test_a_report_at_default_tuning_has_no_tuning_column() -> None: current = {t.name: t.content_hash for t in TASKS} table = render(summarise([record_json()], current)) assert "tuning" not in table + + +def test_runs_from_before_the_driver_read_as_its_defaults() -> None: + # Stored before M2.8, the driver's section was called executor. + current = {t.name: t.content_hash for t in TASKS} + old = record_json() + old["tuning"]["executor"] = old["tuning"].pop("driver") + [row] = summarise([old], current) + assert row.tuning == "" diff --git a/nisse-dev/tests/test_config.py b/nisse-dev/tests/test_config.py index 32ec46d..1b2f9b6 100644 --- a/nisse-dev/tests/test_config.py +++ b/nisse-dev/tests/test_config.py @@ -4,7 +4,7 @@ from typing import Any import pytest -from nisse_dev.config import AgentConfig, ConfigError +from nisse_dev.config import AgentConfig, ConfigError, Project from nisse_dev.llm import DEFAULT_REQUEST_TIMEOUT_S # A complete config as the YAML file would give it, with secrets by reference. @@ -19,9 +19,8 @@ COMPLETE: dict[str, Any] = { }, "llm": {"base_url": "https://litellm.test", "token": "${TEST_LLM_TOKEN}"}, "roles": { - "executor": "claude-haiku-4-5", - "planner": "claude-opus-5", - "helper": "claude-opus-5", + "driver": "claude-opus-5", + "helper": "claude-haiku-4-5", "expert": "claude-haiku-4-5", }, "matrix": { @@ -64,22 +63,27 @@ def build(**changes: Any) -> AgentConfig: return AgentConfig.from_tree(tree(**changes), SECRETS) +def only(config: AgentConfig) -> Project: + [project] = config.projects + return project + + def test_complete_config_builds_with_secrets_by_reference() -> None: config = build() - assert config.project == "greet" + assert only(config).name == "greet" assert config.llm_token == "sk-x" assert config.gitea_token == "gt" - assert config.roles.planner == "claude-opus-5" + assert config.roles.driver == "claude-opus-5" assert config.matrix.room_id == "!room:test" assert config.matrix.homeserver == "https://matrix.test" - assert config.gitea_repo == "don/demo" - assert config.base_branch == "main" - assert config.tuning.executor.max_turns == 40 # absent section: defaults + assert only(config).repo == "don/demo" + assert only(config).base_branch == "main" + assert config.tuning.driver.max_turns == 40 # absent section: defaults def test_paths_expand_the_home_directory() -> None: config = build() - assert config.workspace.is_absolute() + assert only(config).workspace.is_absolute() assert "~" not in str(config.state_db) @@ -119,7 +123,7 @@ def test_an_operator_list_of_only_blanks_is_missing() -> None: def test_base_branch_is_configurable() -> None: - assert build(**{f"{P}base_branch": "trunk"}).base_branch == "trunk" + assert only(build(**{f"{P}base_branch": "trunk"})).base_branch == "trunk" def test_max_output_tokens_is_configurable_and_checked() -> None: @@ -137,8 +141,8 @@ def test_request_timeout_is_configurable_and_checked() -> None: def test_the_request_timeout_reaches_both_clients() -> None: - from nisse_dev.agent import make_llm_clients from nisse_dev.catalog import ModelCatalog + from nisse_dev.loop.build import make_llm_clients normalized, passthrough = make_llm_clients( build(llm__request_timeout_s=42), ModelCatalog({}) @@ -158,7 +162,7 @@ def test_redacted_summary_carries_no_secrets() -> None: def test_tuning_is_read() -> None: config = build( tuning={ - "executor": {"max_turns": 40, "failure_limit": 3}, + "driver": {"max_turns": 40, "failure_limit": 3}, "expert": {"consult_budget": 5}, "delegation": {"max_turns": 20, "max_tokens": 100000, "wall_clock_s": 600}, "tools": {"command_timeout_s": 300, "max_output_chars": 2000}, @@ -166,7 +170,7 @@ def test_tuning_is_read() -> None: } ) tuning = config.tuning - assert (tuning.executor.max_turns, tuning.executor.failure_limit) == (40, 3) + assert (tuning.driver.max_turns, tuning.driver.failure_limit) == (40, 3) assert tuning.expert.consult_budget == 5 assert tuning.delegation.max_tokens == 100_000 assert tuning.delegation.wall_clock_s == 600.0 @@ -175,8 +179,8 @@ def test_tuning_is_read() -> None: def test_a_partly_given_tuning_section_keeps_the_other_defaults() -> None: - tuning = build(tuning={"executor": {"max_turns": 40}}).tuning - assert tuning.executor.failure_limit == 2 + tuning = build(tuning={"driver": {"max_turns": 40}}).tuning + assert tuning.driver.failure_limit == 2 def test_every_problem_is_reported_at_once() -> None: @@ -184,7 +188,7 @@ def test_every_problem_is_reported_at_once() -> None: # with no effect, so unknown keys are errors. bad = tree(llm_extra=1, **{f"{P}workspace": None}) bad["tuning"] = { - "executor": {"max_turn": 40, "failure_limit": 0}, + "driver": {"max_turn": 40, "failure_limit": 0}, "expert": {"consult_budget": True}, "bogus": {"x": 1}, } @@ -194,8 +198,8 @@ def test_every_problem_is_reported_at_once() -> None: for expected in ( "projects.greet: unset or empty — workspace", "llm_extra: unknown setting", - "tuning.executor.max_turn: unknown setting", - "tuning.executor.failure_limit: must be a positive number", + "tuning.driver.max_turn: unknown setting", + "tuning.driver.failure_limit: must be a positive number", "tuning.expert.consult_budget: must be a positive number", "tuning.bogus: unknown section", ): @@ -203,8 +207,8 @@ def test_every_problem_is_reported_at_once() -> None: def test_a_whole_number_is_required_where_one_is_meant() -> None: - with pytest.raises(ConfigError, match="tuning.executor.max_turns"): - build(tuning={"executor": {"max_turns": 2.5}}) + with pytest.raises(ConfigError, match="tuning.driver.max_turns"): + build(tuning={"driver": {"max_turns": 2.5}}) # --- the file -------------------------------------------------------------- @@ -220,8 +224,7 @@ projects: key: ~/.ssh/nisse-dev llm: {base_url: "https://litellm.test", token: "${TEST_LLM_TOKEN}"} roles: - executor: local-fast - planner: claude-opus-5 + driver: local-fast helper: claude-opus-5 expert: claude-haiku-4-5 matrix: @@ -245,7 +248,7 @@ def write(directory: Path, text: str = FILE) -> Path: def test_a_file_is_read(tmp_path: Path) -> None: config = AgentConfig.from_file(write(tmp_path), SECRETS) - assert config.roles.executor == "local-fast" + assert config.roles.driver == "local-fast" assert config.config_file == tmp_path / "nisse-dev.yaml" @@ -256,7 +259,9 @@ def test_invalid_yaml_is_reported(tmp_path: Path) -> None: def test_load_uses_nisse_config(tmp_path: Path) -> None: path = write(tmp_path) - assert AgentConfig.load({**SECRETS, "NISSE_CONFIG": str(path)}).project == "greet" + assert ( + only(AgentConfig.load({**SECRETS, "NISSE_CONFIG": str(path)})).name == "greet" + ) def test_load_defaults_to_the_working_directory( @@ -264,7 +269,7 @@ def test_load_defaults_to_the_working_directory( ) -> None: write(tmp_path) monkeypatch.chdir(tmp_path) - assert AgentConfig.load(SECRETS).project == "greet" + assert only(AgentConfig.load(SECRETS)).name == "greet" def test_load_without_a_file_says_how_to_make_one( @@ -284,8 +289,8 @@ def test_the_example_file_is_valid() -> None: "NISSE_GITEA_TOKEN": "gt", } config = AgentConfig.from_file(example, env) - assert config.tuning.executor.max_turns == 40 - assert config.roles.executor == "qwen3.8-27b" + assert config.tuning.driver.max_turns == 40 + assert config.roles.driver == "claude-opus-5-5" # --- nothing that carries secrets may sit inside the clone ----------------- @@ -294,7 +299,7 @@ def test_the_example_file_is_valid() -> None: def test_state_db_and_workspace_are_separate() -> None: # L8: build artefacts must not share a volume with agent state. config = build() - assert config.state_db.parent != config.workspace + assert config.state_db.parent != only(config).workspace def test_state_db_inside_the_clone_is_refused(tmp_path: Path) -> None: @@ -333,7 +338,7 @@ def test_sibling_paths_are_fine(tmp_path: Path) -> None: ), {**SECRETS, "NISSE_ENV": str(tmp_path / ".env")}, ) - assert config.workspace.name == "project" + assert only(config).workspace.name == "project" def test_context_tuning_is_read_and_checked() -> None: @@ -346,10 +351,10 @@ def test_context_tuning_is_read_and_checked() -> None: def test_the_build_host_is_required_and_defaults_to_root_on_22() -> None: - config = build() - assert (config.build_host.address, config.build_host.user) == ("build.test", "root") - assert config.build_host.port == 22 - assert config.build_host.key.is_absolute() + host = only(build()).build_host + assert (host.address, host.user) == ("build.test", "root") + assert host.port == 22 + assert host.key.is_absolute() with pytest.raises(ConfigError, match="build_host.address"): build(**{f"{P}build_host__address": None}) @@ -360,14 +365,14 @@ def test_a_bad_build_host_port_is_reported() -> None: def test_the_push_key_is_optional_and_a_path() -> None: - assert build().push_key is None - assert build(**{f"{P}push_key": "~/k"}).push_key == Path("~/k").expanduser() + assert only(build()).push_key is None + assert only(build(**{f"{P}push_key": "~/k"})).push_key == Path("~/k").expanduser() def test_a_push_token_needs_its_user_and_excludes_a_key() -> None: token = {f"{P}push_user": "pusher", f"{P}push_token": "${TEST_GITEA_TOKEN}"} - config = build(**token) - assert (config.push_user, config.push_token) == ("pusher", "gt") + project = only(build(**token)) + assert (project.push_user, project.push_token) == ("pusher", "gt") with pytest.raises(ConfigError, match="set both or neither"): build(**{f"{P}push_user": "pusher"}) with pytest.raises(ConfigError, match="not both"): @@ -386,11 +391,21 @@ def test_no_credential_appears_in_the_config_repr() -> None: assert secret not in text -def test_the_loop_defaults_to_old_and_takes_new() -> None: - assert build().loop == "old" - assert build(loop="new").loop == "new" - with pytest.raises(ConfigError, match="loop: must be old or new"): - build(loop="newer") +def test_the_old_loops_settings_are_refused_with_what_to_do_instead() -> None: + old = tree(loop="new", roles__planner="claude-opus-5", roles__driver=None) + old["roles"]["executor"] = "local-fast" + old["tuning"] = {"executor": {"max_turns": 40}} + with pytest.raises(ConfigError) as excinfo: + AgentConfig.from_tree(old, SECRETS) + message = str(excinfo.value) + for expected in ( + "loop: retired — the loop is the only one now; remove it", + "roles.planner: retired — use roles.driver", + "roles.executor: retired", + "tuning.executor: retired — it is tuning.driver now", + "unset or empty — roles.driver", + ): + assert expected in message # --- projects (D8) ----------------------------------------------------------- @@ -410,14 +425,6 @@ def test_a_second_project_is_a_second_entry() -> None: assert other.base_branch == "main" and other.labels == () and other.secrets == {} -def test_the_old_loop_refuses_more_than_one_project() -> None: - config = build(projects__other=SECOND) - with pytest.raises(ConfigError, match="exactly one project"): - config.only_project() - with pytest.raises(ConfigError, match="exactly one project"): - _ = config.gitea_repo - - def test_at_least_one_project_is_needed() -> None: with pytest.raises(ConfigError, match="projects: must name at least one"): build(projects={}) @@ -439,8 +446,8 @@ def test_a_project_name_is_a_slug() -> None: def test_labels_and_egress_are_lists_of_names() -> None: config = build(**{f"{P}labels": ["discuss"], f"{P}egress": ["pypi.org"]}) - assert config.only_project().labels == ("discuss",) - assert config.only_project().egress == ("pypi.org",) + assert only(config).labels == ("discuss",) + assert only(config).egress == ("pypi.org",) with pytest.raises(ConfigError, match="projects.greet.labels: must be a list"): build(**{f"{P}labels": "discuss"}) @@ -454,7 +461,7 @@ def test_secrets_are_by_reference_with_a_visibility() -> None: } } ) - project = config.only_project() + project = only(config) assert project.command_secrets() == {"DB_URL": "sk-x"} assert project.secrets["DEPLOY"].visible == "harness" diff --git a/nisse-dev/tests/test_context.py b/nisse-dev/tests/test_context.py index 7fb6437..51a3854 100644 --- a/nisse-dev/tests/test_context.py +++ b/nisse-dev/tests/test_context.py @@ -9,8 +9,6 @@ import pytest from nisse_dev.canonical import ( AssistantText, CanonicalEvent, - GateOpened, - PlanSubmitted, StoredEvent, ToolCall, ToolResult, @@ -25,8 +23,7 @@ from nisse_dev.context import ( fit, head_and_tail, ) -from nisse_dev.executor import LLMExecutor -from nisse_dev.llm import RoleMap +from nisse_dev.loop.driver import Driver from nisse_dev.oplog import OpLog from nisse_dev.state import StateStore @@ -230,70 +227,27 @@ def test_the_catalog_reads_each_models_limits() -> None: def test_a_role_on_a_model_without_a_window_is_refused() -> None: - CATALOG.require_limits({"executor": "llama3.2", "planner": "claude-haiku-4-5"}) - with pytest.raises(UnknownLimits, match="executor='undeclared'"): - CATALOG.require_limits({"executor": "undeclared", "planner": "llama3.2"}) + CATALOG.require_limits({"driver": "llama3.2", "helper": "claude-haiku-4-5"}) + with pytest.raises(UnknownLimits, match="driver='undeclared'"): + CATALOG.require_limits({"driver": "undeclared", "helper": "llama3.2"}) -# --- the executor sees the approved plan and what came after --------------- - - -def test_the_executor_sees_only_its_milestone(tmp_path: Path) -> None: +def test_a_request_that_cannot_fit_ends_the_run(tmp_path: Path) -> None: + # Never sent: the driver's run fails and the item waits on the operator, + # like any failed call, rather than a server getting a prompt it would + # silently cut. store = StateStore(tmp_path / "state.db", oplog=OpLog("t", io.StringIO())) - for event in ( - UserMessage(text="use greet: a long planning conversation", source="t"), - PlanSubmitted(plan="arch plan", model="m"), - UserMessage(text="approve", source="t"), - PlanSubmitted(plan="milestone 1: write greet.py", model="m"), - GateOpened(gate="milestone-1", reason="plan submitted"), - UserMessage(text="start milestone 1", source="t"), - ): - store.transcript("t", "executor").append(event) - - class Workspace: - def describe(self) -> str: - return "workspace" - - seen: dict[str, Any] = {} - - class Client: - def complete(self, events, tools, model, system=None): - seen["events"] = [e.event for e in events] - seen["system"] = system - return [AssistantText(text="done", model=model)] - - roles = RoleMap(executor="m", planner="m", helper="m", expert="m") - LLMExecutor(Client(), roles, Workspace()).run(store.transcript("t", "executor"), 1) - - texts = [getattr(e, "text", None) for e in seen["events"]] - assert "start milestone 1" in texts - assert "use greet: a long planning conversation" not in texts - assert not any(isinstance(e, PlanSubmitted) for e in seen["events"]) - # The plan rides in the system prompt, which the budget never drops. - assert "The approved plan:\nmilestone 1: write greet.py" in seen["system"] - assert "arch plan" not in seen["system"] - - -def test_a_request_that_cannot_fit_stops_the_milestone(tmp_path: Path) -> None: - # Never sent: the executor stops and asks the operator, like any failed - # call, rather than handing a server a prompt it would silently cut. - store = StateStore(tmp_path / "state.db", oplog=OpLog("t", io.StringIO())) - store.transcript("t", "operator").append( - UserMessage(text="start milestone 1", source="t") - ) - - class Workspace: - def describe(self) -> str: - return "workspace " + "w" * 20_000 # a system prompt too big to fit - + store.transcript("t", "operator").append(UserMessage(text="go", source="t")) inner = RecordingClient() client = BudgetedClient(inner, lambda m: (4096, 1024)) - roles = RoleMap(executor="m", planner="m", helper="m", expert="m") - outcome = LLMExecutor(client, roles, Workspace()).run( - store.transcript("t", "executor"), 1 + outcome = Driver(client, "m").run( + store.transcript("t", "driver"), + system="s" * 20_000, # a system prompt too big to fit + tools=[], + workspace=None, ) - assert outcome.status == "escalate" + assert outcome.kind == "failed" assert "fit" in outcome.note assert inner.calls == [] diff --git a/nisse-dev/tests/test_control.py b/nisse-dev/tests/test_control.py deleted file mode 100644 index 9867401..0000000 --- a/nisse-dev/tests/test_control.py +++ /dev/null @@ -1,289 +0,0 @@ -import io -import json -from pathlib import Path - -import pytest - -from nisse_dev.canonical import UserMessage -from nisse_dev.control import ( - AWAIT_ARCH_SIGNOFF, - AWAIT_INPUT, - AWAIT_MILESTONE_SIGNOFF, - EXECUTING, - IDLE, - PLANNING_ARCH, - PR_OPEN, - AskHuman, - Orchestrator, - PlanningError, - StubAgent, - SubmitPlan, -) -from nisse_dev.oplog import OpLog -from nisse_dev.state import StateStore -from nisse_dev.transport import InMemoryTransport - - -class Harness: - def __init__(self, tmp_path: Path, agent: object | None = None) -> None: - self.stream = io.StringIO() - self.store = StateStore( - tmp_path / "state.db", oplog=OpLog("nisse-dev", self.stream) - ) - self.transport = InMemoryTransport() - self.orch = Orchestrator( - self.store, self.transport, agent or StubAgent(), operators={"don"} - ) - self.orch.start() - - @property - def state(self) -> str: - state = self.store.get_state() - assert state is not None - return state.state - - def events(self, kind: str) -> list[dict]: - return [ - e - for e in map(json.loads, self.stream.getvalue().splitlines()) - if e["event"] == kind - ] - - def outbound_texts(self) -> list[str]: - return [m.text for m in self.transport.outbound] - - -def executed(h: Harness) -> bool: - return any(e["to"] == EXECUTING for e in h.events("state.transition")) - - -@pytest.fixture -def h(tmp_path: Path) -> Harness: - return Harness(tmp_path) - - -def test_boots_idle_and_greets(h: Harness) -> None: - assert h.state == IDLE - assert "ready" in h.outbound_texts()[0] - - -def test_use_project_submits_arch_plan_and_opens_gate(h: Harness) -> None: - h.transport.receive("don", "use demo") - assert h.state == AWAIT_ARCH_SIGNOFF - request = h.transport.outbound[-1] - assert request.kind == "approval_request" - assert "rev-1" in request.text - [opened] = h.events("gate.open") - assert opened["gate"] == "arch" - assert opened["rev"] == "rev-1" - assert opened["task_id"] is not None - - -def test_full_happy_path_through_execution(h: Harness) -> None: - h.transport.receive("don", "use demo") - h.transport.receive("don", "approve") - assert h.state == AWAIT_MILESTONE_SIGNOFF - h.transport.receive("don", "approve") - assert h.state == AWAIT_MILESTONE_SIGNOFF # approved, awaiting start - h.transport.receive("don", "start milestone 1") - assert executed(h) - # The stub completes at once and waits in PR_OPEN, never in EXECUTING. - assert h.state == PR_OPEN - approvals = h.events("gate.approve") - assert [e["gate"] for e in approvals] == ["arch", "milestone-1"] - - -def test_reject_replans_with_feedback_and_bumps_revision(h: Harness) -> None: - h.transport.receive("don", "use demo") - h.transport.receive("don", "too vague, add tests") - assert h.state == AWAIT_ARCH_SIGNOFF - request = h.transport.outbound[-1] - assert "rev-2" in request.text - assert "too vague, add tests" in request.text - [rejected] = h.events("gate.reject") - assert rejected["rev"] == "rev-1" - - -def test_stale_approval_is_refused_then_current_works(h: Harness) -> None: - h.transport.receive("don", "use demo") - h.transport.receive("don", "needs work") # now at rev-2 - h.transport.receive("don", "approve rev-1") - assert h.state == AWAIT_ARCH_SIGNOFF - assert "not applied" in h.outbound_texts()[-1] - assert h.events("gate.approve") == [] - h.transport.receive("don", "approve rev-2") - assert h.state == AWAIT_MILESTONE_SIGNOFF - - -def test_non_operator_messages_are_ignored(h: Harness) -> None: - h.transport.receive("don", "use demo") - before = len(h.transport.outbound) - h.transport.receive("mallory", "approve") - assert h.state == AWAIT_ARCH_SIGNOFF - assert len(h.transport.outbound) == before - - -def test_start_milestone_requires_approval_first(h: Harness) -> None: - h.transport.receive("don", "use demo") - h.transport.receive("don", "approve") - h.transport.receive("don", "start milestone 1") - assert h.state == AWAIT_MILESTONE_SIGNOFF # it was rejection feedback - # Revisions run across the task: arch rev-1, milestone-1 rev-2, and - # after that rejection milestone-1 rev-3. - h.transport.receive("don", "approve rev-3") - h.transport.receive("don", "start milestone 1") - assert executed(h) - - -def test_a_message_is_matched_as_stripped_but_recorded_as_sent( - h: Harness, -) -> None: - h.transport.receive("don", "use demo") - h.transport.receive("don", "approve", original="Nisse Dev: approve") - assert h.state == AWAIT_MILESTONE_SIGNOFF - recorded = [ - e.event.text for e in h.store.all_events() if isinstance(e.event, UserMessage) - ] - assert "Nisse Dev: approve" in recorded - assert "approve" not in recorded - - -def test_status_reports_state_and_gate(h: Harness) -> None: - h.transport.receive("don", "use demo") - h.transport.receive("don", "status") - status = h.outbound_texts()[-1] - assert "AWAIT_ARCH_SIGNOFF" in status - assert "demo" in status - assert "rev-1" in status - - -class QuestionThenPlan: - def plan( - self, - transcript: object, - gate: str, - project: str, - feedback: str | None = None, - answer: str | None = None, - ) -> SubmitPlan | AskHuman: - if answer is None: - return AskHuman("monorepo or polyrepo?") - return SubmitPlan(f"[stub] {gate} plan using {answer}") - - -def test_ask_human_pauses_and_answer_resumes(tmp_path: Path) -> None: - h = Harness(tmp_path, agent=QuestionThenPlan()) - h.transport.receive("don", "use demo") - assert h.state == AWAIT_INPUT - assert h.transport.outbound[-1].kind == "question" - h.transport.receive("don", "monorepo") - assert h.state == AWAIT_ARCH_SIGNOFF - assert "monorepo" in h.transport.outbound[-1].text - - -# --- restarts (L8 boot steps 4 and 5) -------------------------------------------- - - -class FailsOnce: - """A planner whose first turn fails, as Opus 5.5's did on its first live - run when asked for forced tool use.""" - - def __init__(self) -> None: - self.calls = 0 - - def plan( - self, - transcript: object, - gate: str, - project: str, - feedback: str | None = None, - answer: str | None = None, - ) -> SubmitPlan | AskHuman: - self.calls += 1 - if self.calls == 1: - raise PlanningError("planner returned 400") - return SubmitPlan(f"[stub] {gate} plan for {project}") - - -def test_restart_while_awaiting_input_asks_the_same_question( - tmp_path: Path, -) -> None: - agent = FailsOnce() - h = Harness(tmp_path, agent=agent) - h.transport.receive("don", "use demo") - assert h.state == AWAIT_INPUT - h = Harness(tmp_path, agent=agent) # restart on the same state - question = h.transport.outbound[-1] - assert question.kind == "question" - assert "restarted while waiting on you" in question.text - assert "planning for arch failed" in question.text - h.transport.receive("don", "Please retry.") - assert h.state == AWAIT_ARCH_SIGNOFF - - -def test_restart_mid_planning_offers_continue_and_it_replans( - tmp_path: Path, -) -> None: - h = Harness(tmp_path) - h.store.transition(PLANNING_ARCH, cause="test", project="demo", data={}) - h = Harness(tmp_path) - assert h.state == AWAIT_INPUT - assert "'continue'" in h.outbound_texts()[-1] - h.transport.receive("don", "Please retry.") # neither choice: asked again - assert h.state == AWAIT_INPUT - assert h.transport.outbound[-1].kind == "question" - h.transport.receive("don", "continue") - assert h.state == AWAIT_ARCH_SIGNOFF - - -def test_restart_mid_execution_continue_returns_to_the_approved_plan( - h: Harness, tmp_path: Path -) -> None: - h.transport.receive("don", "use demo") - h.transport.receive("don", "approve") - h.transport.receive("don", "approve") - # A crash mid-milestone: the store still says EXECUTING. - state = h.store.get_state() - h.store.transition(EXECUTING, cause="test", project="demo", data=state.data) - h = Harness(tmp_path) - assert h.state == AWAIT_INPUT - h.transport.receive("don", "continue") - assert h.state == AWAIT_MILESTONE_SIGNOFF - assert "start milestone 1" in h.outbound_texts()[-1] - h.transport.receive("don", "start milestone 1") - assert executed(h) - - -def test_restart_question_without_its_interrupted_state_still_replans( - tmp_path: Path, -) -> None: - # What the first live run left behind: a restart question recorded - # before the question kept its prompt or the state it interrupted. - h = Harness(tmp_path) - h.store.transition( - AWAIT_INPUT, cause="test", project="demo", data={"pending": {"kind": "boot"}} - ) - h = Harness(tmp_path) - assert "my last question still stands" in h.outbound_texts()[-1] - h.transport.receive("don", "re-plan") - assert h.state == AWAIT_ARCH_SIGNOFF - - -def test_use_starts_over_when_nothing_is_pending(h: Harness) -> None: - h.store.transition(AWAIT_INPUT, cause="forced", project="demo", data={}) - h.transport.receive("don", "use demo") - assert h.state == AWAIT_ARCH_SIGNOFF - - -def test_commands_ignore_case_but_the_project_name_keeps_it(h: Harness) -> None: - h.transport.receive("don", "Use Demo.") - assert h.state == AWAIT_ARCH_SIGNOFF - assert h.store.get_state().project == "Demo" - h.transport.receive("don", "Status") - assert "AWAIT_ARCH_SIGNOFF" in h.outbound_texts()[-1] - h.transport.receive("don", "Approve") - h.transport.receive("don", "Approve.") - h.transport.receive("don", "Start Milestone 1") - assert h.state == PR_OPEN - h.transport.receive("don", "Project Complete!") - assert h.state == IDLE diff --git a/nisse-dev/tests/test_dump.py b/nisse-dev/tests/test_dump.py index 3b5a836..4c31ac0 100644 --- a/nisse-dev/tests/test_dump.py +++ b/nisse-dev/tests/test_dump.py @@ -12,20 +12,23 @@ def store_with_a_run(path: Path) -> None: store.transcript(None, "operator").append( UserMessage(text="hello before use", source="matrix") ) - store.transition( - "AWAIT_ARCH_SIGNOFF", + store.open_work_item( + "demo-1", + title="Add a truncate helper", + scope="demo", + position="WAITING", + data={"loop": {"wait": "go-ahead", "rev": 1}}, cause="test", - project="demo", - data={"task_id": "demo-1", "gate": {"name": "arch", "rev": 1}}, ) + store.set_active("demo-1", cause="test") log = store.transcript("demo-1", "operator") log.append(UserMessage(text="Add a truncate helper", source="matrix")) - log.as_actor("planner").append(PlanSubmitted(plan="x" * 500, model="opus")) - executor = log.as_actor("executor") - executor.append( + driver = log.as_actor("driver") + driver.append(PlanSubmitted(plan="x" * 500, model="opus")) + driver.append( ToolCall(call_id="c1", tool="run_command", arguments={"command": "ls"}) ) - executor.append(ToolResult(call_id="c1", result="README.md", success=False)) + driver.append(ToolResult(call_id="c1", result="README.md", success=False)) log.begin_intent("open_pr", {"branch": "b"}, cause="test") store.close() @@ -38,14 +41,14 @@ def test_the_dump_shows_items_intents_and_each_event_with_its_actor( text = render(db) assert "active work item: demo-1" in text assert ( - "demo-1 open position: AWAIT_ARCH_SIGNOFF scope: demo gate: arch rev-1" - in (text) + "demo-1 open position: WAITING scope: demo rev-1 waits for: go-ahead" + in text ) assert "pending intent #1 open_pr [demo-1]" in text assert "[agent/operator] USER [matrix]: hello before use" in text assert "[demo-1/operator] USER [matrix]: Add a truncate helper" in text - assert "[demo-1/planner] PLAN (opus, 500 chars): " in text and "[340 more]" in text - assert '[demo-1/executor] CALL run_command {"command": "ls"}' in text + assert "[demo-1/driver] PLAN (opus, 500 chars): " in text and "[340 more]" in text + assert '[demo-1/driver] CALL run_command {"command": "ls"}' in text assert "result (FAILED): README.md" in text diff --git a/nisse-dev/tests/test_executor.py b/nisse-dev/tests/test_executor.py deleted file mode 100644 index fd96529..0000000 --- a/nisse-dev/tests/test_executor.py +++ /dev/null @@ -1,397 +0,0 @@ -import io -from pathlib import Path -from typing import Any - -import pytest - -from nisse_dev.canonical import ( - AssistantText, - CanonicalEvent, - StoredEvent, - ToolCall, - ToolResult, -) -from nisse_dev.execution import SSHHost -from nisse_dev.executor import LLMExecutor -from nisse_dev.llm import LLMError, RoleMap -from nisse_dev.oplog import OpLog -from nisse_dev.state import StateStore, Transcript -from nisse_dev.tools import Workspace - -ROLES = RoleMap( - executor="llama3.2", - planner="claude-haiku-4-5", - helper="claude-haiku-4-5", - expert="claude-haiku-4-5", -) - - -class ScriptedClient: - def __init__(self, turns: list[list[CanonicalEvent]]) -> None: - self._turns = turns - self.calls: list[dict[str, Any]] = [] - - def complete( - self, - events: list[StoredEvent], - tools: list[dict[str, Any]] | None, - model: str, - system: str | None = None, - ) -> list[CanonicalEvent]: - self.calls.append({"tools": tools, "model": model, "system": system}) - if not self._turns: - return [finish(call_id=f"fin-{len(self.calls)}")] - return self._turns.pop(0) - - -class FailingClient: - def complete(self, *args: Any, **kwargs: Any) -> list[CanonicalEvent]: - raise LLMError("proxy exploded") - - -def call(tool: str, call_id: str = "c1", **arguments: Any) -> ToolCall: - return ToolCall(call_id=call_id, tool=tool, arguments=arguments) - - -def finish(summary: str = "done", call_id: str = "fin") -> ToolCall: - return ToolCall(call_id=call_id, tool="finish", arguments={"summary": summary}) - - -@pytest.fixture -def store(tmp_path: Path) -> StateStore: - return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - - -@pytest.fixture -def ws(ssh_host: SSHHost, tmp_path: Path) -> Workspace: - return Workspace(ssh_host, tmp_path / "work", timeout_s=10) - - -ITEM = "t" # the work item these tests run in - - -def log(store: StateStore) -> Transcript: - return store.transcript(ITEM, "executor") - - -class Bound: - """An executor run in the test's work item, as the orchestrator runs it.""" - - def __init__(self, executor: LLMExecutor, store: StateStore) -> None: - self.executor = executor - self.store = store - - def run(self, milestone: int) -> Any: - return self.executor.run(log(self.store), milestone) - - -def build(client: Any, store: StateStore, ws: Workspace, **kw: Any) -> Bound: - return Bound(LLMExecutor(client, ROLES, ws, **kw), store) - - -def test_tool_calls_are_executed_and_results_logged( - store: StateStore, ws: Workspace -) -> None: - client = ScriptedClient( - [ - [call("write_file", path="app.py", content="print(1)\n")], - [finish("wrote the file")], - ] - ) - outcome = build(client, store, ws).run(milestone=1) - - assert outcome.status == "completed" - assert outcome.note == "wrote the file" - assert (ws.root / "app.py").read_text() == "print(1)\n" - kinds = [type(e.event).__name__ for e in log(store).read()] - assert kinds == ["ToolCall", "ToolResult", "ToolCall", "ToolResult"] - - -def test_uses_executor_role_and_reduced_manifest( - store: StateStore, ws: Workspace -) -> None: - client = ScriptedClient([]) - build(client, store, ws).run(milestone=2) - assert client.calls[0]["model"] == "llama3.2" # never the planner - names = [t["function"]["name"] for t in client.calls[0]["tools"]] - assert names == ["read_file", "write_file", "list_dir", "run_command", "finish"] - assert "milestone 2" in client.calls[0]["system"] - - -def test_prose_no_longer_ends_the_milestone(store: StateStore, ws: Workspace) -> None: - # Only finish ends a milestone. Prose breaks the tool contract: the model - # is told once, and here it gets back to work. - client = ScriptedClient( - [ - [call("write_file", path="a.py", content="x")], - [AssistantText(text="all steps done", model="m")], - [finish()], - ] - ) - assert build(client, store, ws).run(milestone=1).status == "completed" - assert "Every reply must be a tool call" in client.calls[2]["system"] - - -def test_persisting_in_prose_escalates(store: StateStore, ws: Workspace) -> None: - prose = [AssistantText(text="let me just chat", model="m")] - client = ScriptedClient([prose, prose]) - outcome = build(client, store, ws).run(milestone=1) - assert outcome.status == "escalate" - assert "prose" in outcome.note - - -def test_finishing_without_any_work_is_refused_once( - store: StateStore, ws: Workspace -) -> None: - # A model that finishes on turn one has done nothing; believing it would - # record an empty milestone as a success. - client = ScriptedClient( - [ - [finish("looks done to me", call_id="f1")], - [call("write_file", path="a.py", content="x")], - [finish("now it is done", call_id="f2")], - ] - ) - outcome = build(client, store, ws).run(milestone=1) - assert outcome.status == "completed" - assert (ws.root / "a.py").exists() # the refusal produced actual work - [refusal] = [ - e.event - for e in log(store).read() - if isinstance(e.event, ToolResult) and e.event.call_id == "f1" - ] - assert not refusal.success - assert "not made any change yet" in refusal.result - - -def test_persisting_in_an_empty_finish_escalates( - store: StateStore, ws: Workspace -) -> None: - client = ScriptedClient([[finish(call_id="f1")], [finish(call_id="f2")]]) - outcome = build(client, store, ws).run(milestone=1) - assert outcome.status == "escalate" - assert "without making any change" in outcome.note - - -def test_calls_after_finish_are_not_run(store: StateStore, ws: Workspace) -> None: - client = ScriptedClient( - [ - [call("write_file", path="a.py", content="x")], - [finish(), call("write_file", call_id="late", path="b.py", content="y")], - ] - ) - assert build(client, store, ws).run(milestone=1).status == "completed" - assert not (ws.root / "b.py").exists() - [late] = [ - e.event - for e in log(store).read() - if isinstance(e.event, ToolResult) and e.event.call_id == "late" - ] - assert "finish had already ended the milestone" in late.result - - -def test_an_invented_tool_counts_toward_escalation( - store: StateStore, ws: Workspace -) -> None: - # Measured: a model on a server that doesn't enforce tool_choice will - # call tools that were never offered ("chat", "respond"). Without this it - # would burn every turn up to the cap. - client = ScriptedClient( - [[call("chat", call_id="x1")], [call("respond_user", call_id="x2")]] - ) - outcome = build(client, store, ws, failure_limit=2).run(milestone=1) - assert outcome.status == "escalate" - assert len(client.calls) == 2 - results = [e.event for e in log(store).read() if isinstance(e.event, ToolResult)] - assert "no such tool: chat" in results[0].result - assert "finish" in results[0].result # it is told what does exist - - -def test_two_consecutive_command_failures_escalate( - store: StateStore, ws: Workspace -) -> None: - client = ScriptedClient( - [ - [call("run_command", command="exit 1")], - [call("run_command", command="exit 1")], - [call("run_command", command="exit 1")], - ] - ) - outcome = build(client, store, ws, failure_limit=2).run(milestone=1) - assert outcome.status == "escalate" - assert "2 consecutive command failures" in outcome.note - assert len(client.calls) == 2 # stopped, did not keep trying - - -def test_a_success_resets_the_failure_count(store: StateStore, ws: Workspace) -> None: - client = ScriptedClient( - [ - [call("run_command", command="exit 1")], - [call("run_command", command="true")], - [call("run_command", command="exit 1")], - [finish()], - ] - ) - assert build(client, store, ws, failure_limit=2).run(milestone=1).status == ( - "completed" - ) - - -def test_failed_reads_do_not_count_toward_escalation( - store: StateStore, ws: Workspace -) -> None: - # Two failed reads in a row must not trip the failure limit: looking for - # a file that is not there is the model orienting itself, not failing. - client = ScriptedClient( - [ - [call("read_file", path="missing-a.txt")], - [call("read_file", path="missing-b.txt")], - [call("write_file", path="a.py", content="x")], - [finish()], - ] - ) - assert build(client, store, ws, failure_limit=2).run(milestone=1).status == ( - "completed" - ) - - -def test_reading_alone_is_not_doing_the_work(store: StateStore, ws: Workspace) -> None: - # Exploration is not evidence a milestone was carried out, so a - # completion claim backed only by reads is still challenged. - client = ScriptedClient( - [ - [call("list_dir", path=".")], - [finish("looks done to me", call_id="f1")], - [finish("really, done", call_id="f2")], - ] - ) - outcome = build(client, store, ws).run(milestone=1) - assert outcome.status == "escalate" - - -def test_turn_cap_is_enforced(store: StateStore, ws: Workspace) -> None: - forever = [[call("list_dir", path=".")] for _ in range(50)] - client = ScriptedClient(forever) - outcome = build(client, store, ws, max_turns=3).run(milestone=1) - assert outcome.status == "capped" - assert "3-turn cap" in outcome.note - assert len(client.calls) == 3 - - -def test_llm_error_escalates_rather_than_crashing( - store: StateStore, ws: Workspace -) -> None: - outcome = build(FailingClient(), store, ws).run(milestone=1) - assert outcome.status == "escalate" - assert "proxy exploded" in outcome.note - - -def test_multiple_calls_in_one_turn_all_run(store: StateStore, ws: Workspace) -> None: - client = ScriptedClient( - [ - [ - call("write_file", call_id="a", path="one.txt", content="1"), - call("write_file", call_id="b", path="two.txt", content="2"), - ], - [finish()], - ] - ) - build(client, store, ws).run(milestone=1) - assert (ws.root / "one.txt").exists() and (ws.root / "two.txt").exists() - results = [ - e.event for e in log(store).read() if type(e.event).__name__ == "ToolResult" - ] - assert {r.call_id for r in results} == {"a", "b", "fin"} - - -class StubDesk: - """Stands in for ExpertDesk without a model behind it.""" - - def __init__(self, answer: str = "use sqlite", error: Exception | None = None): - self.asked: list[str] = [] - self._answer = answer - self._error = error - - def ask(self, transcript: Any, question: str, **kw: Any) -> Any: - self.asked.append(question) - if self._error is not None: - raise self._error - - class _A: - def render(self_inner) -> str: - return self._answer - - return _A() - - -def test_ask_expert_is_offered_only_when_a_desk_exists( - store: StateStore, ws: Workspace -) -> None: - client = ScriptedClient([]) - build(client, store, ws).run(milestone=1) - assert all(t["function"]["name"] != "ask_expert" for t in client.calls[0]["tools"]) - - client2 = ScriptedClient([]) - build(client2, store, ws, expert=StubDesk()).run(milestone=1) - names = [t["function"]["name"] for t in client2.calls[0]["tools"]] - assert "ask_expert" in names - assert names[-1] == "finish" - # L9: in addition to the observable triggers, not instead of them. - assert "run_command" in names - - -def test_ask_expert_answer_comes_back_as_a_tool_result( - store: StateStore, ws: Workspace -) -> None: - desk = StubDesk(answer="prefer sqlite here") - client = ScriptedClient( - [ - [call("ask_expert", question="sqlite or postgres?")], - [call("write_file", path="a.py", content="x")], - [finish()], - ] - ) - build(client, store, ws, expert=desk).run(milestone=1) - - assert desk.asked == ["sqlite or postgres?"] - results = [ - e.event for e in log(store).read() if type(e.event).__name__ == "ToolResult" - ] - assert "prefer sqlite here" in results[0].result - - -def test_consulting_is_not_doing_the_work(store: StateStore, ws: Workspace) -> None: - # Asking is not changing anything, so a completion claim backed only by - # consults is still challenged. - client = ScriptedClient( - [ - [call("ask_expert", question="what should I do?")], - [finish("I think it is done", call_id="f1")], - [finish("really done", call_id="f2")], - ] - ) - outcome = build(client, store, ws, expert=StubDesk()).run(milestone=1) - assert outcome.status == "escalate" - - -def test_exhausted_budget_is_reported_to_the_model_not_raised( - store: StateStore, ws: Workspace -) -> None: - from nisse_dev.expert import BudgetExhausted - - desk = StubDesk(error=BudgetExhausted("consult budget of 3 is spent")) - client = ScriptedClient( - [ - [call("ask_expert", question="again?")], - [call("write_file", path="a.py", content="x")], - [finish()], - ] - ) - outcome = build(client, store, ws, expert=desk).run(milestone=1) - - assert outcome.status == "completed" # it got on with the work - results = [ - e.event for e in log(store).read() if type(e.event).__name__ == "ToolResult" - ] - assert not results[0].success - assert "budget" in results[0].result diff --git a/nisse-dev/tests/test_expert.py b/nisse-dev/tests/test_expert.py index 5097c61..06d5e92 100644 --- a/nisse-dev/tests/test_expert.py +++ b/nisse-dev/tests/test_expert.py @@ -23,8 +23,7 @@ from nisse_dev.oplog import OpLog from nisse_dev.state import StateStore, Transcript ROLES = RoleMap( - executor="llama3.2", - planner="haiku", + driver="haiku", helper="haiku", expert="claude-haiku-4-5", ) diff --git a/nisse-dev/tests/test_gates.py b/nisse-dev/tests/test_gates.py deleted file mode 100644 index 240e14d..0000000 --- a/nisse-dev/tests/test_gates.py +++ /dev/null @@ -1,105 +0,0 @@ -import pytest - -from nisse_dev.gates import Approved, Gate, Ignored, PlanRevision, Rejected - -REVISION = PlanRevision(gate="arch", rev=2, content="the plan") -OPERATORS = {"don"} - - -def gate() -> Gate: - return Gate(REVISION, OPERATORS) - - -def test_bare_approve_binds_to_current_revision() -> None: - result = gate().evaluate("don", "approve", channel="transport") - assert isinstance(result, Approved) - assert result.revision.tag == "rev-2" - assert result.approver == "don" - - -def test_tagged_approve_matching_current_revision() -> None: - result = gate().evaluate("don", "approve rev-2", channel="gitea") - assert isinstance(result, Approved) - assert result.channel == "gitea" - - -def test_stale_revision_tag_is_ignored_with_notice() -> None: - result = gate().evaluate("don", "approve rev-1", channel="transport") - assert isinstance(result, Ignored) - assert result.notice is not None - assert "rev-1" in result.notice - assert "rev-2" in result.notice - - -def test_any_other_reply_is_rejection_with_feedback() -> None: - result = gate().evaluate("don", "too vague, split milestone 1", "transport") - assert isinstance(result, Rejected) - assert result.feedback == "too vague, split milestone 1" - - -def test_unknown_sender_is_silently_ignored() -> None: - result = gate().evaluate("mallory", "approve", channel="transport") - assert isinstance(result, Ignored) - assert result.notice is None - - -def test_approve_with_trailing_words_is_rejection() -> None: - # "approve, but rename the module" is feedback, not approval (L3: the - # typed word approve, nothing else). - result = gate().evaluate("don", "approve, but rename the module", "transport") - assert isinstance(result, Rejected) - - -@pytest.mark.parametrize( - "text", ["Approve", "APPROVE", "Approve.", "approve!", " Approve rev-2 "] -) -def test_approval_ignores_case_and_trailing_punctuation(text: str) -> None: - # A phone keyboard's "Approve" used to count as rejection feedback (#64). - assert isinstance(gate().evaluate("don", text, channel="transport"), Approved) - - -def test_a_capitalised_stale_approval_is_still_refused() -> None: - result = gate().evaluate("don", "Approve REV-1", channel="transport") - assert isinstance(result, Ignored) - - -def test_a_sentence_containing_approve_is_still_feedback() -> None: - result = gate().evaluate("don", "I approve of most of it", channel="transport") - assert isinstance(result, Rejected) - assert result.feedback == "I approve of most of it" - - -# --- binding by arrival (L3): a reply that names no revision is about the one -# that was up when it arrived. - - -def test_a_bare_approval_about_an_earlier_revision_is_not_applied() -> None: - result = gate().evaluate("don", "approve", channel="transport", about=1) - assert isinstance(result, Ignored) - assert "arrived while rev-1 was up" in result.notice - assert "arch rev-2" in result.notice - - -def test_feedback_about_an_earlier_revision_is_not_a_rejection() -> None: - result = gate().evaluate("don", "also add docs", channel="transport", about=1) - assert isinstance(result, Ignored) - assert "not taken as feedback on arch rev-2" in result.notice - - -def test_a_reply_before_any_plan_was_up_is_not_applied() -> None: - result = gate().evaluate("don", "approve", channel="transport", about=0) - assert isinstance(result, Ignored) - assert "no plan" in result.notice - - -def test_replies_about_the_current_revision_apply() -> None: - assert isinstance(gate().evaluate("don", "approve", "transport", about=2), Approved) - result = gate().evaluate("don", "add docs", "transport", about=2) - assert isinstance(result, Rejected) - - -def test_a_named_revision_binds_by_its_number() -> None: - # "approve rev-2" says which plan it means, whenever it arrived. - assert isinstance( - gate().evaluate("don", "approve rev-2", "gitea", about=1), Approved - ) diff --git a/nisse-dev/tests/test_gitea.py b/nisse-dev/tests/test_gitea.py index 108d8ad..cdad09e 100644 --- a/nisse-dev/tests/test_gitea.py +++ b/nisse-dev/tests/test_gitea.py @@ -197,10 +197,7 @@ def test_intent_without_a_branch_is_unverifiable() -> None: assert pull_request_reconciler(client(handler), "o/r")(blank) is None -def test_closing_comments_first_then_closes_issue_and_pr() -> None: - from nisse_dev.approvals import PlanBoard - from nisse_dev.publish import MilestonePublisher - +def test_closing_an_issue_and_a_pr() -> None: seen: list[tuple[str, str, dict]] = [] def handler(request: httpx.Request) -> httpx.Response: @@ -213,15 +210,10 @@ def test_closing_comments_first_then_closes_issue_and_pr() -> None: return httpx.Response(201, json={}) gitea = client(handler) - PlanBoard(gitea, "o/r").close(10, "abandoned by don") - publisher = MilestonePublisher( - None, # type: ignore[arg-type] -- closing touches neither store nor git - gitea, - "o/r", - None, # type: ignore[arg-type] - remote_url="ssh://git@gitea.test/o/r.git", - ) - publisher.close_pull_request(20, "abandoned by don") + gitea.post_comment("o/r", 10, "abandoned by don") + gitea.close_issue("o/r", 10) + gitea.post_comment("o/r", 20, "abandoned by don") + gitea.close_pull_request("o/r", 20) assert seen == [ ("POST", "/api/v1/repos/o/r/issues/10/comments", {"body": "abandoned by don"}), diff --git a/nisse-dev/tests/test_llm.py b/nisse-dev/tests/test_llm.py index b4c22e3..d5bbe7b 100644 --- a/nisse-dev/tests/test_llm.py +++ b/nisse-dev/tests/test_llm.py @@ -183,28 +183,31 @@ def test_malformed_response_raises_llm_error() -> None: ROLES = RoleMap( - executor="local/qwen", - planner="anthropic/claude-opus-5", - helper="anthropic/claude-opus-5", + driver="local/qwen", + helper="anthropic/claude-sonnet-5", expert="anthropic/claude-opus-5", ) def test_role_map_resolves_and_pins() -> None: roles = ROLES - assert roles.resolve("executor") == "local/qwen" + assert roles.resolve("driver") == "local/qwen" assert roles.pinned() == { - "executor": "local/qwen", - "planner": "anthropic/claude-opus-5", - "helper": "anthropic/claude-opus-5", + "driver": "local/qwen", + "helper": "anthropic/claude-sonnet-5", "expert": "anthropic/claude-opus-5", - # Unset, the new loop's roles fall back: driver to the planner's - # model, classifier to the expert's (L9). - "driver": "anthropic/claude-opus-5", + # Unset, the classifier falls back to the expert's model (L9). "classifier": "anthropic/claude-opus-5", } +def test_a_pinned_map_from_before_the_driver_reads_its_own_roles() -> None: + # Pinned before M2.8: the old loop's roles are there, and are ignored. + old = {"planner": "p", "executor": "e", "driver": "d", "helper": "h"} + roles = RoleMap.from_pinned(old, ROLES) + assert (roles.driver, roles.helper, roles.expert) == ("d", "h", ROLES.expert) + + def test_unknown_role_raises() -> None: with pytest.raises(LLMError, match="unknown role"): ROLES.resolve("boss") diff --git a/nisse-dev/tests/test_loop.py b/nisse-dev/tests/test_loop.py index 8f45c89..6c844dc 100644 --- a/nisse-dev/tests/test_loop.py +++ b/nisse-dev/tests/test_loop.py @@ -9,11 +9,19 @@ import pytest from model_contract import assert_renderable from nisse_dev.cancel import Cancel -from nisse_dev.canonical import AssistantText, CanonicalEvent, Fact, ToolCall +from nisse_dev.canonical import ( + AssistantText, + CanonicalEvent, + Fact, + ToolCall, + ToolResult, +) from nisse_dev.gitea import Issue +from nisse_dev.llm import LLMError from nisse_dev.loop.agent import LoopAgent from nisse_dev.loop.driver import Driver from nisse_dev.loop.items import CLOSED, GO_AHEAD, READY, RUNNING, WAITING, loop_of +from nisse_dev.loop.reader import command_form from nisse_dev.oplog import OpLog from nisse_dev.state import StateStore from nisse_dev.transport import InMemoryTransport @@ -205,6 +213,22 @@ def test_before_the_go_ahead_nothing_can_change_or_close(h: Harness) -> None: assert {"restate", "ask_human", "read_file", "run_command"} <= h.model.offered[0] +@pytest.mark.parametrize("text", ["Go", "GO", "Go.", "go!", " go "]) +def test_a_go_ahead_ignores_case_and_trailing_punctuation( + h: Harness, text: str +) -> None: + # A phone keyboard's "Approve" once counted as feedback, not approval (#64). + h.model.turns.append([call("close_item", summary="added and tested")]) + h.agent.intake() + h.say(text) + assert h.store.work_items()[0].status == "closed" + + +def test_the_command_form_keeps_inner_words_apart() -> None: + assert command_form(" Do NEXT.") == "do next" + assert command_form("go ahead, but carefully") == "go ahead, but carefully" + + def test_go_starts_the_work_and_close_item_closes_the_issue(h: Harness) -> None: h.model.turns.append([call("close_item", summary="added and tested")]) h.agent.intake() @@ -368,6 +392,61 @@ def test_stop_ends_a_run_at_the_next_boundary(tmp_path: Path) -> None: assert outcome.kind == "stopped" +def test_a_failed_model_call_fails_the_run(tmp_path: Path) -> None: + class Down: + def complete(self, events, tools, model, system=None): # type: ignore[no-untyped-def] + raise LLMError("proxy down") + + outcome = Driver(Down(), "m").run( # type: ignore[arg-type] + driver_log(tmp_path), system="s", tools=_tools("read_file"), workspace=None + ) + assert outcome.kind == "failed" and "proxy down" in outcome.note + + +def test_stop_mid_turn_skips_the_rest_of_its_calls(tmp_path: Path) -> None: + cancel = Cancel() + + def start(arguments: dict[str, Any]) -> tuple[str, bool]: + cancel.set() # the operator's stop lands while this call runs + return "started", True + + model = ScriptedModel( + [call("start_milestone", name="m"), call("read_file", path="a")] + ) + log = driver_log(tmp_path) + outcome = Driver(model, "m", cancel=cancel).run( + log, + system="s", + tools=_tools("start_milestone", "read_file"), + workspace=None, + handlers={"start_milestone": start}, + ) + assert outcome.kind == "stopped" + results = [e.event for e in log.read() if isinstance(e.event, ToolResult)] + assert [r.success for r in results] == [True, False] # the read never ran + + +def test_a_command_that_works_resets_the_failure_count(tmp_path: Path) -> None: + class Commands: + def dispatch(self, tool: str, arguments: dict[str, Any]) -> Any: + ok = arguments["command"] == "good" + return type("Outcome", (), {"result": "out", "success": ok})() + + def run(command: str) -> list[Any]: + return [call("run_command", command=command)] + + model = ScriptedModel( + run("bad"), run("good"), run("bad"), [call("ask_human", question="?")] + ) + outcome = Driver(model, "m", failure_limit=2).run( + driver_log(tmp_path), + system="s", + tools=_tools("run_command", "ask_human"), + workspace=Commands(), # type: ignore[arg-type] + ) + assert outcome.kind == "wait" # never two failures in a row + + def _tools(*names: str) -> list[dict[str, Any]]: return [ {"type": "function", "function": {"name": n, "parameters": {}}} for n in names diff --git a/nisse-dev/tests/test_loop_help.py b/nisse-dev/tests/test_loop_help.py index ba8346d..41da1c4 100644 --- a/nisse-dev/tests/test_loop_help.py +++ b/nisse-dev/tests/test_loop_help.py @@ -18,14 +18,20 @@ from test_loop_milestones import ( ) from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture -from nisse_dev.canonical import AssistantText, Fact, ToolCall, ToolResult +from nisse_dev.canonical import ( + AssistantText, + DelegationStarted, + Fact, + ToolCall, + ToolResult, +) from nisse_dev.config import Profile from nisse_dev.delegation import DELEGATION_INTENT, Caps from nisse_dev.llm import RoleMap from nisse_dev.loop.help import Helpers from nisse_dev.loop.items import ANSWER, MERGE, loop_of -ROLES = RoleMap(executor="", planner="drv", helper="hlp", expert="exp", driver="drv") +ROLES = RoleMap(driver="drv", helper="hlp", expert="exp") class Spy: @@ -115,6 +121,34 @@ def test_a_helpers_work_lands_on_the_milestone(world: dict[str, Any]) -> None: assert loop_of(item(world)).wait == MERGE +def test_a_delegation_records_what_the_alias_resolved_to( + world: dict[str, Any], +) -> None: + # L9 eval history keyed on an alias alone would average across whatever + # that alias pointed at over time. + helper = Spy( + [call("write_file", path="helped.py", content="H = 1\n")], [hand_back()] + ) + model = ScriptedModel( + RESTATE, + [ + call("start_milestone", name="words"), + call("delegate", task="write helped.py"), + call("finish_milestone", summary="with help", final="yes"), + ], + ) + backed = helpers(helper, backing=lambda alias: f"anthropic/{alias}-backing") + agent, transport = loop_agent(world, model, helpers=backed) + agent.intake() + say(transport, world, "go") + [started] = [ + e.event + for e in world["store"].transcript(item(world).id, "helper").read() + if isinstance(e.event, DelegationStarted) + ] + assert started.backing_model == "anthropic/hlp-backing" + + def test_a_failed_helpers_work_is_discarded(world: dict[str, Any]) -> None: helper = Spy( [call("write_file", path="half.py", content="half\n")], diff --git a/nisse-dev/tests/test_loop_properties.py b/nisse-dev/tests/test_loop_properties.py index 5beea30..97c9ab5 100644 --- a/nisse-dev/tests/test_loop_properties.py +++ b/nisse-dev/tests/test_loop_properties.py @@ -223,7 +223,7 @@ class Walk: reader=self, # type: ignore[arg-type] helpers=Helpers( client=HelperModel(self.rng), # type: ignore[arg-type] - roles=RoleMap(executor="", planner="m", helper="h", expert="x"), + roles=RoleMap(driver="m", helper="h", expert="x"), caps=Caps(turns=4), ), bug_grace_s=600, diff --git a/nisse-dev/tests/test_no_stranding.py b/nisse-dev/tests/test_no_stranding.py deleted file mode 100644 index cc1761e..0000000 --- a/nisse-dev/tests/test_no_stranding.py +++ /dev/null @@ -1,238 +0,0 @@ -"""The orchestrator's resting-state invariant, checked under injected faults. - -Working states (planning, executing) are only right while work runs. After -any message is handled the agent must be somewhere it can wait for the -operator, and some reply must lead on from there. The first live run found -four ways to break this one at a time; this drives a whole project with a -fault at each step instead. -""" - -import io -from pathlib import Path -from typing import Any - -import pytest - -from nisse_dev.control import ( - AWAIT_ARCH_SIGNOFF, - AWAIT_INPUT, - AWAIT_MILESTONE_SIGNOFF, - EXECUTING, - IDLE, - MILESTONE_PLANNING, - PLANNING_ARCH, - PR_OPEN, - AskHuman, - Orchestrator, - PlanningError, - SubmitPlan, -) -from nisse_dev.executor import ExecutorOutcome -from nisse_dev.gitea import GiteaError, PullRequest -from nisse_dev.oplog import OpLog -from nisse_dev.publish import PublishResult -from nisse_dev.state import StateStore -from nisse_dev.transport import InMemoryTransport - -WORKING = {PLANNING_ARCH, MILESTONE_PLANNING, EXECUTING} - - -class Faults: - """One fault, fired once, at the nth call to one site.""" - - def __init__(self, site: str, nth: int, fault: Any) -> None: - self._site, self._nth, self._fault = site, nth, fault - self._calls: dict[str, int] = {} - self.fired = False - - def check(self, site: str) -> Any: - """Raise or return the fault if this call is the one; else None.""" - self._calls[site] = self._calls.get(site, 0) + 1 - if self.fired or site != self._site or self._calls[site] != self._nth: - return None - self.fired = True - if isinstance(self._fault, Exception): - raise self._fault - return self._fault - - -class Planner: - def __init__(self, faults: Faults) -> None: - self._faults = faults - - def plan( - self, - transcript: object, - gate: str, - project: str, - feedback: str | None = None, - answer: str | None = None, - ) -> SubmitPlan | AskHuman: - return self._faults.check("planner") or SubmitPlan(f"{gate} plan") - - -class Executor: - def __init__(self, faults: Faults) -> None: - self._faults = faults - - def run(self, transcript: object, milestone: int) -> ExecutorOutcome: - return self._faults.check("executor") or ExecutorOutcome("completed", "done") - - -class Publisher: - def __init__(self, faults: Faults) -> None: - self._faults = faults - self._opened = 0 - - def reconcilers(self) -> dict[str, Any]: - return {} - - def publish(self, **kwargs: Any) -> PublishResult: - self._faults.check("publisher") - self._opened += 1 - pr = PullRequest(number=self._opened, state="open", head="h", url="u") - return PublishResult(pr, f"opened PR #{self._opened}") - - -def next_reply(store: StateStore, used_project: bool) -> str: - """What an operator following the prompts would say next.""" - state = store.get_state() - assert state is not None - data = state.data - match state.state: - case "IDLE": - return "status" if used_project else "use demo" - case "AWAIT_ARCH_SIGNOFF": - return "approve" - case "AWAIT_MILESTONE_SIGNOFF": - if data["gate"]["approved"]: - return f"start milestone {data['milestone']}" - return "approve" - case "PR_OPEN": - return "next milestone" if data["milestone"] < 2 else "project complete" - case "AWAIT_INPUT": - return "continue" - raise AssertionError(f"stranded in {state.state}") - - -FAULTS = [ - ("planner", 1, RuntimeError("bug in planning")), - ("planner", 1, PlanningError("planner returned 400")), - ("planner", 1, AskHuman("which language?")), - ("planner", 2, RuntimeError("bug in planning")), - ("planner", 3, OSError("disk full")), - ("executor", 1, RuntimeError("bug in the executor")), - ("executor", 1, ExecutorOutcome("escalate", "stuck")), - ("executor", 1, ExecutorOutcome("capped", "max turns")), - ("executor", 2, ExecutorOutcome("stopped", "operator")), - ("publisher", 1, RuntimeError("bug in publishing")), - ("publisher", 1, GiteaError("gitea is down")), - ("publisher", 2, RuntimeError("bug in publishing")), -] - - -@pytest.mark.parametrize( - ("site", "nth", "fault"), FAULTS, ids=[f"{s}{n}-{f!r}" for s, n, f in FAULTS] -) -def test_a_fault_anywhere_never_strands_the_agent( - tmp_path: Path, site: str, nth: int, fault: Any -) -> None: - faults = Faults(site, nth, fault) - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - transport = InMemoryTransport() - Orchestrator( - store, - transport, - Planner(faults), - operators={"don"}, - executor=Executor(faults), - publisher=Publisher(faults), - ).start() - - used_project = False - for _ in range(40): - reply = next_reply(store, used_project) - used_project = used_project or reply.startswith("use ") - transport.receive("don", reply) - state = store.get_state() - assert state is not None - assert state.state not in WORKING, ( - f"after {reply!r}: stranded in {state.state}; " - f"last said: {transport.outbound[-1].text!r}" - ) - if reply == "project complete": - break - else: - raise AssertionError("never reached project complete") - assert faults.fired - assert store.get_state().state == IDLE - - -def test_a_signoff_state_without_its_gate_offers_a_way_out(tmp_path: Path) -> None: - faults = Faults("none", 0, None) - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - transport = InMemoryTransport() - orch = Orchestrator(store, transport, Planner(faults), operators={"don"}) - orch.start() - transport.receive("don", "use demo") - state = store.get_state() - store.transition( - AWAIT_ARCH_SIGNOFF, - cause="test", - project="demo", - data={**state.data, "gate": None}, - ) - orch._data["gate"] = None # as a restart would load it - transport.receive("don", "approve") - assert store.get_state().state == AWAIT_INPUT - transport.receive("don", "continue") - assert store.get_state().state == AWAIT_ARCH_SIGNOFF - - -class RaisingReviewExecutor: - def run(self, transcript: object, milestone: int) -> ExecutorOutcome: - raise RuntimeError("bug while addressing review") - - -def test_a_failing_review_response_keeps_pr_open_and_says_so( - tmp_path: Path, -) -> None: - faults = Faults("none", 0, None) - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - transport = InMemoryTransport() - orch = Orchestrator( - store, - transport, - Planner(faults), - operators={"don"}, - gitea_operators={"don"}, - executor=Executor(faults), - publisher=Publisher(faults), - ) - orch.start() - for reply in ("use demo", "approve", "approve", "start milestone 1"): - transport.receive("don", reply) - assert store.get_state().state == PR_OPEN - orch._executor = RaisingReviewExecutor() - - class Comment: - id, author, body = 1, "don", "add a test" - - orch.handle_review_comment(Comment()) - assert store.get_state().state == PR_OPEN - assert "could not handle a review comment" in transport.outbound[-1].text - transport.receive("don", "project complete") - assert store.get_state().state == IDLE - - -def test_the_stub_configuration_reaches_every_state(tmp_path: Path) -> None: - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - transport = InMemoryTransport() - Orchestrator( - store, transport, Planner(Faults("none", 0, None)), operators={"don"} - ).start() - for reply in ("use demo", "approve", "approve", "start milestone 1"): - transport.receive("don", reply) - assert store.get_state().state == PR_OPEN - transport.receive("don", "next milestone") - assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF diff --git a/nisse-dev/tests/test_oplog.py b/nisse-dev/tests/test_oplog.py index f533fa7..26dc1dc 100644 --- a/nisse-dev/tests/test_oplog.py +++ b/nisse-dev/tests/test_oplog.py @@ -29,10 +29,21 @@ def events(stream: io.StringIO) -> list[dict]: return [json.loads(line) for line in stream.getvalue().splitlines()] +def start_item( + store: StateStore, item_id: str, *, scope: str | None = None, cause: str +) -> None: + """Open a work item, make it the active one and set it running.""" + store.open_work_item( + item_id, title=item_id, scope=scope, position="IDLE", cause=cause + ) + store.set_active(item_id, cause=cause) + store.update_work_item(item_id, position="RUNNING", data={}, cause=cause) + + def test_every_event_carries_the_envelope( store: StateStore, stream: io.StringIO ) -> None: - store.transition("PLANNING_ARCH", cause="operator command: use demo") + start_item(store, "demo-1", cause="issue assigned") intent_id = store.begin_intent("push_branch", {}, cause="milestone execution") store.commit_intent(intent_id, cause="push verified") for event in events(stream): @@ -40,16 +51,20 @@ def test_every_event_carries_the_envelope( assert event["agent"] == "nisse-dev" -def test_transition_logs_from_to_and_cause( +def test_a_position_change_logs_from_to_and_cause( store: StateStore, stream: io.StringIO ) -> None: - store.transition("PLANNING_ARCH", cause="operator command: use demo", project="d") - store.transition("AWAIT_ARCH_SIGNOFF", cause="plan submitted", project="d") + start_item(store, "d-1", scope="d", cause="issue assigned") + store.update_work_item("d-1", position="WAITING", data={}, cause="restated") # Opening the work item logs its own events first (work_item.open, .switch). first, second = [e for e in events(stream) if e["event"] == "state.transition"] - assert (first["from"], first["to"]) == ("IDLE", "PLANNING_ARCH") - assert (second["from"], second["to"]) == ("PLANNING_ARCH", "AWAIT_ARCH_SIGNOFF") - assert second["cause"] == "plan submitted" + assert (first["from"], first["to"]) == ("IDLE", "RUNNING") + assert (second["from"], second["to"], second["project"]) == ( + "RUNNING", + "WAITING", + "d", + ) + assert second["cause"] == "restated" def test_intent_lifecycle_events(store: StateStore, stream: io.StringIO) -> None: @@ -63,9 +78,9 @@ def test_intent_lifecycle_events(store: StateStore, stream: io.StringIO) -> None def test_task_id_threads_through_events(store: StateStore, stream: io.StringIO) -> None: - store.transition("IDLE", cause="boot") + store.open_work_item("general", title="general", position="IDLE", cause="boot") store.oplog.task_id = "task-42" - store.transition("PLANNING_ARCH", cause="operator command") + start_item(store, "task-42", cause="issue assigned") before, *during = events(stream) assert before["task_id"] is None assert {e["task_id"] for e in during} == {"task-42"} @@ -119,12 +134,7 @@ def test_literal_log_lines_for_human_readability( monkeypatch.setattr("nisse_dev.oplog.datetime", _FrozenDatetime) - store.transition( - "PLANNING_ARCH", - cause="operator command: use demo", - project="demo", - data={"task_id": "demo-m2"}, - ) + start_item(store, "demo-m2", scope="demo", cause="issue assigned: demo#2") store.oplog.task_id = "demo-m2" intent_id = store.begin_intent( "push_branch", {"branch": "milestone-2"}, cause="milestone 2 execution" @@ -135,20 +145,20 @@ def test_literal_log_lines_for_human_readability( ( '{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",' ' "event": "work_item.open", "task_id": null,' - ' "cause": "operator command: use demo",' + ' "cause": "issue assigned: demo#2",' ' "work_item": "demo-m2", "scope": "demo"}' ), ( '{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",' ' "event": "work_item.switch", "task_id": null,' - ' "cause": "operator command: use demo",' + ' "cause": "issue assigned: demo#2",' ' "from": null, "to": "demo-m2"}' ), ( '{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",' ' "event": "state.transition", "task_id": null,' - ' "cause": "operator command: use demo",' - ' "from": "IDLE", "to": "PLANNING_ARCH", "project": "demo"}' + ' "cause": "issue assigned: demo#2",' + ' "from": "IDLE", "to": "RUNNING", "project": "demo"}' ), ( '{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",' diff --git a/nisse-dev/tests/test_planner.py b/nisse-dev/tests/test_planner.py deleted file mode 100644 index 3b2ba2a..0000000 --- a/nisse-dev/tests/test_planner.py +++ /dev/null @@ -1,312 +0,0 @@ -import io -from pathlib import Path -from typing import Any - -import pytest - -from nisse_dev.canonical import ( - AssistantText, - CanonicalEvent, - PlanSubmitted, - StoredEvent, - ToolCall, - UserMessage, -) -from nisse_dev.control import ( - AWAIT_ARCH_SIGNOFF, - AWAIT_MILESTONE_SIGNOFF, - AskHuman, - Orchestrator, - PlanningError, - SubmitPlan, -) -from nisse_dev.llm import BudgetExhausted, RoleMap -from nisse_dev.oplog import OpLog -from nisse_dev.planner import PLANNER_TOOLS, LLMPlanningAgent -from nisse_dev.state import StateStore, Transcript -from nisse_dev.transport import InMemoryTransport - -ROLES = RoleMap( - executor="local/qwen", - planner="anthropic/claude-opus-5", - helper="anthropic/claude-opus-5", - expert="anthropic/claude-opus-5", -) - - -class ScriptedClient: - """Returns canned canonical events per call, recording what it was asked.""" - - def __init__(self, turns: list[list[CanonicalEvent]]) -> None: - self._turns = turns - self.calls: list[dict[str, Any]] = [] - - def complete( - self, - events: list[StoredEvent], - tools: list[dict[str, Any]] | None, - model: str, - system: str | None = None, - ) -> list[CanonicalEvent]: - self.calls.append( - {"events": events, "tools": tools, "model": model, "system": system} - ) - if not self._turns: - return [] - return self._turns.pop(0) - - -def plan_call(content: str, gate: str = "arch") -> ToolCall: - return ToolCall( - call_id="call-1", - tool="submit_plan", - arguments={"type": gate, "content": content}, - ) - - -ITEM = "t" # the work item the unit tests plan in - - -def log(store: StateStore) -> Transcript: - return store.transcript(ITEM, "planner") - - -@pytest.fixture -def store(tmp_path: Path) -> StateStore: - return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - - -def test_submit_plan_tool_call_becomes_submit_plan(store: StateStore) -> None: - client = ScriptedClient([[plan_call("step 1: create src/app.py")]]) - agent = LLMPlanningAgent(client, ROLES) - action = agent.plan(log(store), "arch", "demo") - assert action == SubmitPlan("step 1: create src/app.py") - - -def test_ask_human_tool_call_becomes_ask_human(store: StateStore) -> None: - client = ScriptedClient( - [ - [ - ToolCall( - call_id="call-1", - tool="ask_human", - arguments={"question": "postgres or sqlite?"}, - ) - ] - ] - ) - action = LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") - assert action == AskHuman("postgres or sqlite?") - - -def test_planner_role_and_tools_are_used(store: StateStore) -> None: - client = ScriptedClient([[plan_call("plan")]]) - LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") - call = client.calls[0] - assert call["model"] == "anthropic/claude-opus-5" # never the executor - assert [t["function"]["name"] for t in call["tools"]] == [ - "submit_plan", - "ask_human", - ] - assert "demo" in call["system"] and "arch" in call["system"] - - -def test_no_proceed_tool_is_offered() -> None: - names = {tool["function"]["name"] for tool in PLANNER_TOOLS} - assert names == {"submit_plan", "ask_human"} # no self-advancement path - - -def test_model_output_is_appended_to_the_log(store: StateStore) -> None: - client = ScriptedClient( - [[AssistantText(text="thinking out loud", model="m"), plan_call("the plan")]] - ) - LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") - kinds = [type(e.event).__name__ for e in log(store).read()] - assert kinds == ["AssistantText", "ToolCall", "ToolResult"] - - -def test_gate_tool_call_is_answered_in_the_log(store: StateStore) -> None: - # submit_plan ends the turn and has no natural result, but a tool call - # left dangling makes the next render invalid for the provider. - client = ScriptedClient([[plan_call("the plan")]]) - LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") - events = [e.event for e in log(store).read()] - assert [type(e).__name__ for e in events] == ["ToolCall", "ToolResult"] - assert events[1].call_id == events[0].call_id - assert "posted for approval" in events[1].result - - -def test_prose_only_turn_retries_once_then_raises(store: StateStore) -> None: - client = ScriptedClient( - [ - [AssistantText(text="here's what I think", model="m")], - [AssistantText(text="still prose", model="m")], - ] - ) - with pytest.raises(PlanningError, match="no submit_plan or ask_human"): - LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") - assert len(client.calls) == 2 - assert "did not call a tool" in client.calls[1]["system"] - - -def test_retry_succeeds_after_prose(store: StateStore) -> None: - client = ScriptedClient( - [[AssistantText(text="prose", model="m")], [plan_call("the real plan")]] - ) - action = LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") - assert action == SubmitPlan("the real plan") - - -def test_malformed_tool_arguments_are_rejected(store: StateStore) -> None: - blank = ToolCall(call_id="c", tool="submit_plan", arguments={"content": " "}) - client = ScriptedClient([[blank], [blank]]) - with pytest.raises(PlanningError): - LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") - - -def test_planner_sees_prior_transcript(store: StateStore) -> None: - store.transcript(ITEM, "operator").append( - UserMessage(text="use demo", source="transport:@don") - ) - client = ScriptedClient([[plan_call("plan")]]) - LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") - seen = [getattr(e.event, "text", None) for e in client.calls[0]["events"]] - assert "[operator, Matrix] use demo" in seen - - -# --- orchestrator integration (still no network) ------------------------- - - -def build(tmp_path: Path, client: ScriptedClient) -> tuple[Orchestrator, StateStore]: - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - transport = InMemoryTransport() - agent = LLMPlanningAgent(client, ROLES) - orch = Orchestrator(store, transport, agent, operators={"don"}, roles=ROLES) - orch.start() - return orch, store, transport - - -def test_use_project_produces_llm_plan_and_transcript(tmp_path: Path) -> None: - client = ScriptedClient([[plan_call("1. create app.py\n2. add a test")]]) - _, store, transport = build(tmp_path, client) - transport.receive("don", "use demo") - - request = transport.outbound[-1] - assert request.kind == "approval_request" - assert "create app.py" in request.text - - # The work item's own events: `use demo` was said while idle, and is - # carried into the item it opened (D1). - item = store.active_item() - events = [e.event for e in store.transcript(item.id, "operator").read()] - kinds = [type(e).__name__ for e in events] - assert kinds == [ - "UserMessage", - "Fact", - "ToolCall", - "ToolResult", - "PlanSubmitted", - "GateOpened", - ] - submitted = next(e for e in events if isinstance(e, PlanSubmitted)) - assert submitted.model == "anthropic/claude-opus-5" # pinned planner - - -def test_roles_pinned_at_task_start(tmp_path: Path) -> None: - client = ScriptedClient([[plan_call("plan")]]) - _, store, transport = build(tmp_path, client) - transport.receive("don", "use demo") - state = store.get_state() - assert state is not None - assert state.data["roles"]["planner"] == "anthropic/claude-opus-5" - - -def test_planning_failure_gates_to_await_input(tmp_path: Path) -> None: - prose = [AssistantText(text="prose", model="m")] - client = ScriptedClient([prose, prose]) - _, store, transport = build(tmp_path, client) - transport.receive("don", "use demo") - state = store.get_state() - assert state is not None - assert state.state == "AWAIT_INPUT" - assert transport.outbound[-1].kind == "question" - - -def test_operator_can_recover_from_a_planning_failure(tmp_path: Path) -> None: - # Regression: AWAIT_INPUT reached by any path other than ask_human used to - # record no pending item, so every operator reply was refused with "no - # pending question" and the agent was stuck for good. - prose = [AssistantText(text="prose", model="m")] - client = ScriptedClient([prose, prose, [plan_call("recovered plan")]]) - _, store, transport = build(tmp_path, client) - transport.receive("don", "use demo") - assert store.get_state().state == "AWAIT_INPUT" - - transport.receive("don", "try again, keep it short") - - assert store.get_state().state == AWAIT_ARCH_SIGNOFF - assert "recovered plan" in transport.outbound[-1].text - - -def test_operator_can_recover_after_a_stopped_milestone(tmp_path: Path) -> None: - class StoppedExecutor: - def run(self, transcript: object, milestone: int) -> Any: - from nisse_dev.executor import ExecutorOutcome - - return ExecutorOutcome("escalate", "tests failed twice") - - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - transport = InMemoryTransport() - client = ScriptedClient([[plan_call("p1")], [plan_call("p2")]]) - orch = Orchestrator( - store, - transport, - LLMPlanningAgent(client, ROLES), - operators={"don"}, - roles=ROLES, - executor=StoppedExecutor(), - ) - orch.start() - transport.receive("don", "use demo") - transport.receive("don", "approve") - transport.receive("don", "approve") - transport.receive("don", "start milestone 1") - assert store.get_state().state == "AWAIT_INPUT" - - transport.receive("don", "ok, I fixed the dependency") - - assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF - assert "start milestone 1" in transport.outbound[-1].text - - -def test_rejection_feedback_reaches_the_planner_via_the_log( - tmp_path: Path, -) -> None: - client = ScriptedClient([[plan_call("v1")], [plan_call("v2")]]) - _, _store, transport = build(tmp_path, client) - transport.receive("don", "use demo") - transport.receive("don", "too vague, name the files") - - replan_context = [e.event for e in client.calls[1]["events"]] - assert any( - isinstance(e, UserMessage) and "too vague" in e.text for e in replan_context - ) - assert "v2" in transport.outbound[-1].text - - -def test_budget_exhaustion_is_not_retried(store: StateStore) -> None: - """A budget will not reappear within a turn, so a retry buys nothing and - delays telling the operator.""" - - class BrokeClient: - def __init__(self) -> None: - self.calls = 0 - - def complete(self, *args: Any, **kwargs: Any): - self.calls += 1 - raise BudgetExhausted("daily budget exhausted") - - client = BrokeClient() - with pytest.raises(BudgetExhausted): - LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") - assert client.calls == 1 # not two diff --git a/nisse-dev/tests/test_poller.py b/nisse-dev/tests/test_poller.py deleted file mode 100644 index f67d261..0000000 --- a/nisse-dev/tests/test_poller.py +++ /dev/null @@ -1,132 +0,0 @@ -import io -import json -import time - -import pytest - -from nisse_dev.gitea import Comment, GiteaError -from nisse_dev.oplog import OpLog -from nisse_dev.poller import MAX_BACKOFF_S, CommentPoller - - -class FakeGitea: - def __init__(self, pages: list[list[Comment]] | None = None) -> None: - self.pages = pages or [] - self.calls = 0 - self.fail_times = 0 - - def list_comments(self, repo: str, index: int, since: str | None = None): - self.calls += 1 - if self.fail_times > 0: - self.fail_times -= 1 - raise GiteaError("gitea unreachable") - if not self.pages: - return [] - return self.pages.pop(0) - - -def comment(cid: int, author: str = "don", body: str = "please fix") -> Comment: - return Comment(id=cid, author=author, body=body, created_at="t") - - -def build(client, **kw) -> tuple[CommentPoller, list[Comment]]: - seen: list[Comment] = [] - poller = CommentPoller(client, "o/r", seen.append, **kw) - return poller, seen - - -def test_new_comments_are_delivered_once() -> None: - client = FakeGitea([[comment(1)], [comment(1), comment(2)]]) - poller, seen = build(client) - poller.poll_once(1) - poller.poll_once(1) - assert [c.id for c in seen] == [1, 2] # id 1 not redelivered - - -def test_prime_suppresses_the_existing_backlog() -> None: - # Starting a watch must not replay feedback already addressed. - client = FakeGitea([[comment(1), comment(2)], [comment(1), comment(2), comment(3)]]) - poller, seen = build(client) - poller.prime(1) - poller.poll_once(1) - assert [c.id for c in seen] == [3] - - -def test_own_comments_are_ignored() -> None: - client = FakeGitea([[comment(1, author="nisse-dev-testing"), comment(2, "don")]]) - poller, seen = build(client, ignore_authors={"nisse-dev-testing"}) - poller.poll_once(1) - assert [c.id for c in seen] == [2] - - -def test_ignored_comments_are_not_redelivered_later() -> None: - client = FakeGitea( - [ - [comment(1, author="bot")], - [comment(1, author="bot"), comment(2, "don")], - ] - ) - poller, seen = build(client, ignore_authors={"bot"}) - poller.poll_once(1) - poller.poll_once(1) - assert [c.id for c in seen] == [2] - - -def test_outage_does_not_raise_out_of_the_loop() -> None: - stream = io.StringIO() - client = FakeGitea([[comment(1)]]) - client.fail_times = 2 - poller, seen = build( - client, - oplog=OpLog("nisse-dev", stream), - interval_s=0.01, - initial_backoff_s=0.01, - ) - poller.start(1) - deadline = time.monotonic() + 5 - while not seen and time.monotonic() < deadline: - time.sleep(0.05) - poller.stop() - - # It recovered and delivered rather than dying on the outage (L5: an - # outage delays a reaction, never fails a task). - assert [c.id for c in seen] == [1] - assert "gitea.poll_failed" in stream.getvalue() - - -def test_prime_survives_an_outage() -> None: - client = FakeGitea() - client.fail_times = 1 - poller, _ = build(client) - poller.prime(1) # must not raise - - -def test_backoff_grows_then_caps() -> None: - client = FakeGitea() - client.fail_times = 50 - stream = io.StringIO() - poller, _ = build( - client, - oplog=OpLog("nisse-dev", stream), - interval_s=0.001, - initial_backoff_s=0.001, - max_backoff_s=0.008, - ) - poller.start(1) - time.sleep(0.3) - poller.stop() - - delays = [json.loads(line)["retry_in_s"] for line in stream.getvalue().splitlines()] - assert delays[:4] == [0.001, 0.002, 0.004, 0.008] - assert max(delays) == 0.008 # capped, and retried indefinitely (L5) - assert MAX_BACKOFF_S <= 300.0 # the shipped cap is "a few minutes" - - -@pytest.mark.parametrize("author", ["don", "someone-else"]) -def test_poller_does_not_filter_by_author_itself(author: str) -> None: - # Authorisation belongs to the orchestrator's allowlist, not here: the - # poller is pure intake. - client = FakeGitea([[comment(1, author=author)]]) - poller, seen = build(client) - poller.poll_once(1) - assert len(seen) == 1 diff --git a/nisse-dev/tests/test_pr_open.py b/nisse-dev/tests/test_pr_open.py deleted file mode 100644 index 4384c91..0000000 --- a/nisse-dev/tests/test_pr_open.py +++ /dev/null @@ -1,202 +0,0 @@ -"""The loop past EXECUTING (D4, D5): publish, review gate, then next milestone or -project completion.""" - -import io -from pathlib import Path -from typing import Any - -import pytest - -from nisse_dev.canonical import AssistantText, CanonicalEvent, StoredEvent, ToolCall -from nisse_dev.control import ( - AWAIT_INPUT, - AWAIT_MILESTONE_SIGNOFF, - IDLE, - PR_OPEN, - Orchestrator, -) -from nisse_dev.executor import ExecutorOutcome -from nisse_dev.gitea import GiteaError, PullRequest -from nisse_dev.llm import RoleMap -from nisse_dev.oplog import OpLog -from nisse_dev.planner import LLMPlanningAgent -from nisse_dev.publish import PublishResult -from nisse_dev.state import StateStore -from nisse_dev.transport import InMemoryTransport - -ROLES = RoleMap( - executor="llama3.2", - planner="haiku", - helper="haiku", - expert="haiku", -) - - -class ScriptedClient: - def __init__(self, plans: list[str]) -> None: - self._plans = plans - self.calls: list[dict[str, Any]] = [] - - def complete( - self, - events: list[StoredEvent], - tools: list[dict[str, Any]] | None, - model: str, - system: str | None = None, - ) -> list[CanonicalEvent]: - self.calls.append({"system": system}) - if not self._plans: - return [AssistantText(text="no more plans", model=model)] - return [ - ToolCall( - call_id=f"c{len(self.calls)}", - tool="submit_plan", - arguments={"type": "x", "content": self._plans.pop(0)}, - ) - ] - - -class DoneExecutor: - def run(self, transcript: object, milestone: int) -> ExecutorOutcome: - return ExecutorOutcome("completed", f"milestone {milestone} done") - - -class FakePublisher: - def __init__( - self, *, result: PublishResult | None = None, error: Exception | None = None - ) -> None: - self._result = result or PublishResult( - PullRequest(number=7, state="open", head="h", url="https://g/pulls/7"), - "opened PR #7: https://g/pulls/7", - ) - self._error = error - self.calls: list[dict[str, Any]] = [] - - def reconcilers(self) -> dict[str, Any]: - return {"open_pr": lambda intent: True} - - def publish(self, **kwargs: Any) -> PublishResult: - self.calls.append(kwargs) - if self._error is not None: - raise self._error - return self._result - - -def build( - tmp_path: Path, publisher: Any, plans: list[str] | None = None -) -> tuple[StateStore, InMemoryTransport]: - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - transport = InMemoryTransport() - client = ScriptedClient(plans or ["arch plan", "m1 plan", "m2 plan"]) - orch = Orchestrator( - store, - transport, - LLMPlanningAgent(client, ROLES), - operators={"don"}, - roles=ROLES, - executor=DoneExecutor(), - publisher=publisher, - ) - orch.start() - return store, transport - - -def run_to_pr(tmp_path: Path, publisher: Any) -> tuple[StateStore, InMemoryTransport]: - store, transport = build(tmp_path, publisher) - transport.receive("don", "use demo") - transport.receive("don", "approve") # arch - transport.receive("don", "approve") # milestone plan - transport.receive("don", "start milestone 1") - return store, transport - - -def test_completed_milestone_publishes_and_enters_pr_open(tmp_path: Path) -> None: - publisher = FakePublisher() - store, transport = run_to_pr(tmp_path, publisher) - - assert store.get_state().state == PR_OPEN - assert publisher.calls[0]["milestone"] == 1 - assert publisher.calls[0]["project"] == "demo" - assert "m1 plan" in publisher.calls[0]["plan"] # the approved plan travels - assert "PR #7" in transport.outbound[-1].text - - -def test_pr_open_records_the_pull_request(tmp_path: Path) -> None: - store, _ = run_to_pr(tmp_path, FakePublisher()) - assert store.get_state().data["pr"]["number"] == 7 - - -def test_next_milestone_replans_and_bumps_the_counter(tmp_path: Path) -> None: - store, transport = run_to_pr(tmp_path, FakePublisher()) - transport.receive("don", "next milestone") - - state = store.get_state() - assert state.state != PR_OPEN - assert state.data["milestone"] == 2 - assert "m2 plan" in transport.outbound[-1].text - - -def test_project_complete_returns_to_idle(tmp_path: Path) -> None: - store, transport = run_to_pr(tmp_path, FakePublisher()) - transport.receive("don", "project complete") - - state = store.get_state() - assert state.state == IDLE - assert state.project is None - assert "project closed" in transport.outbound[-1].text - - -def test_unknown_command_in_pr_open_explains_the_options(tmp_path: Path) -> None: - store, transport = run_to_pr(tmp_path, FakePublisher()) - transport.receive("don", "looks good!") - assert store.get_state().state == PR_OPEN - assert "next milestone" in transport.outbound[-1].text - - -def test_no_changes_gates_to_the_operator_without_entering_pr_open( - tmp_path: Path, -) -> None: - publisher = FakePublisher(result=PublishResult(None, "no changes to publish")) - store, transport = run_to_pr(tmp_path, publisher) - # Not left in EXECUTING, where every message is answered "busy". - assert store.get_state().state == AWAIT_INPUT - assert "no changes to publish" in transport.outbound[-1].text - transport.receive("don", "ok") - assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF - assert "start milestone 1" in transport.outbound[-1].text - - -def test_publish_failure_gates_to_the_operator(tmp_path: Path) -> None: - publisher = FakePublisher(error=GiteaError("gitea is down")) - store, transport = run_to_pr(tmp_path, publisher) - assert store.get_state().state == AWAIT_INPUT - assert "could not publish" in transport.outbound[-1].text - # And the operator can still get back out (the deadlock regression). - transport.receive("don", "gitea is back") - assert store.get_state().state != AWAIT_INPUT - - -def test_publisher_reconciler_is_registered_at_boot(tmp_path: Path) -> None: - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - store.begin_intent("open_pr", {"branch": "b"}, cause="crash") - transport = InMemoryTransport() - orch = Orchestrator( - store, - transport, - LLMPlanningAgent(ScriptedClient([]), ROLES), - operators={"don"}, - roles=ROLES, - publisher=FakePublisher(), - ) - orch.start() - # The publisher's reconciler answered "already opened", so boot resumed - # instead of gating on an unknown intent kind. - assert store.pending_intents() == [] - assert store.get_state().state != AWAIT_INPUT - - -@pytest.mark.parametrize("command", ["next milestone", "project complete"]) -def test_pr_open_commands_are_operator_only(tmp_path: Path, command: str) -> None: - store, transport = run_to_pr(tmp_path, FakePublisher()) - transport.receive("mallory", command) - assert store.get_state().state == PR_OPEN diff --git a/nisse-dev/tests/test_pr_open_handling.py b/nisse-dev/tests/test_pr_open_handling.py deleted file mode 100644 index 213af6b..0000000 --- a/nisse-dev/tests/test_pr_open_handling.py +++ /dev/null @@ -1,250 +0,0 @@ -"""Gitea threads across restarts and interruptions (#58, #59). - -Uses the real CommentPoller against a fake Gitea, with its thread replaced -by explicit polls, so the cursor logic under test is the real one. -""" - -import io -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from nisse_dev.control import ( - AWAIT_ARCH_SIGNOFF, - AWAIT_INPUT, - AWAIT_MILESTONE_SIGNOFF, - MILESTONE_PLANNING, - PR_OPEN, - Orchestrator, - StubAgent, -) -from nisse_dev.executor import ExecutorOutcome -from nisse_dev.gitea import Comment, PullRequest -from nisse_dev.oplog import OpLog -from nisse_dev.poller import CommentPoller -from nisse_dev.publish import PublishResult -from nisse_dev.state import StateStore -from nisse_dev.transport import InMemoryTransport - -AGENT = "nisse-bot" -PLAN_ISSUE = 10 -PR = 20 - - -class FakeGitea: - """Comment threads by issue number; IDs grow across all threads.""" - - def __init__(self) -> None: - self.threads: dict[int, list[Comment]] = {} - self._next_id = 100 - - def add(self, index: int, author: str, body: str) -> int: - self._next_id += 1 - self.threads.setdefault(index, []).append( - Comment(id=self._next_id, author=author, body=body, created_at="t") - ) - return self._next_id - - def list_comments(self, repo: str, index: int, since: str | None = None) -> list: - return list(self.threads.get(index, [])) - - -@dataclass -class Posted: - comment_id: int - - -class Board: - def __init__(self, gitea: FakeGitea) -> None: - self._gitea = gitea - - def open_project_issue(self, project: str, task_id: str) -> int: - return PLAN_ISSUE - - def post_plan(self, issue: int, gate: str, rev: int, plan: str) -> Posted: - return Posted(self._gitea.add(issue, AGENT, f"{gate} rev-{rev}: {plan}")) - - def echo(self, issue: int, text: str) -> None: - self._gitea.add(issue, AGENT, text) - - -class Executor: - def __init__(self) -> None: - self.runs = 0 - self.outcome = ExecutorOutcome("completed", "done") - - def run(self, transcript: object, milestone: int) -> ExecutorOutcome: - self.runs += 1 - return self.outcome - - -class Publisher: - def __init__(self) -> None: - self.followups = 0 - - def reconcilers(self) -> dict[str, Any]: - return {} - - def publish(self, **kwargs: Any) -> PublishResult: - return PublishResult( - PullRequest(number=PR, state="open", head="h", url="u"), f"opened PR #{PR}" - ) - - def push_followup(self, **kwargs: Any) -> str | None: - self.followups += 1 - return "abcdef1234" - - def comment(self, index: int, body: str) -> None: - pass - - -class Agent: - """One orchestrator process: a restart builds a new one on the same - store and the same Gitea.""" - - def __init__(self, tmp_path: Path, gitea: FakeGitea, executor: Executor) -> None: - self.store = StateStore( - tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()) - ) - self.transport = InMemoryTransport() - self.pollers: dict[int, CommentPoller] = {} - self.publisher = Publisher() - - def factory(handler: Any) -> CommentPoller: - poller = CommentPoller(gitea, "o/r", handler, ignore_authors=[AGENT]) - agent = self - - def start(index: int) -> None: # no thread: tests poll explicitly - agent.pollers[index] = poller - - poller.start = start # type: ignore[method-assign] - poller.stop = lambda: None # type: ignore[method-assign] - return poller - - self.orch = Orchestrator( - self.store, - self.transport, - StubAgent(), - operators={"don"}, - gitea_operators={"don"}, - board=Board(gitea), - executor=executor, - publisher=self.publisher, - poller_factory=factory, - ) - self.orch.start() - - @property - def state(self) -> str: - state = self.store.get_state() - assert state is not None - return state.state - - def poll(self, index: int) -> list[Comment]: - return self.pollers[index].poll_once(index) - - -def to_pr_open(agent: Agent) -> None: - for reply in ("use demo", "approve", "approve", "start milestone 1"): - agent.transport.receive("don", reply) - assert agent.state == PR_OPEN - - -def test_an_approval_posted_while_down_is_applied_after_a_restart( - tmp_path: Path, -) -> None: - gitea = FakeGitea() - agent = Agent(tmp_path, gitea, Executor()) - agent.transport.receive("don", "use demo") - assert agent.state == AWAIT_ARCH_SIGNOFF - - gitea.add(PLAN_ISSUE, "don", "approve") # while the agent is down - agent = Agent(tmp_path, gitea, Executor()) - [delivered] = agent.poll(PLAN_ISSUE) - assert delivered.body == "approve" - assert agent.state == AWAIT_MILESTONE_SIGNOFF - - -def test_a_handled_approval_is_not_replayed_after_a_restart(tmp_path: Path) -> None: - gitea = FakeGitea() - agent = Agent(tmp_path, gitea, Executor()) - agent.transport.receive("don", "use demo") - gitea.add(PLAN_ISSUE, "don", "approve") - agent.poll(PLAN_ISSUE) # arch approved; the milestone plan is posted - assert agent.state == AWAIT_MILESTONE_SIGNOFF - - agent = Agent(tmp_path, gitea, Executor()) - assert agent.poll(PLAN_ISSUE) == [] - assert agent.state == AWAIT_MILESTONE_SIGNOFF - - -def test_a_new_revision_ignores_an_approval_posted_before_it(tmp_path: Path) -> None: - gitea = FakeGitea() - agent = Agent(tmp_path, gitea, Executor()) - agent.transport.receive("don", "use demo") - agent.transport.receive("don", "needs tests") # rejects: rev-2 is posted - assert agent.state == AWAIT_ARCH_SIGNOFF - stale = gitea.add(PLAN_ISSUE, "don", "approve") - agent.transport.receive("don", "and docs") # rev-3 posted after it - assert all(c.id != stale for c in agent.poll(PLAN_ISSUE)) - assert agent.state == AWAIT_ARCH_SIGNOFF - - -def test_data_from_before_cursors_skips_old_comments_on_restart( - tmp_path: Path, -) -> None: - gitea = FakeGitea() - agent = Agent(tmp_path, gitea, Executor()) - agent.transport.receive("don", "use demo") - gitea.add(PLAN_ISSUE, "don", "approve") # an old approval, say of rev-1 - state = agent.store.get_state() - state.data["board"].pop("last_comment") # as saved before this change - agent.store.transition( - state.state, cause="test", project=state.project, data=state.data - ) - agent = Agent(tmp_path, gitea, Executor()) - assert agent.poll(PLAN_ISSUE) == [] - assert agent.state == AWAIT_ARCH_SIGNOFF - - -def test_a_restart_in_pr_open_resumes_and_catches_up_on_review( - tmp_path: Path, -) -> None: - gitea = FakeGitea() - agent = Agent(tmp_path, gitea, Executor()) - to_pr_open(agent) - - gitea.add(PR, "don", "please add a test") # while the agent is down - agent = Agent(tmp_path, gitea, Executor()) - assert agent.state == PR_OPEN # no restart question - assert "restarted with PR #20 open" in agent.transport.outbound[-1].text - [delivered] = agent.poll(PR) - assert delivered.body == "please add a test" - assert agent.publisher.followups == 1 - assert agent.state == PR_OPEN - - -def test_a_failed_review_response_returns_to_the_open_pr(tmp_path: Path) -> None: - gitea = FakeGitea() - executor = Executor() - agent = Agent(tmp_path, gitea, executor) - to_pr_open(agent) - - executor.outcome = ExecutorOutcome("escalate", "stuck") - gitea.add(PR, "don", "please add a test") - agent.poll(PR) - assert agent.state == AWAIT_INPUT - - # A comment while it waits is not lost: back in PR_OPEN it is delivered. - executor.outcome = ExecutorOutcome("completed", "done") - gitea.add(PR, "don", "try again") - agent.poll(PR) - assert agent.state == AWAIT_INPUT - agent.transport.receive("don", "ok") - assert agent.state == PR_OPEN - [delivered] = agent.poll(PR) - assert delivered.body == "try again" - assert agent.publisher.followups == 1 - - agent.transport.receive("don", "next milestone") - assert agent.state in (MILESTONE_PLANNING, AWAIT_MILESTONE_SIGNOFF) diff --git a/nisse-dev/tests/test_publish.py b/nisse-dev/tests/test_publish.py deleted file mode 100644 index 234158d..0000000 --- a/nisse-dev/tests/test_publish.py +++ /dev/null @@ -1,289 +0,0 @@ -import io -import subprocess -from pathlib import Path -from typing import Any - -import pytest - -from nisse_dev.execution import SSHHost -from nisse_dev.git import GitRepo, HTTPSToken -from nisse_dev.gitea import GiteaError, PullRequest -from nisse_dev.oplog import OpLog -from nisse_dev.publish import OPEN_PR_INTENT, MilestonePublisher -from nisse_dev.state import StateStore - -# The build VM for every test here: the session's throwaway sshd. -HOST: SSHHost - - -@pytest.fixture(autouse=True) -def _build_host(ssh_host: SSHHost) -> None: - global HOST - HOST = ssh_host - - -class FakeGitea: - def __init__(self, *, fail: bool = False) -> None: - self.opened: list[dict[str, Any]] = [] - self.existing: list[str] = [] - self.closed: set[str] = set() - self._fail = fail - - def open_pull_request( - self, repo: str, *, head: str, base: str, title: str, body: str - ) -> PullRequest: - if self._fail: - raise GiteaError("gitea is down") - self.opened.append( - {"repo": repo, "head": head, "base": base, "title": title, "body": body} - ) - self.existing.append(head) - return PullRequest( - number=len(self.opened), state="open", head=head, url=f"u/{head}" - ) - - def find_pull_request(self, repo: str, head: str) -> PullRequest | None: - if head in self.existing: - state = "closed" if head in self.closed else "open" - return PullRequest(number=1, state=state, head=head, url="u") - return None - - -class FakeGit: - """A real repository whose origin/main is a local ref; pushes are - recorded, not sent.""" - - def __init__( - self, root: Path, *, changes: bool = True, committed: bool = False - ) -> None: - self.repo = GitRepo(HOST, root) - self.repo.init() - self.repo.configure_identity("bot", "bot@example.test") - (root / "README.md").write_text("sandbox") - self.repo.commit_all("Initial commit") - git(root, "update-ref", "refs/remotes/origin/main", "HEAD") - if changes: - (root / "work.txt").write_text("done") - if committed: - # The executor committing its own work, as on the first live run. - self.repo.commit_all("Add the work") - self.pushed: list[str] = [] - self.remote: dict[str, str] = {} - - def head_sha(self) -> str: - return self.repo.head_sha() - - def commits_since(self, ref: str) -> int: - return self.repo.commits_since(ref) - - def remote_sha( - self, remote_url: str, branch: str, *, credential: object = None - ) -> str | None: - return self.remote.get(branch) - - def current_branch(self) -> str: - return self.repo.current_branch() - - def checkout_new_branch(self, name: str) -> None: - self.repo.checkout_new_branch(name) - - def commit_all(self, message: str) -> str | None: - return self.repo.commit_all(message) - - def left_out(self) -> list[str]: - return self.repo.left_out() - - def push(self, remote_url: str, branch: str, *, credential: object = None) -> None: - self.pushed.append(branch) - self.remote[branch] = self.repo.head_sha() - - -def git(root: Path, *args: str) -> None: - subprocess.run(["git", *args], cwd=root, check=True, capture_output=True) - - -@pytest.fixture -def store(tmp_path: Path) -> StateStore: - return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - - -def build(store: StateStore, git: Any, client: Any) -> MilestonePublisher: - return MilestonePublisher( - store, - client, - "o/sandbox", - git, - remote_url="ssh://git@gitea.test/o/sandbox.git", - push_credential=HTTPSToken("pusher", "tok"), - ) - - -def test_publish_commits_pushes_and_opens_a_pr( - store: StateStore, tmp_path: Path -) -> None: - git = FakeGit(tmp_path / "work") - client = FakeGitea() - result = build(store, git, client).publish( - task_id="demo-abc", milestone=2, project="demo", plan="the plan" - ) - - assert git.pushed == ["demo-abc-milestone-2"] - assert client.opened[0]["head"] == "demo-abc-milestone-2" - assert client.opened[0]["base"] == "main" - assert "the plan" in client.opened[0]["body"] - assert result.pull_request is not None - assert "opened PR #1" in result.note - - -def test_a_branch_named_after_the_task_does_not_block_publishing( - store: StateStore, tmp_path: Path -) -> None: - # Live run 3: the executor ran `git checkout -b ` itself. With - # "/milestone-1" as the harness's branch, git refused it, since a - # branch name can't also be a directory of other branch names. - git = FakeGit(tmp_path / "work") - git.repo.checkout_new_branch("demo-abc") - result = build(store, git, FakeGitea()).publish( - task_id="demo-abc", milestone=1, project="demo", plan="p" - ) - assert result.pull_request is not None - assert git.pushed == ["demo-abc-milestone-1"] - - -def test_left_out_artifacts_are_reported_not_published( - store: StateStore, tmp_path: Path -) -> None: - git = FakeGit(tmp_path / "work") - (tmp_path / "work" / "__pycache__").mkdir() - (tmp_path / "work" / "__pycache__" / "work.cpython-310.pyc").write_bytes(b"x") - client = FakeGitea() - result = build(store, git, client).publish( - task_id="t", milestone=1, project="demo", plan="p" - ) - assert "left out 1 build artifact(s): __pycache__/work.cpython-310.pyc" in ( - result.note - ) - assert "## Left out" in client.opened[0]["body"] - assert "`__pycache__/work.cpython-310.pyc`" in client.opened[0]["body"] - - -def test_intent_is_recorded_then_committed(store: StateStore, tmp_path: Path) -> None: - build(store, FakeGit(tmp_path / "work"), FakeGitea()).publish( - task_id="t", milestone=1, project="demo", plan="p" - ) - # Nothing left pending: the intent opened before the push was settled - # once the PR existed. - assert store.pending_intents() == [] - - -def test_failure_after_intent_leaves_it_pending_for_reconciliation( - store: StateStore, tmp_path: Path -) -> None: - publisher = build(store, FakeGit(tmp_path / "work"), FakeGitea(fail=True)) - with pytest.raises(GiteaError): - publisher.publish(task_id="t", milestone=1, project="demo", plan="p") - - [pending] = store.pending_intents() - assert pending.kind == OPEN_PR_INTENT - assert pending.payload["branch"] == "t-milestone-1" - assert pending.payload["sha"] - - -def test_nothing_to_commit_is_reported_not_raised( - store: StateStore, tmp_path: Path -) -> None: - git = FakeGit(tmp_path / "work", changes=False) - client = FakeGitea() - result = build(store, git, client).publish( - task_id="t", milestone=1, project="demo", plan="p" - ) - assert result.pull_request is None - assert result.note == "no changes to publish" - assert client.opened == [] - # No externally visible act was attempted, so no intent was opened. - assert store.pending_intents() == [] - - -def test_branch_is_namespaced_by_task(store: StateStore, tmp_path: Path) -> None: - publisher = build(store, FakeGit(tmp_path / "work"), FakeGitea()) - assert publisher.branch_for("greet-9f2", 3) == "greet-9f2-milestone-3" - - -def test_reconciler_settles_an_intent_left_by_a_crash( - store: StateStore, tmp_path: Path -) -> None: - # Simulate the dangerous window: the PR was opened, but the process died - # before the intent could be committed. On the next boot the reconciler - # must say "already done" so the milestone is not published twice. - client = FakeGitea() - publisher = build(store, FakeGit(tmp_path / "work"), client) - intent_id = store.begin_intent( - OPEN_PR_INTENT, {"branch": "t-milestone-1"}, cause="crash simulation" - ) - client.existing.append("t-milestone-1") - - reconcilers = publisher.reconcilers() - assert set(reconcilers) == {OPEN_PR_INTENT} - [pending] = store.pending_intents() - assert reconcilers[OPEN_PR_INTENT](pending) is True - - store.commit_intent(intent_id, cause="verified") - assert client.opened == [] # nothing re-published - - -def test_work_the_executor_committed_itself_is_published( - store: StateStore, tmp_path: Path -) -> None: - git = FakeGit(tmp_path / "work", committed=True) - client = FakeGitea() - result = build(store, git, client).publish( - task_id="t", milestone=1, project="demo", plan="p" - ) - assert result.pull_request is not None - assert git.pushed == ["t-milestone-1"] - assert git.remote["t-milestone-1"] == git.head_sha() - - -def test_a_followup_the_executor_committed_itself_is_pushed( - store: StateStore, tmp_path: Path -) -> None: - git = FakeGit(tmp_path / "work") - publisher = build(store, git, FakeGitea()) - publisher.publish(task_id="t", milestone=1, project="demo", plan="p") - (tmp_path / "work" / "work.txt").write_text("fixed") - git.repo.commit_all("Address review") - sha = publisher.push_followup(task_id="t", milestone=1, message="m") - assert sha == git.head_sha() - assert git.pushed == ["t-milestone-1", "t-milestone-1"] - # Nothing new since that push: no second push, and it says so. - assert publisher.push_followup(task_id="t", milestone=1, message="m") is None - assert len(git.pushed) == 2 - - -def test_publishing_again_adopts_the_open_pr_instead_of_opening_another( - store: StateStore, tmp_path: Path -) -> None: - # E.g. Gitea opened the PR but the response was lost, and the milestone - # was published again: the PR is adopted and updated, never duplicated. - git = FakeGit(tmp_path / "work") - client = FakeGitea() - publisher = build(store, git, client) - publisher.publish(task_id="t", milestone=1, project="demo", plan="p") - (tmp_path / "work" / "more.txt").write_text("more") - result = publisher.publish(task_id="t", milestone=1, project="demo", plan="p") - assert len(client.opened) == 1 - assert result.pull_request is not None and result.pull_request.number == 1 - assert "already open" in result.note - assert git.pushed == ["t-milestone-1", "t-milestone-1"] # updated it - assert store.pending_intents() == [] - - -def test_a_closed_pr_is_not_adopted(store: StateStore, tmp_path: Path) -> None: - git = FakeGit(tmp_path / "work") - client = FakeGitea() - client.existing.append("t-milestone-1") - client.closed.add("t-milestone-1") - publisher = build(store, git, client) - assert publisher.published_pull_request("t", 1) is None - publisher.publish(task_id="t", milestone=1, project="demo", plan="p") - assert len(client.opened) == 1 # the operator closed it; a new one opens diff --git a/nisse-dev/tests/test_quint_counterexamples.py b/nisse-dev/tests/test_quint_counterexamples.py deleted file mode 100644 index 59eb697..0000000 --- a/nisse-dev/tests/test_quint_counterexamples.py +++ /dev/null @@ -1,400 +0,0 @@ -"""The counterexamples found by specs/orchestrator.qnt, replayed against the -real orchestrator. Each names the scenario in specs/orchestrator_scenarios.qnt -it reproduces; all seven are fixed, and the tests stay as regressions. - -The model is a hand-written reading of control.py; reproducing each trace -here is what shows the flaw is in the design, not in the model. -""" - -import io -from pathlib import Path -from typing import Any - -import pytest - -from nisse_dev.control import ( - AWAIT_INPUT, - AWAIT_MILESTONE_SIGNOFF, - EXECUTING, - IDLE, - MILESTONE_PLANNING, - PR_OPEN, - Orchestrator, - StubAgent, -) -from nisse_dev.executor import ExecutorOutcome -from nisse_dev.gitea import Comment, GiteaError, PullRequest -from nisse_dev.oplog import OpLog -from nisse_dev.poller import CommentPoller -from nisse_dev.publish import PublishResult -from nisse_dev.state import StateStore -from nisse_dev.transport import InMemoryTransport -from nisse_dev.worker import SerialWorker - -AGENT = "nisse-bot" - - -class QueueingWorker(SerialWorker): - """Holds submitted jobs until released, so a test can queue two - messages before either runs, as a real worker does under load.""" - - def __init__(self) -> None: - super().__init__() - self.held: list[Any] = [] - self.holding = True - - def submit(self, job: Any) -> None: - if self.holding: - self.held.append(job) - else: - job() - - def call(self, job: Any, timeout: float | None = None) -> Any: - return job() - - @property - def busy(self) -> bool: - return False - - def release(self) -> None: - self.holding = False - jobs, self.held = self.held, [] - for job in jobs: - job() - - -class FakeGitea: - def __init__(self) -> None: - self.threads: dict[int, list[Comment]] = {} - self.prs: set[str] = set() - self._next = 100 - - def add(self, index: int, author: str, body: str) -> int: - self._next += 1 - self.threads.setdefault(index, []).append( - Comment(id=self._next, author=author, body=body, created_at="t") - ) - return self._next - - def list_comments(self, repo: str, index: int, since: str | None = None) -> list: - return list(self.threads.get(index, [])) - - -class Board: - def __init__(self, gitea: FakeGitea) -> None: - self._gitea = gitea - - def open_project_issue(self, project: str, task_id: str) -> int: - return 10 - - def post_plan(self, issue: int, gate: str, rev: int, plan: str) -> Any: - class Posted: - comment_id = self._gitea.add(issue, AGENT, f"{gate} rev-{rev}") - - return Posted() - - def echo(self, issue: int, text: str) -> None: - pass - - def close(self, issue: int, note: str) -> None: - pass - - -class Executor: - def __init__(self) -> None: - self.outcome = ExecutorOutcome("completed", "done") - - def run(self, transcript: object, milestone: int) -> ExecutorOutcome: - return self.outcome - - -class Publisher: - """Opens one PR per branch, as Gitea does; a second attempt fails.""" - - def __init__(self, gitea: FakeGitea, store: StateStore) -> None: - self._gitea = gitea - self._store = store - self.crash_after_opening = False - self.attempts = 0 - - def reconcilers(self) -> dict[str, Any]: - return {"open_pr": lambda intent: intent.payload["branch"] in self._gitea.prs} - - def publish(self, **kwargs: Any) -> PublishResult: - branch = f"{kwargs['task_id']}-milestone-{kwargs['milestone']}" - self.attempts += 1 - intent = self._store.begin_intent("open_pr", {"branch": branch}, cause="t") - if branch in self._gitea.prs: - raise GiteaError("pull request already exists for these targets") - self._gitea.prs.add(branch) - if self.crash_after_opening: - raise SystemExit("crash") # the process dies here - self._store.commit_intent(intent, cause="opened") - return PublishResult( - PullRequest(number=20, state="open", head=branch, url="u"), "opened PR #20" - ) - - def published_pull_request(self, task_id: str, milestone: int) -> Any: - branch = f"{task_id}-milestone-{milestone}" - if branch in self._gitea.prs: - return PullRequest(number=20, state="open", head=branch, url="u") - return None - - def push_followup(self, **kwargs: Any) -> str | None: - return "abc12345" - - def comment(self, index: int, body: str) -> None: - pass - - -class Agent: - def __init__(self, tmp_path: Path, gitea: FakeGitea, *, worker: Any = None) -> None: - self.store = StateStore( - tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()) - ) - self.transport = InMemoryTransport() - self.gitea = gitea - self.executor = Executor() - self.publisher = Publisher(gitea, self.store) - self.pollers: dict[int, CommentPoller] = {} - - def factory(handler: Any) -> CommentPoller: - poller = CommentPoller(gitea, "o/r", handler, ignore_authors=[AGENT]) - - def start(index: int) -> None: - self.pollers[index] = poller - - poller.start = start # type: ignore[method-assign] - poller.stop = lambda: None # type: ignore[method-assign] - return poller - - self.orch = Orchestrator( - self.store, - self.transport, - StubAgent(), - operators={"don"}, - gitea_operators={"don"}, - board=Board(gitea), - executor=self.executor, - publisher=self.publisher, - poller_factory=factory, - **({"worker": worker} if worker else {}), - ) - self.orch.start() - - @property - def state(self) -> str: - return self.store.get_state().state - - def gate(self) -> dict: - return self.store.get_state().data.get("gate") or {} - - def say(self, *texts: str) -> None: - for text in texts: - self.transport.receive("don", text) - - -def test_double_approve_approves_an_unseen_plan(tmp_path: Path) -> None: - """doubleApproveApprovesAnUnseenPlanTest""" - worker = QueueingWorker() - agent = Agent(tmp_path, FakeGitea(), worker=worker) - worker.holding = False - agent.say("use demo") # arch rev-1 is up - worker.holding = True - agent.say("approve", "approve") # both queued, looking at arch rev-1 - worker.release() - # The second approval must not approve the milestone plan, never seen. - assert not agent.gate().get("approved") - - -def test_named_revision_approves_the_wrong_gate(tmp_path: Path) -> None: - """namedRevisionApprovesTheWrongGateTest""" - worker = QueueingWorker() - agent = Agent(tmp_path, FakeGitea(), worker=worker) - worker.holding = False - agent.say("use demo") - worker.holding = True - agent.say("approve", "approve rev-1") # "rev-1" meant the architecture - worker.release() - assert not agent.gate().get("approved") - - -def test_approval_on_both_channels_approves_the_next_plan(tmp_path: Path) -> None: - """approvalOnBothChannelsApprovesTheNextPlanTest""" - gitea = FakeGitea() - worker = QueueingWorker() - agent = Agent(tmp_path, gitea, worker=worker) - worker.holding = False - agent.say("use demo") - worker.holding = True - agent.say("approve") # Matrix: queued - gitea.add(10, "don", "approve") # and on the issue, about the same plan - # The poller delivers it; it waits behind the Matrix approval. - delivered = agent.pollers[10] - worker.held.append(lambda: delivered.poll_once(10)) - worker.release() - assert agent.state == AWAIT_MILESTONE_SIGNOFF - assert not agent.gate().get("approved") - - -def test_an_afterthought_rejects_the_next_plan(tmp_path: Path) -> None: - """anAfterthoughtRejectsTheNextPlanTest""" - worker = QueueingWorker() - agent = Agent(tmp_path, FakeGitea(), worker=worker) - worker.holding = False - agent.say("use demo") - worker.holding = True - agent.say("approve", "also, keep it in one file") - worker.release() - # The note was about the architecture; the milestone plan was not - # rejected by it. - assert agent.state == AWAIT_MILESTONE_SIGNOFF - assert agent.gate().get("rev") == 2 # arch was rev-1; not re-planned - - -def test_crash_after_opening_the_pr_publishes_again(tmp_path: Path) -> None: - """crashAfterOpeningThePrPublishesAgainTest""" - gitea = FakeGitea() - agent = Agent(tmp_path, gitea) - agent.say("use demo", "approve", "approve") - agent.publisher.crash_after_opening = True - with pytest.raises(SystemExit): - agent.say("start milestone 1") - assert agent.state == EXECUTING # the PR exists; PR_OPEN was never recorded - - agent = Agent(tmp_path, gitea) # restart: the intent is verified - # There is a PR for this milestone: the agent adopts it on boot and ends - # up watching it, never trying to open another. - assert agent.state == PR_OPEN - assert "PR #20 is open" in agent.transport.outbound[-1].text - agent.say("continue", "start milestone 1") # no longer means anything - assert agent.publisher.attempts == 0 - assert agent.state == PR_OPEN - - -def test_crash_during_a_review_loses_the_comment(tmp_path: Path) -> None: - """crashDuringAReviewLosesTheCommentTest""" - gitea = FakeGitea() - agent = Agent(tmp_path, gitea) - agent.say("use demo", "approve", "approve", "start milestone 1") - assert agent.state == PR_OPEN - gitea.add(20, "don", "please add a test") - - class Crash(BaseException): - pass - - def dies(transcript: object, milestone: int) -> ExecutorOutcome: - raise Crash() # the process dies mid-review - - agent.executor.run = dies # type: ignore[method-assign] - with pytest.raises(Crash): - agent.pollers[20].poll_once(20) - - agent = Agent(tmp_path, gitea) # restart - assert agent.state == PR_OPEN - # The comment was taken but never answered: after the restart the - # operator is told, and it is delivered and answered again. - assert "answering it again" in agent.transport.outbound[-1].text - [delivered] = agent.pollers[20].poll_once(20) - assert delivered.body == "please add a test" - assert "pushed" in agent.transport.outbound[-1].text - assert "reviewing" not in agent.store.get_state().data["pr"] - - -def test_project_complete_leaves_the_plan_issue_open(tmp_path: Path) -> None: - """projectCompleteLeavesThePlanIssueOpenTest""" - closed: list[int] = [] - gitea = FakeGitea() - agent = Agent(tmp_path, gitea) - agent.orch._board.close = lambda issue, note: closed.append(issue) # type: ignore[attr-defined] - agent.say("use demo", "approve", "approve", "start milestone 1", "project complete") - assert agent.state == IDLE - assert closed == [10] - - -def test_the_harness_reaches_pr_open_without_faults(tmp_path: Path) -> None: - """Guard for the harness itself: with nothing queued or crashed, the - same flow is correct.""" - agent = Agent(tmp_path, FakeGitea()) - agent.say("use demo", "approve") - assert agent.state == AWAIT_MILESTONE_SIGNOFF - agent.say("approve", "start milestone 1") - assert agent.state == PR_OPEN - assert agent.publisher.attempts == 1 - _ = (AWAIT_INPUT, MILESTONE_PLANNING) - - -def test_boot_asks_as_before_when_gitea_cannot_say( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - gitea = FakeGitea() - agent = Agent(tmp_path, gitea) - agent.say("use demo", "approve", "approve") - agent.publisher.crash_after_opening = True - with pytest.raises(SystemExit): - agent.say("start milestone 1") - - def unreachable(self: Publisher, task_id: str, milestone: int) -> Any: - raise GiteaError("gitea is down") - - monkeypatch.setattr(Publisher, "published_pull_request", unreachable) - agent = Agent(tmp_path, gitea) - assert agent.state == AWAIT_INPUT - assert "restarted mid-EXECUTING" in agent.transport.outbound[-1].text - - -class Crash(BaseException): - """The process dying: no handler runs.""" - - -def crash_mid_review(agent: Agent, gitea: FakeGitea) -> None: - def dies(transcript: object, milestone: int) -> ExecutorOutcome: - raise Crash() - - agent.executor.run = dies # type: ignore[method-assign] - with pytest.raises(Crash): - agent.pollers[20].poll_once(20) - - -def test_a_second_crash_while_answering_again_still_redelivers(tmp_path: Path) -> None: - gitea = FakeGitea() - agent = Agent(tmp_path, gitea) - agent.say("use demo", "approve", "approve", "start milestone 1") - gitea.add(20, "don", "please add a test") - crash_mid_review(agent, gitea) - agent = Agent(tmp_path, gitea) - crash_mid_review(agent, gitea) # dies again while answering it again - agent = Agent(tmp_path, gitea) - [delivered] = agent.pollers[20].poll_once(20) - assert delivered.body == "please add a test" - - -def test_an_executor_error_answers_the_comment_so_it_is_not_redelivered( - tmp_path: Path, -) -> None: - gitea = FakeGitea() - agent = Agent(tmp_path, gitea) - agent.say("use demo", "approve", "approve", "start milestone 1") - gitea.add(20, "don", "please add a test") - - def fails(transcript: object, milestone: int) -> ExecutorOutcome: - raise RuntimeError("bug in the executor") - - agent.executor.run = fails # type: ignore[method-assign] - agent.pollers[20].poll_once(20) - assert "could not handle a review comment" in agent.transport.outbound[-1].text - agent = Agent(tmp_path, gitea) # restart: nothing to answer again - assert "still watching" in agent.transport.outbound[-1].text - assert agent.pollers[20].poll_once(20) == [] - - -def test_project_complete_still_completes_when_gitea_is_down(tmp_path: Path) -> None: - agent = Agent(tmp_path, FakeGitea()) - - def down(issue: int, note: str) -> None: - raise GiteaError("gitea is down") - - agent.orch._board.close = down # type: ignore[attr-defined] - agent.say("use demo", "approve", "approve", "start milestone 1", "project complete") - assert agent.state == IDLE - assert "could not close plan issue #10" in agent.transport.outbound[-1].text diff --git a/nisse-dev/tests/test_review_loop.py b/nisse-dev/tests/test_review_loop.py deleted file mode 100644 index 5bedce9..0000000 --- a/nisse-dev/tests/test_review_loop.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Reacting to PR review comments while in PR_OPEN (architecture D5).""" - -import io -from pathlib import Path -from typing import Any - -from nisse_dev.canonical import CanonicalEvent, StoredEvent, ToolCall -from nisse_dev.control import AWAIT_INPUT, PR_OPEN, Orchestrator -from nisse_dev.executor import ExecutorOutcome -from nisse_dev.gitea import Comment, GiteaError, PullRequest -from nisse_dev.llm import RoleMap -from nisse_dev.oplog import OpLog -from nisse_dev.planner import LLMPlanningAgent -from nisse_dev.publish import PublishResult -from nisse_dev.state import StateStore -from nisse_dev.transport import InMemoryTransport - -ROLES = RoleMap(executor="x", planner="y", helper="y", expert="y") - - -class ScriptedClient: - def __init__(self) -> None: - self.calls = 0 - - def complete(self, events, tools, model, system=None) -> list[CanonicalEvent]: - self.calls += 1 - return [ - ToolCall( - call_id=f"c{self.calls}", - tool="submit_plan", - arguments={"type": "x", "content": f"plan {self.calls}"}, - ) - ] - - -class RecordingExecutor: - def __init__(self, outcomes: list[str] | None = None) -> None: - self.runs: list[int] = [] - self.seen_events: list[list[StoredEvent]] = [] - # One entry per run, so a test can let the milestone succeed and - # only the review response fail. - self._outcomes = outcomes or [] - self.store: StateStore | None = None - - def run(self, transcript: Any, milestone: int) -> ExecutorOutcome: - self.runs.append(milestone) - # What it is given: the work item's transcript, nothing wider. - self.seen_events.append(transcript.read()) - status = self._outcomes.pop(0) if self._outcomes else "completed" - return ExecutorOutcome(status, f"executor said {status}") - - -class RecordingPublisher: - def __init__(self, *, sha: str | None = "abc12345", error: Exception | None = None): - self._sha = sha - self._error = error - self.followups: list[dict[str, Any]] = [] - self.comments: list[tuple[int, str]] = [] - - def reconcilers(self) -> dict[str, Any]: - return {} - - def publish(self, **kwargs: Any) -> PublishResult: - return PublishResult( - PullRequest(number=42, state="open", head="h", url="u"), "opened PR #42: u" - ) - - def push_followup(self, **kwargs: Any) -> str | None: - self.followups.append(kwargs) - if self._error is not None: - raise self._error - return self._sha - - def comment(self, index: int, body: str) -> None: - self.comments.append((index, body)) - - -def build( - tmp_path: Path, - executor: Any, - publisher: Any, - gitea_operators: set[str] | None = None, -) -> tuple[Orchestrator, StateStore, InMemoryTransport]: - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - if hasattr(executor, "store"): - executor.store = store - transport = InMemoryTransport() - orch = Orchestrator( - store, - transport, - LLMPlanningAgent(ScriptedClient(), ROLES), - operators={"don"}, - roles=ROLES, - executor=executor, - publisher=publisher, - gitea_operators=gitea_operators if gitea_operators is not None else {"don"}, - ) - orch.start() - transport.receive("don", "use demo") - transport.receive("don", "approve") - transport.receive("don", "approve") - transport.receive("don", "start milestone 1") - return orch, store, transport - - -def review( - cid: int = 5, author: str = "don", body: str = "rename the module" -) -> Comment: - return Comment(id=cid, author=author, body=body, created_at="t") - - -def test_review_comment_triggers_a_fix_and_a_push(tmp_path: Path) -> None: - executor, publisher = RecordingExecutor(), RecordingPublisher() - orch, store, transport = build(tmp_path, executor, publisher) - assert store.get_state().state == PR_OPEN - - orch.handle_review_comment(review()) - - assert executor.runs == [1, 1] # milestone run, then the review response - assert publisher.followups[0]["milestone"] == 1 - assert "abc12345" in transport.outbound[-1].text - assert publisher.comments[0][0] == 42 # replied on the PR - - -def test_comment_enters_the_transcript_before_the_executor_runs( - tmp_path: Path, -) -> None: - executor, publisher = RecordingExecutor(), RecordingPublisher() - orch, _, _ = build(tmp_path, executor, publisher) - orch.handle_review_comment(review(body="please add a test")) - - # The executor must see the feedback as conversation, not as an - # out-of-band instruction it was never shown. - texts = [getattr(e.event, "text", "") for e in executor.seen_events[-1]] - assert any("please add a test" in t for t in texts) - - -def test_non_operator_comments_are_ignored(tmp_path: Path) -> None: - executor, publisher = RecordingExecutor(), RecordingPublisher() - orch, store, _ = build(tmp_path, executor, publisher) - orch.handle_review_comment(review(author="drive-by")) - - assert executor.runs == [1] # only the original milestone run - assert publisher.followups == [] - assert store.get_state().state == PR_OPEN - - -def test_comments_outside_pr_open_are_ignored(tmp_path: Path) -> None: - executor, publisher = RecordingExecutor(), RecordingPublisher() - orch, _store, transport = build(tmp_path, executor, publisher) - transport.receive("don", "project complete") # back to IDLE - - orch.handle_review_comment(review()) - assert executor.runs == [1] - - -def test_failed_review_response_gates_to_the_operator(tmp_path: Path) -> None: - # Milestone succeeds; the response to the review comment does not. - executor = RecordingExecutor(outcomes=["completed", "escalate"]) - publisher = RecordingPublisher() - orch, store, transport = build(tmp_path, executor, publisher) - orch.handle_review_comment(review()) - - assert store.get_state().state == AWAIT_INPUT - assert "could not address the review comment" in transport.outbound[-1].text - assert publisher.followups == [] - - -def test_no_code_change_needed_is_reported(tmp_path: Path) -> None: - executor, publisher = RecordingExecutor(), RecordingPublisher(sha=None) - orch, _, transport = build(tmp_path, executor, publisher) - orch.handle_review_comment(review(body="just curious, why sqlite?")) - - assert "no code change was needed" in transport.outbound[-1].text - - -def test_push_failure_is_reported_without_losing_pr_open(tmp_path: Path) -> None: - executor = RecordingExecutor() - publisher = RecordingPublisher(error=GiteaError("push rejected")) - orch, store, transport = build(tmp_path, executor, publisher) - orch.handle_review_comment(review()) - - assert "could not push the fix" in transport.outbound[-1].text - assert store.get_state().state == PR_OPEN # still reviewable, can retry diff --git a/nisse-dev/tests/test_state.py b/nisse-dev/tests/test_state.py index 29a61c6..b392135 100644 --- a/nisse-dev/tests/test_state.py +++ b/nisse-dev/tests/test_state.py @@ -24,22 +24,25 @@ def test_fresh_store_stamps_schema_version(db: Path) -> None: assert store.schema_version == 1 -def test_transition_persists_across_reopen(db: Path) -> None: +def test_a_work_item_persists_across_reopen(db: Path) -> None: with StateStore(db) as store: - store.transition( - "EXECUTING", cause="test", project="demo", data={"milestone": 2} + store.open_work_item( + "demo-1", title="t", scope="demo", position="WAITING", cause="test" + ) + store.set_active("demo-1", cause="test") + store.update_work_item( + "demo-1", position="RUNNING", data={"milestone": 2}, cause="test" ) with StateStore(db) as store: - state = store.get_state() - assert state is not None - assert state.state == "EXECUTING" - assert state.project == "demo" - assert state.data == {"milestone": 2} + item = store.active_item() + assert item is not None + assert (item.id, item.scope, item.position) == ("demo-1", "demo", "RUNNING") + assert item.data == {"milestone": 2} -def test_get_state_on_fresh_store_is_none(db: Path) -> None: +def test_a_fresh_store_has_no_active_item(db: Path) -> None: with StateStore(db) as store: - assert store.get_state() is None + assert store.active_item() is None def test_intent_lifecycle(db: Path) -> None: diff --git a/nisse-dev/tests/test_stop.py b/nisse-dev/tests/test_stop.py deleted file mode 100644 index 97a25e4..0000000 --- a/nisse-dev/tests/test_stop.py +++ /dev/null @@ -1,365 +0,0 @@ -"""The operator's kill switch, `stop` (architecture L2).""" - -import io -import threading -from pathlib import Path -from typing import Any - -import pytest - -from nisse_dev.cancel import STOPPED_RESULT, Cancel -from nisse_dev.canonical import AssistantText, ToolCall, ToolResult, UserMessage -from nisse_dev.control import ( - AWAIT_ARCH_SIGNOFF, - AWAIT_INPUT, - AWAIT_MILESTONE_SIGNOFF, - Orchestrator, - PlanningStopped, - StubAgent, - SubmitPlan, -) -from nisse_dev.delegation import DelegationResult -from nisse_dev.executor import ExecutorOutcome, LLMExecutor -from nisse_dev.llm import RoleMap -from nisse_dev.oplog import OpLog -from nisse_dev.planner import LLMPlanningAgent -from nisse_dev.state import StateStore, Transcript -from nisse_dev.tools import ToolOutcome -from nisse_dev.transport import InMemoryTransport -from nisse_dev.worker import SerialWorker - -ROLES = RoleMap( - executor="llama3.2", - planner="claude-haiku-4-5", - helper="claude-haiku-4-5", - expert="claude-haiku-4-5", -) - - -def log(store: StateStore, actor: str = "executor") -> Transcript: - return store.transcript("t", actor) - - -def new_store(path: Path) -> StateStore: - return StateStore(path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - - -def user_messages(store: StateStore) -> list[str]: - return [ - e.event.text for e in store.all_events() if isinstance(e.event, UserMessage) - ] - - -# --- the orchestrator ------------------------------------------------------ - - -class OperatorDuringWork: - """An executor that hears from the operator mid-milestone, then does what - the real loop does at its next tool boundary.""" - - def __init__(self, cancel: Cancel, says: str) -> None: - self.cancel = cancel - self.says = says - self.transport: InMemoryTransport | None = None - - def run(self, transcript: object, milestone: int) -> ExecutorOutcome: - assert self.transport is not None - self.transport.receive("don", self.says) - if self.cancel.requested: - return ExecutorOutcome("stopped", "during turn 3") - return ExecutorOutcome("completed", "done") - - -def at_start_of_milestone( - tmp_path: Path, says: str -) -> tuple[StateStore, InMemoryTransport, Cancel]: - store = new_store(tmp_path) - transport = InMemoryTransport() - cancel = Cancel() - executor = OperatorDuringWork(cancel, says) - executor.transport = transport - orch = Orchestrator( - store, - transport, - StubAgent(), - operators={"don"}, - executor=executor, - cancel=cancel, - ) - orch.start() - for text in ("use demo", "approve", "approve"): - transport.receive("don", text) - return store, transport, cancel - - -def test_stop_halts_a_running_milestone_and_asks_why(tmp_path: Path) -> None: - store, transport, cancel = at_start_of_milestone(tmp_path, "stop") - transport.receive("don", "start milestone 1") - - assert store.get_state().state == AWAIT_INPUT - prompt = transport.outbound[-1] - assert prompt.kind == "question" - assert "stopped milestone 1 (during turn 3)" in prompt.text - assert "Why did you stop it?" in prompt.text - assert user_messages(store)[-1] == "stop" - assert not cancel.requested # the next piece of work is not pre-stopped - - -def test_the_reason_is_recorded_and_the_plan_stays_approved(tmp_path: Path) -> None: - store, transport, _ = at_start_of_milestone(tmp_path, "stop") - transport.receive("don", "start milestone 1") - transport.receive("don", "it was editing the wrong module") - - assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF - # The next run renders this, so the model hears why it was stopped. - assert user_messages(store)[-1] == "it was editing the wrong module" - assert "start milestone 1" in transport.outbound[-1].text - - -def test_stop_with_nothing_running_says_so_and_changes_nothing( - tmp_path: Path, -) -> None: - store = new_store(tmp_path) - transport = InMemoryTransport() - orch = Orchestrator(store, transport, StubAgent(), operators={"don"}) - orch.start() - transport.receive("don", "use demo") - transport.receive("don", "stop") - - # At a gate, "stop" is the command, not rejection feedback. - assert store.get_state().state == AWAIT_ARCH_SIGNOFF - assert orch._data["revs"]["arch"] == 1 - assert ( - transport.outbound[-1].text == f"nothing to stop (state={AWAIT_ARCH_SIGNOFF})" - ) - - -def test_stop_from_a_non_operator_is_ignored(tmp_path: Path) -> None: - _, transport, cancel = at_start_of_milestone(tmp_path, "approve") - before = len(transport.outbound) - transport.receive("mallory", "stop") - assert len(transport.outbound) == before - assert not cancel.requested - - -def test_other_messages_during_work_are_answered_not_acted_on( - tmp_path: Path, -) -> None: - store, transport, _ = at_start_of_milestone(tmp_path, "approve") - transport.receive("don", "start milestone 1") - - texts = [m.text for m in transport.outbound] - assert "busy (EXECUTING); say 'stop' to halt" in texts - # Recorded, as every operator message is, but not applied to anything. - assert user_messages(store)[-1] == "approve" - - -class StopsBeforePlanning: - def __init__(self, cancel: Cancel) -> None: - self.cancel = cancel - self.transport: InMemoryTransport | None = None - self.answers: list[str | None] = [] - - def plan( - self, - transcript: object, - gate: str, - project: str, - feedback: str | None = None, - answer: str | None = None, - ) -> SubmitPlan: - self.answers.append(answer) - if len(self.answers) == 1: - assert self.transport is not None - self.transport.receive("don", "stop") - if self.cancel.requested: - raise PlanningStopped("stopped by the operator") - return SubmitPlan(f"{gate} plan, after: {answer}") - - -def test_stop_during_planning_resumes_planning_with_the_reason( - tmp_path: Path, -) -> None: - store = new_store(tmp_path) - transport = InMemoryTransport() - cancel = Cancel() - planner = StopsBeforePlanning(cancel) - planner.transport = transport - orch = Orchestrator(store, transport, planner, operators={"don"}, cancel=cancel) - orch.start() - transport.receive("don", "use demo") - - assert store.get_state().state == AWAIT_INPUT - assert "stopped planning arch" in transport.outbound[-1].text - - transport.receive("don", "keep it to one module") - assert store.get_state().state == AWAIT_ARCH_SIGNOFF - assert planner.answers[-1] == "keep it to one module" - - -# --- on real threads ------------------------------------------------------- - - -class BlocksUntilStopped: - def __init__(self, cancel: Cancel) -> None: - self.cancel = cancel - self.running = threading.Event() - - def run(self, transcript: object, milestone: int) -> ExecutorOutcome: - self.running.set() - for _ in range(500): - if self.cancel.requested: - return ExecutorOutcome("stopped", "at a tool boundary") - threading.Event().wait(0.01) - return ExecutorOutcome("completed", "never stopped") - - -@pytest.mark.parametrize("word", ["stop", "Stop!", " STOP "]) -def test_stop_reaches_work_running_on_the_worker(tmp_path: Path, word: str) -> None: - # The bug this design fixes: the work used to run on the transport's own - # thread, so no message, stop included, was read until it finished. - worker = SerialWorker() - worker.start() - store = worker.call(lambda: new_store(tmp_path)) - transport = InMemoryTransport() - cancel = Cancel() - executor = BlocksUntilStopped(cancel) - orch = Orchestrator( - store, - transport, - StubAgent(), - operators={"don"}, - executor=executor, - worker=worker, - cancel=cancel, - ) - worker.call(orch.start) - for text in ("use demo", "approve", "approve"): - transport.receive("don", text) - # Reply only once the plan is up: a reply stamped with an earlier - # revision is not applied to a later one. - worker.call(lambda: None) - transport.receive("don", "start milestone 1") - assert executor.running.wait(5) - - transport.receive("don", "status") # delivered while the milestone runs - transport.receive("don", word) # a phone may capitalise it (#64) - worker.call(lambda: None) # wait for the stop to be recorded - - texts = [m.text for m in transport.outbound] - assert "busy (EXECUTING); say 'stop' to halt" in texts - assert worker.call(store.get_state).state == AWAIT_INPUT - assert "stopped milestone 1 (at a tool boundary)" in texts[-1] - worker.call(store.close) - worker.stop(timeout=5) - - -# --- the loops ------------------------------------------------------------- - - -class ScriptedClient: - def __init__(self, turns: list[list[Any]], on_call: Any = None) -> None: - self.turns = turns - self.on_call = on_call - self.calls = 0 - - def complete(self, events: Any, tools: Any, model: str, system: Any = None): - self.calls += 1 - if self.on_call: - self.on_call() - return self.turns.pop(0) if self.turns else [AssistantText("done", model)] - - -class StopsAfterFirstCall: - """A workspace whose first tool call is where the operator says stop.""" - - def __init__(self, cancel: Cancel) -> None: - self.cancel = cancel - self.ran: list[str] = [] - - def describe(self) -> str: - return "workspace" - - def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome: - self.ran.append(tool) - self.cancel.set() - return ToolOutcome("ok", success=True) - - -def test_executor_stops_before_a_turn_without_calling_the_model( - tmp_path: Path, -) -> None: - cancel = Cancel() - cancel.set() - client = ScriptedClient([]) - executor = LLMExecutor(client, ROLES, StopsAfterFirstCall(cancel), cancel=cancel) - outcome = executor.run(log(new_store(tmp_path)), 1) - assert outcome == ExecutorOutcome("stopped", "before turn 1") - assert client.calls == 0 - - -def test_executor_stops_between_tool_calls_and_records_the_skipped_ones( - tmp_path: Path, -) -> None: - cancel = Cancel() - store = new_store(tmp_path) - ws = StopsAfterFirstCall(cancel) - calls = [ - ToolCall(call_id="a", tool="run_command", arguments={"command": "make"}), - ToolCall(call_id="b", tool="write_file", arguments={"path": "x"}), - ] - executor = LLMExecutor(ScriptedClient([calls]), ROLES, ws, cancel=cancel) - outcome = executor.run(log(store), 1) - - assert outcome == ExecutorOutcome("stopped", "during turn 1") - assert ws.ran == ["run_command"] - results = { - e.event.call_id: e.event.result - for e in log(store).read() - if isinstance(e.event, ToolResult) - } - # Every call has a result, so the transcript still renders. - assert results == {"a": "ok", "b": STOPPED_RESULT} - - -class StoppedDelegate: - def run(self, transcript: object, *, brief: str, branch: str) -> DelegationResult: - return DelegationResult("stopped", "stopped by the operator during turn 2") - - -def test_a_stopped_delegation_stops_the_milestone(tmp_path: Path) -> None: - failing = [ - [ToolCall(call_id=f"c{n}", tool="run_command", arguments={"command": "x"})] - for n in range(2) - ] - - class Failing: - def describe(self) -> str: - return "workspace" - - def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome: - return ToolOutcome("exit 1", success=False) - - executor = LLMExecutor( - ScriptedClient(failing), - ROLES, - Failing(), - delegate=StoppedDelegate(), - ) - outcome = executor.run(log(new_store(tmp_path)), 1) - assert outcome.status == "stopped" - assert "during a delegation" in outcome.note - - -def test_planner_does_not_start_a_turn_once_stopped(tmp_path: Path) -> None: - cancel = Cancel() - cancel.set() - client = ScriptedClient([]) - planner = LLMPlanningAgent(client, ROLES, cancel=cancel) - try: - planner.plan(log(new_store(tmp_path), "planner"), "arch", "demo") - except PlanningStopped: - pass - else: - raise AssertionError("planning ran after stop") - assert client.calls == 0 diff --git a/nisse-dev/tests/test_tools.py b/nisse-dev/tests/test_tools.py index b5fef45..7e652e9 100644 --- a/nisse-dev/tests/test_tools.py +++ b/nisse-dev/tests/test_tools.py @@ -5,7 +5,7 @@ from pathlib import Path import pytest from nisse_dev.execution import REDACTED, SSHHost -from nisse_dev.tools import EXECUTOR_TOOLS, MAX_OUTPUT_CHARS, Workspace +from nisse_dev.tools import EXECUTION_TOOLS, MAX_OUTPUT_CHARS, Workspace @pytest.fixture @@ -15,9 +15,9 @@ def ws(ssh_host: SSHHost, tmp_path: Path) -> Workspace: def test_manifest_is_small_and_flat() -> None: # L9: fewer tools, flatter arguments for the local executor. - names = [t["function"]["name"] for t in EXECUTOR_TOOLS] + names = [t["function"]["name"] for t in EXECUTION_TOOLS] assert names == ["read_file", "write_file", "list_dir", "run_command"] - for tool in EXECUTOR_TOOLS: + for tool in EXECUTION_TOOLS: props = tool["function"]["parameters"]["properties"] assert all(p["type"] == "string" for p in props.values()) diff --git a/nisse-dev/tests/test_views.py b/nisse-dev/tests/test_views.py index 838ad1a..b5ae05a 100644 --- a/nisse-dev/tests/test_views.py +++ b/nisse-dev/tests/test_views.py @@ -2,11 +2,9 @@ import io from pathlib import Path -from typing import Any from nisse_dev.canonical import ( AssistantText, - CanonicalEvent, DelegationCompleted, DelegationStarted, Fact, @@ -16,19 +14,10 @@ from nisse_dev.canonical import ( ToolResult, UserMessage, ) -from nisse_dev.control import MILESTONE_PLANNING, PR_OPEN, Orchestrator -from nisse_dev.executor import ExecutorOutcome -from nisse_dev.gitea import Comment, PullRequest -from nisse_dev.llm import RoleMap from nisse_dev.oplog import OpLog -from nisse_dev.planner import LLMPlanningAgent -from nisse_dev.publish import PublishResult from nisse_dev.render import OpenAIChatRenderer from nisse_dev.render_anthropic import AnthropicMessagesRenderer from nisse_dev.state import StateStore -from nisse_dev.transport import InMemoryTransport - -ROLES = RoleMap(executor="x", planner="y", helper="y", expert="y") def new_store(tmp_path: Path) -> StateStore: @@ -39,155 +28,18 @@ def texts(events: list[StoredEvent]) -> list[str]: return [getattr(e.event, "text", "") for e in events] -# --- the orchestrator: run 2's planner, after a milestone ---------------------- - - -class PlanningClient: - """Always submits a plan, recording the events of each request.""" - - def __init__(self) -> None: - self.requests: list[list[StoredEvent]] = [] - - def complete(self, events, tools, model, system=None) -> list[CanonicalEvent]: - self.requests.append(events) - n = len(self.requests) - return [ - ToolCall( - call_id=f"plan-{n}", - tool="submit_plan", - arguments={"content": f"plan {n}"}, - ) - ] - - -class WorkingExecutor: - """Leaves the trail a real executor run does: text, calls, results.""" - - def __init__(self) -> None: - self.views: list[list[StoredEvent]] = [] - - def run(self, transcript: Any, milestone: int) -> ExecutorOutcome: - self.views.append(transcript.view()) - transcript.append(AssistantText(text="I'll write the helper", model="x")) - call = ToolCall( - call_id=f"run-{len(self.views)}", - tool="run_command", - arguments={"command": "pytest"}, - ) - transcript.append(call) - transcript.append( - ToolResult(call_id=call.call_id, result="1 passed", success=True) - ) - return ExecutorOutcome("completed", "added truncate() with a test") - - -class Publisher: - def reconcilers(self) -> dict[str, Any]: - return {} - - def publish(self, **kwargs: Any) -> PublishResult: - return PublishResult( - PullRequest(number=165, state="open", head="h", url="https://g/pr/165"), - "opened PR #165", - ) - - def push_followup(self, **kwargs: Any) -> str | None: - return None - - def comment(self, index: int, body: str) -> None: - pass - - -def run_one_milestone( - tmp_path: Path, -) -> tuple[Orchestrator, StateStore, InMemoryTransport, PlanningClient, Any]: - store = new_store(tmp_path) - transport = InMemoryTransport() - client = PlanningClient() - executor = WorkingExecutor() - orch = Orchestrator( - store, - transport, - LLMPlanningAgent(client, ROLES), - operators={"don"}, - roles=ROLES, - executor=executor, - publisher=Publisher(), - gitea_operators={"don"}, - ) - orch.start() - for text in ("add a truncate helper", "use demo", "approve", "approve"): - transport.receive("don", text) - transport.receive("don", "start milestone 1") - assert store.get_state().state == PR_OPEN - return orch, store, transport, client, executor - - -def test_the_planner_after_a_milestone_sees_facts_not_executor_turns( - tmp_path: Path, -) -> None: - # Run 2: shown the executor's calls as its own turns, the planner took - # milestone 1's work for its own and planned around it (#78). - _, store, transport, client, _ = run_one_milestone(tmp_path) - transport.receive("don", "next milestone") - assert store.get_state().state != MILESTONE_PLANNING # it planned - - events = client.requests[-1] - body = OpenAIChatRenderer().render(events, tools=None, model="y") - assistant = [m for m in body["messages"] if m["role"] == "assistant"] - called = [c["function"]["name"] for m in assistant for c in m["tool_calls"]] - assert set(called) == {"submit_plan"} # its own turns, and only those - seen = "\n".join(texts(events)) - assert "I'll write the helper" not in seen - assert "[operator, Matrix, sent while idle] add a truncate helper" in seen - assert "[orchestrator] arch plan rev-1 approved by don" in seen - assert "milestone 1 finished: added truncate() with a test" in seen - assert "PR #165" in seen and "https://g/pr/165" in seen - assert "planning milestone 2" in seen - - -def test_the_executor_sees_the_milestone_start_and_the_comment_source( - tmp_path: Path, -) -> None: - orch, _, _, _, executor = run_one_milestone(tmp_path) - assert "[orchestrator] milestone 1 started" in texts(executor.views[0]) - - orch.handle_review_comment( - Comment(id=7, author="don", body="add a docstring", created_at="t") - ) - review_view = texts(executor.views[-1]) - assert "[operator, comment on PR #165] add a docstring" in review_view - # Its own earlier turn is still its own, not relabelled. - own = [e for e in executor.views[-1] if isinstance(e.event, AssistantText)] - assert [e.event.text for e in own] == ["I'll write the helper"] - - -def test_the_outcome_is_recorded_as_a_fact_when_the_task_closes( - tmp_path: Path, -) -> None: - _, store, transport, _, _ = run_one_milestone(tmp_path) - item = store.active_item() - transport.receive("don", "project complete") - facts = [ - e.event.text - for e in store.transcript(item.id, "orchestrator").read() - if isinstance(e.event, Fact) - ] - assert facts[-1] == "work item completed: don said project complete" - - # --- the projection, per role ---------------------------------------------------- def delegated_milestone(store: StateStore) -> None: - """A milestone in which the executor failed and handed off to a helper.""" - store.transcript("a", "planner").append(PlanSubmitted(plan="1. fix it", model="y")) - executor = store.transcript("a", "executor") - executor.append( + """A milestone in which the driver failed and handed off to a helper.""" + store.transcript("a", "driver").append(PlanSubmitted(plan="1. fix it", model="y")) + driver = store.transcript("a", "driver") + driver.append( ToolCall(call_id="e1", tool="run_command", arguments={"command": "make"}) ) - executor.append(ToolResult(call_id="e1", result="error: missing ;", success=False)) - helper = executor.as_actor("helper") + driver.append(ToolResult(call_id="e1", result="error: missing ;", success=False)) + helper = driver.as_actor("helper") helper.append( DelegationStarted(brief="make fails", role="helper", model="y", caps={}) ) @@ -207,10 +59,10 @@ def delegated_milestone(store: StateStore) -> None: ) -def test_the_executor_sees_the_helpers_report_not_its_turns(tmp_path: Path) -> None: +def test_the_driver_sees_the_helpers_report_not_its_turns(tmp_path: Path) -> None: store = new_store(tmp_path) delegated_milestone(store) - view = store.transcript("a", "executor").view() + view = store.transcript("a", "driver").view() assert [type(e.event).__name__ for e in view] == [ "ToolCall", "ToolResult", @@ -221,7 +73,7 @@ def test_the_executor_sees_the_helpers_report_not_its_turns(tmp_path: Path) -> N ) -def test_the_helper_sees_the_plan_and_the_executors_work_as_narration( +def test_the_helper_sees_the_plan_and_the_drivers_work_as_narration( tmp_path: Path, ) -> None: store = new_store(tmp_path) @@ -229,7 +81,7 @@ def test_the_helper_sees_the_plan_and_the_executors_work_as_narration( view = store.transcript("a", "helper").view() said = texts(view) assert said[0] == "[approved plan] 1. fix it" - assert said[1] == "[executor] run_command(make) → failed: error: missing ;" + assert said[1] == "[driver] run_command(make) → failed: error: missing ;" # Its own events are unchanged, tool calls and all. assert any(isinstance(e.event, ToolCall) and e.event.call_id == "h1" for e in view) assert not any( @@ -237,20 +89,11 @@ def test_the_helper_sees_the_plan_and_the_executors_work_as_narration( ) -def test_the_planner_sees_no_other_roles_work(tmp_path: Path) -> None: - store = new_store(tmp_path) - delegated_milestone(store) - store.transcript("a", "orchestrator").append(Fact(text="milestone 1 finished")) - assert texts(store.transcript("a", "planner").view()) == [ - "[orchestrator] milestone 1 finished" - ] - - def test_a_view_keeps_log_ids_so_cursors_still_work(tmp_path: Path) -> None: store = new_store(tmp_path) delegated_milestone(store) - raw = store.transcript("a", "executor").read() - view = store.transcript("a", "executor").view(after=raw[2].id) + raw = store.transcript("a", "driver").read() + view = store.transcript("a", "driver").view(after=raw[2].id) assert [e.id for e in view] == [raw[-1].id] # the helper's report @@ -277,7 +120,7 @@ def test_operator_messages_carry_their_channel(tmp_path: Path) -> None: operator = store.transcript("a", "operator") for source in ("transport:@don", "gitea:issue:9:comment:4", "gitea:comment:5"): operator.append(UserMessage(text="hi", source=source)) - assert texts(store.transcript("a", "planner").view()) == [ + assert texts(store.transcript("a", "driver").view()) == [ "[operator, Matrix] hi", "[operator, comment on plan issue #9] hi", "[operator, Gitea comment] hi", diff --git a/nisse-dev/tests/test_work_items.py b/nisse-dev/tests/test_work_items.py deleted file mode 100644 index be03177..0000000 --- a/nisse-dev/tests/test_work_items.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Work items in the state store (architecture L1, L7; Development Plan M1.1).""" - -import io -from pathlib import Path -from typing import Any - -import pytest - -from nisse_dev.canonical import AssistantText, UserMessage -from nisse_dev.control import ( - AWAIT_ARCH_SIGNOFF, - PR_OPEN, - AskHuman, - Orchestrator, - SubmitPlan, -) -from nisse_dev.oplog import OpLog -from nisse_dev.state import IDLE, StateStore -from nisse_dev.transport import InMemoryTransport - - -def new_store(path: Path) -> StateStore: - return StateStore(path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - - -# --- the store --------------------------------------------------------------- - - -def test_a_transcript_reads_only_its_own_work_item(tmp_path: Path) -> None: - store = new_store(tmp_path) - store.transcript("a", "operator").append(UserMessage(text="for a", source="m")) - store.transcript("b", "operator").append(UserMessage(text="for b", source="m")) - store.transcript(None, "operator").append(UserMessage(text="idle", source="m")) - assert [e.event.text for e in store.transcript("a", "planner").read()] == ["for a"] - assert [e.event.text for e in store.transcript(None, "planner").read()] == ["idle"] - - -def test_every_event_carries_its_work_item_and_actor(tmp_path: Path) -> None: - store = new_store(tmp_path) - log = store.transcript("a", "executor") - log.append(AssistantText(text="working", model="m")) - log.as_actor("helper").append(AssistantText(text="helping", model="f")) - assert [(e.work_item, e.actor) for e in store.all_events()] == [ - ("a", "executor"), - ("a", "helper"), - ] - - -def test_an_unknown_actor_is_refused(tmp_path: Path) -> None: - with pytest.raises(ValueError, match="unknown actor"): - new_store(tmp_path).transcript("a", "someone") - - -def test_intents_are_owned_by_their_work_item(tmp_path: Path) -> None: - store = new_store(tmp_path) - store.transcript("a", "helper").begin_intent("delegation", {}, cause="t") - [intent] = store.pending_intents() - assert intent.work_item == "a" - - -def test_work_items_and_the_active_one_survive_a_restart(tmp_path: Path) -> None: - store = new_store(tmp_path) - store.open_work_item("a", title="demo", scope="demo", position=IDLE, cause="t") - store.set_active("a", cause="t") - store.update_work_item("a", position="PLANNING_ARCH", data={"x": 1}, cause="t") - store.close() - store = new_store(tmp_path) - item = store.active_item() - assert item is not None - assert (item.id, item.position, item.data, item.status) == ( - "a", - "PLANNING_ARCH", - {"x": 1}, - "open", - ) - - -def test_closing_records_the_outcome_and_going_idle_clears_active( - tmp_path: Path, -) -> None: - store = new_store(tmp_path) - store.open_work_item("a", title="demo", position=IDLE, cause="t") - store.set_active("a", cause="t") - store.close_work_item("a", outcome="completed", cause="t") - store.set_active(None, cause="t") - assert store.active_item() is None - item = store.work_item("a") - assert item is not None and (item.status, item.outcome) == ("closed", "completed") - with pytest.raises(KeyError): - store.close_work_item("a", outcome="abandoned", cause="t") # already closed - - -def test_messages_sent_while_idle_are_carried_with_their_origin( - tmp_path: Path, -) -> None: - store = new_store(tmp_path) - store.set_active(None, cause="boot") - idle = store.transcript(None, "operator") - first = idle.append(UserMessage(text="add a truncate helper", source="m:1")) - store.open_work_item("a", title="demo", position=IDLE, cause="t") - assert store.carry_idle_messages("a") == 1 - [carried] = store.transcript("a", "planner").read() - assert carried.event.text == "add a truncate helper" - assert f"#{first}" in carried.event.source # where it came from - assert len(store.all_events()) == 2 # a copy; the original is untouched - - -# --- the orchestrator ---------------------------------------------------------- - - -class RecordingPlanner: - """Records what each planning turn could read.""" - - def __init__(self) -> None: - self.seen: list[list[str]] = [] - - def plan( - self, - transcript: Any, - gate: str, - project: str, - feedback: str | None = None, - answer: str | None = None, - ) -> SubmitPlan | AskHuman: - self.seen.append( - [ - e.event.text - for e in transcript.read() - if isinstance(e.event, UserMessage) - ] - ) - return SubmitPlan(f"{gate} plan for {project}") - - -def build(tmp_path: Path) -> tuple[StateStore, InMemoryTransport, RecordingPlanner]: - store = new_store(tmp_path) - transport = InMemoryTransport() - planner = RecordingPlanner() - Orchestrator(store, transport, planner, operators={"don"}).start() - return store, transport, planner - - -def test_a_second_task_never_sees_the_first(tmp_path: Path) -> None: - # The second live run's planner read every earlier task's events (#78). - store, transport, planner = build(tmp_path) - for text in ("build the first thing", "use first", "approve", "approve"): - transport.receive("don", text) - transport.receive("don", "start milestone 1") - assert store.get_state().state == PR_OPEN - transport.receive("don", "project complete") - - transport.receive("don", "build the second thing") - transport.receive("don", "use second") - assert store.get_state().state == AWAIT_ARCH_SIGNOFF - second_task_view = planner.seen[-1] - assert "build the second thing" in second_task_view - assert not any("first" in text for text in second_task_view) - - -def test_each_task_is_its_own_work_item_with_its_outcome(tmp_path: Path) -> None: - store, transport, _ = build(tmp_path) - for text in ("use first", "approve", "approve", "start milestone 1"): - transport.receive("don", text) - transport.receive("don", "project complete") - transport.receive("don", "use second") - transport.receive("don", "abandon") - outcomes = [(item.title, item.outcome) for item in store.work_items()] - assert outcomes == [("first", "completed"), ("second", "abandoned")] - assert store.active_item() is None - - -def test_the_task_message_before_use_reaches_the_planner(tmp_path: Path) -> None: - # Run 2's flow: the task, then `use`. It must still be planned from. - _, transport, planner = build(tmp_path) - transport.receive("don", "add a truncate helper") - transport.receive("don", "use demo") - assert "add a truncate helper" in planner.seen[0] diff --git a/nisse-dev/tests/test_worker.py b/nisse-dev/tests/test_worker.py index 4b0a643..41f2ee4 100644 --- a/nisse-dev/tests/test_worker.py +++ b/nisse-dev/tests/test_worker.py @@ -62,10 +62,10 @@ def test_the_store_is_usable_only_from_the_thread_that_owns_it( store = worker.call( lambda: StateStore(tmp_path / "state.db", oplog=OpLog("t", io.StringIO())) ) - assert worker.call(store.get_state) is None + assert worker.call(store.active_item) is None # The tripwire the worker design keeps on: any other thread fails loudly. with pytest.raises(Exception, match="same thread"): - store.get_state() + store.active_item() worker.call(store.close) diff --git a/scripts/benchmark.py b/scripts/benchmark.py index bac0802..77e86d2 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -1,4 +1,4 @@ -"""Benchmark executor models on canned milestones, and compare the results. +"""Benchmark driver models on canned milestones, and compare the results. uv run python scripts/benchmark.py run --models A,B [--runs 5] [--tasks T,...] [--note "vLLM 0.29, AWQ, fp8 KV"] [--config PATH] @@ -14,13 +14,13 @@ import os import sys from pathlib import Path -from nisse_dev.agent import build_host, make_llm_clients from nisse_dev.bench.results import append, load, render, results_file, summarise from nisse_dev.bench.runner import agent_commit, run_task from nisse_dev.bench.tasks import load_tasks from nisse_dev.catalog import ModelCatalog from nisse_dev.config import AgentConfig, ConfigError from nisse_dev.envfile import load_env_file +from nisse_dev.loop.build import build_host, make_llm_clients REPO = Path(__file__).resolve().parents[1] RESULTS = REPO / "bench" / "results" @@ -46,7 +46,8 @@ def run(args: argparse.Namespace) -> int: catalog = ModelCatalog.discover(config.llm_base_url, config.llm_token) catalog.require_limits({m: m for m in models}) normalized, _passthrough = make_llm_clients(config, catalog) - host = build_host(config) # runs happen on the build VM, as the agent's do + # Runs happen on the first project's build VM, as the agent's do. + host = build_host(config, config.projects[0]) commit = agent_commit(REPO) for model in models: @@ -54,7 +55,7 @@ def run(args: argparse.Namespace) -> int: out = results_file(args.results, model) for task in tasks: for n in range(args.runs): - # The executor always takes the normalized route in the agent, + # The driver always takes the normalized route in the agent, # so it does here too. record = run_task( task, diff --git a/scripts/e2e.sh b/scripts/e2e.sh index ced432b..c2bdf10 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -53,7 +53,7 @@ print(i["spend"], i.get("max_budget"))' 2>/dev/null) || echo "? ?" } read -r before budget < <(spend) status=0 -uv run pytest -m e2e -rs tests/integration/test_end_to_end.py tests/integration/test_end_to_end_loop.py "$@" || status=$? +uv run pytest -m e2e -rs tests/integration/test_end_to_end_loop.py "$@" || status=$? read -r after _ < <(spend) python3 -c " b, a, cap = '$before', '$after', '$budget' diff --git a/scripts/evaluate_model.sh b/scripts/evaluate_model.sh index 81dcf22..30362db 100755 --- a/scripts/evaluate_model.sh +++ b/scripts/evaluate_model.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Evaluate a model for the executor role, from a declared LiteLLM alias to a +# Evaluate a model for the driver role, from a declared LiteLLM alias to a # benchmark report next to every model measured before it. # # scripts/evaluate_model.sh ALIAS "how it is served" [RUNS] [--config PATH] diff --git a/specs/README.md b/specs/README.md index 5b071e5..2252ad2 100644 --- a/specs/README.md +++ b/specs/README.md @@ -6,41 +6,19 @@ it finds flaws the design and the code share, not places where the code strays from the design. So every counterexample is also reproduced as a failing test against the real code before anything is changed. -## `orchestrator.qnt`: nisse-dev's control loop - -Models `nisse-dev/src/nisse_dev/control.py`: the states, the gate, pending -questions, Gitea comment cursors, publish intents, and the worker queue. -Operator messages, Gitea comments, planner/executor/publisher outcomes -(exceptions included) and **a crash at any point** all interleave freely. -Each pure function is named after the method it mirrors. - -The properties are at the end of the file (`all_properties` is their -conjunction). Counterexamples are written out as short traces in -`orchestrator_scenarios.qnt`, and replayed against the real code in -`nisse-dev/tests/test_quint_counterexamples.py`. - ```sh npm install -g @informalsystems/quint # 0.33 or later - -# Random simulation, one property at a time: -quint run specs/orchestrator.qnt --invariant=no_stranding \ - --max-steps=40 --max-samples=200000 - -# The counterexample traces (each expects its property to break): -quint test specs/orchestrator_scenarios.qnt ``` Exhaustive bounded checking (`quint verify`) uses Apalache and needs Java 17. -**Keeping it true.** When `control.py` changes how a state is entered or -left, change the matching function here in the same PR. When a -counterexample's fix lands, flip its scenario's `expect` to the property -holding, and drop the `xfail` from its Python reproduction. +The gated state machine the agent ran before the switch-over (M2.8) had its +own model, `orchestrator.qnt`; it went with that code, and is in the history. ## `loop.qnt`: the agent loop, as the architecture describes it -Models the design in `docs/Architecture.md` rather than code, since the new -loop doesn't exist yet: +Models the design in `docs/Architecture.md` rather than the code (it was +written before the loop was): - work items from assigned issues and conversations; - restatements and revision-bound go-aheads; - park-and-switch, with branches pushed on parking; @@ -66,7 +44,7 @@ end are reachability witnesses: each should be *violated*, which shows the state it names is reachable. **Against the real code.** `nisse-dev/tests/test_loop_properties.py` checks -the same properties against the new loop itself: a seeded random walk over +the same properties against the loop itself: a seeded random walk over what the operator and Gitea can do (go-aheads, answers, comments, merges, closes, abandon, a crash and restart), checking every property after every step. It found two bugs the design model couldn't, because they were in the diff --git a/specs/orchestrator.qnt b/specs/orchestrator.qnt deleted file mode 100644 index 69c6145..0000000 --- a/specs/orchestrator.qnt +++ /dev/null @@ -1,488 +0,0 @@ -// A model of nisse-dev's orchestrator (nisse-dev/src/nisse_dev/control.py), -// written to look for counterexamples: interleavings of operator messages, -// Gitea comments, work outcomes and crashes that break a property the -// design relies on. -// -// Everything lives in one record, `s`, and each handler is a pure function -// named after the method it mirrors. The model keeps what matters for the -// properties below and drops the rest (texts, transcripts, models, tools). -// -// Durable state (the state store) survives a crash; volatile state (the -// worker queue, work in flight, pollers, the cancel flag) does not. Gitea is -// the outside world and survives everything. -// -// Run: quint run specs/orchestrator.qnt --invariant=all_properties -// quint run specs/orchestrator.qnt --invariant= -module orchestrator { - - // ---- bounds -------------------------------------------------------------- - - pure val MAX_MS = 2 // milestones per task - pure val MAX_TASKS = 2 // tasks per run ('use' / 'abandon' / 'complete') - pure val MAX_QUEUE = 3 // jobs waiting on the worker - pure val MAX_IDS = 8 // Gitea comments per run - - // ---- vocabulary ---------------------------------------------------------- - - pure val WORKING = Set("PLANNING_ARCH", "MILESTONE_PLANNING", "EXECUTING") - pure val SIGNOFF = Set("AWAIT_ARCH_SIGNOFF", "AWAIT_MILESTONE_SIGNOFF") - // Work that sets _working, so a message then gets "busy" (control._handle). - pure val LONG_WORK = Set("plan", "exec", "review") - - // What the operator had in front of them: the plan last put up for sign-off. - type Plan = { g: str, rev: int } - pure val NO_PLAN: Plan = { g: "", rev: 0 } - - type Gate = { on: bool, g: str, rev: int, approved: bool } - pure val NO_GATE: Gate = { on: false, g: "", rev: 0, approved: false } - - type Pending = { kind: str, g: str, interrupted: str } - pure val NO_PENDING: Pending = { kind: "", g: "", interrupted: "" } - - // A message or comment waiting on the worker. `saw` is what the operator - // had seen when they sent it; the agent never knows it, the properties do. - type Job = { src: str, kind: str, saw: Plan, cid: int } - - type PlanComment = { id: int, tid: int, kind: str, saw: Plan } - type PrComment = { id: int, key: int } - - type Watch = { on: bool, floor: int } - pure val OFF: Watch = { on: false, floor: 0 } - - type Sys = { - // -- durable: the state store (control._data and the state) -- - st: str, - tid: int, // task counter; a task's branches are its own - task: bool, // a project is in use - gate: Gate, - revs: str -> int, // latest revision per gate - lastRev: int, // revisions run across the task (L3) - ms: int, - pending: Pending, - prOn: bool, prKey: int, prLast: int, // _data["pr"] - reviewing: int, // pr["reviewing"]: the comment being answered, 0 if none - boardOn: bool, boardPosted: bool, boardLast: int, // _data["board"] - intents: Set[int], // pending open_pr intents, by PR key - // -- Gitea -- - planThread: List[PlanComment], - prThread: List[PrComment], - nextId: int, - giteaPr: int -> str, // PR key -> "open" / "closed" - opened: int -> int, // PR key -> how many PRs Gitea ever opened for it - openIssues: Set[int], // plan issues still open, by task - // -- volatile: lost in a crash -- - queue: List[Job], - work: str, // none / plan / exec / publish / review - workGate: str, - workCid: int, - pubStage: int, - cancel: bool, - planWatch: Watch, - prWatch: Watch, - // -- the operator's view, and history the properties read -- - shown: Plan, - taken: Set[int], // review comments the agent took up - answered: Set[int], // ... and answered (pushed, asked, or reported) - badApproval: bool, - staleFeedback: bool, - } - - // ---- helpers ------------------------------------------------------------ - - pure def msGate(ms: int): str = if (ms == 1) "ms1" else "ms2" - pure def prKeyOf(s: Sys): int = s.tid * 10 + s.ms - pure def prState(s: Sys, key: int): str = - if (s.giteaPr.keys().contains(key)) s.giteaPr.get(key) else "none" - pure def planIds(s: Sys): int = - s.planThread.foldl(0, (acc, c) => if (c.tid == s.tid and c.id > acc) c.id else acc) - pure def prIds(s: Sys, key: int): int = - s.prThread.foldl(0, (acc, c) => if (c.key == key and c.id > acc) c.id else acc) - pure def max(a: int, b: int): int = if (a > b) a else b - // Gitea opens a PR for a branch. - pure def openPr(s: Sys, key: int): Sys = - { ...s, giteaPr: s.giteaPr.put(key, "open"), - opened: s.opened.put(key, (if (s.opened.keys().contains(key)) s.opened.get(key) else 0) + 1) } - - pure def queued(s: Sys, src: str): bool = s.queue.foldl(false, (acc, j) => acc or j.src == src) - - // _await_input - pure def awaitInput(s: Sys, p: Pending): Sys = { ...s, st: "AWAIT_INPUT", pending: p } - - // _settled: a job that leaves a working state behind with nothing running - // moves to AWAIT_INPUT with the restart choice. - pure def settle(s: Sys): Sys = - if (s.work == "none" and WORKING.contains(s.st)) - awaitInput(s, { kind: "boot", g: "", interrupted: s.st }) - else s - - pure def endWork(s: Sys): Sys = { ...s, work: "none", workGate: "", workCid: 0, cancel: false } - - // _run_planning_turn, first half: the turn is now in flight. - pure def startPlanning(s: Sys, g: str): Sys = - { ...s, st: if (g == "arch") "PLANNING_ARCH" else "MILESTONE_PLANNING", - work: "plan", workGate: g } - - // _begin_project - pure def beginProject(s: Sys, giteaUp: bool): Sys = - val t = s.tid + 1 - startPlanning( - { ...s, tid: t, task: true, gate: NO_GATE, - revs: Map("arch" -> 0, "ms1" -> 0, "ms2" -> 0), lastRev: 0, ms: 1, pending: NO_PENDING, - prOn: false, prKey: 0, prLast: 0, reviewing: 0, - boardOn: giteaUp, boardPosted: false, boardLast: 0, - openIssues: if (giteaUp) s.openIssues.union(Set(t)) else s.openIssues, - planWatch: OFF, prWatch: OFF }, - "arch") - - // _run_planning_turn, second half: the planner submitted a plan. - // _post_plan_to_board then watches the issue from here (a fresh watch - // skips what is already there). - pure def planSubmitted(s: Sys, giteaUp: bool): Sys = - val g = s.workGate - val rev = s.lastRev + 1 - val posted = s.boardOn and giteaUp - val t = endWork({ ...s, - revs: s.revs.set(g, rev), lastRev: rev, - gate: { on: true, g: g, rev: rev, approved: false }, - st: if (g == "arch") "AWAIT_ARCH_SIGNOFF" else "AWAIT_MILESTONE_SIGNOFF", - shown: { g: g, rev: rev } }) - if (posted) - { ...t, boardPosted: true, boardLast: planIds(t), planWatch: { on: true, floor: planIds(t) } } - else t - - // _execute_milestone: start - pure def startExec(s: Sys): Sys = { ...s, st: "EXECUTING", work: "exec" } - - // _apply_gate_reply and Gate.evaluate, for Matrix and Gitea alike. - // A job's `saw` doubles as its arrival stamp: the revision up when it was - // sent (Matrix) or the latest plan posted before it (Gitea). A reply that - // names no revision applies only if that is the current one. - pure def applyGate(s: Sys, j: Job): Sys = - val current = { g: s.gate.g, rev: s.gate.rev } - if (s.gate.approved) - if (j.kind == "start") startExec(s) else s - else if (j.kind == "approveRev" and j.saw.rev != s.gate.rev) - s // names another revision: ignored with a notice - else if (j.kind != "approveRev" and j.saw.rev != s.gate.rev) - s // about another revision: not applied, the agent says so - else if (j.kind == "approve" or j.kind == "approveRev") - val s2 = { ...s, badApproval: s.badApproval or j.saw != current } - if (s.gate.g == "arch") - startPlanning({ ...s2, gate: NO_GATE }, msGate(s.ms)) - else - { ...s2, gate: { ...s.gate, approved: true }, st: "AWAIT_MILESTONE_SIGNOFF" } - else - // Anything else is rejection feedback, and the gate is planned again. - startPlanning({ ...s, staleFeedback: s.staleFeedback or j.saw != current, gate: NO_GATE }, - s.gate.g) - - // _return_to_pr_open / _watch_pr - pure def returnToPrOpen(s: Sys): Sys = - { ...s, st: "PR_OPEN", prWatch: if (s.prOn) { on: true, floor: s.prLast } else OFF } - - // _abandon: IDLE first, then close the issue and the PR, abort intents. - pure def abandon(s: Sys): Sys = - { ...s, st: "IDLE", task: false, gate: NO_GATE, pending: NO_PENDING, - prOn: false, reviewing: 0, boardOn: false, boardPosted: false, intents: Set(), - openIssues: s.openIssues.exclude(Set(s.tid)), - giteaPr: if (s.prOn) s.giteaPr.put(s.prKey, "closed") else s.giteaPr, - planWatch: OFF, prWatch: OFF } - - // _handle_pr_open: 'project complete' closes the plan issue; the PR is - // the operator's to merge. - pure def complete(s: Sys): Sys = - { ...s, st: "IDLE", task: false, gate: NO_GATE, pending: NO_PENDING, - prOn: false, reviewing: 0, boardOn: false, boardPosted: false, - openIssues: s.openIssues.exclude(Set(s.tid)), planWatch: OFF, prWatch: OFF } - - // _resume_after_restart - pure def resume(s: Sys, p: Pending, j: Job): Sys = - if (j.kind != "continue" and j.kind != "replan") - { ...s, pending: p } // asked again - else if (p.interrupted == "PR_OPEN") - returnToPrOpen(s) - else if (j.kind == "continue" and p.interrupted == "EXECUTING" and s.gate.approved) - { ...s, st: "AWAIT_MILESTONE_SIGNOFF" } - else - val g = - if (p.interrupted == "PLANNING_ARCH") "arch" - else if (p.interrupted == "MILESTONE_PLANNING" or p.interrupted == "EXECUTING") msGate(s.ms) - else if (s.gate.g != "") s.gate.g - else if (s.revs.get("arch") == 0) "arch" - else msGate(s.ms) - startPlanning({ ...s, gate: NO_GATE }, g) - - // _handle_answer - pure def answer(s: Sys, j: Job, giteaUp: bool): Sys = - val p = s.pending - val s0 = { ...s, pending: NO_PENDING } - if (p.kind == "") - if (j.kind == "use") beginProject({ ...s, planWatch: OFF, prWatch: OFF }, giteaUp) else s - else if (p.kind == "question" or p.kind == "planning_failed") startPlanning(s0, p.g) - else if (p.kind == "review_stopped") returnToPrOpen(s0) - else if (p.kind == "milestone_stopped") { ...s0, st: "AWAIT_MILESTONE_SIGNOFF" } - else if (p.kind == "boot") resume(s0, p, j) - else s0 - - // _handle_now, for a Matrix message the worker has reached. - pure def handleMatrix(s: Sys, j: Job, giteaUp: bool): Sys = - if (j.kind == "abandon") - if (s.st == "IDLE") s else abandon(s) - else if (s.st == "IDLE") - if (j.kind == "use") beginProject(s, giteaUp) else s - else if (SIGNOFF.contains(s.st)) - if (not(s.gate.on)) - awaitInput(s, { kind: "boot", g: "", - interrupted: if (s.st == "AWAIT_ARCH_SIGNOFF") "PLANNING_ARCH" else "MILESTONE_PLANNING" }) - else applyGate(s, j) - else if (s.st == "PR_OPEN") - if (j.kind == "next" and s.ms < MAX_MS) - startPlanning({ ...s, ms: s.ms + 1, gate: NO_GATE, prOn: false, prWatch: OFF }, msGate(s.ms + 1)) - else if (j.kind == "complete") complete(s) - else s - else if (s.st == "AWAIT_INPUT") answer(s, j, giteaUp) - else s // busy - - // _handle_plan_comment_now - pure def handlePlanComment(s: Sys, j: Job): Sys = - if (not(SIGNOFF.contains(s.st))) s // not taken - else applyGate({ ...s, boardLast: max(s.boardLast, j.cid) }, j) - - // _handle_review_comment_now, first half: the executor starts on it, - // with the comment marked (durably) until it returns. - pure def handleReviewComment(s: Sys, j: Job): Sys = - if (s.st != "PR_OPEN") s // not taken - else { ...s, prLast: max(s.prLast, j.cid), taken: s.taken.union(Set(j.cid)), - reviewing: j.cid, work: "review", workCid: j.cid } - - pure def handleJob(s: Sys, j: Job, giteaUp: bool): Sys = - if (j.src == "matrix") handleMatrix(s, j, giteaUp) - else if (j.src == "plan") handlePlanComment(s, j) - else handleReviewComment(s, j) - - // Orchestrator.start, after a crash: reconcile intents against Gitea, - // then resume, re-ask, or ask the restart question. - pure def boot(s: Sys, giteaUp: bool): Sys = - val unresolved = s.intents.size() > 0 and not(giteaUp) - val s0 = { ...s, intents: if (giteaUp) Set() else s.intents } - if (unresolved) - awaitInput(s0, { kind: "boot", g: "", interrupted: s.st }) - else if (s.st == "AWAIT_INPUT" and s.pending.kind != "") - s0 - else if (s.st == "EXECUTING" and giteaUp and prState(s, prKeyOf(s)) == "open") - // L8 boot step 3: the milestone's PR is open, so it was published and only - // PR_OPEN went unrecorded. Adopt it; nothing watched it, so from 0. - { ...s0, st: "PR_OPEN", prOn: true, prKey: prKeyOf(s), prLast: 0, - prWatch: { on: true, floor: 0 }, planWatch: OFF } - else if (s.st == "PR_OPEN") - // A comment whose answer never finished is delivered again. - { ...s0, prWatch: if (s.prOn) - { on: true, floor: if (s.reviewing > 0) s.reviewing - 1 else s.prLast } - else OFF } - else if (s.st == "IDLE" or (SIGNOFF.contains(s.st) and s.gate.on)) - if (s.gate.on and s.boardPosted) { ...s0, planWatch: { on: true, floor: s.boardLast } } else s0 - else - awaitInput(s0, { kind: "boot", g: "", interrupted: s.st }) - - // ---- the system --------------------------------------------------------- - - var s: Sys - - action init = s' = { - st: "IDLE", tid: 0, task: false, gate: NO_GATE, - revs: Map("arch" -> 0, "ms1" -> 0, "ms2" -> 0), lastRev: 0, ms: 1, pending: NO_PENDING, - prOn: false, prKey: 0, prLast: 0, reviewing: 0, - boardOn: false, boardPosted: false, boardLast: 0, - intents: Set(), - planThread: List(), prThread: List(), nextId: 1, - giteaPr: Map(), opened: Map(), openIssues: Set(), - queue: List(), work: "none", workGate: "", workCid: 0, pubStage: 0, cancel: false, - planWatch: OFF, prWatch: OFF, - shown: NO_PLAN, taken: Set(), answered: Set(), - badApproval: false, staleFeedback: false, - } - - // Each action has a deterministic form taking its choices as parameters, - // so scenarios (orchestrator_scenarios.qnt) can replay a trace exactly; - // `step` picks the parameters at random. - - pure val MATRIX_KINDS = Set("use", "approve", "approveRev", "text", "start", "next", - "complete", "abandon", "continue", "replan", "stop") - pure val COMMENT_KINDS = Set("approve", "approveRev", "text", "start") - - // The operator sends a Matrix message (control._handle, on the - // transport's thread). While long work runs it only gets "busy"; `stop` - // reaches the cancel flag whenever the worker is busy. - action send(kind: str): bool = all { - s.tid < MAX_TASKS or kind != "use", - if (kind == "stop") - s' = { ...s, cancel: s.cancel or s.work != "none" } - else if (LONG_WORK.contains(s.work)) - s' = s - else all { - s.queue.length() < MAX_QUEUE, - s' = { ...s, queue: s.queue.append({ src: "matrix", kind: kind, saw: s.shown, cid: 0 }) }, - }, - } - - // The operator comments on the plan issue. - action commentPlan(kind: str): bool = all { - s.task, s.openIssues.contains(s.tid), s.nextId <= MAX_IDS, - s' = { ...s, nextId: s.nextId + 1, - planThread: s.planThread.append({ id: s.nextId, tid: s.tid, kind: kind, saw: s.shown }) }, - } - - // The operator comments on the open PR. - action commentPr: bool = all { - s.prOn, s.nextId <= MAX_IDS, prState(s, s.prKey) == "open", - s' = { ...s, nextId: s.nextId + 1, prThread: s.prThread.append({ id: s.nextId, key: s.prKey }) }, - } - - // A poller delivers the next new comment (CommentPoller.poll_once). It - // waits on the worker, so one job per poller at a time. - action pollPlan: bool = { - val c = s.planThread.foldl({ id: 0, tid: 0, kind: "", saw: NO_PLAN }, - (acc, x) => if (x.tid == s.tid and x.id > s.planWatch.floor and (acc.id == 0 or x.id < acc.id)) x else acc) - all { - s.planWatch.on, c.id > 0, not(queued(s, "plan")), s.queue.length() < MAX_QUEUE, - s' = { ...s, planWatch: { on: true, floor: c.id }, - queue: s.queue.append({ src: "plan", kind: c.kind, saw: c.saw, cid: c.id }) }, - } - } - - action pollPr: bool = { - val c = s.prThread.foldl({ id: 0, key: 0 }, - (acc, x) => if (x.key == s.prKey and x.id > s.prWatch.floor and (acc.id == 0 or x.id < acc.id)) x else acc) - all { - s.prWatch.on, c.id > 0, not(queued(s, "pr")), s.queue.length() < MAX_QUEUE, - s' = { ...s, prWatch: { on: true, floor: c.id }, - queue: s.queue.append({ src: "pr", kind: "review", saw: NO_PLAN, cid: c.id }) }, - } - } - - // The worker takes the next job, one at a time, and settles it. - action runJob(giteaUp: bool): bool = all { - s.work == "none", s.queue.length() > 0, - s' = settle(handleJob({ ...s, queue: s.queue.tail() }, s.queue.head(), giteaUp)), - } - - // A planning turn ends. - action finishPlanning(outcome: str, giteaUp: bool): bool = all { - s.work == "plan", outcome != "stopped" or s.cancel, - s' = - if (outcome == "submit") planSubmitted(s, giteaUp) - else if (outcome == "ask" or outcome == "stopped") - awaitInput(endWork(s), { kind: "question", g: s.workGate, interrupted: "" }) - else if (outcome == "error") - awaitInput(endWork(s), { kind: "planning_failed", g: s.workGate, interrupted: "" }) - else settle(endWork(s)), - } - - // The executor finishes a milestone. - action finishExec(outcome: str): bool = all { - s.work == "exec", outcome != "stopped" or s.cancel, - s' = - if (outcome == "completed") { ...s, work: "publish", pubStage: 0, cancel: false } - else if (outcome == "exception") settle(endWork(s)) - else awaitInput(endWork(s), { kind: "milestone_stopped", g: "", interrupted: "" }), - } - - // Publishing, one step at a time, so a crash can land between them - // (MilestonePublisher.publish and control._publish_milestone): - // 0 commit, begin_intent; 1 push, then adopt an open PR for the branch or - // open one ("lost": Gitea opened it but the response never came back); - // 2 commit_intent; 3 record the PR, watch it, PR_OPEN. - action publishStep(outcome: str): bool = { - val key = prKeyOf(s) - all { - s.work == "publish", - s' = - if (s.pubStage == 0) - if (outcome == "nochanges" or outcome == "fail") - awaitInput(endWork(s), { kind: "milestone_stopped", g: "", interrupted: "" }) - else if (outcome == "exception") settle(endWork(s)) - else { ...s, intents: s.intents.union(Set(key)), pubStage: 1 } - else if (s.pubStage == 1) - if (outcome == "exception") settle(endWork(s)) - else if (outcome == "fail" or outcome == "nochanges") - awaitInput(endWork(s), { kind: "milestone_stopped", g: "", interrupted: "" }) - else if (prState(s, key) == "open") - { ...s, pubStage: 2 } // adopted, never a second - else if (outcome == "lost") - awaitInput(endWork(openPr(s, key)), - { kind: "milestone_stopped", g: "", interrupted: "" }) - else { ...openPr(s, key), pubStage: 2 } - else if (s.pubStage == 2) - { ...s, intents: s.intents.exclude(Set(key)), pubStage: 3 } - else - endWork({ ...s, prOn: true, prKey: key, prLast: prIds(s, key), - prWatch: { on: true, floor: prIds(s, key) }, planWatch: OFF, - st: "PR_OPEN" }), - } - } - - // The executor finishes responding to a review comment. - action finishReview(outcome: str): bool = all { - s.work == "review", outcome != "stopped" or s.cancel, - s' = - val t = endWork({ ...s, answered: s.answered.union(Set(s.workCid)), reviewing: 0 }) - if (outcome == "stopped" or outcome == "escalate") - awaitInput(t, { kind: "review_stopped", g: "", interrupted: "" }) - else t, - } - - // The process dies and starts again: volatile state is gone. - action crash(giteaUp: bool): bool = - s' = boot({ ...s, queue: List(), work: "none", workGate: "", workCid: 0, pubStage: 0, - cancel: false, planWatch: OFF, prWatch: OFF }, giteaUp) - - action step = { - nondet kind = MATRIX_KINDS.oneOf() - nondet ckind = COMMENT_KINDS.oneOf() - nondet up = Set(true, false).oneOf() - nondet planOutcome = Set("submit", "ask", "error", "stopped", "exception").oneOf() - nondet workOutcome = Set("completed", "stopped", "escalate", "exception").oneOf() - nondet pubOutcome = Set("ok", "nochanges", "fail", "lost", "exception").oneOf() - any { - send(kind), commentPlan(ckind), commentPr, pollPlan, pollPr, runJob(up), - finishPlanning(planOutcome, up), finishExec(workOutcome), publishStep(pubOutcome), - finishReview(workOutcome), crash(up), - } - } - - // ---- properties --------------------------------------------------------- - - // #66: a working state only while its work is running. - val no_stranding = WORKING.contains(s.st) implies s.work != "none" - - // Waiting on the operator always means a question to answer. - val await_input_has_a_question = s.st == "AWAIT_INPUT" implies s.pending.kind != "" - - // A milestone runs only on an approved plan for that milestone. - val execution_needs_approval = - s.st == "EXECUTING" implies (s.gate.on and s.gate.approved and s.gate.g == msGate(s.ms)) - - // An approval only ever approves the plan the operator was looking at. - val approval_only_of_what_was_seen = not(s.badApproval) - - // Rejection feedback only ever lands on the plan it was written about. - val feedback_only_on_what_was_seen = not(s.staleFeedback) - - // Never more than one PR opened for a milestone. - val one_pr_per_milestone = s.opened.keys().forall(k => s.opened.get(k) <= 1) - - // A review comment the agent took up is answered, being worked on, or - // marked to be delivered again: never silently dropped. - val review_comments_answered = - s.taken.forall(id => s.answered.contains(id) or (s.work == "review" and s.workCid == id) - or s.reviewing == id) - - // With no task, no plan issue is left open. - val no_open_issue_without_a_task = not(s.task) implies s.openIssues.size() == 0 - - val all_properties = and { - no_stranding, await_input_has_a_question, execution_needs_approval, - approval_only_of_what_was_seen, feedback_only_on_what_was_seen, - one_pr_per_milestone, review_comments_answered, no_open_issue_without_a_task, - } -} diff --git a/specs/orchestrator_scenarios.qnt b/specs/orchestrator_scenarios.qnt deleted file mode 100644 index 88307a1..0000000 --- a/specs/orchestrator_scenarios.qnt +++ /dev/null @@ -1,125 +0,0 @@ -// Counterexamples found in orchestrator.qnt, written out as short traces. -// Each ends by expecting the property it breaks to be false; once the design -// is fixed, the expectation is flipped and the trace kept as a regression. -// All fixed: 1-4 (replies bind to the revision up when they arrived), 5 (boot -// and publishing adopt an open PR), 6 (redelivered), 7 (issue closed). -// -// Run: quint test specs/orchestrator_scenarios.qnt (runs every *Test) -module orchestrator_scenarios { - import orchestrator.* from "./orchestrator" - - // Architecture plan rev-1 is up for sign-off, posted to Gitea. - action toArchSignoff = init - .then(send("use")).then(runJob(true)) - .then(finishPlanning("submit", true)) - - // The milestone plan is approved; the next word starts it. - action toMilestoneApproved = toArchSignoff - .then(send("approve")).then(runJob(true)) - .then(finishPlanning("submit", true)) - .then(send("approve")).then(runJob(true)) - - action toPrOpen = toMilestoneApproved - .then(send("start")).then(runJob(true)) - .then(finishExec("completed")) - .then(publishStep("ok")).then(publishStep("ok")) - .then(publishStep("ok")).then(publishStep("ok")) - - // 1. The operator sends "approve" twice, say a double tap or an impatient - // repeat, while looking at the architecture plan. Both are queued - // before either runs. The first approves the architecture; planning - // the milestone runs; the second then approved the milestone plan, - // which the operator had not seen. Fixed: it is about rev-1, not - // applied. - run doubleApproveApprovesAnUnseenPlanTest = - toArchSignoff - .then(send("approve")).then(send("approve")) - .then(runJob(true)) // approves arch rev-1 - .then(finishPlanning("submit", true)) // milestone-1 rev-1 is up - .then(runJob(true)) // the second "approve" - .expect(s.gate.g == "ms1" and not(s.gate.approved) and approval_only_of_what_was_seen) - - // 2. The same with the revision named: "approve rev-1" meant for the - // architecture's rev-1 also matched the milestone's rev-1, because - // revisions were numbered per gate. Fixed: numbered across the task, - // the milestone plan is rev-2. - run namedRevisionApprovesTheWrongGateTest = - toArchSignoff - .then(send("approve")).then(send("approveRev")) - .then(runJob(true)) - .then(finishPlanning("submit", true)) - .then(runJob(true)) - .expect(s.gate.rev == 2 and not(s.gate.approved) and approval_only_of_what_was_seen) - - // 3. Approval on both channels: the operator approves in Matrix and on - // the Gitea issue. The Gitea approval is delivered and queued behind - // the Matrix one, which approves the architecture and posts the - // milestone plan. Re-attaching the plan watch does not recall a job - // already queued, so the Gitea approval approved the milestone plan. - // Fixed: it is about the plan posted before it, arch rev-1. - run approvalOnBothChannelsApprovesTheNextPlanTest = - toArchSignoff - .then(send("approve")).then(commentPlan("approve")).then(pollPlan) - .then(runJob(true)) - .then(finishPlanning("submit", true)) - .then(runJob(true)) - .expect(s.gate.g == "ms1" and not(s.gate.approved) and approval_only_of_what_was_seen) - - // 4. An afterthought becomes a rejection: "approve", then "also, keep it - // in one file", sent together. The note landed on the milestone plan - // as rejection feedback and it was planned again. Fixed: recorded, - // but it rejects nothing. - run anAfterthoughtRejectsTheNextPlanTest = - toArchSignoff - .then(send("approve")).then(send("text")) - .then(runJob(true)) - .then(finishPlanning("submit", true)) - .then(runJob(true)) - .expect(s.st == "AWAIT_MILESTONE_SIGNOFF" and feedback_only_on_what_was_seen) - - // 5. A crash after the PR is opened but before PR_OPEN is recorded. - // Boot found the intent's PR and marked it done, but the state still - // said EXECUTING, so it asked the restart question; 'continue' and - // 'start' then published again, which Gitea refused, with no path back - // to PR_OPEN. Fixed: boot sees the open PR and adopts it. - run crashAfterOpeningThePrPublishesAgainTest = - toMilestoneApproved - .then(send("start")).then(runJob(true)) - .then(finishExec("completed")) - .then(publishStep("ok")) // begin_intent - .then(publishStep("ok")) // push, open the PR - .then(crash(true)) // adopted on boot - .expect(s.st == "PR_OPEN" and s.prOn and one_pr_per_milestone) - - // 5b. Gitea opens the PR but the response is lost; the operator starts - // the milestone again. Publishing adopts the open PR. - run lostResponseThenRetryAdoptsThePrTest = - toMilestoneApproved - .then(send("start")).then(runJob(true)) - .then(finishExec("completed")) - .then(publishStep("ok")).then(publishStep("lost")) - .then(send("continue")).then(runJob(true)) // back to the approved plan - .then(send("start")).then(runJob(true)) - .then(finishExec("completed")) - .then(publishStep("ok")).then(publishStep("ok")) - .then(publishStep("ok")).then(publishStep("ok")) - .expect(s.st == "PR_OPEN" and one_pr_per_milestone) - - // 6. A crash while the executor answers a review comment. The comment's - // cursor was saved before the work started (at most once), and a - // restart in PR_OPEN resumed quietly, so it was never answered. Fixed: - // the comment is marked while answered, and delivered again. - run crashDuringAReviewLosesTheCommentTest = - toPrOpen - .then(commentPr).then(pollPr).then(runJob(true)) // the executor starts on it - .then(crash(true)) - .expect(s.st == "PR_OPEN" and review_comments_answered) - .then(pollPr).then(runJob(true)).then(finishReview("completed")) - .expect(s.answered.size() == 1 and s.reviewing == 0) - - // 7. 'project complete' left the plan issue open. Fixed: it closes it. - run projectCompleteLeavesThePlanIssueOpenTest = - toPrOpen - .then(send("complete")).then(runJob(true)) - .expect(s.st == "IDLE" and no_open_issue_without_a_task) -} diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 67c2866..1ab9f15 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -33,8 +33,8 @@ def live_config() -> AgentConfig: @pytest.fixture def live_model(live_config: AgentConfig) -> str: - """The model the live tests plan, execute and delegate with.""" - return live_config.roles.planner + """The model the live tests drive and delegate with.""" + return live_config.roles.driver @pytest.fixture diff --git a/tests/integration/e2e_support.py b/tests/integration/e2e_support.py index 8b649fc..f2d31f2 100644 --- a/tests/integration/e2e_support.py +++ b/tests/integration/e2e_support.py @@ -10,6 +10,8 @@ from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path +from nisse_dev.config import BuildHost +from nisse_dev.execution import SSHHost from nisse_dev.gitea import GiteaClient, GiteaError from nisse_dev.llm import RoleMap @@ -23,7 +25,7 @@ E2E_MODEL = "qwen3.8-27b-nothink" def e2e_roles() -> RoleMap: model = os.environ.get("NISSE_E2E_MODEL", "").strip() or E2E_MODEL - return RoleMap(executor=model, planner=model, helper=model, expert=model) + return RoleMap(driver=model, helper=model, expert=model) def task(run: str) -> str: @@ -177,3 +179,18 @@ def fresh_repo( finally: if not os.environ.get("NISSE_E2E_KEEP_REPOS"): delete_repo(operator, repo.name) + + +def build_host(ssh_host: SSHHost) -> BuildHost: + """The operator's build VM when scripts/e2e.sh names one, otherwise the + throwaway local sshd.""" + address = os.environ.get("NISSE_E2E_BUILD_HOST") + if address: + key = Path(os.environ["NISSE_E2E_BUILD_KEY"]) + return BuildHost(address=address, key=key) + return BuildHost( + address=ssh_host.address, + key=ssh_host._key, + user=ssh_host.user, + port=ssh_host.port, + ) diff --git a/tests/integration/nisse-test.yaml b/tests/integration/nisse-test.yaml index c450a92..46ab6ee 100644 --- a/tests/integration/nisse-test.yaml +++ b/tests/integration/nisse-test.yaml @@ -36,8 +36,7 @@ llm: # not local-model competence (measured separately by test_model_capability), # so every role uses a model that holds the tool-call contract. roles: - executor: claude-haiku-4-5 - planner: claude-haiku-4-5 + driver: claude-haiku-4-5 helper: claude-haiku-4-5 expert: claude-haiku-4-5 diff --git a/tests/integration/test_benchmark_live.py b/tests/integration/test_benchmark_live.py index 206b885..e74a77a 100644 --- a/tests/integration/test_benchmark_live.py +++ b/tests/integration/test_benchmark_live.py @@ -1,16 +1,16 @@ """One live benchmark run: the simplest task, through the agent's own -client wiring, with the test config's executor.""" +client wiring, with the test config's driver.""" from pathlib import Path import pytest -from nisse_dev.agent import make_llm_clients from nisse_dev.bench.runner import run_task from nisse_dev.bench.tasks import load_tasks from nisse_dev.catalog import ModelCatalog from nisse_dev.config import AgentConfig from nisse_dev.execution import SSHHost +from nisse_dev.loop.build import make_llm_clients pytestmark = pytest.mark.integration @@ -20,7 +20,7 @@ def test_the_simplest_task_passes_live( ) -> None: catalog = ModelCatalog.discover(live_config.llm_base_url, live_config.llm_token) normalized, _ = make_llm_clients(live_config, catalog) - facts = catalog.facts(live_config.roles.executor) + facts = catalog.facts(live_config.roles.driver) [task] = load_tasks(["write-function"]) record = run_task( diff --git a/tests/integration/test_crash_durability.py b/tests/integration/test_crash_durability.py index dad16db..e0eeb02 100644 --- a/tests/integration/test_crash_durability.py +++ b/tests/integration/test_crash_durability.py @@ -19,7 +19,8 @@ from nisse_dev.canonical import UserMessage from nisse_dev.state import StateStore store = StateStore(sys.argv[1]) -store.transition("EXECUTING", cause="test", project="demo", data={"milestone": 2}) +store.open_work_item("demo-1", title="t", scope="demo", position="RUNNING", data={"milestone": 2}, cause="test") +store.set_active("demo-1", cause="test") store.begin_intent("push_branch", {"branch": "milestone-2"}, cause="test") store.transcript("t", "operator").append(UserMessage(text="start milestone 2", source="m:1")) os._exit(1) # die without closing anything, as a power cut would @@ -38,10 +39,10 @@ def test_state_survives_process_death(tmp_path: Path) -> None: assert result.returncode == 1, result.stderr with StateStore(db) as store: - state = store.get_state() - assert state is not None - assert state.state == "EXECUTING" - assert state.data == {"milestone": 2} + item = store.active_item() + assert item is not None + assert item.position == "RUNNING" + assert item.data == {"milestone": 2} report = reconcile_on_boot(store, {"push_branch": lambda intent: None}) assert report.action == "await_input" diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py deleted file mode 100644 index 408d90a..0000000 --- a/tests/integration/test_end_to_end.py +++ /dev/null @@ -1,214 +0,0 @@ -"""The end-to-end test (Development Plan M1.6): the whole agent, assembled as -it runs in production, against real models, the Gitea sandbox and a real -build host, driven by a scripted operator. - -It automates the checks of bring-up run 3: -- a milestone reaches a PR on the sandbox that contains only the work, no - build artifacts (#80), pushed with the push credential; -- the clone on the build host holds no credential (#89); -- after the milestone, the planner's view carries the orchestrator's facts - and none of the executor's turns (#78). - -Only the Matrix transport is replaced, by the in-memory one: the operator's -Matrix credentials are not something a test should hold. Run it with -scripts/e2e.sh after each milestone; it spends model tokens and takes minutes. -""" - -import contextlib -import os -import shlex -import uuid -from collections.abc import Iterator -from dataclasses import replace -from fnmatch import fnmatch -from pathlib import Path -from typing import Any - -import pytest -from e2e_support import e2e_roles, fresh_repo, operator_gitea -from e2e_support import task as make_task - -from nisse_dev.agent import Agent, build_agent -from nisse_dev.canonical import UserMessage -from nisse_dev.config import AgentConfig, BuildHost -from nisse_dev.control import ( - AWAIT_ARCH_SIGNOFF, - AWAIT_INPUT, - AWAIT_MILESTONE_SIGNOFF, - PR_OPEN, -) -from nisse_dev.execution import SSHHost -from nisse_dev.gitea import GiteaClient, GiteaError -from nisse_dev.transport import InMemoryTransport - -pytestmark = [pytest.mark.integration, pytest.mark.e2e] - -OPERATOR = "@dongar:dongar.net" # an allowlisted operator in nisse-test.yaml -SETTLE_S = 1200 # a milestone, with real models, inside one message - - -def build_host(ssh_host: SSHHost) -> BuildHost: - """The operator's build VM when scripts/e2e.sh names one, otherwise the - throwaway local sshd.""" - address = os.environ.get("NISSE_E2E_BUILD_HOST") - if address: - key = Path(os.environ["NISSE_E2E_BUILD_KEY"]) - return BuildHost(address=address, key=key) - return BuildHost( - address=ssh_host.address, - key=ssh_host._key, - user=ssh_host.user, - port=ssh_host.port, - ) - - -@pytest.fixture -def agent( - live_config: AgentConfig, - ssh_host: SSHHost, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> Iterator[tuple[Agent, InMemoryTransport, str]]: - run = uuid.uuid4().hex[:8] - stack = contextlib.ExitStack() - operator = operator_gitea(live_config.gitea_url) - account = GiteaClient(live_config.gitea_url, live_config.gitea_token).whoami() - fresh = stack.enter_context( - fresh_repo(operator, run, tmp_path / "keys", agent=account) - ) - config = replace( - live_config.with_project( - repo=fresh.name, - base_branch="main", - push_key=fresh.push_key, - workspace=Path(f"/tmp/nisse-e2e-{run}/sandbox"), - build_host=build_host(ssh_host), - ), - roles=e2e_roles(), - state_db=tmp_path / "state.db", - ) - transport = InMemoryTransport() - monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: transport) - agent = None - try: - agent = build_agent(config) - agent.start() - yield agent, transport, run - finally: - if agent is not None: - agent.stop() - # The run's clone on the build host goes too, even if startup failed. - host = SSHHost( - config.build_host.address, - key=config.build_host.key, - user=config.build_host.user, - port=config.build_host.port, - state_dir=tmp_path / "ssh", - ) - host.execute(f"rm -rf /tmp/nisse-e2e-{run}") - stack.close() # the run's repository - - -class Operator: - """Says things to the agent and waits for it to finish with each.""" - - def __init__(self, agent: Agent, transport: InMemoryTransport) -> None: - self.agent = agent - self.transport = transport - - def say(self, text: str) -> None: - self.transport.receive(OPERATOR, text) - # Messages are handled in order on the agent's worker, so a no-op - # behind this one returns when it, and the work it started, is done. - self.agent.worker.call(lambda: None, timeout=SETTLE_S) - - def state(self) -> Any: - return self.agent.worker.call(self.agent.store.get_state) - - def drive_to_pr(self, max_steps: int = 12) -> Any: - """Answer whatever the agent actually does: a live model may ask a - question at any gate, so a fixed script would break.""" - for _ in range(max_steps): - state = self.state() - gate = state.data.get("gate") or {} - if state.state == PR_OPEN: - return state - if state.state == AWAIT_INPUT: - self.say("Use your best judgement and proceed.") - elif state.state == AWAIT_MILESTONE_SIGNOFF and gate.get("approved"): - self.say("start milestone 1") - elif state.state in (AWAIT_ARCH_SIGNOFF, AWAIT_MILESTONE_SIGNOFF): - self.say("approve") - else: - raise AssertionError(f"stuck in {state.state}: {self.last()}") - raise AssertionError(f"no PR after {max_steps} steps: {self.last()}") - - def last(self) -> str: - return self.transport.outbound[-1].text if self.transport.outbound else "" - - -# The test's own idea of an artifact, independent of the code's list, so -# emptying that list can't make this pass. -def is_artifact(path: str) -> bool: - parts = path.split("/") - return ( - "__pycache__" in parts - or ".pytest_cache" in parts - or fnmatch(parts[-1], "*.py[co]") - ) - - -def test_a_milestone_end_to_end( - agent: tuple[Agent, InMemoryTransport, str], -) -> None: - running, transport, run = agent - operator = Operator(running, transport) - gitea, repo = running.gitea, running.config.gitea_repo - pr: dict[str, Any] = {} - try: - # The task first, before any project: it must be kept, not lost. - operator.say(make_task(run)) - operator.say(f"use {running.config.project}") - state = operator.drive_to_pr() - pr = state.data["pr"] - task = state.data["task_id"] - - # --- the PR holds the work and only the work (D4, #80) --- - files = gitea._request("GET", f"/repos/{repo}/pulls/{pr['number']}/files") - changed = [f["filename"] for f in files] - assert changed, "the PR is empty" - assert not [p for p in changed if is_artifact(p)], changed - found = gitea._request("GET", f"/repos/{repo}/pulls/{pr['number']}") - assert found["head"]["ref"] == f"{task}-milestone-1" - - # --- no credential in the clone on the build host (#89) --- - host = SSHHost( - running.config.build_host.address, - key=running.config.build_host.key, - user=running.config.build_host.user, - port=running.config.build_host.port, - state_dir=running.config.state_db.parent / "ssh", - ) - config = host.execute( - f"cat {shlex.quote(str(running.config.workspace / '.git' / 'config'))}" - ).stdout - assert "PRIVATE KEY" not in config - assert running.config.gitea_token not in config - - # --- the planner sees facts, not the executor's turns (#78) --- - operator.say("next milestone") - store = running.store - view = running.worker.call(lambda: store.transcript(task, "planner").view()) - said = [e.event.text for e in view if isinstance(e.event, UserMessage)] - assert any("[orchestrator] milestone 1 finished" in t for t in said), said - assert any("[orchestrator] milestone 1 published" in t for t in said), said - assert not [e for e in view if e.actor == "executor"], "executor turns leaked" - finally: - # Leave the sandbox as it was: closes the PR and the plan issue. - operator.say("abandon") - if pr: - branch = gitea._request("GET", f"/repos/{repo}/pulls/{pr['number']}")[ - "head" - ]["ref"] - with contextlib.suppress(GiteaError): # cleanup is best effort - gitea._request("DELETE", f"/repos/{repo}/branches/{branch}") diff --git a/tests/integration/test_end_to_end_loop.py b/tests/integration/test_end_to_end_loop.py index f1fd533..a17d4cc 100644 --- a/tests/integration/test_end_to_end_loop.py +++ b/tests/integration/test_end_to_end_loop.py @@ -22,7 +22,14 @@ from pathlib import Path from typing import Any import pytest -from e2e_support import dependent_task, e2e_roles, fresh_repo, operator_gitea, task +from e2e_support import ( + build_host, + dependent_task, + e2e_roles, + fresh_repo, + operator_gitea, + task, +) from nisse_dev.config import AgentConfig, LoopTuning from nisse_dev.execution import SSHHost @@ -54,8 +61,6 @@ def running( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> Iterator[tuple[RunningLoop, InMemoryTransport, str]]: - from test_end_to_end import build_host - run = uuid.uuid4().hex[:8] stack = contextlib.ExitStack() # Kept outside pytest's temp directories, which pytest prunes, so a @@ -66,15 +71,18 @@ def running( operator = operator_gitea(live_config.gitea_url) account = GiteaClient(live_config.gitea_url, live_config.gitea_token).whoami() fresh = stack.enter_context(fresh_repo(operator, run, keep / "keys", agent=account)) + [project] = live_config.projects + project = replace( + project, + repo=fresh.name, + base_branch="main", + push_key=fresh.push_key, + workspace=Path(f"/tmp/nisse-e2e-{run}/sandbox"), + build_host=build_host(ssh_host), + ) config = replace( - live_config.with_project( - repo=fresh.name, - base_branch="main", - push_key=fresh.push_key, - workspace=Path(f"/tmp/nisse-e2e-{run}/sandbox"), - build_host=build_host(ssh_host), - ), - loop="new", + live_config, + projects=(project,), roles=e2e_roles(), state_db=keep / "state.db", # Short, so a bug's grace period runs out within the test. @@ -97,10 +105,10 @@ def running( if agent is not None: agent.stop() host = SSHHost( - config.build_host.address, - key=config.build_host.key, - user=config.build_host.user, - port=config.build_host.port, + project.build_host.address, + key=project.build_host.key, + user=project.build_host.user, + port=project.build_host.port, state_dir=tmp_path / "ssh", ) host.execute(f"rm -rf /tmp/nisse-e2e-{run}") @@ -197,7 +205,7 @@ class Operator: def merge_and_close(self, issue: int) -> None: """Approve and merge the item's PR, and see the agent close the item.""" - gitea, repo = self.agent.gitea, self.agent.config.gitea_repo + gitea, repo = self.agent.gitea, self.agent.config.projects[0].repo as_operator = operator_gitea(self.agent.config.gitea_url) pr = loop_of(self.item(issue)).pr with contextlib.suppress(GiteaError): # approval may not be required @@ -222,7 +230,7 @@ class Operator: def tidy(self, issues: list[int], prs: list[int]) -> None: """Leave the sandbox tidy: whatever is still open gets closed.""" - gitea, repo = self.agent.gitea, self.agent.config.gitea_repo + gitea, repo = self.agent.gitea, self.agent.config.projects[0].repo as_operator = operator_gitea(self.agent.config.gitea_url) for issue in issues: item = self.item(issue) @@ -245,7 +253,7 @@ def test_the_slice_end_to_end( ) -> None: agent, transport, run = running operator = Operator(agent, transport) - gitea, repo = agent.gitea, agent.config.gitea_repo + gitea, repo = agent.gitea, agent.config.projects[0].repo as_operator = operator_gitea(agent.config.gitea_url) issue = None pr = 0 @@ -297,7 +305,7 @@ def test_an_issue_blocked_on_another_waits_for_it( closes; then B starts from a workspace that has A's work.""" agent, transport, run = running operator = Operator(agent, transport) - gitea, repo = agent.gitea, agent.config.gitea_repo + gitea, repo = agent.gitea, agent.config.projects[0].repo as_operator = operator_gitea(agent.config.gitea_url) owner, name = repo.split("/") issues: list[int] = [] @@ -360,7 +368,7 @@ def test_a_bug_goes_ahead_after_its_grace_period( the grace period runs out, with no go from the operator.""" agent, transport, run = running operator = Operator(agent, transport) - repo, base = agent.config.gitea_repo, agent.config.base_branch + repo, base = agent.config.projects[0].repo, agent.config.projects[0].base_branch as_operator = operator_gitea(agent.config.gitea_url) issues: list[int] = [] prs: list[int] = [] @@ -412,7 +420,7 @@ def test_a_plain_yes_is_a_go_ahead( and the agent says so before acting.""" agent, transport, run = running operator = Operator(agent, transport) - repo = agent.config.gitea_repo + repo = agent.config.projects[0].repo as_operator = operator_gitea(agent.config.gitea_url) issues: list[int] = [] prs: list[int] = [] @@ -454,7 +462,7 @@ def test_a_two_milestone_feature( it; the item closes after both merge.""" agent, transport, run = running operator = Operator(agent, transport) - gitea, repo = agent.gitea, agent.config.gitea_repo + gitea, repo = agent.gitea, agent.config.projects[0].repo as_operator = operator_gitea(agent.config.gitea_url) issues: list[int] = [] prs: list[int] = [] @@ -510,7 +518,7 @@ def test_a_conversation_becomes_filed_work( is proposed, agreed, filed as a record issue, and started as agreed.""" agent, transport, run = running operator = Operator(agent, transport) - gitea, repo = agent.gitea, agent.config.gitea_repo + gitea, repo = agent.gitea, agent.config.projects[0].repo issues: list[int] = [] prs: list[int] = [] @@ -602,7 +610,7 @@ def test_a_conversation_becomes_a_new_project( with fresh_repo( as_operator, f"{run}-new", keep / "keys", agent=agent.account, seed=False ) as repo: - host = agent.config.build_host + host = agent.config.projects[0].build_host filled = ( entry.group(1) .replace(f"/{name}", repo.name) @@ -646,7 +654,7 @@ def test_a_question_leaves_the_restatement_up( nothing: the same revision stays up, and a go binds to it.""" agent, transport, run = running operator = Operator(agent, transport) - repo = agent.config.gitea_repo + repo = agent.config.projects[0].repo as_operator = operator_gitea(agent.config.gitea_url) issues: list[int] = [] try: diff --git a/tests/integration/test_expert_live.py b/tests/integration/test_expert_live.py index a61fb6b..0df266a 100644 --- a/tests/integration/test_expert_live.py +++ b/tests/integration/test_expert_live.py @@ -22,7 +22,7 @@ pytestmark = pytest.mark.integration def desk(tmp_path: Path, litellm: LiteLLMClient, live_model: str): def make(budget: int = 2) -> tuple[ExpertDesk, StateStore]: model = live_model - roles = RoleMap(executor=model, planner=model, helper=model, expert=model) + roles = RoleMap(driver=model, helper=model, expert=model) store = StateStore( tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()) ) diff --git a/tests/integration/test_gate_restart.py b/tests/integration/test_gate_restart.py deleted file mode 100644 index f3e6d8f..0000000 --- a/tests/integration/test_gate_restart.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Integration: a restart mid-gate resumes the open gate with its revision -history intact; unresolved intents gate the whole boot to AWAIT_INPUT -(architecture L8 and L3 together).""" - -import io -from pathlib import Path - -import pytest - -from nisse_dev.control import ( - AWAIT_ARCH_SIGNOFF, - AWAIT_INPUT, - AWAIT_MILESTONE_SIGNOFF, - Orchestrator, - StubAgent, -) -from nisse_dev.oplog import OpLog -from nisse_dev.state import StateStore -from nisse_dev.transport import InMemoryTransport - -pytestmark = pytest.mark.integration - - -def boot(db: Path) -> tuple[Orchestrator, StateStore, InMemoryTransport]: - store = StateStore(db, oplog=OpLog("nisse-dev", io.StringIO())) - transport = InMemoryTransport() - orch = Orchestrator(store, transport, StubAgent(), operators={"don"}) - orch.start() - return orch, store, transport - - -def test_restart_mid_gate_resumes_with_revisions_intact(tmp_path: Path) -> None: - db = tmp_path / "state.db" - _, store, transport = boot(db) - transport.receive("don", "use demo") - transport.receive("don", "needs work") # arch gate now at rev-2 - task_id = store.oplog.task_id - store.close() - - _, store2, transport2 = boot(db) - state = store2.get_state() - assert state is not None - assert state.state == AWAIT_ARCH_SIGNOFF - assert store2.oplog.task_id == task_id # correlation survives restart - notices = [m.text for m in transport2.outbound] - assert any("rev-2" in n and "still open" in n for n in notices) - - transport2.receive("don", "approve rev-1") # stale across the restart too - assert store2.get_state().state == AWAIT_ARCH_SIGNOFF - transport2.receive("don", "approve rev-2") - assert store2.get_state().state == AWAIT_MILESTONE_SIGNOFF - - -def test_restart_with_unresolved_intent_gates_boot(tmp_path: Path) -> None: - db = tmp_path / "state.db" - _, store, transport = boot(db) - transport.receive("don", "use demo") - store.begin_intent("push_branch", {"branch": "m1"}, cause="test") - store.close() - - _, store2, transport2 = boot(db) - assert store2.get_state().state == AWAIT_INPUT - question = transport2.outbound[-1] - assert question.kind == "question" - assert "unresolved work" in question.text diff --git a/tests/integration/test_gitea_live.py b/tests/integration/test_gitea_live.py index f1c6e29..2b9beef 100644 --- a/tests/integration/test_gitea_live.py +++ b/tests/integration/test_gitea_live.py @@ -9,11 +9,11 @@ from pathlib import Path import pytest from e2e_support import fresh_repo, operator_gitea -from nisse_dev.agent import remote_url from nisse_dev.config import AgentConfig from nisse_dev.execution import SSHHost from nisse_dev.git import GitError, GitRepo, SSHKey from nisse_dev.gitea import GiteaClient, pull_request_reconciler +from nisse_dev.loop.build import remote_url from nisse_dev.state import Intent pytestmark = pytest.mark.integration diff --git a/tests/integration/test_llm_live.py b/tests/integration/test_llm_live.py index e2821ab..b596ae9 100644 --- a/tests/integration/test_llm_live.py +++ b/tests/integration/test_llm_live.py @@ -11,7 +11,7 @@ from nisse_dev.llm import LiteLLMClient pytestmark = pytest.mark.integration -def test_executor_model_completes_via_litellm( +def test_the_driver_model_completes_via_litellm( litellm: LiteLLMClient, live_config: AgentConfig ) -> None: events = [ @@ -23,7 +23,7 @@ def test_executor_model_completes_via_litellm( ), ) ] - result = litellm.complete(events, tools=None, model=live_config.roles.executor) + result = litellm.complete(events, tools=None, model=live_config.roles.driver) texts = [e.text for e in result if isinstance(e, AssistantText)] assert texts, f"no assistant text in {result!r}" # A plumbing check, so any answer will do: what the model chooses to say @@ -41,7 +41,7 @@ def test_declared_limits_reach_a_budgeted_request( roles = live_config.roles.pinned() catalog.require_limits(roles) # every test role has a declared window - model = live_config.roles.executor + model = live_config.roles.driver facts = catalog.facts(model) client = BudgetedClient( litellm, diff --git a/tests/integration/test_milestone_live.py b/tests/integration/test_milestone_live.py deleted file mode 100644 index f58c3b3..0000000 --- a/tests/integration/test_milestone_live.py +++ /dev/null @@ -1,99 +0,0 @@ -"""End-to-end milestone against real models: plan with the planner role, -approve through the gate, then let the executor role actually create and test -a file on disk. - -Uses the test config's planner model for every role: it needs a model that -holds the tool-call contract on both ends. -""" - -import io -from pathlib import Path - -import pytest - -from nisse_dev.control import ( - AWAIT_ARCH_SIGNOFF, - AWAIT_INPUT, - AWAIT_MILESTONE_SIGNOFF, - EXECUTING, - PR_OPEN, - Orchestrator, -) -from nisse_dev.execution import SSHHost -from nisse_dev.executor import LLMExecutor -from nisse_dev.llm import LiteLLMClient, RoleMap -from nisse_dev.oplog import OpLog -from nisse_dev.planner import LLMPlanningAgent -from nisse_dev.state import StateStore -from nisse_dev.tools import Workspace -from nisse_dev.transport import InMemoryTransport - -pytestmark = pytest.mark.integration - - -def _drive_to_execution( - store: StateStore, transport: InMemoryTransport, max_steps: int = 10 -) -> None: - """Play the operator. A scripted approve/approve/start sequence is not - robust against a live model: ask_human is a legitimate move at any gate, - and a question shifts the state machine under a fixed script. So respond - to whatever the agent actually did.""" - for _ in range(max_steps): - state = store.get_state() - assert state is not None - if state.state == EXECUTING: - return - gate = state.data.get("gate") or {} - if state.state == AWAIT_INPUT: - transport.receive("don", "Use your best judgement and proceed.") - elif state.state == AWAIT_MILESTONE_SIGNOFF and gate.get("approved"): - # The milestone runs to completion inside this command. - transport.receive("don", "start milestone 1") - return - elif state.state in (AWAIT_ARCH_SIGNOFF, AWAIT_MILESTONE_SIGNOFF): - transport.receive("don", "approve") - else: - raise AssertionError(f"unexpected state {state.state}") - raise AssertionError( - f"never reached execution; last message: {transport.outbound[-1]}" - ) - - -def test_plan_approve_execute_produces_a_real_file( - tmp_path: Path, litellm: LiteLLMClient, live_model: str, ssh_host: SSHHost -) -> None: - # Both roles use the capable model here: the point is the whole gated - # loop over a real proxy, not local-model competence (measured separately). - model = live_model - roles = RoleMap(executor=model, planner=model, helper=model, expert=model) - client = litellm - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - workspace = Workspace(ssh_host, tmp_path / "work", timeout_s=60) - transport = InMemoryTransport() - orchestrator = Orchestrator( - store, - transport, - LLMPlanningAgent(client, roles), - operators={"don"}, - roles=roles, - executor=LLMExecutor(client, roles, workspace, max_turns=10), - ) - orchestrator.start() - - transport.receive( - "don", - "use greet. Milestone 1: create greet.py in the workspace root defining " - "greet(name) returning 'Hello, !', and verify it by running python.", - ) - _drive_to_execution(store, transport) - - state = store.get_state() - assert state is not None - # Never EXECUTING at rest: with no publisher a finished milestone waits in - # PR_OPEN, and one that stopped short waits for the operator. - assert state.state in (PR_OPEN, AWAIT_INPUT) - - created = list(workspace.root.rglob("*.py")) - assert created, f"executor created no Python file; said: {transport.outbound[-1]}" - combined = "\n".join(p.read_text() for p in created) - assert "def greet" in combined diff --git a/tests/integration/test_model_capability.py b/tests/integration/test_model_capability.py index 3703cbe..dbb5a0a 100644 --- a/tests/integration/test_model_capability.py +++ b/tests/integration/test_model_capability.py @@ -30,11 +30,10 @@ from nisse_dev.canonical import ( ) from nisse_dev.catalog import ModelCatalog, select_client from nisse_dev.config import AgentConfig -from nisse_dev.executor import FINISH_TOOL from nisse_dev.expert import parse_answer from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient, LLMError -from nisse_dev.planner import PLANNER_TOOLS -from nisse_dev.tools import EXECUTOR_TOOLS +from nisse_dev.loop.tools import ASK_HUMAN, FINISH_MILESTONE, RESTATE +from nisse_dev.tools import EXECUTION_TOOLS pytestmark = pytest.mark.integration @@ -99,44 +98,42 @@ def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None assert texts, f"{model} produced no text: {produced!r}" -def test_model_holds_the_planner_contract( +def test_model_holds_the_restatement_contract( model: str, catalog: ModelCatalog, routes ) -> None: - """L3's approvals are structural at the tool layer: a model that answers in - prose cannot be gated, because there is no plan to approve.""" + """L3's go-ahead is structural at the tool layer: a driver that answers + in prose cannot be gated, because there is no restatement to agree to.""" normalized, passthrough = routes client = select_client(model, normalized, passthrough, catalog) produced = client.complete( - user("Draft an architecture plan for a command-line slug generator."), - PLANNER_TOOLS, + user("Issue #1: add a command-line slug generator."), + [RESTATE, ASK_HUMAN], model, system=( - "You must answer by calling exactly one tool: submit_plan or " + "You must answer by calling exactly one tool: restate or " "ask_human. Plain prose is not accepted." ), ) assert_answered(client, model, catalog) calls = [e for e in produced if isinstance(e, ToolCall)] assert calls, f"{model} answered in prose instead of calling a tool" - assert calls[0].tool in {"submit_plan", "ask_human"} - if calls[0].tool == "submit_plan": - content = calls[0].arguments.get("content", "") + assert calls[0].tool in {"restate", "ask_human"} + if calls[0].tool == "restate": + content = calls[0].arguments.get("text", "") # llama3.2 emitted "{" here — structurally a tool call, substantively # nothing. Length is a crude proxy, but it separates the two. - assert len(str(content)) > 80, ( - f"{model} submitted a degenerate plan: {content!r}" - ) + assert len(str(content)) > 80, f"{model} restated degenerately: {content!r}" def test_model_round_trips_a_tool_result( model: str, catalog: ModelCatalog, routes ) -> None: - """The executor loop feeds results back; a model that cannot consume its + """The driver's run feeds results back; a model that cannot consume its own tool result cannot run a milestone.""" normalized, passthrough = routes client = select_client(model, normalized, passthrough, catalog) events = user("List the files in the current directory using your tools.") - produced = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.") + produced = client.complete(events, EXECUTION_TOOLS, model, system="Use a tool.") calls = [e for e in produced if isinstance(e, ToolCall)] assert calls, f"{model} made no tool call" @@ -154,7 +151,7 @@ def test_model_round_trips_a_tool_result( ) ) assert_answered(client, model, catalog) - follow_up = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.") + follow_up = client.complete(events, EXECUTION_TOOLS, model, system="Use a tool.") assert follow_up, f"{model} produced nothing after its tool result" @@ -203,7 +200,7 @@ def test_route_selection_matches_the_model( assert_answered(chosen, model, catalog) -FINISH_TOOLS = [*EXECUTOR_TOOLS, FINISH_TOOL] +FINISH_TOOLS = [*EXECUTION_TOOLS, FINISH_MILESTONE] OFFERED = {tool["function"]["name"] for tool in FINISH_TOOLS} @@ -212,7 +209,7 @@ def test_model_calls_only_the_tools_it_was_offered( ) -> None: """Measured on a server that did not enforce tool_choice: a model asked to chat called "chat", "say_hello" and "respond", none of them offered. - The executor counts those as failures; this measures how often.""" + The driver answers those as failures; this measures how often.""" normalized, passthrough = routes client = select_client(model, normalized, passthrough, catalog) produced = client.complete( diff --git a/tests/integration/test_passthrough_live.py b/tests/integration/test_passthrough_live.py index 8ec328a..55f4f60 100644 --- a/tests/integration/test_passthrough_live.py +++ b/tests/integration/test_passthrough_live.py @@ -16,7 +16,7 @@ from nisse_dev.config import AgentConfig from nisse_dev.llm import AnthropicPassthroughClient from nisse_dev.oplog import OpLog from nisse_dev.state import StateStore -from nisse_dev.tools import EXECUTOR_TOOLS +from nisse_dev.tools import EXECUTION_TOOLS pytestmark = pytest.mark.integration @@ -54,7 +54,7 @@ def test_passthrough_round_trips_a_tool_call( client = passthrough produced = client.complete( store.transcript("t", "helper").read(), - EXECUTOR_TOOLS, + EXECUTION_TOOLS, live_model, system="Use a tool. Do not answer in prose.", ) @@ -69,7 +69,7 @@ def test_passthrough_round_trips_a_tool_call( ToolResult(call_id=calls[0].call_id, result="app.py\ntests/", success=True) ) follow_up = client.complete( - store.transcript("t", "helper").read(), EXECUTOR_TOOLS, live_model + store.transcript("t", "helper").read(), EXECUTION_TOOLS, live_model ) assert follow_up, "the vendor rejected or ignored the tool_result turn" diff --git a/tests/integration/test_planning_live.py b/tests/integration/test_planning_live.py deleted file mode 100644 index 0ee41ae..0000000 --- a/tests/integration/test_planning_live.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Live planning turn: a real model must answer with a submit_plan or -ask_human tool call, not prose. - -Plans with the test config's planner role. The planner role is a frontier -model today (Development Plan §2), and small local models do not hold the tool-call -contract reliably, so that role must be one that does. -""" - -import io -from pathlib import Path - -import pytest - -from nisse_dev.canonical import UserMessage -from nisse_dev.control import AskHuman, SubmitPlan -from nisse_dev.llm import LiteLLMClient, RoleMap -from nisse_dev.oplog import OpLog -from nisse_dev.planner import LLMPlanningAgent -from nisse_dev.state import StateStore - -pytestmark = pytest.mark.integration - - -def test_live_planning_turn_returns_a_tool_action( - tmp_path: Path, litellm: LiteLLMClient, live_model: str -) -> None: - model = live_model - roles = RoleMap(executor=model, planner=model, helper=model, expert=model) - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - store.transcript("t", "operator").append( - UserMessage( - text=( - "Project 'hello-cli': a Python script that prints a greeting. " - "Draft the architecture plan." - ), - source="live-test", - ) - ) - agent = LLMPlanningAgent(litellm, roles) - - action = agent.plan(store.transcript("t", "planner"), "arch", "hello-cli") - - assert isinstance(action, SubmitPlan | AskHuman) - text = action.content if isinstance(action, SubmitPlan) else action.question - assert text.strip(), "tool call carried empty content" - # The turn must be durable regardless of what the model said (L8/L7). - assert len(store.transcript("t", "planner").read()) > 1