diff --git a/.env.example b/.env.example index 57cb427..bda36a0 100644 --- a/.env.example +++ b/.env.example @@ -1,59 +1,17 @@ -# Copy to .env (chmod 600). Real environment variables win over this file. -# .env is gitignored; never commit real values. - -# --- LLM access (A5): the LiteLLM proxy is the single LLM destination --- -NISSE_LLM_BASE_URL=http://litellm.example:4000 -NISSE_LLM_TOKEN=sk-your-litellm-virtual-key - -# --- Role mapping (A11): roles resolve to concrete model strings --- -NISSE_ROLE_EXECUTOR=anthropic/claude-haiku-4-5-20251001 -NISSE_ROLE_PLANNER=anthropic/claude-opus-5 -NISSE_ROLE_DELEGATE=anthropic/claude-opus-5 -NISSE_ROLE_CONSULTANT=anthropic/claude-opus-5 - -# Opt-in for the live planning test: a model the virtual key can reach that -# holds the tool-call contract. Planning escalates unconditionally (A12), and -# small local models do not hold the contract reliably — llama3.2 returns -# truncated tool arguments, Haiku-class and up are solid. -NISSE_LIVE_PLANNER_MODEL=claude-haiku-4-5 - -# --- Matrix transport (A1), used by nisse-dev's MatrixTransport --- -NISSE_MATRIX_HOMESERVER=https://matrix.example -NISSE_MATRIX_USER_ID=@nisse-dev:example -NISSE_MATRIX_TOKEN=syt_your_matrix_access_token -NISSE_MATRIX_ROOM_ID=!roomid:example -# Dedicated room for the live integration test; never the production room. -NISSE_MATRIX_TEST_ROOM_ID=!testroomid:example -NISSE_MATRIX_OPERATORS=@you:example - -# --- Gitea (B2): service account for branches, PRs, and comment intake --- -NISSE_GITEA_URL=https://gitea.example -NISSE_GITEA_TOKEN=your-scoped-service-account-token -# Sandbox repo for live tests; never the real project repo. -NISSE_GITEA_TEST_REPO=owner/sandbox-repo -# Gitea logins allowed to drive the agent via PR comments (A4 per-channel -# allowlist; a Matrix MXID and a Gitea login are different names). -NISSE_GITEA_OPERATORS=your-gitea-login - -# --- Running the agent (the demo, not the tests) --- -# The tests use NISSE_GITEA_TEST_REPO; a running agent uses these. +# Secrets only. Copy to .env (chmod 600); it is gitignored, so never commit +# real values. Real environment variables win over this file. # -# Layout. NISSE_WORKSPACE is the *clone the agent works in*, not the directory -# the agent runs from — they must not be the same place: -# -# ~/nisse-demo-workspace/ deployment root, not a git repo -# ├── .env this file -# ├── state.db the A2 state store -# └── project/ NISSE_WORKSPACE, the clone -# -# The agent commits its workspace with `git add -A` and pushes it, so anything -# inside the clone reaches the project repository. A .env or state store placed -# there would publish credentials as a side effect of the agent working -# correctly; startup refuses to proceed if either is inside the workspace. -NISSE_STATE_DB=~/nisse-demo-workspace/state.db -NISSE_WORKSPACE=~/nisse-demo-workspace/project -NISSE_PROJECT=demo -NISSE_GITEA_REPO=owner/demo-repo -# NISSE_BASE_BRANCH=main -# Path to this file, if not ./.env -# NISSE_ENV=~/nisse-demo-workspace/.env +# Everything else lives in the YAML config (see nisse-dev.example.yaml), +# which refers to these by name, e.g. `token: ${NISSE_LLM_TOKEN}`. The live +# integration tests use the same names, from tests/integration/nisse-test.yaml. + +# llm.token: the LiteLLM virtual key (A5) +NISSE_LLM_TOKEN=replace-me +# matrix.token: the Matrix access token for the agent's account (A1) +NISSE_MATRIX_TOKEN=replace-me +# gitea.token: the Gitea service account's scoped token (B2) +NISSE_GITEA_TOKEN=replace-me + +# Where the config file is, if not ./nisse-dev.yaml. Where this file is, if +# not ./.env, is NISSE_ENV, which has to be set in the real environment. +# NISSE_CONFIG=~/nisse-demo-workspace/nisse-dev.yaml diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 6f7d82c..07030d1 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -34,19 +34,11 @@ jobs: run: uv run pytest -m "not integration" - name: Integration tests - # Live-path credentials come from repo Actions secrets; the tests - # skip-guard on these, so a missing secret or unreachable service - # degrades to "skipped", never a failure on unrelated work. + # Everything but the secrets is in tests/integration/nisse-test.yaml, + # which refers to these by name. A missing secret fails the live + # tests that need it; it is never quietly turned into a skip. env: - NISSE_LLM_BASE_URL: ${{ secrets.NISSE_LLM_BASE_URL }} NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }} - NISSE_ROLE_EXECUTOR: ${{ secrets.NISSE_ROLE_EXECUTOR }} - NISSE_LIVE_PLANNER_MODEL: ${{ secrets.NISSE_LIVE_PLANNER_MODEL }} - NISSE_MATRIX_HOMESERVER: ${{ secrets.NISSE_MATRIX_HOMESERVER }} - NISSE_MATRIX_USER_ID: ${{ secrets.NISSE_MATRIX_USER_ID }} NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }} - NISSE_MATRIX_TEST_ROOM_ID: ${{ secrets.NISSE_MATRIX_TEST_ROOM_ID }} - NISSE_GITEA_URL: ${{ secrets.NISSE_GITEA_URL }} NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }} - NISSE_GITEA_TEST_REPO: ${{ secrets.NISSE_GITEA_TEST_REPO }} run: uv run pytest -m integration -rs diff --git a/.gitignore b/.gitignore index 7efcb3d..46559d1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ __pycache__/ .pytest_cache/ .ruff_cache/ dist/ +nisse-dev.yaml diff --git a/docs/Architecture-Build-Spec.md b/docs/Architecture-Build-Spec.md index 405ba34..d7ce157 100644 --- a/docs/Architecture-Build-Spec.md +++ b/docs/Architecture-Build-Spec.md @@ -389,8 +389,10 @@ Requirements: - **No vendor credentials on the agent VM.** LiteLLM holds every key, sourced from Bitwarden machine accounts (A7); the agent holds one LiteLLM token. Adding a vendor does not touch the agent VM, and a compromised agent leaks no provider keys. -- Config from env/`.env`, **nothing hardcoded**: base URL, auth token, and the role - mapping (A11). Changing any of it is an env change + restart, nothing more. +- Config from the agent's config file, **nothing hardcoded**: base URL, auth token, + and the role mapping (A11). Secrets in it are references to environment + variables, never values. Changing any of it is a config edit + restart, + nothing more. - **Budget caps and spend alerts live on the LiteLLM virtual key**, not in agent code. ### A6. Browser Tool diff --git a/docs/Tuning-Guide.md b/docs/Tuning-Guide.md new file mode 100644 index 0000000..49fb34f --- /dev/null +++ b/docs/Tuning-Guide.md @@ -0,0 +1,288 @@ +# Tuning Guide + +How to fit a local model onto the inference host, and how to set the agent's +tuning knobs. Part 1 covers the ideas you need to reason about the hardware. +Part 2 covers each knob in `nisse-dev.yaml`. Part 3 gives an order to tune +in. + +The knobs live under `tuning:` in the config file (see +`nisse-dev.example.yaml`). They are read at startup, so a change takes effect +on the next restart. Unknown or misspelt keys are startup errors. + +--- + +## Part 1 — How a local model spends VRAM + +A model server holding a model in GPU memory spends it on three things: + +1. **Weights**, which are fixed for a given model and quantization. +2. **The KV cache**, which grows with context length. +3. **Working memory** for activations and runtime buffers. Budget 1–2 GB of + headroom. + +On a 24 GB card, the whole game is trading (1) against (2). + +### Weights and quantization + +A model's weights take roughly *parameters × bits per weight ÷ 8* bytes. + +| | 16-bit (full) | 8-bit | ~4.5-bit (Q4_K_M, AWQ) | +|---|---|---|---| +| 24B | 48 GB | 24 GB | ~14 GB | +| 32B | 64 GB | 32 GB | ~19 GB | + +Quantization stores each weight in fewer bits. Common 4-bit formats come out +nearer 4.5–5 bits per weight once scaling factors are counted, so check the +actual file size. The quality cost is small at 5–6 bits, usually acceptable at +4, and it climbs quickly below that. For agent work the cost shows up first as +**worse tool-call discipline**, not as worse prose. The capability matrix +(Part 3) measures exactly that. + +**Mixture-of-experts** models such as Qwen3-30B-A3B are different. Memory is +set by *total* parameters (30B), but speed is set by the parameters *active* +for each token (3B). An MoE model of a given size therefore generates much +faster than a dense model of the same size. It's often weaker than a dense +model of the same total size, but not always for tool use. + +### The KV cache, and why context costs memory + +As the model reads the prompt, every layer stores a key and a value vector for +every token, so that later tokens can attend to them without recomputing. +That store is the KV cache. Its size per token is: + + 2 (key and value) × layers × KV heads × head size × bytes per value + +The numbers come from the model's `config.json` (`num_hidden_layers`, +`num_key_value_heads`, `head_dim`). Some worked examples at 16-bit: + +| Model | Layers × KV heads × head size | Per token | 32k tokens | +|---|---|---|---| +| Qwen3-32B | 64 × 8 × 128 | 256 KB | ~8 GB | +| Mistral/Devstral Small 24B | 40 × 8 × 128 | 160 KB | ~5 GB | +| Qwen3-30B-A3B (MoE) | 48 × 4 × 128 | 96 KB | ~3 GB | + +(Taken from the published configs; recheck against the model card before +relying on them.) + +Two consequences: + +- **Context length is a memory decision, not a free setting.** The spec's + estimate (Part H) was about 5 GB left for KV cache beside a 4-bit 32B model. + At 16-bit, that's only about 20k tokens of context for Qwen3-32B. +- **KV-cache quantization** stores keys and values in 8 bits (or 4). At 8 + bits it halves the cache and is close to lossless. At 4 bits it is + noticeably worse. Quantizing the cache to 8-bit is usually the first lever + to pull, since it buys twice the context for almost nothing. + +The server allocates the cache for the context length you configure. It +doesn't matter how much of it a request uses. So set the context length on the +server deliberately: it is the ceiling for *prompt plus response* on every +request. + +### Prompt processing vs generation + +A request has two phases, and they run at very different speeds: + +- **Prefill** reads the prompt. It is parallel and fast: thousands of tokens + per second on a 4090. +- **Decode** writes the response one token at a time. Roughly 30–40 tokens per + second for a dense 32B at 4-bit, and several times that for a 3B-active MoE. + Treat these numbers as ballpark; measure your own. + +An agent loop re-sends the whole transcript on every turn, which makes the +prompt long and the responses short (a tool call is often under 200 tokens). +So prefill matters more than the headline tokens-per-second suggests. + +**Prefix caching** is the fix. If the start of a prompt matches the previous +request, the server reuses the KV cache for it and prefills only the new part. +llama.cpp does this per slot, and vLLM does it automatically. For this +workload it is often the single largest speed win. It works here because the +rendered transcript only grows at the end, so consecutive turns share +everything but their newest events. + +### Tokens and characters + +For English and code, a token is roughly **4 characters**. That's useful for +converting the character-based knobs below into context cost. + +--- + +## Part 2 — The knobs + +For each knob: what it controls, what goes wrong at either extreme, and what +to look at when deciding. + +### `llm.max_output_tokens` — default 16384 + +The ceiling on a single response. It is sent with every request, for every +role. + +- **Why it exists:** a response that hits the ceiling is truncated + mid-sentence, or worse, mid-tool-call, where it can still parse. The agent + detects truncation and treats it as an error, but a ceiling that's too + tight still costs a failed turn. Planning with a frontier model was observed + using over 4,000 output tokens. +- **Too high, on a local model:** the server reserves room for the whole + response inside the context window. With a 32k context and a 16k ceiling, + only 16k is left for the prompt. Some servers reject any request where + prompt plus ceiling exceeds the context. +- **Too low:** truncated plans and tool calls. +- **Deciding:** executor turns are short, so 2–4k is plenty for a local + executor. Frontier planning wants the large default. This one knob covers + every role for now; a per-model setting is coming with the context budget + (see *Coming next*). + +### `tuning.executor.max_turns` — default 24 + +How many model calls one milestone may make before the executor stops with +`capped` and asks you what to do. + +- **Too low:** milestones that were making progress stop short. You'll see + `capped` outcomes on work that was going well. +- **Too high:** a confused model keeps going. Every turn also adds its tool + calls and results to the transcript, so a long run spends context as well + as time. +- **Deciding:** look at how many turns *successful* milestones take, and set + the cap comfortably above the high end, around 1.5–2× the turns of your + longer successful runs. A stronger model usually needs fewer turns, not + more. + +### `tuning.executor.failure_limit` — default 2 + +How many consecutive failed `run_command` calls trigger escalation: the +sub-task is handed to a frontier model as a delegation (spec A12). A +successful command resets the count. Failed file reads don't count, since +reading is looking around, not doing the work. + +- **Why it's a count and not a judgment:** small models are poorly calibrated + about their own competence. Asked "are you stuck?", they say no until far + too late, or yes constantly. A count of observable failures is a signal + that doesn't depend on the model's opinion of itself. +- **Too low:** normal iteration escalates. Test-first work *expects* a failing + test run before the fix, so a limit of 1 hands off routine work and spends + frontier tokens on it. +- **Too high:** a weak model circles, re-running the same failing command + with small variations, while the transcript fills up. +- **Deciding:** raise it as local models get stronger. Watch two things + together. First, how often delegations fix what the local model couldn't + (escalation is doing its job). Second, how often the local model recovers + on its own after its second failure (the limit may be too low). + +### `tuning.expert.consult_budget` — default 3 + +How many `ask_expert` calls one milestone may make. `ask_expert` is a single +stateless question to a stronger model, with no access to the repository +(spec A14), for things like "is this API deprecated?". + +- **Too low:** the executor can't get unstuck on a knowledge question it + can't answer from the code. +- **Too high:** a confused executor asks variations of the same question + instead of doing the work or failing. That's the circling A12 warns about. +- **Deciding:** count consults per milestone, and check whether the answers + changed what the executor did next. If it regularly exhausts the budget + without progress, the budget is fine and the model or the plan is the + problem. + +### `tuning.delegation` — hard caps on one delegation + +A delegation is a frontier model working a sub-task with full tool access +(spec A13). When a cap is hit, the delegation **fails**, and its branch and +work are discarded, because resuming on a half-edited tree isn't safe. + +| Knob | Default | Caps | +|---|---|---| +| `max_turns` | 30 | model calls | +| `max_tokens` | 200000 | input + output tokens, as reported by the provider | +| `wall_clock_s` | 900 | elapsed time | + +- **`max_tokens` is a scope limit, not the money limit.** Spending limits live + on the LiteLLM key (spec A5). This cap stops one delegation from growing + without bound. An unbounded one could still exhaust a daily budget and take + every frontier model away until it resets. +- **Too low:** delegations fail just short of finishing and their work is + thrown away. That is the most expensive way to be wrong, because you paid + for the work. +- **Too high:** a stuck delegation burns tokens and time before failing. +- **Deciding:** every delegation records the tokens it used and how long it + ran (the `delegation_completed` event, captured for evaluation per spec + A16). Set each cap above what successful delegations actually use, with + room to spare. + +### `tuning.tools.command_timeout_s` — default 120 + +How long one `run_command` may run before its whole process group is killed. + +- **Too low:** slow test suites, dependency installs and Docker builds get + killed and reported as failures. That also counts toward `failure_limit`. +- **Too high:** a command that never exits blocks the turn for the whole + timeout, such as a dev server started in the foreground or a test waiting on + input. `stop` waits for a running command to finish too, so this is also the + worst-case delay before a stop takes effect. +- **Deciding:** set it just above the slowest legitimate command in the + projects the agent works on. + +### `tuning.tools.max_output_chars` — default 4000 + +How much of a tool's output (command output, file contents, directory +listings) the model sees. Anything past this is cut, with a note saying how +much was dropped. + +- **This is a context knob.** Every tool result stays in the transcript and is + re-sent every turn. 4,000 characters is about 1,000 tokens, so 20 turns of + maximal output is about 20k tokens, which is already most of a local + model's context. +- **Too low:** the model doesn't see the part of the output that matters. +- **Too high:** the transcript outgrows the context window sooner. +- **A known weakness:** truncation keeps the *start* of the output, but test + runners put their failure summary at the *end*. A long, failing pytest run + can lose exactly the lines the model needed. Keeping the head and the tail + would fix this, and is a candidate for the context-budget work. + +--- + +## Part 3 — An order to tune in + +Upstream choices change what's right downstream, so tune in this order and +change one thing at a time. + +1. **Model and quantization.** Start with whether a candidate holds the + tool-call contract at all: + + uv run pytest -m integration tests/integration/test_model_capability.py \ + --capability-models=alias-a,alias-b + + A model that falls back to prose instead of calling tools is disqualified, + however good its benchmarks look. Before comparing aliases, check what + each one actually points at in LiteLLM (`/model/info`, the + `litellm_params.model` field). Two aliases believed to be the same model + once weren't, and that cost a lot of debugging. +2. **Context length and KV-cache precision** on the server. Take the largest + context that fits beside the weights with 1–2 GB of headroom. Try an 8-bit + KV cache before giving up context. +3. **`llm.max_output_tokens`**, so the prompt keeps most of that context. +4. **Real milestones.** Run representative work and look at the evidence: + turns used, escalations, delegation outcomes, and time taken. +5. **The loop knobs**: `failure_limit`, `max_turns`, `consult_budget`, the + delegation caps, then the tool limits, one at a time. + +**Expect noise.** Local model behaviour varies a lot from run to run. One +early comparison measured 12 of 20 runs holding the tool contract against 20 +of 20, a real difference (p ≈ 0.003). Three runs each could easily have +pointed the other way. Before changing a knob because of a result, repeat the +run enough times that the result would survive a coin flip. + +--- + +## Coming next + +The next changes in the 4090-readiness work add knobs, and this guide gains a +section for each: + +- **A context budget.** Rendering a transcript that fits the model's context + window, with what to drop or summarize when it doesn't, and per-model output + ceilings. +- **Enforced tool calls.** An explicit `finish` tool, so the server can be told + a tool call is required and hold the model to the contract, where today it + can fall back to prose. +- **A benchmark script** that runs the capability matrix and representative + milestones across candidate models and prints a comparison table. diff --git a/nisse-dev.example.yaml b/nisse-dev.example.yaml new file mode 100644 index 0000000..8f61011 --- /dev/null +++ b/nisse-dev.example.yaml @@ -0,0 +1,57 @@ +# Example configuration for nisse-dev. Copy to nisse-dev.yaml (or point +# NISSE_CONFIG at your copy) and edit. +# +# - Structure lives here. Secrets do not: write them as ${VARIABLE} references, +# resolved from the environment (or .env, see .env.example) at startup. +# - Keep this file outside the workspace clone. Startup refuses otherwise, +# because the agent commits and pushes everything in the clone. +# - Unknown keys are errors, so a misspelt knob fails at startup instead of +# silently doing nothing. + +state_db: ~/nisse-demo-workspace/state.db +workspace: ~/nisse-demo-workspace/project # the clone +project: demo + +llm: # A5: LiteLLM is the only LLM destination + base_url: http://litellm.example:4000 + token: ${NISSE_LLM_TOKEN} + # Ceiling on each response. Must fit inside a local model's context window + # alongside the prompt; see docs/Tuning-Guide.md. + max_output_tokens: 16384 + +roles: # A11: role -> LiteLLM model alias + executor: local-fast + planner: claude-opus-5 + delegate: claude-opus-5 + consultant: claude-opus-5 + +matrix: # A1 + homeserver: https://matrix.example + user_id: "@nisse-dev:example" + token: ${NISSE_MATRIX_TOKEN} + room_id: "!roomid:example" + operators: ["@you:example"] + +gitea: # B2 + url: https://gitea.example + token: ${NISSE_GITEA_TOKEN} + repo: owner/demo-repo + operators: [your-gitea-login] + base_branch: main + +# What each knob does, and how to choose a value: docs/Tuning-Guide.md. +# These are the defaults; the section can be left out entirely. The file is +# read at startup, so a change takes effect on the next restart. +tuning: + executor: + max_turns: 24 # model calls per milestone before it stops as capped + failure_limit: 2 # consecutive failed commands before escalating (A12) + expert: + consult_budget: 3 # ask_expert calls per milestone (A14) + delegation: # hard caps on one delegation (A13) + max_turns: 30 + max_tokens: 200000 + wall_clock_s: 900 + tools: + command_timeout_s: 120 # one run_command before it is killed + max_output_chars: 4000 # tool output kept for the model; the rest is cut diff --git a/nisse-dev/pyproject.toml b/nisse-dev/pyproject.toml index 050c4b4..21971be 100644 --- a/nisse-dev/pyproject.toml +++ b/nisse-dev/pyproject.toml @@ -3,7 +3,11 @@ name = "nisse-dev" version = "0.1.0" description = "Dev agent: plans and implements software projects, produces Gitea PRs" requires-python = ">=3.12" -dependencies = ["nisse-core", "httpx>=0.27"] +dependencies = [ + "nisse-core", + "httpx>=0.27", + "pyyaml>=6", +] [project.scripts] nisse-dev = "nisse_dev.main:main" diff --git a/nisse-dev/src/nisse_dev/agent.py b/nisse-dev/src/nisse_dev/agent.py index 2510e4f..7aa0f6f 100644 --- a/nisse-dev/src/nisse_dev/agent.py +++ b/nisse-dev/src/nisse_dev/agent.py @@ -16,7 +16,7 @@ from nisse_dev.cancel import Cancel from nisse_dev.catalog import ModelCatalog, select_client from nisse_dev.config import AgentConfig from nisse_dev.control import Orchestrator -from nisse_dev.delegation import Delegation +from nisse_dev.delegation import Caps, Delegation from nisse_dev.executor import LLMExecutor from nisse_dev.expert import ExpertDesk from nisse_dev.git import GitRepo, authenticated_url @@ -149,18 +149,29 @@ def build_agent( account = gitea.whoami() git_repo = prepare_workspace(config, account) - workspace = Workspace(config.workspace) + tuning = config.tuning + workspace = Workspace( + config.workspace, + timeout_s=tuning.tools.command_timeout_s, + max_output_chars=tuning.tools.max_output_chars, + ) - normalized = LiteLLMClient(config.llm_base_url, config.llm_token) + normalized = LiteLLMClient( + config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens + ) # Delegations prefer the vendor-native route; fidelity is cheap # elsewhere (A5). Which route a given model gets is decided by # select_client, not by where the caller happens to sit. - passthrough = AnthropicPassthroughClient(config.llm_base_url, config.llm_token) + passthrough = AnthropicPassthroughClient( + config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens + ) transport = MatrixTransport(config.matrix) announce = notify or transport.send_message - expert = ExpertDesk(normalized, store, config.roles) + expert = ExpertDesk( + normalized, store, config.roles, budget=tuning.expert.consult_budget + ) delegate_model = config.roles.resolve("delegate") delegation = Delegation( select_client(delegate_model, normalized, passthrough, catalog), @@ -169,6 +180,11 @@ def build_agent( git_repo, model=delegate_model, backing_model=catalog.facts(delegate_model).backing, + caps=Caps( + turns=tuning.delegation.max_turns, + tokens=tuning.delegation.max_tokens, + wall_clock_s=tuning.delegation.wall_clock_s, + ), cancel=cancel, ) executor = LLMExecutor( @@ -180,6 +196,8 @@ def build_agent( delegate=delegation, notify=announce, cancel=cancel, + max_turns=tuning.executor.max_turns, + failure_limit=tuning.executor.failure_limit, ) publisher = MilestonePublisher( store, diff --git a/nisse-dev/src/nisse_dev/catalog.py b/nisse-dev/src/nisse_dev/catalog.py index ce22d9d..175684d 100644 --- a/nisse-dev/src/nisse_dev/catalog.py +++ b/nisse-dev/src/nisse_dev/catalog.py @@ -121,8 +121,8 @@ class ModelCatalog: Reported together rather than one per boot: an operator fixing a config wants the whole list, not to rediscover the next typo on the - next start (the same reason ``AgentConfig.from_env`` batches its - missing variables). + next start (the same reason ``AgentConfig`` reports every missing + setting at once). """ unknown = { role: alias for role, alias in roles.items() if alias not in self._facts diff --git a/nisse-dev/src/nisse_dev/config.py b/nisse-dev/src/nisse_dev/config.py index 1b3dd51..03829f0 100644 --- a/nisse-dev/src/nisse_dev/config.py +++ b/nisse-dev/src/nisse_dev/config.py @@ -1,49 +1,198 @@ -"""Agent configuration (spec A5: from env/.env, nothing hardcoded). +"""Agent configuration (spec A5: nothing hardcoded). + +Configuration is a YAML file: roles, connections, and the tuning knobs (see +docs/Tuning-Guide.md). It holds no secrets. A credential is written as a +reference, ``token: ${NISSE_LLM_TOKEN}``, and resolved from the environment, +so the file can be reviewed, shared, or committed without leaking. The +environment supplies those references and nothing else: every setting has +exactly one home. Every value is collected and validated **before** anything is built, and a -single error names every missing variable at once. The alternative — failing -at first use — surfaces a missing token twenty minutes into a milestone, on -the call that mattered. +single error names every problem at once. The alternative — failing at first +use — surfaces a missing token twenty minutes into a milestone, on the call +that mattered. Unknown keys are errors too: a misspelt tuning knob that was +silently ignored would look exactly like a knob that has no effect. """ import os +import re from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field, fields from pathlib import Path +from typing import Any -from nisse_dev.llm import RoleMap +import yaml + +from nisse_dev.llm import DEFAULT_MAX_OUTPUT_TOKENS, RoleMap from nisse_dev.matrix import MatrixConfig -REQUIRED = ( - "NISSE_STATE_DB", - "NISSE_WORKSPACE", - "NISSE_PROJECT", - "NISSE_LLM_BASE_URL", - "NISSE_LLM_TOKEN", - "NISSE_ROLE_EXECUTOR", - "NISSE_ROLE_PLANNER", - "NISSE_ROLE_DELEGATE", - "NISSE_ROLE_CONSULTANT", - "NISSE_MATRIX_HOMESERVER", - "NISSE_MATRIX_USER_ID", - "NISSE_MATRIX_TOKEN", - "NISSE_MATRIX_ROOM_ID", - "NISSE_MATRIX_OPERATORS", - "NISSE_GITEA_URL", - "NISSE_GITEA_TOKEN", - "NISSE_GITEA_REPO", - "NISSE_GITEA_OPERATORS", +DEFAULT_CONFIG_FILE = "nisse-dev.yaml" + +REQUIRED_KEYS = ( + "state_db", + "workspace", + "project", + "llm.base_url", + "llm.token", + "roles.executor", + "roles.planner", + "roles.delegate", + "roles.consultant", + "matrix.homeserver", + "matrix.user_id", + "matrix.token", + "matrix.room_id", + "matrix.operators", + "gitea.url", + "gitea.token", + "gitea.repo", + "gitea.operators", ) +OPTIONAL_KEYS = ("llm.max_output_tokens", "gitea.base_branch") + +_REFERENCE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") class ConfigError(RuntimeError): pass -def _people(raw: str) -> frozenset[str]: - """Operator allowlists are comma-separated; blanks are dropped so a - trailing comma cannot admit an empty identity.""" - return frozenset(part.strip() for part in raw.split(",") if part.strip()) +# --- tuning ---------------------------------------------------------------- +# The defaults are the values the agent ran with before they were +# configurable, so an absent `tuning:` section changes nothing. + + +@dataclass(frozen=True) +class ExecutorTuning: + max_turns: int = 24 + failure_limit: int = 2 + + +@dataclass(frozen=True) +class ExpertTuning: + consult_budget: int = 3 + + +@dataclass(frozen=True) +class DelegationTuning: + max_turns: int = 30 + max_tokens: int = 200_000 + wall_clock_s: float = 900.0 + + +@dataclass(frozen=True) +class ToolsTuning: + command_timeout_s: int = 120 + max_output_chars: int = 4000 + + +@dataclass(frozen=True) +class Tuning: + executor: ExecutorTuning = field(default_factory=ExecutorTuning) + expert: ExpertTuning = field(default_factory=ExpertTuning) + delegation: DelegationTuning = field(default_factory=DelegationTuning) + tools: ToolsTuning = field(default_factory=ToolsTuning) + + +_TUNING_SECTIONS: dict[str, type] = { + f.name: f.default_factory # type: ignore[misc] + for f in fields(Tuning) +} + + +def _parse_tuning(raw: Any, errors: list[str]) -> Tuning: + if raw is None: + return Tuning() + if not isinstance(raw, dict): + errors.append("tuning: must be a mapping") + return Tuning() + sections: dict[str, Any] = {} + for name, section in raw.items(): + cls = _TUNING_SECTIONS.get(name) + if cls is None: + errors.append(f"tuning.{name}: unknown section") + continue + if not isinstance(section, dict): + errors.append(f"tuning.{name}: must be a mapping") + continue + known = {f.name: f for f in fields(cls)} + values: dict[str, Any] = {} + for key, value in section.items(): + spec = known.get(key) + if spec is None: + errors.append(f"tuning.{name}.{key}: unknown setting") + continue + number = _positive(value, float if spec.type in (float, "float") else int) + if number is None: + errors.append(f"tuning.{name}.{key}: must be a positive number") + continue + values[key] = number + sections[name] = cls(**values) + return Tuning(**sections) + + +def _positive(value: Any, kind: type) -> int | float | None: + # bool is an int in Python; `true` for a turn limit is a mistake. + if isinstance(value, bool) or not isinstance(value, int | float): + return None + if kind is int and not float(value).is_integer(): + return None + if value <= 0: + return None + return kind(value) + + +# --- loading --------------------------------------------------------------- + + +def _people(raw: Any) -> frozenset[str]: + """Operator allowlists: a YAML list, or comma-separated in the + environment. Blanks are dropped so a trailing comma cannot admit an empty + identity.""" + parts = raw if isinstance(raw, list) else str(raw).split(",") + return frozenset(str(part).strip() for part in parts if str(part).strip()) + + +def _flatten(tree: Mapping[str, Any], prefix: str = "") -> dict[str, Any]: + flat: dict[str, Any] = {} + for key, value in tree.items(): + dotted = f"{prefix}{key}" + if isinstance(value, dict) and dotted != "tuning": + flat.update(_flatten(value, f"{dotted}.")) + else: + flat[dotted] = value + return flat + + +def _resolve(value: Any, key: str, env: Mapping[str, str], errors: list[str]) -> Any: + """Replace ${VAR} references with the environment's value. A reference to + an unset variable is an error, never an empty string: an empty token would + pass for a value and fail at the first request.""" + if not isinstance(value, str): + return value + + def lookup(match: re.Match[str]) -> str: + name = match.group(1) + found = (env.get(name) or "").strip() + if not found: + errors.append(f"{key}: references ${{{name}}}, which is unset or empty") + return found + + return _REFERENCE.sub(lookup, value) + + +def _read_file(path: Path, errors: list[str]) -> dict[str, Any]: + try: + loaded = yaml.safe_load(path.read_text()) + except yaml.YAMLError as exc: + errors.append(f"{path}: not valid YAML ({exc})") + return {} + if loaded is None: + return {} + if not isinstance(loaded, dict): + errors.append(f"{path}: must be a mapping at the top level") + return {} + return loaded @dataclass(frozen=True) @@ -61,30 +210,107 @@ class AgentConfig: gitea_repo: str gitea_operators: frozenset[str] base_branch: str = "main" + max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS + tuning: Tuning = field(default_factory=Tuning) + config_file: Path | None = None @classmethod - def from_env(cls, env: Mapping[str, str] = os.environ) -> "AgentConfig": + def load(cls, env: Mapping[str, str] = os.environ) -> "AgentConfig": + """The file named by NISSE_CONFIG, or ./nisse-dev.yaml.""" + path = Path(env.get("NISSE_CONFIG") or DEFAULT_CONFIG_FILE).expanduser() + if not path.is_file(): + raise ConfigError( + f"cannot start: no config file at {path}; create it from " + "nisse-dev.example.yaml, or point NISSE_CONFIG at one" + ) + return cls.from_file(path, env) + + @classmethod + def from_file( + cls, path: Path, env: Mapping[str, str] = os.environ + ) -> "AgentConfig": + errors: list[str] = [] + tree = _read_file(path, errors) + return cls._build(tree, env, errors, config_file=path) + + @classmethod + def from_tree( + cls, tree: Mapping[str, Any], env: Mapping[str, str] = os.environ + ) -> "AgentConfig": + """From an already-parsed config, as the file would give it.""" + return cls._build(tree, env, [], config_file=None) + + @classmethod + def _build( + cls, + tree: Mapping[str, Any], + env: Mapping[str, str], + errors: list[str], + *, + config_file: Path | None, + ) -> "AgentConfig": + tuning = _parse_tuning(tree.get("tuning"), errors) + flat = _flatten(tree) + flat.pop("tuning", None) + + known = set(REQUIRED_KEYS) | set(OPTIONAL_KEYS) + for key in sorted(set(flat) - known): + errors.append(f"{key}: unknown setting") + values = { + key: _resolve(value, key, env, errors) + for key, value in flat.items() + if key in known + } + # Whitespace-only counts as missing: a token of spaces would pass a # truthiness check and fail at the first request instead. - missing = [name for name in REQUIRED if not (env.get(name) or "").strip()] + missing = [ + key + for key in REQUIRED_KEYS + if values.get(key) is None + or (isinstance(values[key], str) and not values[key].strip()) + or (isinstance(values[key], list) and not _people(values[key])) + ] if missing: - raise ConfigError( - "cannot start: these are unset or empty — " + ", ".join(missing) - ) + errors.append("unset or empty — " + ", ".join(missing)) + + max_output = values.get("llm.max_output_tokens", DEFAULT_MAX_OUTPUT_TOKENS) + try: + max_output = int(max_output) + if max_output <= 0: + raise ValueError + except (TypeError, ValueError): + errors.append("llm.max_output_tokens: must be a positive integer") + if errors: + raise ConfigError("cannot start: " + "; ".join(errors)) + config = cls( - state_db=Path(env["NISSE_STATE_DB"]).expanduser(), - workspace=Path(env["NISSE_WORKSPACE"]).expanduser(), - project=env["NISSE_PROJECT"], - llm_base_url=env["NISSE_LLM_BASE_URL"], - llm_token=env["NISSE_LLM_TOKEN"], - roles=RoleMap.from_env(env), - matrix=MatrixConfig.from_env(env), - operators=_people(env["NISSE_MATRIX_OPERATORS"]), - gitea_url=env["NISSE_GITEA_URL"], - gitea_token=env["NISSE_GITEA_TOKEN"], - gitea_repo=env["NISSE_GITEA_REPO"], - gitea_operators=_people(env["NISSE_GITEA_OPERATORS"]), - base_branch=env.get("NISSE_BASE_BRANCH", "main"), + state_db=Path(values["state_db"]).expanduser(), + workspace=Path(values["workspace"]).expanduser(), + project=str(values["project"]), + llm_base_url=str(values["llm.base_url"]), + llm_token=str(values["llm.token"]), + roles=RoleMap( + executor=str(values["roles.executor"]), + planner=str(values["roles.planner"]), + delegate=str(values["roles.delegate"]), + consultant=str(values["roles.consultant"]), + ), + matrix=MatrixConfig( + homeserver=str(values["matrix.homeserver"]).rstrip("/"), + user_id=str(values["matrix.user_id"]), + token=str(values["matrix.token"]), + room_id=str(values["matrix.room_id"]), + ), + operators=_people(values["matrix.operators"]), + gitea_url=str(values["gitea.url"]), + gitea_token=str(values["gitea.token"]), + gitea_repo=str(values["gitea.repo"]), + gitea_operators=_people(values["gitea.operators"]), + base_branch=str(values.get("gitea.base_branch") or "main"), + max_output_tokens=max_output, + tuning=tuning, + config_file=config_file, ) config._refuse_secrets_inside_the_clone(env) return config @@ -93,14 +319,20 @@ class AgentConfig: """The workspace is a git clone the agent commits with ``git add -A`` and pushes. Anything inside it reaches the project repository, so a state store or an env file placed there would publish credentials as a - side effect of the agent working correctly — silently, and once. + side effect of the agent working correctly — silently, and once. The + config file holds no secrets itself, but it names where they come + from and how the agent is deployed, which is not the project's + business either. """ workspace = self.workspace.expanduser().resolve() - offenders = [] - for label, path in ( - ("NISSE_STATE_DB", self.state_db), + candidates = [ + ("state_db", self.state_db), ("NISSE_ENV", Path(env.get("NISSE_ENV", ".env"))), - ): + ] + if self.config_file is not None: + candidates.append(("NISSE_CONFIG", self.config_file)) + offenders = [] + for label, path in candidates: resolved = path.expanduser().resolve() if resolved == workspace or workspace in resolved.parents: offenders.append(f"{label} ({resolved})") diff --git a/nisse-dev/src/nisse_dev/gitea.py b/nisse-dev/src/nisse_dev/gitea.py index d5fb62b..6a0678f 100644 --- a/nisse-dev/src/nisse_dev/gitea.py +++ b/nisse-dev/src/nisse_dev/gitea.py @@ -10,8 +10,6 @@ idempotent across a crash: boot reconciliation (A3) asks Gitea whether the effect already happened rather than repeating it. """ -import os -from collections.abc import Mapping from dataclasses import dataclass from typing import Any @@ -60,14 +58,6 @@ class GiteaClient: transport=transport, ) - @classmethod - def from_env(cls, env: Mapping[str, str] = os.environ) -> "GiteaClient": - base_url = env.get("NISSE_GITEA_URL") - token = env.get("NISSE_GITEA_TOKEN") - if not base_url or not token: - raise GiteaError("set NISSE_GITEA_URL and NISSE_GITEA_TOKEN") - return cls(base_url, token) - def _request(self, method: str, path: str, **kwargs: Any) -> Any: try: response = self._http.request(method, path, **kwargs) diff --git a/nisse-dev/src/nisse_dev/llm.py b/nisse-dev/src/nisse_dev/llm.py index 77c4841..7f1542d 100644 --- a/nisse-dev/src/nisse_dev/llm.py +++ b/nisse-dev/src/nisse_dev/llm.py @@ -2,18 +2,16 @@ One client interface, present from the first real LLM call. All traffic goes to the LiteLLM proxy — the VM's single LLM destination; the agent holds one -LiteLLM token and no vendor credentials. Config comes from the environment -(``NISSE_LLM_BASE_URL``, ``NISSE_LLM_TOKEN``), nothing hardcoded. +LiteLLM token and no vendor credentials. The base URL and token come from the +agent's config (nisse_dev.config), nothing hardcoded. Model identity is expressed as roles (A11), resolved to concrete model strings -from env config; the resolved mapping is pinned at task start so a mid-task +from config; the resolved mapping is pinned at task start so a mid-task config change cannot split a task across two planners. """ import json -import os import time -from collections.abc import Mapping from dataclasses import asdict, dataclass from typing import Any, ClassVar, Protocol @@ -49,11 +47,6 @@ RATE_LIMIT_RETRY_S = 20.0 DEFAULT_MAX_OUTPUT_TOKENS = 16384 -def max_output_tokens(env: Mapping[str, str] = os.environ) -> int: - raw = env.get("NISSE_MAX_OUTPUT_TOKENS") - return int(raw) if raw else DEFAULT_MAX_OUTPUT_TOKENS - - class UpstreamUnavailable(LLMError): """The proxy or provider could not be reached, or refused for reasons unrelated to what was asked: a network blip, a timeout, a 5xx, a rate @@ -113,22 +106,10 @@ class RoleMap: delegate: str consultant: str - _ENV_VARS: ClassVar[dict[str, str]] = { - "executor": "NISSE_ROLE_EXECUTOR", - "planner": "NISSE_ROLE_PLANNER", - "delegate": "NISSE_ROLE_DELEGATE", - "consultant": "NISSE_ROLE_CONSULTANT", - } - - @classmethod - def from_env(cls, env: Mapping[str, str] = os.environ) -> "RoleMap": - missing = [var for var in cls._ENV_VARS.values() if not env.get(var)] - if missing: - raise LLMError(f"role mapping incomplete; set: {', '.join(missing)}") - return cls(**{role: env[var] for role, var in cls._ENV_VARS.items()}) + ROLES: ClassVar[tuple[str, ...]] = ("executor", "planner", "delegate", "consultant") def resolve(self, role: str) -> str: - if role not in self._ENV_VARS: + if role not in self.ROLES: raise LLMError(f"unknown role: {role!r}") return str(getattr(self, role)) @@ -156,20 +137,12 @@ class LiteLLMClient: ) self._renderer = OpenAIChatRenderer() self._sleep = sleep - self._max_tokens = max_tokens or max_output_tokens() + self._max_tokens = max_tokens or DEFAULT_MAX_OUTPUT_TOKENS # Set after each call so a caller can meter itself (A13 caps); # a side channel rather than a return value, so the LLMClient # protocol stays the same for every implementation. self.last_usage: dict[str, int] = {} - @classmethod - def from_env(cls, env: Mapping[str, str] = os.environ) -> "LiteLLMClient": - base_url = env.get("NISSE_LLM_BASE_URL") - token = env.get("NISSE_LLM_TOKEN") - if not base_url or not token: - raise LLMError("set NISSE_LLM_BASE_URL and NISSE_LLM_TOKEN") - return cls(base_url, token) - def complete( self, events: list[StoredEvent], @@ -203,7 +176,7 @@ class LiteLLMClient: if choice.get("finish_reason") == "length": raise TruncatedResponse( f"{requested_model} hit its output limit mid-answer; " - "raise NISSE_MAX_OUTPUT_TOKENS" + "raise llm.max_output_tokens" ) produced_by = data.get("model") or requested_model usage = data.get("usage") or {} @@ -275,20 +248,10 @@ class AnthropicPassthroughClient: transport=transport, ) self._renderer = AnthropicMessagesRenderer() - self._max_tokens = max_tokens or max_output_tokens() + self._max_tokens = max_tokens or DEFAULT_MAX_OUTPUT_TOKENS self._sleep = sleep self.last_usage: dict[str, int] = {} - @classmethod - def from_env( - cls, env: Mapping[str, str] = os.environ - ) -> "AnthropicPassthroughClient": - base_url = env.get("NISSE_LLM_BASE_URL") - token = env.get("NISSE_LLM_TOKEN") - if not base_url or not token: - raise LLMError("set NISSE_LLM_BASE_URL and NISSE_LLM_TOKEN") - return cls(base_url, token) - def complete( self, events: list[StoredEvent], @@ -315,7 +278,7 @@ class AnthropicPassthroughClient: if data.get("stop_reason") == "max_tokens": raise TruncatedResponse( f"{requested_model} hit the output limit of {self._max_tokens} " - f"tokens mid-answer; raise NISSE_MAX_OUTPUT_TOKENS" + f"tokens mid-answer; raise llm.max_output_tokens" ) content = data.get("content") if not isinstance(content, list): diff --git a/nisse-dev/src/nisse_dev/main.py b/nisse-dev/src/nisse_dev/main.py index da9971f..6e2591c 100644 --- a/nisse-dev/src/nisse_dev/main.py +++ b/nisse-dev/src/nisse_dev/main.py @@ -20,7 +20,7 @@ from nisse_dev.envfile import load_env_file def main() -> int: load_env_file(Path(os.environ.get("NISSE_ENV", ".env")).expanduser()) try: - config = AgentConfig.from_env() + config = AgentConfig.load() except ConfigError as exc: # Fail here rather than twenty minutes into a milestone on the one # call that needed the missing value. diff --git a/nisse-dev/src/nisse_dev/matrix.py b/nisse-dev/src/nisse_dev/matrix.py index 19366b5..dde3e2b 100644 --- a/nisse-dev/src/nisse_dev/matrix.py +++ b/nisse-dev/src/nisse_dev/matrix.py @@ -11,12 +11,10 @@ loop backs off (capped) and retries indefinitely — an outage delays reactions but never fails anything. """ -import os import threading import uuid -from collections.abc import Callable, Mapping +from collections.abc import Callable from dataclasses import dataclass -from typing import ClassVar import httpx @@ -34,29 +32,6 @@ class MatrixConfig: token: str room_id: str - _ENV: ClassVar[tuple[str, ...]] = ( - "NISSE_MATRIX_HOMESERVER", - "NISSE_MATRIX_USER_ID", - "NISSE_MATRIX_TOKEN", - ) - - @classmethod - def from_env( - cls, - env: Mapping[str, str] = os.environ, - room_var: str = "NISSE_MATRIX_ROOM_ID", - ) -> "MatrixConfig": - needed = (*cls._ENV, room_var) - missing = [var for var in needed if not env.get(var)] - if missing: - raise TransportError(f"matrix config incomplete; set: {', '.join(missing)}") - return cls( - homeserver=env["NISSE_MATRIX_HOMESERVER"].rstrip("/"), - user_id=env["NISSE_MATRIX_USER_ID"], - token=env["NISSE_MATRIX_TOKEN"], - room_id=env[room_var], - ) - class MatrixTransport: def __init__( diff --git a/nisse-dev/src/nisse_dev/tools.py b/nisse-dev/src/nisse_dev/tools.py index ada5607..4468d47 100644 --- a/nisse-dev/src/nisse_dev/tools.py +++ b/nisse-dev/src/nisse_dev/tools.py @@ -86,13 +86,6 @@ EXECUTOR_TOOLS: list[dict[str, Any]] = [ ] -def _truncate(text: str) -> str: - if len(text) <= MAX_OUTPUT_CHARS: - return text - dropped = len(text) - MAX_OUTPUT_CHARS - return f"{text[:MAX_OUTPUT_CHARS]}\n[... {dropped} characters truncated]" - - def _child_env() -> dict[str, str]: """The project's build and test code is arbitrary; it has no business reading the agent's LiteLLM, Matrix, or Gitea credentials.""" @@ -106,10 +99,24 @@ class ToolOutcome: class Workspace: - def __init__(self, root: Path, *, timeout_s: int = DEFAULT_TIMEOUT_S) -> None: + def __init__( + self, + root: Path, + *, + timeout_s: int = DEFAULT_TIMEOUT_S, + max_output_chars: int = MAX_OUTPUT_CHARS, + ) -> None: self.root = Path(root) self.root.mkdir(parents=True, exist_ok=True) self._timeout_s = timeout_s + self._max_output_chars = max_output_chars + + def _truncate(self, text: str) -> str: + limit = self._max_output_chars + if len(text) <= limit: + return text + dropped = len(text) - limit + return f"{text[:limit]}\n[... {dropped} characters truncated]" def _resolve(self, path: str) -> Path: candidate = Path(path) @@ -142,7 +149,7 @@ class Workspace: def read_file(self, path: str) -> ToolOutcome: target = self._resolve(path) try: - return ToolOutcome(_truncate(target.read_text()), success=True) + return ToolOutcome(self._truncate(target.read_text()), success=True) except (OSError, UnicodeDecodeError) as exc: return ToolOutcome(f"could not read {path}: {exc}", success=False) @@ -175,7 +182,9 @@ class Workspace: ) except OSError as exc: return ToolOutcome(f"could not list {path}: {exc}", success=False) - return ToolOutcome(_truncate("\n".join(entries) or "(empty)"), success=True) + return ToolOutcome( + self._truncate("\n".join(entries) or "(empty)"), success=True + ) def describe(self) -> str: """Told to the model, so it does not have to guess where it is.""" @@ -212,7 +221,7 @@ class Workspace: f"command timed out after {self._timeout_s}s: {command}", success=False, ) - body = _truncate( + body = self._truncate( f"exit code: {process.returncode}\nstdout:\n{stdout}\nstderr:\n{stderr}" ) return ToolOutcome(body, success=process.returncode == 0) diff --git a/nisse-dev/tests/test_agent_build.py b/nisse-dev/tests/test_agent_build.py index d30797f..655309e 100644 --- a/nisse-dev/tests/test_agent_build.py +++ b/nisse-dev/tests/test_agent_build.py @@ -8,7 +8,7 @@ from typing import Any import httpx import pytest -from test_config import COMPLETE +from test_config import SECRETS, tree from nisse_dev.agent import build_agent, discover_models, prepare_workspace from nisse_dev.catalog import ModelCatalog, UnknownModel @@ -34,14 +34,15 @@ TEST_CATALOG = ModelCatalog.from_payload( ) -def config_for(tmp_path: Path, **overrides: str) -> AgentConfig: - env = { - **COMPLETE, - "NISSE_STATE_DB": str(tmp_path / "state.db"), - "NISSE_WORKSPACE": str(tmp_path / "work"), - **overrides, - } - return AgentConfig.from_env(env) +def config_for(tmp_path: Path, **changes: str) -> AgentConfig: + return AgentConfig.from_tree( + tree( + state_db=str(tmp_path / "state.db"), + workspace=str(tmp_path / "work"), + **changes, + ), + SECRETS, + ) @pytest.fixture @@ -59,7 +60,7 @@ def origin(tmp_path: Path) -> Path: def test_workspace_is_cloned_when_absent(tmp_path: Path, origin: Path) -> None: - config = config_for(tmp_path, NISSE_GITEA_URL=f"file://{origin}") + config = config_for(tmp_path, gitea__url=f"file://{origin}") # authenticated_url would mangle a file:// remote, so clone directly. GitRepo(tmp_path).clone(str(origin), config.workspace) repo = prepare_workspace(config, "nisse-dev-testing") @@ -206,7 +207,7 @@ def test_httpx_clients_are_not_shared_between_routes( def test_a_typod_role_refuses_to_boot(tmp_path: Path, monkeypatch) -> None: # Misconfiguration should cost a second at startup, not be discovered ten # minutes into a milestone. - config = config_for(tmp_path, NISSE_ROLE_PLANNER="clause-opus-5") + config = config_for(tmp_path, roles__planner="clause-opus-5") monkeypatch.setattr( "nisse_dev.agent.ModelCatalog.discover", classmethod(lambda cls, *a, **kw: TEST_CATALOG), @@ -220,7 +221,7 @@ def test_a_role_this_key_cannot_reach_refuses_to_boot( ) -> None: # /model/info is scoped per key, so the same check catches a role pointed # at a real model that this token has no access to. - config = config_for(tmp_path, NISSE_ROLE_EXECUTOR="local-fast") + config = config_for(tmp_path, roles__executor="local-fast") monkeypatch.setattr( "nisse_dev.agent.ModelCatalog.discover", classmethod(lambda cls, *a, **kw: TEST_CATALOG), @@ -286,3 +287,57 @@ def test_delegation_records_what_the_alias_resolved_to( agent = build_offline(tmp_path, origin, monkeypatch) assert agent.orchestrator._delegate._backing_model == "anthropic/claude-opus-5" agent.stop() + + +def test_tuning_reaches_every_component( + tmp_path: Path, origin: Path, monkeypatch +) -> None: + from dataclasses import replace + + from nisse_dev.config import ( + DelegationTuning, + ExecutorTuning, + ExpertTuning, + ToolsTuning, + Tuning, + ) + + config = replace( + config_for(tmp_path), + max_output_tokens=8192, + tuning=Tuning( + executor=ExecutorTuning(max_turns=40, failure_limit=3), + expert=ExpertTuning(consult_budget=5), + delegation=DelegationTuning( + max_turns=12, max_tokens=50_000, wall_clock_s=60 + ), + tools=ToolsTuning(command_timeout_s=300, max_output_chars=2000), + ), + ) + GitRepo(tmp_path).clone(str(origin), config.workspace) + monkeypatch.setattr( + "nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing" + ) + monkeypatch.setattr( + "nisse_dev.agent.ModelCatalog.discover", + classmethod(lambda cls, *a, **kw: TEST_CATALOG), + ) + monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport()) + monkeypatch.setattr( + "nisse_dev.agent.authenticated_url", + lambda base, repo, user, token: str(origin), + ) + agent = build_agent(config) + executor = agent.orchestrator._executor + delegation = agent.orchestrator._delegate + + assert (executor._max_turns, executor._failure_limit) == (40, 3) + assert executor._expert._budget == 5 + assert delegation._caps.turns == 12 + assert delegation._caps.tokens == 50_000 + assert delegation._caps.wall_clock_s == 60 + assert executor._workspace._timeout_s == 300 + assert executor._workspace._max_output_chars == 2000 + assert executor._client._max_tokens == 8192 + assert delegation._client._max_tokens == 8192 + agent.stop() diff --git a/nisse-dev/tests/test_config.py b/nisse-dev/tests/test_config.py index 87f8d92..03c27ad 100644 --- a/nisse-dev/tests/test_config.py +++ b/nisse-dev/tests/test_config.py @@ -1,93 +1,266 @@ +import copy from pathlib import Path +from typing import Any import pytest from nisse_dev.config import AgentConfig, ConfigError -COMPLETE = { - "NISSE_STATE_DB": "~/demo/state.db", - "NISSE_WORKSPACE": "~/demo/work", - "NISSE_PROJECT": "greet", - "NISSE_LLM_BASE_URL": "https://litellm.test", - "NISSE_LLM_TOKEN": "sk-x", - "NISSE_ROLE_EXECUTOR": "claude-haiku-4-5", - "NISSE_ROLE_PLANNER": "claude-opus-5", - "NISSE_ROLE_DELEGATE": "claude-opus-5", - "NISSE_ROLE_CONSULTANT": "claude-haiku-4-5", - "NISSE_MATRIX_HOMESERVER": "https://matrix.test", - "NISSE_MATRIX_USER_ID": "@nisse-dev-testing:test", - "NISSE_MATRIX_TOKEN": "syt_x", - "NISSE_MATRIX_ROOM_ID": "!room:test", - "NISSE_MATRIX_OPERATORS": "@don:test", - "NISSE_GITEA_URL": "https://gitea.test", - "NISSE_GITEA_TOKEN": "gt", - "NISSE_GITEA_REPO": "don/demo", - "NISSE_GITEA_OPERATORS": "don", +# A complete config as the YAML file would give it, with secrets by reference. +COMPLETE: dict[str, Any] = { + "state_db": "~/demo/state.db", + "workspace": "~/demo/work", + "project": "greet", + "llm": {"base_url": "https://litellm.test", "token": "${TEST_LLM_TOKEN}"}, + "roles": { + "executor": "claude-haiku-4-5", + "planner": "claude-opus-5", + "delegate": "claude-opus-5", + "consultant": "claude-haiku-4-5", + }, + "matrix": { + "homeserver": "https://matrix.test/", + "user_id": "@nisse-dev-testing:test", + "token": "${TEST_MATRIX_TOKEN}", + "room_id": "!room:test", + "operators": ["@don:test"], + }, + "gitea": { + "url": "https://gitea.test", + "token": "${TEST_GITEA_TOKEN}", + "repo": "don/demo", + "operators": ["don"], + }, +} +SECRETS = { + "TEST_LLM_TOKEN": "sk-x", + "TEST_MATRIX_TOKEN": "syt_x", + "TEST_GITEA_TOKEN": "gt", } -def test_complete_environment_builds() -> None: - config = AgentConfig.from_env(COMPLETE) +def tree(**changes: Any) -> dict[str, Any]: + """COMPLETE with dotted keys changed; a value of None removes the key.""" + result = copy.deepcopy(COMPLETE) + for dotted, value in changes.items(): + *parents, leaf = dotted.split("__") + node = result + for part in parents: + node = node.setdefault(part, {}) + if value is None: + node.pop(leaf, None) + else: + node[leaf] = value + return result + + +def build(**changes: Any) -> AgentConfig: + return AgentConfig.from_tree(tree(**changes), SECRETS) + + +def test_complete_config_builds_with_secrets_by_reference() -> None: + config = build() assert config.project == "greet" + assert config.llm_token == "sk-x" + assert config.gitea_token == "gt" assert config.roles.planner == "claude-opus-5" assert config.matrix.room_id == "!room:test" + assert config.matrix.homeserver == "https://matrix.test" assert config.gitea_repo == "don/demo" assert config.base_branch == "main" + assert config.tuning.executor.max_turns == 24 # absent section: defaults def test_paths_expand_the_home_directory() -> None: - config = AgentConfig.from_env(COMPLETE) + config = build() assert config.workspace.is_absolute() assert "~" not in str(config.state_db) -def test_every_missing_variable_is_reported_at_once() -> None: +def test_every_missing_setting_is_reported_at_once() -> None: # Failing at first use would surface a missing token twenty minutes into # a milestone, on the call that mattered. - partial = {k: v for k, v in COMPLETE.items() if k != "NISSE_MATRIX_TOKEN"} - partial.pop("NISSE_GITEA_REPO") with pytest.raises(ConfigError) as excinfo: - AgentConfig.from_env(partial) + build(matrix__token=None, gitea__repo=None) message = str(excinfo.value) - assert "NISSE_MATRIX_TOKEN" in message - assert "NISSE_GITEA_REPO" in message + assert "matrix.token" in message + assert "gitea.repo" in message def test_empty_is_treated_as_missing() -> None: - with pytest.raises(ConfigError, match="NISSE_GITEA_TOKEN"): - AgentConfig.from_env({**COMPLETE, "NISSE_GITEA_TOKEN": " "}) + with pytest.raises(ConfigError, match="gitea.repo"): + build(gitea__repo=" ") -def test_operator_lists_are_split_and_trimmed() -> None: - config = AgentConfig.from_env( - { - **COMPLETE, - "NISSE_MATRIX_OPERATORS": " @don:test , @other:test ,", - "NISSE_GITEA_OPERATORS": "don,", - } - ) +def test_a_reference_to_an_unset_variable_is_an_error() -> None: + # Never an empty string: an empty token would pass for a value and fail + # at the first request. + env = {k: v for k, v in SECRETS.items() if k != "TEST_GITEA_TOKEN"} + with pytest.raises(ConfigError, match=r"gitea.token: references \$\{TEST_GITEA"): + AgentConfig.from_tree(tree(), env) + + +def test_operator_lists_drop_blanks() -> None: + config = build(matrix__operators=[" @don:test ", "@other:test", " "]) assert config.operators == {"@don:test", "@other:test"} - # A trailing comma must not admit an empty identity into an allowlist. + # A blank entry must not admit an empty identity into an allowlist. assert "" not in config.operators - assert config.gitea_operators == {"don"} -def test_base_branch_is_overridable() -> None: - config = AgentConfig.from_env({**COMPLETE, "NISSE_BASE_BRANCH": "trunk"}) - assert config.base_branch == "trunk" +def test_an_operator_list_of_only_blanks_is_missing() -> None: + with pytest.raises(ConfigError, match="gitea.operators"): + build(gitea__operators=[" "]) + + +def test_base_branch_is_configurable() -> None: + assert build(gitea__base_branch="trunk").base_branch == "trunk" + + +def test_max_output_tokens_is_configurable_and_checked() -> None: + assert build(llm__max_output_tokens=8192).max_output_tokens == 8192 + with pytest.raises(ConfigError, match="llm.max_output_tokens"): + build(llm__max_output_tokens=0) def test_redacted_summary_carries_no_secrets() -> None: - summary = AgentConfig.from_env(COMPLETE).redacted() + summary = build().redacted() flat = " ".join(summary.values()) - for secret in ("sk-x", "syt_x", "gt"): + for secret in SECRETS.values(): assert secret not in flat assert summary["project"] == "greet" -def test_state_db_and_workspace_are_separate(tmp_path: Path) -> None: +def test_tuning_is_read() -> None: + config = build( + tuning={ + "executor": {"max_turns": 40, "failure_limit": 3}, + "expert": {"consult_budget": 5}, + "delegation": {"max_turns": 20, "max_tokens": 100000, "wall_clock_s": 600}, + "tools": {"command_timeout_s": 300, "max_output_chars": 2000}, + } + ) + tuning = config.tuning + assert (tuning.executor.max_turns, tuning.executor.failure_limit) == (40, 3) + assert tuning.expert.consult_budget == 5 + assert tuning.delegation.max_tokens == 100_000 + assert tuning.delegation.wall_clock_s == 600.0 + assert tuning.tools.max_output_chars == 2000 + + +def test_a_partly_given_tuning_section_keeps_the_other_defaults() -> None: + tuning = build(tuning={"executor": {"max_turns": 40}}).tuning + assert tuning.executor.failure_limit == 2 + + +def test_every_problem_is_reported_at_once() -> None: + # A misspelt knob that was silently ignored would look exactly like a knob + # with no effect, so unknown keys are errors. + bad = tree(project=None, llm_extra=1) + bad["tuning"] = { + "executor": {"max_turn": 40, "failure_limit": 0}, + "expert": {"consult_budget": True}, + "bogus": {"x": 1}, + } + with pytest.raises(ConfigError) as excinfo: + AgentConfig.from_tree(bad, SECRETS) + message = str(excinfo.value) + for expected in ( + "project", + "llm_extra: unknown setting", + "tuning.executor.max_turn: unknown setting", + "tuning.executor.failure_limit: must be a positive number", + "tuning.expert.consult_budget: must be a positive number", + "tuning.bogus: unknown section", + ): + assert expected in message + + +def test_a_whole_number_is_required_where_one_is_meant() -> None: + with pytest.raises(ConfigError, match="tuning.executor.max_turns"): + build(tuning={"executor": {"max_turns": 2.5}}) + + +# --- the file -------------------------------------------------------------- + +FILE = """ +state_db: ~/demo/state.db +workspace: ~/demo/work +project: greet +llm: {base_url: "https://litellm.test", token: "${TEST_LLM_TOKEN}"} +roles: + executor: local-fast + planner: claude-opus-5 + delegate: claude-opus-5 + consultant: claude-haiku-4-5 +matrix: + homeserver: https://matrix.test + user_id: "@nisse-dev-testing:test" + token: ${TEST_MATRIX_TOKEN} + room_id: "!room:test" + operators: ["@don:test"] +gitea: + url: https://gitea.test + token: ${TEST_GITEA_TOKEN} + repo: don/demo + operators: [don] +""" + + +def write(directory: Path, text: str = FILE) -> Path: + path = directory / "nisse-dev.yaml" + path.write_text(text) + return path + + +def test_a_file_is_read(tmp_path: Path) -> None: + config = AgentConfig.from_file(write(tmp_path), SECRETS) + assert config.roles.executor == "local-fast" + assert config.config_file == tmp_path / "nisse-dev.yaml" + + +def test_invalid_yaml_is_reported(tmp_path: Path) -> None: + with pytest.raises(ConfigError, match="not valid YAML"): + AgentConfig.from_file(write(tmp_path, "roles: [unclosed"), SECRETS) + + +def test_load_uses_nisse_config(tmp_path: Path) -> None: + path = write(tmp_path) + assert AgentConfig.load({**SECRETS, "NISSE_CONFIG": str(path)}).project == "greet" + + +def test_load_defaults_to_the_working_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write(tmp_path) + monkeypatch.chdir(tmp_path) + assert AgentConfig.load(SECRETS).project == "greet" + + +def test_load_without_a_file_says_how_to_make_one( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + with pytest.raises(ConfigError, match="nisse-dev.example.yaml"): + AgentConfig.load(SECRETS) + + +def test_the_example_file_is_valid() -> None: + # The example documents every knob; it must not drift out of loading. + example = Path(__file__).resolve().parents[2] / "nisse-dev.example.yaml" + env = { + "NISSE_LLM_TOKEN": "sk-x", + "NISSE_MATRIX_TOKEN": "syt_x", + "NISSE_GITEA_TOKEN": "gt", + } + assert AgentConfig.from_file(example, env).tuning.executor.max_turns == 24 + + +# --- nothing that carries secrets may sit inside the clone ----------------- + + +def test_state_db_and_workspace_are_separate() -> None: # A2: build artefacts must not share a volume with agent state. - config = AgentConfig.from_env(COMPLETE) + config = build() assert config.state_db.parent != config.workspace @@ -96,13 +269,7 @@ def test_state_db_inside_the_clone_is_refused(tmp_path: Path) -> None: # own state alongside the project. work = tmp_path / "project" with pytest.raises(ConfigError, match="inside the workspace"): - AgentConfig.from_env( - { - **COMPLETE, - "NISSE_WORKSPACE": str(work), - "NISSE_STATE_DB": str(work / "state.db"), - } - ) + build(workspace=str(work), state_db=str(work / "state.db")) def test_env_file_inside_the_clone_is_refused(tmp_path: Path) -> None: @@ -110,23 +277,23 @@ def test_env_file_inside_the_clone_is_refused(tmp_path: Path) -> None: # as a side effect of working correctly. work = tmp_path / "project" with pytest.raises(ConfigError, match="NISSE_ENV"): - AgentConfig.from_env( - { - **COMPLETE, - "NISSE_WORKSPACE": str(work), - "NISSE_STATE_DB": str(tmp_path / "state.db"), - "NISSE_ENV": str(work / ".env"), - } + AgentConfig.from_tree( + tree(workspace=str(work), state_db=str(tmp_path / "state.db")), + {**SECRETS, "NISSE_ENV": str(work / ".env")}, ) +def test_a_config_file_inside_the_clone_is_refused(tmp_path: Path) -> None: + work = tmp_path / "project" + work.mkdir() + text = FILE.replace("workspace: ~/demo/work", f"workspace: {work}") + with pytest.raises(ConfigError, match="NISSE_CONFIG"): + AgentConfig.from_file(write(work, text), SECRETS) + + def test_sibling_paths_are_fine(tmp_path: Path) -> None: - config = AgentConfig.from_env( - { - **COMPLETE, - "NISSE_WORKSPACE": str(tmp_path / "project"), - "NISSE_STATE_DB": str(tmp_path / "state.db"), - "NISSE_ENV": str(tmp_path / ".env"), - } + config = AgentConfig.from_tree( + tree(workspace=str(tmp_path / "project"), state_db=str(tmp_path / "state.db")), + {**SECRETS, "NISSE_ENV": str(tmp_path / ".env")}, ) assert config.workspace.name == "project" diff --git a/nisse-dev/tests/test_dev_main.py b/nisse-dev/tests/test_dev_main.py index 2304939..600252f 100644 --- a/nisse-dev/tests/test_dev_main.py +++ b/nisse-dev/tests/test_dev_main.py @@ -2,25 +2,23 @@ from pathlib import Path import pytest -from nisse_dev.config import REQUIRED from nisse_dev.main import main -def test_main_refuses_to_start_without_configuration( +def test_main_refuses_to_start_without_a_config_file( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], tmp_path: Path, ) -> None: - # Clear the real variables rather than swapping os.environ: from_env binds - # the environment object as a default argument, so a replacement would be - # ignored. And point NISSE_ENV at a file that does not exist, because the - # working directory's .env holds live credentials. - for name in REQUIRED: - monkeypatch.delenv(name, raising=False) + # Run from an empty directory, and point NISSE_ENV at a file that does not + # exist: the repository's .env holds live credentials, and a test must + # never be one cwd away from starting a real agent. + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("NISSE_CONFIG", raising=False) monkeypatch.setenv("NISSE_ENV", str(tmp_path / "absent.env")) assert main() == 2 err = capsys.readouterr().err assert "cannot start" in err - assert "NISSE_LLM_TOKEN" in err + assert "nisse-dev.example.yaml" in err diff --git a/nisse-dev/tests/test_gitea.py b/nisse-dev/tests/test_gitea.py index 2f2190c..17a4d20 100644 --- a/nisse-dev/tests/test_gitea.py +++ b/nisse-dev/tests/test_gitea.py @@ -162,11 +162,6 @@ def test_list_comments_passes_since() -> None: assert seen["since"] == "2026-09-07T00:00:00Z" -def test_from_env_requires_config() -> None: - with pytest.raises(GiteaError, match="NISSE_GITEA_URL"): - GiteaClient.from_env({}) - - # --- reconciler (A3 external truth) -------------------------------------- diff --git a/nisse-dev/tests/test_llm.py b/nisse-dev/tests/test_llm.py index 746ad53..5b1238e 100644 --- a/nisse-dev/tests/test_llm.py +++ b/nisse-dev/tests/test_llm.py @@ -21,7 +21,6 @@ from nisse_dev.llm import ( RoleMap, TruncatedResponse, UpstreamUnavailable, - max_output_tokens, ) EVENTS = [StoredEvent(id=1, ts="t", event=UserMessage(text="hi", source="m:1"))] @@ -165,16 +164,16 @@ def test_malformed_response_raises_llm_error() -> None: client.complete(EVENTS, tools=None, model="local/qwen") -ROLE_ENV = { - "NISSE_ROLE_EXECUTOR": "local/qwen", - "NISSE_ROLE_PLANNER": "anthropic/claude-opus-5", - "NISSE_ROLE_DELEGATE": "anthropic/claude-opus-5", - "NISSE_ROLE_CONSULTANT": "anthropic/claude-opus-5", -} +ROLES = RoleMap( + executor="local/qwen", + planner="anthropic/claude-opus-5", + delegate="anthropic/claude-opus-5", + consultant="anthropic/claude-opus-5", +) def test_role_map_resolves_and_pins() -> None: - roles = RoleMap.from_env(ROLE_ENV) + roles = ROLES assert roles.resolve("executor") == "local/qwen" assert roles.pinned() == { "executor": "local/qwen", @@ -184,19 +183,9 @@ def test_role_map_resolves_and_pins() -> None: } -def test_role_map_missing_env_raises() -> None: - with pytest.raises(LLMError, match="NISSE_ROLE_PLANNER"): - RoleMap.from_env({"NISSE_ROLE_EXECUTOR": "local/qwen"}) - - def test_unknown_role_raises() -> None: with pytest.raises(LLMError, match="unknown role"): - RoleMap.from_env(ROLE_ENV).resolve("boss") - - -def test_client_from_env_requires_config() -> None: - with pytest.raises(LLMError, match="NISSE_LLM_BASE_URL"): - LiteLLMClient.from_env({}) + ROLES.resolve("boss") def test_load_env_file(tmp_path: Path) -> None: @@ -306,12 +295,11 @@ def test_a_complete_answer_is_not_mistaken_for_truncation() -> None: assert call.arguments["content"] == "a real plan" -def test_output_ceiling_is_configurable_and_generous_by_default() -> None: +def test_output_ceiling_is_generous_by_default() -> None: # Output is billed by what is produced, so a low ceiling buys nothing and - # costs silent truncation. + # costs silent truncation. The configured value is tested in test_config. assert DEFAULT_MAX_OUTPUT_TOKENS >= 8192 - assert max_output_tokens({}) == DEFAULT_MAX_OUTPUT_TOKENS - assert max_output_tokens({"NISSE_MAX_OUTPUT_TOKENS": "2048"}) == 2048 + assert LiteLLMClient("http://x", "t")._max_tokens == DEFAULT_MAX_OUTPUT_TOKENS def test_truncation_is_an_llm_error_so_callers_already_handle_it() -> None: diff --git a/nisse-dev/tests/test_matrix.py b/nisse-dev/tests/test_matrix.py index 2e1fd69..049651b 100644 --- a/nisse-dev/tests/test_matrix.py +++ b/nisse-dev/tests/test_matrix.py @@ -143,20 +143,3 @@ def test_start_without_handler_raises() -> None: ) with pytest.raises(TransportError, match="no message handler"): transport.start() - - -def test_config_from_env_reports_missing() -> None: - with pytest.raises(TransportError, match="NISSE_MATRIX_TOKEN"): - MatrixConfig.from_env({"NISSE_MATRIX_HOMESERVER": "https://m"}) - - -def test_config_from_env_custom_room_var() -> None: - env = { - "NISSE_MATRIX_HOMESERVER": "https://m/", - "NISSE_MATRIX_USER_ID": "@nisse-dev:test", - "NISSE_MATRIX_TOKEN": "t", - "NISSE_MATRIX_TEST_ROOM_ID": "!t:test", - } - config = MatrixConfig.from_env(env, room_var="NISSE_MATRIX_TEST_ROOM_ID") - assert config.room_id == "!t:test" - assert config.homeserver == "https://m" # trailing slash stripped diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a7d20d5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,20 @@ +"""Command-line options for the repository-level test suites. + +Declared here rather than beside the tests that use them: pytest only reads +options from conftest files it loads at startup, and this directory is one +of the configured test paths. +""" + +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--capability-models", + default="", + help=( + "comma-separated LiteLLM aliases to measure with the capability " + "matrix (tests/integration/test_model_capability.py). Spends real " + "tokens on every model listed; the matrix skips when this is empty." + ), + ) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..67c2866 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,62 @@ +"""Shared setup for the live integration tests. + +They run against real services, configured by nisse-test.yaml beside this +file: an ordinary agent config aimed at test resources, committed to the +repository. Its secrets are ${…} references, supplied by CI as environment +variables and locally by .env at the repository root. + +A missing secret is a failure, not a skip. A half-configured environment must +not pass quietly as a skip: that is how a broken CI credential hides. +""" + +from pathlib import Path + +import pytest + +from nisse_dev.config import AgentConfig +from nisse_dev.envfile import load_env_file +from nisse_dev.gitea import GiteaClient +from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient + +REPO = Path(__file__).resolve().parents[2] +TEST_CONFIG = Path(__file__).parent / "nisse-test.yaml" + +# The secrets nisse-test.yaml refers to, for local runs. CI sets them as real +# environment variables, which win over the file. +load_env_file(REPO / ".env") + + +@pytest.fixture(scope="session") +def live_config() -> AgentConfig: + return AgentConfig.from_file(TEST_CONFIG) + + +@pytest.fixture +def live_model(live_config: AgentConfig) -> str: + """The model the live tests plan, execute and delegate with.""" + return live_config.roles.planner + + +@pytest.fixture +def litellm(live_config: AgentConfig) -> LiteLLMClient: + return LiteLLMClient( + live_config.llm_base_url, + live_config.llm_token, + max_tokens=live_config.max_output_tokens, + ) + + +@pytest.fixture +def passthrough(live_config: AgentConfig) -> AnthropicPassthroughClient: + return AnthropicPassthroughClient( + live_config.llm_base_url, + live_config.llm_token, + max_tokens=live_config.max_output_tokens, + ) + + +@pytest.fixture +def gitea(live_config: AgentConfig) -> GiteaClient: + client = GiteaClient(live_config.gitea_url, live_config.gitea_token) + yield client + client.close() diff --git a/tests/integration/nisse-test.yaml b/tests/integration/nisse-test.yaml new file mode 100644 index 0000000..2d4451f --- /dev/null +++ b/tests/integration/nisse-test.yaml @@ -0,0 +1,40 @@ +# Agent config for the live integration tests, aimed at test resources: the +# testing identities, the dedicated test room, and the sandbox repo. Never +# point it at production. +# +# Secrets are ${…} references. CI passes them as environment variables; +# locally they come from .env at the repository root. A missing one fails +# the live tests that need it, rather than skipping them, so a broken +# credential cannot hide as a skip. + +# Unused by the live tests, which work in pytest's temporary directories; +# present because every agent config has them. +state_db: /tmp/nisse-test/state.db +workspace: /tmp/nisse-test/work +project: nisse-dev-testing + +llm: + base_url: https://litellm.dongar.net + token: ${NISSE_LLM_TOKEN} + +# The live tests exercise the gated loop and the routes over a real proxy, +# not local-model competence (measured separately by test_model_capability), +# so every role uses a model that holds the tool-call contract. +roles: + executor: claude-haiku-4-5 + planner: claude-haiku-4-5 + delegate: claude-haiku-4-5 + consultant: claude-haiku-4-5 + +matrix: + homeserver: https://matrix.dongar.net + user_id: "@nisse-dev-testing:dongar.net" + token: ${NISSE_MATRIX_TOKEN} + room_id: "!UxpXhTlQsFIeNZwkgv:dongar.net" # the test room, never production + operators: ["@dongar:dongar.net"] + +gitea: + url: https://gitea.dongar.net + token: ${NISSE_GITEA_TOKEN} + repo: dongar/nisse-dev-testing-sandbox # the sandbox, never a real project + operators: [dongar] diff --git a/tests/integration/test_delegation_live.py b/tests/integration/test_delegation_live.py index 3ccc9d8..4c73f6e 100644 --- a/tests/integration/test_delegation_live.py +++ b/tests/integration/test_delegation_live.py @@ -2,30 +2,18 @@ working code and a structured handback out.""" import io -import os from pathlib import Path import pytest from nisse_dev.delegation import Caps, Delegation -from nisse_dev.envfile import load_env_file from nisse_dev.git import GitRepo from nisse_dev.llm import AnthropicPassthroughClient from nisse_dev.oplog import OpLog from nisse_dev.state import StateStore from nisse_dev.tools import Workspace -load_env_file(Path(__file__).resolve().parents[2] / ".env") - -SENTINEL = "NISSE_LIVE_PLANNER_MODEL" - -pytestmark = [ - pytest.mark.integration, - pytest.mark.skipif( - not os.environ.get(SENTINEL), - reason=f"no live frontier route configured ({SENTINEL} unset)", - ), -] +pytestmark = pytest.mark.integration BRIEF = """Create `slugify.py` in the workspace root containing a function `slugify(text)` that lowercases the text and replaces every run of @@ -34,7 +22,9 @@ hyphen. Then create `test_slugify.py` with at least two assertions covering it, and run `python -m pytest test_slugify.py -q` until it passes.""" -def build(tmp_path: Path, caps: Caps) -> tuple[Delegation, GitRepo, StateStore]: +def build( + tmp_path: Path, caps: Caps, client: AnthropicPassthroughClient, model: str +) -> tuple[Delegation, GitRepo, StateStore]: repo = GitRepo(tmp_path / "work") repo.init() repo.configure_identity("nisse-dev-testing", "n@t.test") @@ -42,18 +32,22 @@ def build(tmp_path: Path, caps: Caps) -> tuple[Delegation, GitRepo, StateStore]: repo.commit_all("seed") store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) delegation = Delegation( - AnthropicPassthroughClient.from_env(), + client, store, Workspace(repo.root, timeout_s=60), repo, - model=os.environ[SENTINEL], + model=model, caps=caps, ) return delegation, repo, store -def test_delegate_completes_a_scoped_subtask(tmp_path: Path) -> None: - delegation, repo, _store = build(tmp_path, Caps(turns=12, wall_clock_s=300.0)) +def test_delegate_completes_a_scoped_subtask( + tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str +) -> None: + delegation, repo, _store = build( + tmp_path, Caps(turns=12, wall_clock_s=300.0), passthrough, live_model + ) result = delegation.run(brief=BRIEF, branch="live/delegation-1") @@ -76,10 +70,14 @@ def test_delegate_completes_a_scoped_subtask(tmp_path: Path) -> None: assert check.success, f"delegate's own tests do not pass: {check.result[:400]}" -def test_a_cap_breach_discards_the_work(tmp_path: Path) -> None: +def test_a_cap_breach_discards_the_work( + tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str +) -> None: # One turn cannot finish the brief, so this exercises the abort path # against a real model rather than a scripted one. - delegation, repo, _ = build(tmp_path, Caps(turns=1, wall_clock_s=120.0)) + delegation, repo, _ = build( + tmp_path, Caps(turns=1, wall_clock_s=120.0), passthrough, live_model + ) result = delegation.run(brief=BRIEF, branch="live/delegation-2") diff --git a/tests/integration/test_entry_points.py b/tests/integration/test_entry_points.py index dc6d951..5a75521 100644 --- a/tests/integration/test_entry_points.py +++ b/tests/integration/test_entry_points.py @@ -2,6 +2,7 @@ import os import subprocess +from pathlib import Path import pytest @@ -16,27 +17,31 @@ def test_ops_entry_point_runs_and_greets() -> None: assert result.stdout == "Hello from nisse-ops, the house spirit is awake.\n" -def test_dev_entry_point_refuses_to_start_unconfigured() -> None: - """Missing configuration must fail at startup, naming everything absent. +def test_dev_entry_point_refuses_to_start_unconfigured(tmp_path: Path) -> None: + """An incomplete config must fail at startup, naming everything absent. - Run with a deliberately empty environment and a .env path that does not - exist: the real .env in the working directory holds live credentials, and - a test must never be one cwd away from starting a real agent. + Run from an empty directory, with an empty config file and a .env path + that does not exist: the repository's .env holds live credentials, and a + test must never be one cwd away from starting a real agent. """ + config = tmp_path / "nisse-dev.yaml" + config.write_text("") result = subprocess.run( ["nisse-dev"], capture_output=True, text=True, timeout=30, check=False, + cwd=tmp_path, env={ "PATH": os.environ["PATH"], "HOME": os.environ["HOME"], "NISSE_ENV": "/nonexistent/.env", + "NISSE_CONFIG": str(config), }, ) assert result.returncode == 2 assert "cannot start" in result.stderr # All of them at once, not the first one it happened to reach. - for expected in ("NISSE_STATE_DB", "NISSE_MATRIX_TOKEN", "NISSE_GITEA_REPO"): + for expected in ("state_db", "matrix.token", "gitea.repo"): assert expected in result.stderr diff --git a/tests/integration/test_expert_live.py b/tests/integration/test_expert_live.py index 8197efd..0205152 100644 --- a/tests/integration/test_expert_live.py +++ b/tests/integration/test_expert_live.py @@ -5,40 +5,34 @@ that parses against our local schema, since no API-level enforcement exists. """ import io -import os from pathlib import Path import pytest -from nisse_dev.envfile import load_env_file from nisse_dev.evals import collect from nisse_dev.expert import BudgetExhausted, ExpertDesk from nisse_dev.llm import LiteLLMClient, RoleMap from nisse_dev.oplog import OpLog from nisse_dev.state import StateStore -load_env_file(Path(__file__).resolve().parents[2] / ".env") - -SENTINEL = "NISSE_LIVE_PLANNER_MODEL" - -pytestmark = [ - pytest.mark.integration, - pytest.mark.skipif( - not os.environ.get(SENTINEL), - reason=f"no live frontier route configured ({SENTINEL} unset)", - ), -] +pytestmark = pytest.mark.integration -def desk(tmp_path: Path, budget: int = 2) -> tuple[ExpertDesk, StateStore]: - model = os.environ[SENTINEL] - roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model) - store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) - return ExpertDesk(LiteLLMClient.from_env(), store, roles, budget=budget), store +@pytest.fixture +def desk(tmp_path: Path, litellm: LiteLLMClient, live_model: str): + def make(budget: int = 2) -> tuple[ExpertDesk, StateStore]: + model = live_model + roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model) + store = StateStore( + tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()) + ) + return ExpertDesk(litellm, store, roles, budget=budget), store + + return make -def test_live_consult_parses_against_the_local_schema(tmp_path: Path) -> None: - expert, store = desk(tmp_path) +def test_live_consult_parses_against_the_local_schema(desk) -> None: + expert, store = desk() answer = expert.ask( "For a single-process agent storing a few thousand rows of task state, " @@ -53,8 +47,8 @@ def test_live_consult_parses_against_the_local_schema(tmp_path: Path) -> None: assert record.exit_status == "answered" -def test_live_budget_stops_a_circling_caller(tmp_path: Path) -> None: - expert, _ = desk(tmp_path, budget=1) +def test_live_budget_stops_a_circling_caller(desk) -> None: + expert, _ = desk(budget=1) expert.ask("Name one advantage of WAL mode in SQLite. Be brief.") with pytest.raises(BudgetExhausted): expert.ask("And another?") diff --git a/tests/integration/test_gitea_live.py b/tests/integration/test_gitea_live.py index c1f3d0f..27270ab 100644 --- a/tests/integration/test_gitea_live.py +++ b/tests/integration/test_gitea_live.py @@ -5,29 +5,17 @@ Cleans up after itself (closes the PR, deletes the branch) so repeated runs do not silt up the sandbox. """ -import os import uuid from pathlib import Path import pytest -from nisse_dev.envfile import load_env_file +from nisse_dev.config import AgentConfig from nisse_dev.git import GitError, GitRepo, authenticated_url from nisse_dev.gitea import GiteaClient, pull_request_reconciler from nisse_dev.state import Intent -load_env_file(Path(__file__).resolve().parents[2] / ".env") - -SENTINEL = "NISSE_GITEA_URL" -ALSO_REQUIRED = ("NISSE_GITEA_TOKEN", "NISSE_GITEA_TEST_REPO") - -pytestmark = [ - pytest.mark.integration, - pytest.mark.skipif( - not os.environ.get(SENTINEL), - reason=f"no live Gitea configured ({SENTINEL} unset)", - ), -] +pytestmark = pytest.mark.integration def intent_for(branch: str) -> Intent: @@ -41,16 +29,13 @@ def intent_for(branch: str) -> Intent: ) -def test_push_open_pr_and_reconcile(tmp_path: Path) -> None: - missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)] - assert not missing, ( - f"{SENTINEL} is set, so the live Gitea path is expected to work, " - f"but these are missing: {', '.join(missing)}" - ) - base_url = os.environ[SENTINEL] - token = os.environ["NISSE_GITEA_TOKEN"] - repo_path = os.environ["NISSE_GITEA_TEST_REPO"] - client = GiteaClient.from_env() +def test_push_open_pr_and_reconcile( + tmp_path: Path, live_config: AgentConfig, gitea: GiteaClient +) -> None: + base_url = live_config.gitea_url + token = live_config.gitea_token + repo_path = live_config.gitea_repo # the sandbox + client = gitea account = client.whoami() assert "testing" in account, ( diff --git a/tests/integration/test_llm_live.py b/tests/integration/test_llm_live.py index 196487e..db0ef5c 100644 --- a/tests/integration/test_llm_live.py +++ b/tests/integration/test_llm_live.py @@ -1,41 +1,17 @@ -"""Live smoke test against the real LiteLLM proxy. - -NISSE_LLM_BASE_URL alone decides whether this runs: if it is set, the live -path is expected to work, and anything else missing is a *failure*, not a -skip. A half-configured environment must not pass quietly as a skip — that is -how a broken CI credential hides. -""" - -import os -from pathlib import Path +"""Live smoke test against the real LiteLLM proxy.""" import pytest from nisse_dev.canonical import AssistantText, StoredEvent, UserMessage -from nisse_dev.envfile import load_env_file +from nisse_dev.config import AgentConfig from nisse_dev.llm import LiteLLMClient -load_env_file(Path(__file__).resolve().parents[2] / ".env") - -SENTINEL = "NISSE_LLM_BASE_URL" -ALSO_REQUIRED = ("NISSE_LLM_TOKEN", "NISSE_ROLE_EXECUTOR") - -pytestmark = [ - pytest.mark.integration, - pytest.mark.skipif( - not os.environ.get(SENTINEL), - reason=f"no live LLM configured ({SENTINEL} unset)", - ), -] +pytestmark = pytest.mark.integration -def test_executor_model_completes_via_litellm() -> None: - missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)] - assert not missing, ( - f"{SENTINEL} is set, so the live LLM path is expected to work, " - f"but these are missing: {', '.join(missing)}" - ) - client = LiteLLMClient.from_env() +def test_executor_model_completes_via_litellm( + litellm: LiteLLMClient, live_config: AgentConfig +) -> None: events = [ StoredEvent( id=1, @@ -45,9 +21,7 @@ def test_executor_model_completes_via_litellm() -> None: ), ) ] - result = client.complete( - events, tools=None, model=os.environ["NISSE_ROLE_EXECUTOR"] - ) + result = litellm.complete(events, tools=None, model=live_config.roles.executor) texts = [e.text for e in result if isinstance(e, AssistantText)] assert texts, f"no assistant text in {result!r}" assert "pong" in texts[0].lower() diff --git a/tests/integration/test_matrix_live.py b/tests/integration/test_matrix_live.py index 1bf8922..018cf55 100644 --- a/tests/integration/test_matrix_live.py +++ b/tests/integration/test_matrix_live.py @@ -1,36 +1,14 @@ """Live smoke test against the real homeserver, bound strictly to the -dedicated test room (never the production room). +dedicated test room that nisse-test.yaml names (never the production room).""" -NISSE_MATRIX_HOMESERVER alone decides whether this runs: if it is set, the -live path is expected to work, and anything else missing is a *failure*, not -a skip. A half-configured environment must not pass quietly as a skip — that -is how a broken CI credential hides.""" - -import os import uuid -from pathlib import Path import pytest -from nisse_dev.envfile import load_env_file +from nisse_dev.config import AgentConfig from nisse_dev.matrix import MatrixConfig, MatrixTransport -load_env_file(Path(__file__).resolve().parents[2] / ".env") - -SENTINEL = "NISSE_MATRIX_HOMESERVER" -ALSO_REQUIRED = ( - "NISSE_MATRIX_USER_ID", - "NISSE_MATRIX_TOKEN", - "NISSE_MATRIX_TEST_ROOM_ID", -) - -pytestmark = [ - pytest.mark.integration, - pytest.mark.skipif( - not os.environ.get(SENTINEL), - reason=f"no live homeserver configured ({SENTINEL} unset)", - ), -] +pytestmark = pytest.mark.integration def _assert_testing_identity(transport: MatrixTransport, config: MatrixConfig) -> None: @@ -46,7 +24,7 @@ def _assert_testing_identity(transport: MatrixTransport, config: MatrixConfig) - whoami.raise_for_status() actual = whoami.json()["user_id"] assert actual == config.user_id, ( - f"NISSE_MATRIX_USER_ID is {config.user_id} but the token belongs to " + f"matrix.user_id is {config.user_id} but the token belongs to " f"{actual}; the self-message filter would not match" ) assert "testing" in actual, ( @@ -54,13 +32,8 @@ def _assert_testing_identity(transport: MatrixTransport, config: MatrixConfig) - ) -def test_send_lands_in_test_room() -> None: - missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)] - assert not missing, ( - f"{SENTINEL} is set, so the live Matrix path is expected to work, " - f"but these are missing: {', '.join(missing)}" - ) - config = MatrixConfig.from_env(room_var="NISSE_MATRIX_TEST_ROOM_ID") +def test_send_lands_in_test_room(live_config: AgentConfig) -> None: + config = live_config.matrix marker = f"transport smoke test {uuid.uuid4().hex}" transport = MatrixTransport(config) try: diff --git a/tests/integration/test_milestone_live.py b/tests/integration/test_milestone_live.py index 8a8ea62..a10e63c 100644 --- a/tests/integration/test_milestone_live.py +++ b/tests/integration/test_milestone_live.py @@ -2,12 +2,11 @@ approve through the gate, then let the executor role actually create and test a file on disk. -Gated on NISSE_LIVE_PLANNER_MODEL like the planning test — it needs a model -that holds the tool-call contract on both ends. +Uses the test config's planner model for every role: it needs a model that +holds the tool-call contract on both ends. """ import io -import os from pathlib import Path import pytest @@ -19,7 +18,6 @@ from nisse_dev.control import ( EXECUTING, Orchestrator, ) -from nisse_dev.envfile import load_env_file from nisse_dev.executor import LLMExecutor from nisse_dev.llm import LiteLLMClient, RoleMap from nisse_dev.oplog import OpLog @@ -28,17 +26,7 @@ from nisse_dev.state import StateStore from nisse_dev.tools import Workspace from nisse_dev.transport import InMemoryTransport -load_env_file(Path(__file__).resolve().parents[2] / ".env") - -SENTINEL = "NISSE_LIVE_PLANNER_MODEL" - -pytestmark = [ - pytest.mark.integration, - pytest.mark.skipif( - not os.environ.get(SENTINEL), - reason=f"no live planner route configured ({SENTINEL} unset)", - ), -] +pytestmark = pytest.mark.integration def _drive_to_execution( @@ -67,16 +55,14 @@ def _drive_to_execution( ) -def test_plan_approve_execute_produces_a_real_file(tmp_path: Path) -> None: - assert os.environ.get("NISSE_LLM_TOKEN"), ( - f"{SENTINEL} is set, so the live path is expected to work, " - "but NISSE_LLM_TOKEN is missing" - ) +def test_plan_approve_execute_produces_a_real_file( + tmp_path: Path, litellm: LiteLLMClient, live_model: str +) -> None: # Both roles use the capable model here: the point is the whole gated # loop over a real proxy, not local-model competence (measured separately). - model = os.environ[SENTINEL] + model = live_model roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model) - client = LiteLLMClient.from_env() + client = litellm store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) workspace = Workspace(tmp_path / "work", timeout_s=60) transport = InMemoryTransport() diff --git a/tests/integration/test_model_capability.py b/tests/integration/test_model_capability.py index f5f3ff4..8205cf5 100644 --- a/tests/integration/test_model_capability.py +++ b/tests/integration/test_model_capability.py @@ -9,12 +9,15 @@ What it measures is the thing that actually breaks: whether a model can hold the tool-call contract our loops depend on. llama3.2 could not — it returned `{` as an entire plan — and no amount of prompt work fixed it. -Opt in with NISSE_CAPABILITY_MODELS, comma separated. Skipped otherwise, -because it spends real tokens across several models. -""" +Opt in by naming the models to measure, for one run: -import os -from pathlib import Path + uv run pytest -m integration tests/integration/test_model_capability.py \ + --capability-models=local-fast,local-coder + +Skipped otherwise, because it spends real tokens across every model listed. +Which models to measure is a question for a particular run, so it is an +argument, not configuration. +""" import pytest @@ -26,45 +29,56 @@ from nisse_dev.canonical import ( UserMessage, ) from nisse_dev.catalog import ModelCatalog, select_client -from nisse_dev.envfile import load_env_file +from nisse_dev.config import AgentConfig from nisse_dev.expert import parse_answer from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient, LLMError from nisse_dev.planner import PLANNER_TOOLS from nisse_dev.tools import EXECUTOR_TOOLS -load_env_file(Path(__file__).resolve().parents[2] / ".env") - -SENTINEL = "NISSE_CAPABILITY_MODELS" -MODELS = [m.strip() for m in os.environ.get(SENTINEL, "").split(",") if m.strip()] - -pytestmark = [ - pytest.mark.integration, - pytest.mark.skipif( - not MODELS, reason=f"set {SENTINEL} to a comma-separated model list" - ), -] +pytestmark = pytest.mark.integration -def clients() -> tuple[LiteLLMClient, AnthropicPassthroughClient]: - return LiteLLMClient.from_env(), AnthropicPassthroughClient.from_env() +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "model" not in metafunc.fixturenames: + return + raw = metafunc.config.getoption("--capability-models") + models = [m.strip() for m in raw.split(",") if m.strip()] + if models: + metafunc.parametrize("model", models) + else: + metafunc.parametrize( + "model", + [ + pytest.param( + None, + marks=pytest.mark.skip( + reason="no models to measure; pass --capability-models" + ), + ) + ], + ) + + +@pytest.fixture +def routes( + litellm: LiteLLMClient, passthrough: AnthropicPassthroughClient +) -> tuple[LiteLLMClient, AnthropicPassthroughClient]: + return litellm, passthrough @pytest.fixture(scope="module") -def catalog() -> ModelCatalog: +def catalog(live_config: AgentConfig) -> ModelCatalog: """The live catalog this key can see. /model/info is scoped per key, so the probe list and the key have to agree.""" - return ModelCatalog.discover( - os.environ["NISSE_LLM_BASE_URL"], os.environ["NISSE_LLM_TOKEN"] - ) + return ModelCatalog.discover(live_config.llm_base_url, live_config.llm_token) def user(text: str) -> list[StoredEvent]: return [StoredEvent(id=1, ts="t", event=UserMessage(text=text, source="probe"))] -@pytest.mark.parametrize("model", MODELS) -def test_model_answers_at_all(model: str, catalog: ModelCatalog) -> None: - normalized, passthrough = clients() +def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None: + normalized, passthrough = routes client = select_client(model, normalized, passthrough, catalog) produced = client.complete( user("Reply with exactly one word: pong"), None, model, system="Be terse." @@ -73,11 +87,12 @@ def test_model_answers_at_all(model: str, catalog: ModelCatalog) -> None: assert texts, f"{model} produced no text: {produced!r}" -@pytest.mark.parametrize("model", MODELS) -def test_model_holds_the_planner_contract(model: str, catalog: ModelCatalog) -> None: +def test_model_holds_the_planner_contract( + model: str, catalog: ModelCatalog, routes +) -> None: """A4's gating is structural at the tool layer: a model that answers in prose cannot be gated, because there is no plan to approve.""" - normalized, passthrough = clients() + normalized, passthrough = routes client = select_client(model, normalized, passthrough, catalog) produced = client.complete( user("Draft an architecture plan for a command-line slug generator."), @@ -100,11 +115,12 @@ def test_model_holds_the_planner_contract(model: str, catalog: ModelCatalog) -> ) -@pytest.mark.parametrize("model", MODELS) -def test_model_round_trips_a_tool_result(model: str, catalog: ModelCatalog) -> None: +def test_model_round_trips_a_tool_result( + model: str, catalog: ModelCatalog, routes +) -> None: """The executor loop feeds results back; a model that cannot consume its own tool result cannot run a milestone.""" - normalized, passthrough = clients() + normalized, passthrough = routes client = select_client(model, normalized, passthrough, catalog) events = user("List the files in the current directory using your tools.") produced = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.") @@ -128,13 +144,12 @@ def test_model_round_trips_a_tool_result(model: str, catalog: ModelCatalog) -> N assert follow_up, f"{model} produced nothing after its tool result" -@pytest.mark.parametrize("model", MODELS) def test_model_holds_the_prompt_level_json_contract( - model: str, catalog: ModelCatalog + model: str, catalog: ModelCatalog, routes ) -> None: """A14 contracts structured output at the prompt level because vendors differ on JSON enforcement. This measures whether that holds per model.""" - normalized, passthrough = clients() + normalized, passthrough = routes client = select_client(model, normalized, passthrough, catalog) produced = client.complete( user("Is SQLite sufficient for a single-process agent's task state?"), @@ -152,12 +167,13 @@ def test_model_holds_the_prompt_level_json_contract( ) -@pytest.mark.parametrize("model", MODELS) -def test_route_selection_matches_the_model(model: str, catalog: ModelCatalog) -> None: +def test_route_selection_matches_the_model( + model: str, catalog: ModelCatalog, routes +) -> None: """A non-Anthropic model must not be sent Anthropic-shaped bodies. This was hardcoded until gpt-6-astra made it observable, and name-sniffed until semantic aliases made the name stop meaning anything.""" - normalized, passthrough = clients() + normalized, passthrough = routes facts = catalog.facts(model) chosen = select_client(model, normalized, passthrough, catalog) if facts.provider == "anthropic": diff --git a/tests/integration/test_passthrough_live.py b/tests/integration/test_passthrough_live.py index f5c9191..e5d0812 100644 --- a/tests/integration/test_passthrough_live.py +++ b/tests/integration/test_passthrough_live.py @@ -6,30 +6,19 @@ passthrough rather than being flattened by the proxy. """ import io -import os from pathlib import Path import httpx import pytest from nisse_dev.canonical import AssistantText, ToolCall, ToolResult, UserMessage -from nisse_dev.envfile import load_env_file +from nisse_dev.config import AgentConfig from nisse_dev.llm import AnthropicPassthroughClient from nisse_dev.oplog import OpLog from nisse_dev.state import StateStore from nisse_dev.tools import EXECUTOR_TOOLS -load_env_file(Path(__file__).resolve().parents[2] / ".env") - -SENTINEL = "NISSE_LIVE_PLANNER_MODEL" - -pytestmark = [ - pytest.mark.integration, - pytest.mark.skipif( - not os.environ.get(SENTINEL), - reason=f"no live frontier route configured ({SENTINEL} unset)", - ), -] +pytestmark = pytest.mark.integration def store_with(tmp_path: Path, *events) -> StateStore: @@ -39,18 +28,22 @@ def store_with(tmp_path: Path, *events) -> StateStore: return store -def test_passthrough_completes_a_plain_turn(tmp_path: Path) -> None: +def test_passthrough_completes_a_plain_turn( + tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str +) -> None: store = store_with( tmp_path, UserMessage(text="Reply with exactly: pong", source="live") ) - result = AnthropicPassthroughClient.from_env().complete( - store.read_events(), None, os.environ[SENTINEL], system="Be terse." + result = passthrough.complete( + store.read_events(), None, live_model, system="Be terse." ) texts = [e.text for e in result if isinstance(e, AssistantText)] assert texts and "pong" in texts[0].lower() -def test_passthrough_round_trips_a_tool_call(tmp_path: Path) -> None: +def test_passthrough_round_trips_a_tool_call( + tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str +) -> None: store = store_with( tmp_path, UserMessage( @@ -58,11 +51,11 @@ def test_passthrough_round_trips_a_tool_call(tmp_path: Path) -> None: source="live", ), ) - client = AnthropicPassthroughClient.from_env() + client = passthrough produced = client.complete( store.read_events(), EXECUTOR_TOOLS, - os.environ[SENTINEL], + live_model, system="Use a tool. Do not answer in prose.", ) calls = [e for e in produced if isinstance(e, ToolCall)] @@ -75,18 +68,18 @@ def test_passthrough_round_trips_a_tool_call(tmp_path: Path) -> None: store.append_event( ToolResult(call_id=calls[0].call_id, result="app.py\ntests/", success=True) ) - follow_up = client.complete( - store.read_events(), EXECUTOR_TOOLS, os.environ[SENTINEL] - ) + follow_up = client.complete(store.read_events(), EXECUTOR_TOOLS, live_model) assert follow_up, "the vendor rejected or ignored the tool_result turn" -def test_prompt_caching_survives_the_passthrough() -> None: +def test_prompt_caching_survives_the_passthrough( + live_config: AgentConfig, live_model: str +) -> None: # A5 asks for this to be verified rather than assumed: a proxy that # flattened the request would silently drop cache control and turn a # cost optimisation into a per-call expense. - base = os.environ["NISSE_LLM_BASE_URL"].rstrip("/") - token = os.environ["NISSE_LLM_TOKEN"] + base = live_config.llm_base_url.rstrip("/") + token = live_config.llm_token response = httpx.post( f"{base}/v1/messages", headers={ @@ -95,7 +88,7 @@ def test_prompt_caching_survives_the_passthrough() -> None: "anthropic-version": "2023-06-01", }, json={ - "model": os.environ[SENTINEL], + "model": live_model, "max_tokens": 16, "messages": [{"role": "user", "content": "Say pong."}], }, diff --git a/tests/integration/test_planning_live.py b/tests/integration/test_planning_live.py index e49044f..821ccec 100644 --- a/tests/integration/test_planning_live.py +++ b/tests/integration/test_planning_live.py @@ -1,47 +1,30 @@ """Live planning turn: a real model must answer with a submit_plan or ask_human tool call, not prose. -Deliberately opt-in via NISSE_LIVE_PLANNER_MODEL rather than the usual -NISSE_LLM_BASE_URL sentinel. Planning escalates to a frontier model -unconditionally (A12), and the small local model does not hold the tool-call -contract reliably — measured, see the PR — so this test needs a frontier -route that does not exist yet. Set the variable to the model to plan with -once one is configured; until then this skips on purpose, not by accident. +Plans with the test config's planner role. Planning escalates to a frontier +model unconditionally (A12), and small local models do not hold the tool-call +contract reliably, so that role must be one that does. """ import io -import os from pathlib import Path import pytest from nisse_dev.canonical import UserMessage from nisse_dev.control import AskHuman, SubmitPlan -from nisse_dev.envfile import load_env_file from nisse_dev.llm import LiteLLMClient, RoleMap from nisse_dev.oplog import OpLog from nisse_dev.planner import LLMPlanningAgent from nisse_dev.state import StateStore -load_env_file(Path(__file__).resolve().parents[2] / ".env") - -SENTINEL = "NISSE_LIVE_PLANNER_MODEL" - -pytestmark = [ - pytest.mark.integration, - pytest.mark.skipif( - not os.environ.get(SENTINEL), - reason=f"no frontier planner route configured ({SENTINEL} unset)", - ), -] +pytestmark = pytest.mark.integration -def test_live_planning_turn_returns_a_tool_action(tmp_path: Path) -> None: - assert os.environ.get("NISSE_LLM_TOKEN"), ( - f"{SENTINEL} is set, so the live planning path is expected to work, " - "but NISSE_LLM_TOKEN is missing" - ) - model = os.environ[SENTINEL] +def test_live_planning_turn_returns_a_tool_action( + tmp_path: Path, litellm: LiteLLMClient, live_model: str +) -> None: + model = live_model roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model) store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) store.append_event( @@ -53,7 +36,7 @@ def test_live_planning_turn_returns_a_tool_action(tmp_path: Path) -> None: source="live-test", ) ) - agent = LLMPlanningAgent(LiteLLMClient.from_env(), store, roles) + agent = LLMPlanningAgent(litellm, store, roles) action = agent.plan("arch", "hello-cli") diff --git a/uv.lock b/uv.lock index c86747e..c9cbbdb 100644 --- a/uv.lock +++ b/uv.lock @@ -113,12 +113,14 @@ source = { editable = "nisse-dev" } dependencies = [ { name = "httpx" }, { name = "nisse-core" }, + { name = "pyyaml" }, ] [package.metadata] requires-dist = [ { name = "httpx", specifier = ">=0.27" }, { name = "nisse-core", editable = "nisse-core" }, + { name = "pyyaml", specifier = ">=6" }, ] [[package]] @@ -175,6 +177,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + [[package]] name = "ruff" version = "0.16.6"