Merge pull request 'Move configuration to YAML, and make the tuning knobs configurable' (#43) from yaml-config into main
CI / checks (push) Failing after 51s
CI / checks (push) Failing after 51s
Reviewed-on: dongar/nisse#43 Reviewed-by: Don Garrett <dgarrett@acm.org>
This commit was merged in pull request #43.
This commit is contained in:
commit
31851b7396
35 files changed
+1357
-605
No files matched your search
+16
-58
@@ -1,59 +1,17 @@
|
||||
# Copy to .env (chmod 600). Real environment variables win over this file.
|
||||
# .env is gitignored; never commit real values.
|
||||
|
||||
# --- LLM access (A5): the LiteLLM proxy is the single LLM destination ---
|
||||
NISSE_LLM_BASE_URL=http://litellm.example:4000
|
||||
NISSE_LLM_TOKEN=sk-your-litellm-virtual-key
|
||||
|
||||
# --- Role mapping (A11): roles resolve to concrete model strings ---
|
||||
NISSE_ROLE_EXECUTOR=anthropic/claude-haiku-4-5-20251001
|
||||
NISSE_ROLE_PLANNER=anthropic/claude-opus-5
|
||||
NISSE_ROLE_DELEGATE=anthropic/claude-opus-5
|
||||
NISSE_ROLE_CONSULTANT=anthropic/claude-opus-5
|
||||
|
||||
# Opt-in for the live planning test: a model the virtual key can reach that
|
||||
# holds the tool-call contract. Planning escalates unconditionally (A12), and
|
||||
# small local models do not hold the contract reliably — llama3.2 returns
|
||||
# truncated tool arguments, Haiku-class and up are solid.
|
||||
NISSE_LIVE_PLANNER_MODEL=claude-haiku-4-5
|
||||
|
||||
# --- Matrix transport (A1), used by nisse-dev's MatrixTransport ---
|
||||
NISSE_MATRIX_HOMESERVER=https://matrix.example
|
||||
NISSE_MATRIX_USER_ID=@nisse-dev:example
|
||||
NISSE_MATRIX_TOKEN=syt_your_matrix_access_token
|
||||
NISSE_MATRIX_ROOM_ID=!roomid:example
|
||||
# Dedicated room for the live integration test; never the production room.
|
||||
NISSE_MATRIX_TEST_ROOM_ID=!testroomid:example
|
||||
NISSE_MATRIX_OPERATORS=@you:example
|
||||
|
||||
# --- Gitea (B2): service account for branches, PRs, and comment intake ---
|
||||
NISSE_GITEA_URL=https://gitea.example
|
||||
NISSE_GITEA_TOKEN=your-scoped-service-account-token
|
||||
# Sandbox repo for live tests; never the real project repo.
|
||||
NISSE_GITEA_TEST_REPO=owner/sandbox-repo
|
||||
# Gitea logins allowed to drive the agent via PR comments (A4 per-channel
|
||||
# allowlist; a Matrix MXID and a Gitea login are different names).
|
||||
NISSE_GITEA_OPERATORS=your-gitea-login
|
||||
|
||||
# --- Running the agent (the demo, not the tests) ---
|
||||
# The tests use NISSE_GITEA_TEST_REPO; a running agent uses these.
|
||||
# Secrets only. Copy to .env (chmod 600); it is gitignored, so never commit
|
||||
# real values. Real environment variables win over this file.
|
||||
#
|
||||
# Layout. NISSE_WORKSPACE is the *clone the agent works in*, not the directory
|
||||
# the agent runs from — they must not be the same place:
|
||||
#
|
||||
# ~/nisse-demo-workspace/ deployment root, not a git repo
|
||||
# ├── .env this file
|
||||
# ├── state.db the A2 state store
|
||||
# └── project/ NISSE_WORKSPACE, the clone
|
||||
#
|
||||
# The agent commits its workspace with `git add -A` and pushes it, so anything
|
||||
# inside the clone reaches the project repository. A .env or state store placed
|
||||
# there would publish credentials as a side effect of the agent working
|
||||
# correctly; startup refuses to proceed if either is inside the workspace.
|
||||
NISSE_STATE_DB=~/nisse-demo-workspace/state.db
|
||||
NISSE_WORKSPACE=~/nisse-demo-workspace/project
|
||||
NISSE_PROJECT=demo
|
||||
NISSE_GITEA_REPO=owner/demo-repo
|
||||
# NISSE_BASE_BRANCH=main
|
||||
# Path to this file, if not ./.env
|
||||
# NISSE_ENV=~/nisse-demo-workspace/.env
|
||||
# Everything else lives in the YAML config (see nisse-dev.example.yaml),
|
||||
# which refers to these by name, e.g. `token: ${NISSE_LLM_TOKEN}`. The live
|
||||
# integration tests use the same names, from tests/integration/nisse-test.yaml.
|
||||
|
||||
# llm.token: the LiteLLM virtual key (A5)
|
||||
NISSE_LLM_TOKEN=replace-me
|
||||
# matrix.token: the Matrix access token for the agent's account (A1)
|
||||
NISSE_MATRIX_TOKEN=replace-me
|
||||
# gitea.token: the Gitea service account's scoped token (B2)
|
||||
NISSE_GITEA_TOKEN=replace-me
|
||||
|
||||
# Where the config file is, if not ./nisse-dev.yaml. Where this file is, if
|
||||
# not ./.env, is NISSE_ENV, which has to be set in the real environment.
|
||||
# NISSE_CONFIG=~/nisse-demo-workspace/nisse-dev.yaml
|
||||
+3
-11
@@ -34,19 +34,11 @@ jobs:
|
||||
run: uv run pytest -m "not integration"
|
||||
|
||||
- name: Integration tests
|
||||
# Live-path credentials come from repo Actions secrets; the tests
|
||||
# skip-guard on these, so a missing secret or unreachable service
|
||||
# degrades to "skipped", never a failure on unrelated work.
|
||||
# Everything but the secrets is in tests/integration/nisse-test.yaml,
|
||||
# which refers to these by name. A missing secret fails the live
|
||||
# tests that need it; it is never quietly turned into a skip.
|
||||
env:
|
||||
NISSE_LLM_BASE_URL: ${{ secrets.NISSE_LLM_BASE_URL }}
|
||||
NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }}
|
||||
NISSE_ROLE_EXECUTOR: ${{ secrets.NISSE_ROLE_EXECUTOR }}
|
||||
NISSE_LIVE_PLANNER_MODEL: ${{ secrets.NISSE_LIVE_PLANNER_MODEL }}
|
||||
NISSE_MATRIX_HOMESERVER: ${{ secrets.NISSE_MATRIX_HOMESERVER }}
|
||||
NISSE_MATRIX_USER_ID: ${{ secrets.NISSE_MATRIX_USER_ID }}
|
||||
NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }}
|
||||
NISSE_MATRIX_TEST_ROOM_ID: ${{ secrets.NISSE_MATRIX_TEST_ROOM_ID }}
|
||||
NISSE_GITEA_URL: ${{ secrets.NISSE_GITEA_URL }}
|
||||
NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }}
|
||||
NISSE_GITEA_TEST_REPO: ${{ secrets.NISSE_GITEA_TEST_REPO }}
|
||||
run: uv run pytest -m integration -rs
|
||||
@@ -6,3 +6,4 @@ __pycache__/
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
dist/
|
||||
nisse-dev.yaml
|
||||
@@ -389,8 +389,10 @@ Requirements:
|
||||
- **No vendor credentials on the agent VM.** LiteLLM holds every key, sourced from
|
||||
Bitwarden machine accounts (A7); the agent holds one LiteLLM token. Adding a vendor
|
||||
does not touch the agent VM, and a compromised agent leaks no provider keys.
|
||||
- Config from env/`.env`, **nothing hardcoded**: base URL, auth token, and the role
|
||||
mapping (A11). Changing any of it is an env change + restart, nothing more.
|
||||
- Config from the agent's config file, **nothing hardcoded**: base URL, auth token,
|
||||
and the role mapping (A11). Secrets in it are references to environment
|
||||
variables, never values. Changing any of it is a config edit + restart,
|
||||
nothing more.
|
||||
- **Budget caps and spend alerts live on the LiteLLM virtual key**, not in agent code.
|
||||
|
||||
### A6. Browser Tool
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
# Tuning Guide
|
||||
|
||||
How to fit a local model onto the inference host, and how to set the agent's
|
||||
tuning knobs. Part 1 covers the ideas you need to reason about the hardware.
|
||||
Part 2 covers each knob in `nisse-dev.yaml`. Part 3 gives an order to tune
|
||||
in.
|
||||
|
||||
The knobs live under `tuning:` in the config file (see
|
||||
`nisse-dev.example.yaml`). They are read at startup, so a change takes effect
|
||||
on the next restart. Unknown or misspelt keys are startup errors.
|
||||
|
||||
---
|
||||
|
||||
## Part 1 — How a local model spends VRAM
|
||||
|
||||
A model server holding a model in GPU memory spends it on three things:
|
||||
|
||||
1. **Weights**, which are fixed for a given model and quantization.
|
||||
2. **The KV cache**, which grows with context length.
|
||||
3. **Working memory** for activations and runtime buffers. Budget 1–2 GB of
|
||||
headroom.
|
||||
|
||||
On a 24 GB card, the whole game is trading (1) against (2).
|
||||
|
||||
### Weights and quantization
|
||||
|
||||
A model's weights take roughly *parameters × bits per weight ÷ 8* bytes.
|
||||
|
||||
| | 16-bit (full) | 8-bit | ~4.5-bit (Q4_K_M, AWQ) |
|
||||
|---|---|---|---|
|
||||
| 24B | 48 GB | 24 GB | ~14 GB |
|
||||
| 32B | 64 GB | 32 GB | ~19 GB |
|
||||
|
||||
Quantization stores each weight in fewer bits. Common 4-bit formats come out
|
||||
nearer 4.5–5 bits per weight once scaling factors are counted, so check the
|
||||
actual file size. The quality cost is small at 5–6 bits, usually acceptable at
|
||||
4, and it climbs quickly below that. For agent work the cost shows up first as
|
||||
**worse tool-call discipline**, not as worse prose. The capability matrix
|
||||
(Part 3) measures exactly that.
|
||||
|
||||
**Mixture-of-experts** models such as Qwen3-30B-A3B are different. Memory is
|
||||
set by *total* parameters (30B), but speed is set by the parameters *active*
|
||||
for each token (3B). An MoE model of a given size therefore generates much
|
||||
faster than a dense model of the same size. It's often weaker than a dense
|
||||
model of the same total size, but not always for tool use.
|
||||
|
||||
### The KV cache, and why context costs memory
|
||||
|
||||
As the model reads the prompt, every layer stores a key and a value vector for
|
||||
every token, so that later tokens can attend to them without recomputing.
|
||||
That store is the KV cache. Its size per token is:
|
||||
|
||||
2 (key and value) × layers × KV heads × head size × bytes per value
|
||||
|
||||
The numbers come from the model's `config.json` (`num_hidden_layers`,
|
||||
`num_key_value_heads`, `head_dim`). Some worked examples at 16-bit:
|
||||
|
||||
| Model | Layers × KV heads × head size | Per token | 32k tokens |
|
||||
|---|---|---|---|
|
||||
| Qwen3-32B | 64 × 8 × 128 | 256 KB | ~8 GB |
|
||||
| Mistral/Devstral Small 24B | 40 × 8 × 128 | 160 KB | ~5 GB |
|
||||
| Qwen3-30B-A3B (MoE) | 48 × 4 × 128 | 96 KB | ~3 GB |
|
||||
|
||||
(Taken from the published configs; recheck against the model card before
|
||||
relying on them.)
|
||||
|
||||
Two consequences:
|
||||
|
||||
- **Context length is a memory decision, not a free setting.** The spec's
|
||||
estimate (Part H) was about 5 GB left for KV cache beside a 4-bit 32B model.
|
||||
At 16-bit, that's only about 20k tokens of context for Qwen3-32B.
|
||||
- **KV-cache quantization** stores keys and values in 8 bits (or 4). At 8
|
||||
bits it halves the cache and is close to lossless. At 4 bits it is
|
||||
noticeably worse. Quantizing the cache to 8-bit is usually the first lever
|
||||
to pull, since it buys twice the context for almost nothing.
|
||||
|
||||
The server allocates the cache for the context length you configure. It
|
||||
doesn't matter how much of it a request uses. So set the context length on the
|
||||
server deliberately: it is the ceiling for *prompt plus response* on every
|
||||
request.
|
||||
|
||||
### Prompt processing vs generation
|
||||
|
||||
A request has two phases, and they run at very different speeds:
|
||||
|
||||
- **Prefill** reads the prompt. It is parallel and fast: thousands of tokens
|
||||
per second on a 4090.
|
||||
- **Decode** writes the response one token at a time. Roughly 30–40 tokens per
|
||||
second for a dense 32B at 4-bit, and several times that for a 3B-active MoE.
|
||||
Treat these numbers as ballpark; measure your own.
|
||||
|
||||
An agent loop re-sends the whole transcript on every turn, which makes the
|
||||
prompt long and the responses short (a tool call is often under 200 tokens).
|
||||
So prefill matters more than the headline tokens-per-second suggests.
|
||||
|
||||
**Prefix caching** is the fix. If the start of a prompt matches the previous
|
||||
request, the server reuses the KV cache for it and prefills only the new part.
|
||||
llama.cpp does this per slot, and vLLM does it automatically. For this
|
||||
workload it is often the single largest speed win. It works here because the
|
||||
rendered transcript only grows at the end, so consecutive turns share
|
||||
everything but their newest events.
|
||||
|
||||
### Tokens and characters
|
||||
|
||||
For English and code, a token is roughly **4 characters**. That's useful for
|
||||
converting the character-based knobs below into context cost.
|
||||
|
||||
---
|
||||
|
||||
## Part 2 — The knobs
|
||||
|
||||
For each knob: what it controls, what goes wrong at either extreme, and what
|
||||
to look at when deciding.
|
||||
|
||||
### `llm.max_output_tokens` — default 16384
|
||||
|
||||
The ceiling on a single response. It is sent with every request, for every
|
||||
role.
|
||||
|
||||
- **Why it exists:** a response that hits the ceiling is truncated
|
||||
mid-sentence, or worse, mid-tool-call, where it can still parse. The agent
|
||||
detects truncation and treats it as an error, but a ceiling that's too
|
||||
tight still costs a failed turn. Planning with a frontier model was observed
|
||||
using over 4,000 output tokens.
|
||||
- **Too high, on a local model:** the server reserves room for the whole
|
||||
response inside the context window. With a 32k context and a 16k ceiling,
|
||||
only 16k is left for the prompt. Some servers reject any request where
|
||||
prompt plus ceiling exceeds the context.
|
||||
- **Too low:** truncated plans and tool calls.
|
||||
- **Deciding:** executor turns are short, so 2–4k is plenty for a local
|
||||
executor. Frontier planning wants the large default. This one knob covers
|
||||
every role for now; a per-model setting is coming with the context budget
|
||||
(see *Coming next*).
|
||||
|
||||
### `tuning.executor.max_turns` — default 24
|
||||
|
||||
How many model calls one milestone may make before the executor stops with
|
||||
`capped` and asks you what to do.
|
||||
|
||||
- **Too low:** milestones that were making progress stop short. You'll see
|
||||
`capped` outcomes on work that was going well.
|
||||
- **Too high:** a confused model keeps going. Every turn also adds its tool
|
||||
calls and results to the transcript, so a long run spends context as well
|
||||
as time.
|
||||
- **Deciding:** look at how many turns *successful* milestones take, and set
|
||||
the cap comfortably above the high end, around 1.5–2× the turns of your
|
||||
longer successful runs. A stronger model usually needs fewer turns, not
|
||||
more.
|
||||
|
||||
### `tuning.executor.failure_limit` — default 2
|
||||
|
||||
How many consecutive failed `run_command` calls trigger escalation: the
|
||||
sub-task is handed to a frontier model as a delegation (spec A12). A
|
||||
successful command resets the count. Failed file reads don't count, since
|
||||
reading is looking around, not doing the work.
|
||||
|
||||
- **Why it's a count and not a judgment:** small models are poorly calibrated
|
||||
about their own competence. Asked "are you stuck?", they say no until far
|
||||
too late, or yes constantly. A count of observable failures is a signal
|
||||
that doesn't depend on the model's opinion of itself.
|
||||
- **Too low:** normal iteration escalates. Test-first work *expects* a failing
|
||||
test run before the fix, so a limit of 1 hands off routine work and spends
|
||||
frontier tokens on it.
|
||||
- **Too high:** a weak model circles, re-running the same failing command
|
||||
with small variations, while the transcript fills up.
|
||||
- **Deciding:** raise it as local models get stronger. Watch two things
|
||||
together. First, how often delegations fix what the local model couldn't
|
||||
(escalation is doing its job). Second, how often the local model recovers
|
||||
on its own after its second failure (the limit may be too low).
|
||||
|
||||
### `tuning.expert.consult_budget` — default 3
|
||||
|
||||
How many `ask_expert` calls one milestone may make. `ask_expert` is a single
|
||||
stateless question to a stronger model, with no access to the repository
|
||||
(spec A14), for things like "is this API deprecated?".
|
||||
|
||||
- **Too low:** the executor can't get unstuck on a knowledge question it
|
||||
can't answer from the code.
|
||||
- **Too high:** a confused executor asks variations of the same question
|
||||
instead of doing the work or failing. That's the circling A12 warns about.
|
||||
- **Deciding:** count consults per milestone, and check whether the answers
|
||||
changed what the executor did next. If it regularly exhausts the budget
|
||||
without progress, the budget is fine and the model or the plan is the
|
||||
problem.
|
||||
|
||||
### `tuning.delegation` — hard caps on one delegation
|
||||
|
||||
A delegation is a frontier model working a sub-task with full tool access
|
||||
(spec A13). When a cap is hit, the delegation **fails**, and its branch and
|
||||
work are discarded, because resuming on a half-edited tree isn't safe.
|
||||
|
||||
| Knob | Default | Caps |
|
||||
|---|---|---|
|
||||
| `max_turns` | 30 | model calls |
|
||||
| `max_tokens` | 200000 | input + output tokens, as reported by the provider |
|
||||
| `wall_clock_s` | 900 | elapsed time |
|
||||
|
||||
- **`max_tokens` is a scope limit, not the money limit.** Spending limits live
|
||||
on the LiteLLM key (spec A5). This cap stops one delegation from growing
|
||||
without bound. An unbounded one could still exhaust a daily budget and take
|
||||
every frontier model away until it resets.
|
||||
- **Too low:** delegations fail just short of finishing and their work is
|
||||
thrown away. That is the most expensive way to be wrong, because you paid
|
||||
for the work.
|
||||
- **Too high:** a stuck delegation burns tokens and time before failing.
|
||||
- **Deciding:** every delegation records the tokens it used and how long it
|
||||
ran (the `delegation_completed` event, captured for evaluation per spec
|
||||
A16). Set each cap above what successful delegations actually use, with
|
||||
room to spare.
|
||||
|
||||
### `tuning.tools.command_timeout_s` — default 120
|
||||
|
||||
How long one `run_command` may run before its whole process group is killed.
|
||||
|
||||
- **Too low:** slow test suites, dependency installs and Docker builds get
|
||||
killed and reported as failures. That also counts toward `failure_limit`.
|
||||
- **Too high:** a command that never exits blocks the turn for the whole
|
||||
timeout, such as a dev server started in the foreground or a test waiting on
|
||||
input. `stop` waits for a running command to finish too, so this is also the
|
||||
worst-case delay before a stop takes effect.
|
||||
- **Deciding:** set it just above the slowest legitimate command in the
|
||||
projects the agent works on.
|
||||
|
||||
### `tuning.tools.max_output_chars` — default 4000
|
||||
|
||||
How much of a tool's output (command output, file contents, directory
|
||||
listings) the model sees. Anything past this is cut, with a note saying how
|
||||
much was dropped.
|
||||
|
||||
- **This is a context knob.** Every tool result stays in the transcript and is
|
||||
re-sent every turn. 4,000 characters is about 1,000 tokens, so 20 turns of
|
||||
maximal output is about 20k tokens, which is already most of a local
|
||||
model's context.
|
||||
- **Too low:** the model doesn't see the part of the output that matters.
|
||||
- **Too high:** the transcript outgrows the context window sooner.
|
||||
- **A known weakness:** truncation keeps the *start* of the output, but test
|
||||
runners put their failure summary at the *end*. A long, failing pytest run
|
||||
can lose exactly the lines the model needed. Keeping the head and the tail
|
||||
would fix this, and is a candidate for the context-budget work.
|
||||
|
||||
---
|
||||
|
||||
## Part 3 — An order to tune in
|
||||
|
||||
Upstream choices change what's right downstream, so tune in this order and
|
||||
change one thing at a time.
|
||||
|
||||
1. **Model and quantization.** Start with whether a candidate holds the
|
||||
tool-call contract at all:
|
||||
|
||||
uv run pytest -m integration tests/integration/test_model_capability.py \
|
||||
--capability-models=alias-a,alias-b
|
||||
|
||||
A model that falls back to prose instead of calling tools is disqualified,
|
||||
however good its benchmarks look. Before comparing aliases, check what
|
||||
each one actually points at in LiteLLM (`/model/info`, the
|
||||
`litellm_params.model` field). Two aliases believed to be the same model
|
||||
once weren't, and that cost a lot of debugging.
|
||||
2. **Context length and KV-cache precision** on the server. Take the largest
|
||||
context that fits beside the weights with 1–2 GB of headroom. Try an 8-bit
|
||||
KV cache before giving up context.
|
||||
3. **`llm.max_output_tokens`**, so the prompt keeps most of that context.
|
||||
4. **Real milestones.** Run representative work and look at the evidence:
|
||||
turns used, escalations, delegation outcomes, and time taken.
|
||||
5. **The loop knobs**: `failure_limit`, `max_turns`, `consult_budget`, the
|
||||
delegation caps, then the tool limits, one at a time.
|
||||
|
||||
**Expect noise.** Local model behaviour varies a lot from run to run. One
|
||||
early comparison measured 12 of 20 runs holding the tool contract against 20
|
||||
of 20, a real difference (p ≈ 0.003). Three runs each could easily have
|
||||
pointed the other way. Before changing a knob because of a result, repeat the
|
||||
run enough times that the result would survive a coin flip.
|
||||
|
||||
---
|
||||
|
||||
## Coming next
|
||||
|
||||
The next changes in the 4090-readiness work add knobs, and this guide gains a
|
||||
section for each:
|
||||
|
||||
- **A context budget.** Rendering a transcript that fits the model's context
|
||||
window, with what to drop or summarize when it doesn't, and per-model output
|
||||
ceilings.
|
||||
- **Enforced tool calls.** An explicit `finish` tool, so the server can be told
|
||||
a tool call is required and hold the model to the contract, where today it
|
||||
can fall back to prose.
|
||||
- **A benchmark script** that runs the capability matrix and representative
|
||||
milestones across candidate models and prints a comparison table.
|
||||
@@ -0,0 +1,57 @@
|
||||
# Example configuration for nisse-dev. Copy to nisse-dev.yaml (or point
|
||||
# NISSE_CONFIG at your copy) and edit.
|
||||
#
|
||||
# - Structure lives here. Secrets do not: write them as ${VARIABLE} references,
|
||||
# resolved from the environment (or .env, see .env.example) at startup.
|
||||
# - Keep this file outside the workspace clone. Startup refuses otherwise,
|
||||
# because the agent commits and pushes everything in the clone.
|
||||
# - Unknown keys are errors, so a misspelt knob fails at startup instead of
|
||||
# silently doing nothing.
|
||||
|
||||
state_db: ~/nisse-demo-workspace/state.db
|
||||
workspace: ~/nisse-demo-workspace/project # the clone
|
||||
project: demo
|
||||
|
||||
llm: # A5: LiteLLM is the only LLM destination
|
||||
base_url: http://litellm.example:4000
|
||||
token: ${NISSE_LLM_TOKEN}
|
||||
# Ceiling on each response. Must fit inside a local model's context window
|
||||
# alongside the prompt; see docs/Tuning-Guide.md.
|
||||
max_output_tokens: 16384
|
||||
|
||||
roles: # A11: role -> LiteLLM model alias
|
||||
executor: local-fast
|
||||
planner: claude-opus-5
|
||||
delegate: claude-opus-5
|
||||
consultant: claude-opus-5
|
||||
|
||||
matrix: # A1
|
||||
homeserver: https://matrix.example
|
||||
user_id: "@nisse-dev:example"
|
||||
token: ${NISSE_MATRIX_TOKEN}
|
||||
room_id: "!roomid:example"
|
||||
operators: ["@you:example"]
|
||||
|
||||
gitea: # B2
|
||||
url: https://gitea.example
|
||||
token: ${NISSE_GITEA_TOKEN}
|
||||
repo: owner/demo-repo
|
||||
operators: [your-gitea-login]
|
||||
base_branch: main
|
||||
|
||||
# What each knob does, and how to choose a value: docs/Tuning-Guide.md.
|
||||
# These are the defaults; the section can be left out entirely. The file is
|
||||
# read at startup, so a change takes effect on the next restart.
|
||||
tuning:
|
||||
executor:
|
||||
max_turns: 24 # model calls per milestone before it stops as capped
|
||||
failure_limit: 2 # consecutive failed commands before escalating (A12)
|
||||
expert:
|
||||
consult_budget: 3 # ask_expert calls per milestone (A14)
|
||||
delegation: # hard caps on one delegation (A13)
|
||||
max_turns: 30
|
||||
max_tokens: 200000
|
||||
wall_clock_s: 900
|
||||
tools:
|
||||
command_timeout_s: 120 # one run_command before it is killed
|
||||
max_output_chars: 4000 # tool output kept for the model; the rest is cut
|
||||
@@ -3,7 +3,11 @@ name = "nisse-dev"
|
||||
version = "0.1.0"
|
||||
description = "Dev agent: plans and implements software projects, produces Gitea PRs"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = ["nisse-core", "httpx>=0.27"]
|
||||
dependencies = [
|
||||
"nisse-core",
|
||||
"httpx>=0.27",
|
||||
"pyyaml>=6",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
nisse-dev = "nisse_dev.main:main"
|
||||
|
||||
@@ -16,7 +16,7 @@ from nisse_dev.cancel import Cancel
|
||||
from nisse_dev.catalog import ModelCatalog, select_client
|
||||
from nisse_dev.config import AgentConfig
|
||||
from nisse_dev.control import Orchestrator
|
||||
from nisse_dev.delegation import Delegation
|
||||
from nisse_dev.delegation import Caps, Delegation
|
||||
from nisse_dev.executor import LLMExecutor
|
||||
from nisse_dev.expert import ExpertDesk
|
||||
from nisse_dev.git import GitRepo, authenticated_url
|
||||
@@ -149,18 +149,29 @@ def build_agent(
|
||||
account = gitea.whoami()
|
||||
|
||||
git_repo = prepare_workspace(config, account)
|
||||
workspace = Workspace(config.workspace)
|
||||
tuning = config.tuning
|
||||
workspace = Workspace(
|
||||
config.workspace,
|
||||
timeout_s=tuning.tools.command_timeout_s,
|
||||
max_output_chars=tuning.tools.max_output_chars,
|
||||
)
|
||||
|
||||
normalized = LiteLLMClient(config.llm_base_url, config.llm_token)
|
||||
normalized = LiteLLMClient(
|
||||
config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
|
||||
)
|
||||
# Delegations prefer the vendor-native route; fidelity is cheap
|
||||
# elsewhere (A5). Which route a given model gets is decided by
|
||||
# select_client, not by where the caller happens to sit.
|
||||
passthrough = AnthropicPassthroughClient(config.llm_base_url, config.llm_token)
|
||||
passthrough = AnthropicPassthroughClient(
|
||||
config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
|
||||
)
|
||||
|
||||
transport = MatrixTransport(config.matrix)
|
||||
announce = notify or transport.send_message
|
||||
|
||||
expert = ExpertDesk(normalized, store, config.roles)
|
||||
expert = ExpertDesk(
|
||||
normalized, store, config.roles, budget=tuning.expert.consult_budget
|
||||
)
|
||||
delegate_model = config.roles.resolve("delegate")
|
||||
delegation = Delegation(
|
||||
select_client(delegate_model, normalized, passthrough, catalog),
|
||||
@@ -169,6 +180,11 @@ def build_agent(
|
||||
git_repo,
|
||||
model=delegate_model,
|
||||
backing_model=catalog.facts(delegate_model).backing,
|
||||
caps=Caps(
|
||||
turns=tuning.delegation.max_turns,
|
||||
tokens=tuning.delegation.max_tokens,
|
||||
wall_clock_s=tuning.delegation.wall_clock_s,
|
||||
),
|
||||
cancel=cancel,
|
||||
)
|
||||
executor = LLMExecutor(
|
||||
@@ -180,6 +196,8 @@ def build_agent(
|
||||
delegate=delegation,
|
||||
notify=announce,
|
||||
cancel=cancel,
|
||||
max_turns=tuning.executor.max_turns,
|
||||
failure_limit=tuning.executor.failure_limit,
|
||||
)
|
||||
publisher = MilestonePublisher(
|
||||
store,
|
||||
|
||||
@@ -121,8 +121,8 @@ class ModelCatalog:
|
||||
|
||||
Reported together rather than one per boot: an operator fixing a
|
||||
config wants the whole list, not to rediscover the next typo on the
|
||||
next start (the same reason ``AgentConfig.from_env`` batches its
|
||||
missing variables).
|
||||
next start (the same reason ``AgentConfig`` reports every missing
|
||||
setting at once).
|
||||
"""
|
||||
unknown = {
|
||||
role: alias for role, alias in roles.items() if alias not in self._facts
|
||||
|
||||
@@ -1,49 +1,198 @@
|
||||
"""Agent configuration (spec A5: from env/.env, nothing hardcoded).
|
||||
"""Agent configuration (spec A5: nothing hardcoded).
|
||||
|
||||
Configuration is a YAML file: roles, connections, and the tuning knobs (see
|
||||
docs/Tuning-Guide.md). It holds no secrets. A credential is written as a
|
||||
reference, ``token: ${NISSE_LLM_TOKEN}``, and resolved from the environment,
|
||||
so the file can be reviewed, shared, or committed without leaking. The
|
||||
environment supplies those references and nothing else: every setting has
|
||||
exactly one home.
|
||||
|
||||
Every value is collected and validated **before** anything is built, and a
|
||||
single error names every missing variable at once. The alternative — failing
|
||||
at first use — surfaces a missing token twenty minutes into a milestone, on
|
||||
the call that mattered.
|
||||
single error names every problem at once. The alternative — failing at first
|
||||
use — surfaces a missing token twenty minutes into a milestone, on the call
|
||||
that mattered. Unknown keys are errors too: a misspelt tuning knob that was
|
||||
silently ignored would look exactly like a knob that has no effect.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field, fields
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from nisse_dev.llm import RoleMap
|
||||
import yaml
|
||||
|
||||
from nisse_dev.llm import DEFAULT_MAX_OUTPUT_TOKENS, RoleMap
|
||||
from nisse_dev.matrix import MatrixConfig
|
||||
|
||||
REQUIRED = (
|
||||
"NISSE_STATE_DB",
|
||||
"NISSE_WORKSPACE",
|
||||
"NISSE_PROJECT",
|
||||
"NISSE_LLM_BASE_URL",
|
||||
"NISSE_LLM_TOKEN",
|
||||
"NISSE_ROLE_EXECUTOR",
|
||||
"NISSE_ROLE_PLANNER",
|
||||
"NISSE_ROLE_DELEGATE",
|
||||
"NISSE_ROLE_CONSULTANT",
|
||||
"NISSE_MATRIX_HOMESERVER",
|
||||
"NISSE_MATRIX_USER_ID",
|
||||
"NISSE_MATRIX_TOKEN",
|
||||
"NISSE_MATRIX_ROOM_ID",
|
||||
"NISSE_MATRIX_OPERATORS",
|
||||
"NISSE_GITEA_URL",
|
||||
"NISSE_GITEA_TOKEN",
|
||||
"NISSE_GITEA_REPO",
|
||||
"NISSE_GITEA_OPERATORS",
|
||||
DEFAULT_CONFIG_FILE = "nisse-dev.yaml"
|
||||
|
||||
REQUIRED_KEYS = (
|
||||
"state_db",
|
||||
"workspace",
|
||||
"project",
|
||||
"llm.base_url",
|
||||
"llm.token",
|
||||
"roles.executor",
|
||||
"roles.planner",
|
||||
"roles.delegate",
|
||||
"roles.consultant",
|
||||
"matrix.homeserver",
|
||||
"matrix.user_id",
|
||||
"matrix.token",
|
||||
"matrix.room_id",
|
||||
"matrix.operators",
|
||||
"gitea.url",
|
||||
"gitea.token",
|
||||
"gitea.repo",
|
||||
"gitea.operators",
|
||||
)
|
||||
OPTIONAL_KEYS = ("llm.max_output_tokens", "gitea.base_branch")
|
||||
|
||||
_REFERENCE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
||||
|
||||
|
||||
class ConfigError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _people(raw: str) -> frozenset[str]:
|
||||
"""Operator allowlists are comma-separated; blanks are dropped so a
|
||||
trailing comma cannot admit an empty identity."""
|
||||
return frozenset(part.strip() for part in raw.split(",") if part.strip())
|
||||
# --- tuning ----------------------------------------------------------------
|
||||
# The defaults are the values the agent ran with before they were
|
||||
# configurable, so an absent `tuning:` section changes nothing.
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExecutorTuning:
|
||||
max_turns: int = 24
|
||||
failure_limit: int = 2
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExpertTuning:
|
||||
consult_budget: int = 3
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DelegationTuning:
|
||||
max_turns: int = 30
|
||||
max_tokens: int = 200_000
|
||||
wall_clock_s: float = 900.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToolsTuning:
|
||||
command_timeout_s: int = 120
|
||||
max_output_chars: int = 4000
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Tuning:
|
||||
executor: ExecutorTuning = field(default_factory=ExecutorTuning)
|
||||
expert: ExpertTuning = field(default_factory=ExpertTuning)
|
||||
delegation: DelegationTuning = field(default_factory=DelegationTuning)
|
||||
tools: ToolsTuning = field(default_factory=ToolsTuning)
|
||||
|
||||
|
||||
_TUNING_SECTIONS: dict[str, type] = {
|
||||
f.name: f.default_factory # type: ignore[misc]
|
||||
for f in fields(Tuning)
|
||||
}
|
||||
|
||||
|
||||
def _parse_tuning(raw: Any, errors: list[str]) -> Tuning:
|
||||
if raw is None:
|
||||
return Tuning()
|
||||
if not isinstance(raw, dict):
|
||||
errors.append("tuning: must be a mapping")
|
||||
return Tuning()
|
||||
sections: dict[str, Any] = {}
|
||||
for name, section in raw.items():
|
||||
cls = _TUNING_SECTIONS.get(name)
|
||||
if cls is None:
|
||||
errors.append(f"tuning.{name}: unknown section")
|
||||
continue
|
||||
if not isinstance(section, dict):
|
||||
errors.append(f"tuning.{name}: must be a mapping")
|
||||
continue
|
||||
known = {f.name: f for f in fields(cls)}
|
||||
values: dict[str, Any] = {}
|
||||
for key, value in section.items():
|
||||
spec = known.get(key)
|
||||
if spec is None:
|
||||
errors.append(f"tuning.{name}.{key}: unknown setting")
|
||||
continue
|
||||
number = _positive(value, float if spec.type in (float, "float") else int)
|
||||
if number is None:
|
||||
errors.append(f"tuning.{name}.{key}: must be a positive number")
|
||||
continue
|
||||
values[key] = number
|
||||
sections[name] = cls(**values)
|
||||
return Tuning(**sections)
|
||||
|
||||
|
||||
def _positive(value: Any, kind: type) -> int | float | None:
|
||||
# bool is an int in Python; `true` for a turn limit is a mistake.
|
||||
if isinstance(value, bool) or not isinstance(value, int | float):
|
||||
return None
|
||||
if kind is int and not float(value).is_integer():
|
||||
return None
|
||||
if value <= 0:
|
||||
return None
|
||||
return kind(value)
|
||||
|
||||
|
||||
# --- loading ---------------------------------------------------------------
|
||||
|
||||
|
||||
def _people(raw: Any) -> frozenset[str]:
|
||||
"""Operator allowlists: a YAML list, or comma-separated in the
|
||||
environment. Blanks are dropped so a trailing comma cannot admit an empty
|
||||
identity."""
|
||||
parts = raw if isinstance(raw, list) else str(raw).split(",")
|
||||
return frozenset(str(part).strip() for part in parts if str(part).strip())
|
||||
|
||||
|
||||
def _flatten(tree: Mapping[str, Any], prefix: str = "") -> dict[str, Any]:
|
||||
flat: dict[str, Any] = {}
|
||||
for key, value in tree.items():
|
||||
dotted = f"{prefix}{key}"
|
||||
if isinstance(value, dict) and dotted != "tuning":
|
||||
flat.update(_flatten(value, f"{dotted}."))
|
||||
else:
|
||||
flat[dotted] = value
|
||||
return flat
|
||||
|
||||
|
||||
def _resolve(value: Any, key: str, env: Mapping[str, str], errors: list[str]) -> Any:
|
||||
"""Replace ${VAR} references with the environment's value. A reference to
|
||||
an unset variable is an error, never an empty string: an empty token would
|
||||
pass for a value and fail at the first request."""
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
|
||||
def lookup(match: re.Match[str]) -> str:
|
||||
name = match.group(1)
|
||||
found = (env.get(name) or "").strip()
|
||||
if not found:
|
||||
errors.append(f"{key}: references ${{{name}}}, which is unset or empty")
|
||||
return found
|
||||
|
||||
return _REFERENCE.sub(lookup, value)
|
||||
|
||||
|
||||
def _read_file(path: Path, errors: list[str]) -> dict[str, Any]:
|
||||
try:
|
||||
loaded = yaml.safe_load(path.read_text())
|
||||
except yaml.YAMLError as exc:
|
||||
errors.append(f"{path}: not valid YAML ({exc})")
|
||||
return {}
|
||||
if loaded is None:
|
||||
return {}
|
||||
if not isinstance(loaded, dict):
|
||||
errors.append(f"{path}: must be a mapping at the top level")
|
||||
return {}
|
||||
return loaded
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -61,30 +210,107 @@ class AgentConfig:
|
||||
gitea_repo: str
|
||||
gitea_operators: frozenset[str]
|
||||
base_branch: str = "main"
|
||||
max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS
|
||||
tuning: Tuning = field(default_factory=Tuning)
|
||||
config_file: Path | None = None
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, env: Mapping[str, str] = os.environ) -> "AgentConfig":
|
||||
def load(cls, env: Mapping[str, str] = os.environ) -> "AgentConfig":
|
||||
"""The file named by NISSE_CONFIG, or ./nisse-dev.yaml."""
|
||||
path = Path(env.get("NISSE_CONFIG") or DEFAULT_CONFIG_FILE).expanduser()
|
||||
if not path.is_file():
|
||||
raise ConfigError(
|
||||
f"cannot start: no config file at {path}; create it from "
|
||||
"nisse-dev.example.yaml, or point NISSE_CONFIG at one"
|
||||
)
|
||||
return cls.from_file(path, env)
|
||||
|
||||
@classmethod
|
||||
def from_file(
|
||||
cls, path: Path, env: Mapping[str, str] = os.environ
|
||||
) -> "AgentConfig":
|
||||
errors: list[str] = []
|
||||
tree = _read_file(path, errors)
|
||||
return cls._build(tree, env, errors, config_file=path)
|
||||
|
||||
@classmethod
|
||||
def from_tree(
|
||||
cls, tree: Mapping[str, Any], env: Mapping[str, str] = os.environ
|
||||
) -> "AgentConfig":
|
||||
"""From an already-parsed config, as the file would give it."""
|
||||
return cls._build(tree, env, [], config_file=None)
|
||||
|
||||
@classmethod
|
||||
def _build(
|
||||
cls,
|
||||
tree: Mapping[str, Any],
|
||||
env: Mapping[str, str],
|
||||
errors: list[str],
|
||||
*,
|
||||
config_file: Path | None,
|
||||
) -> "AgentConfig":
|
||||
tuning = _parse_tuning(tree.get("tuning"), errors)
|
||||
flat = _flatten(tree)
|
||||
flat.pop("tuning", None)
|
||||
|
||||
known = set(REQUIRED_KEYS) | set(OPTIONAL_KEYS)
|
||||
for key in sorted(set(flat) - known):
|
||||
errors.append(f"{key}: unknown setting")
|
||||
values = {
|
||||
key: _resolve(value, key, env, errors)
|
||||
for key, value in flat.items()
|
||||
if key in known
|
||||
}
|
||||
|
||||
# Whitespace-only counts as missing: a token of spaces would pass a
|
||||
# truthiness check and fail at the first request instead.
|
||||
missing = [name for name in REQUIRED if not (env.get(name) or "").strip()]
|
||||
missing = [
|
||||
key
|
||||
for key in REQUIRED_KEYS
|
||||
if values.get(key) is None
|
||||
or (isinstance(values[key], str) and not values[key].strip())
|
||||
or (isinstance(values[key], list) and not _people(values[key]))
|
||||
]
|
||||
if missing:
|
||||
raise ConfigError(
|
||||
"cannot start: these are unset or empty — " + ", ".join(missing)
|
||||
)
|
||||
errors.append("unset or empty — " + ", ".join(missing))
|
||||
|
||||
max_output = values.get("llm.max_output_tokens", DEFAULT_MAX_OUTPUT_TOKENS)
|
||||
try:
|
||||
max_output = int(max_output)
|
||||
if max_output <= 0:
|
||||
raise ValueError
|
||||
except (TypeError, ValueError):
|
||||
errors.append("llm.max_output_tokens: must be a positive integer")
|
||||
if errors:
|
||||
raise ConfigError("cannot start: " + "; ".join(errors))
|
||||
|
||||
config = cls(
|
||||
state_db=Path(env["NISSE_STATE_DB"]).expanduser(),
|
||||
workspace=Path(env["NISSE_WORKSPACE"]).expanduser(),
|
||||
project=env["NISSE_PROJECT"],
|
||||
llm_base_url=env["NISSE_LLM_BASE_URL"],
|
||||
llm_token=env["NISSE_LLM_TOKEN"],
|
||||
roles=RoleMap.from_env(env),
|
||||
matrix=MatrixConfig.from_env(env),
|
||||
operators=_people(env["NISSE_MATRIX_OPERATORS"]),
|
||||
gitea_url=env["NISSE_GITEA_URL"],
|
||||
gitea_token=env["NISSE_GITEA_TOKEN"],
|
||||
gitea_repo=env["NISSE_GITEA_REPO"],
|
||||
gitea_operators=_people(env["NISSE_GITEA_OPERATORS"]),
|
||||
base_branch=env.get("NISSE_BASE_BRANCH", "main"),
|
||||
state_db=Path(values["state_db"]).expanduser(),
|
||||
workspace=Path(values["workspace"]).expanduser(),
|
||||
project=str(values["project"]),
|
||||
llm_base_url=str(values["llm.base_url"]),
|
||||
llm_token=str(values["llm.token"]),
|
||||
roles=RoleMap(
|
||||
executor=str(values["roles.executor"]),
|
||||
planner=str(values["roles.planner"]),
|
||||
delegate=str(values["roles.delegate"]),
|
||||
consultant=str(values["roles.consultant"]),
|
||||
),
|
||||
matrix=MatrixConfig(
|
||||
homeserver=str(values["matrix.homeserver"]).rstrip("/"),
|
||||
user_id=str(values["matrix.user_id"]),
|
||||
token=str(values["matrix.token"]),
|
||||
room_id=str(values["matrix.room_id"]),
|
||||
),
|
||||
operators=_people(values["matrix.operators"]),
|
||||
gitea_url=str(values["gitea.url"]),
|
||||
gitea_token=str(values["gitea.token"]),
|
||||
gitea_repo=str(values["gitea.repo"]),
|
||||
gitea_operators=_people(values["gitea.operators"]),
|
||||
base_branch=str(values.get("gitea.base_branch") or "main"),
|
||||
max_output_tokens=max_output,
|
||||
tuning=tuning,
|
||||
config_file=config_file,
|
||||
)
|
||||
config._refuse_secrets_inside_the_clone(env)
|
||||
return config
|
||||
@@ -93,14 +319,20 @@ class AgentConfig:
|
||||
"""The workspace is a git clone the agent commits with ``git add -A``
|
||||
and pushes. Anything inside it reaches the project repository, so a
|
||||
state store or an env file placed there would publish credentials as a
|
||||
side effect of the agent working correctly — silently, and once.
|
||||
side effect of the agent working correctly — silently, and once. The
|
||||
config file holds no secrets itself, but it names where they come
|
||||
from and how the agent is deployed, which is not the project's
|
||||
business either.
|
||||
"""
|
||||
workspace = self.workspace.expanduser().resolve()
|
||||
offenders = []
|
||||
for label, path in (
|
||||
("NISSE_STATE_DB", self.state_db),
|
||||
candidates = [
|
||||
("state_db", self.state_db),
|
||||
("NISSE_ENV", Path(env.get("NISSE_ENV", ".env"))),
|
||||
):
|
||||
]
|
||||
if self.config_file is not None:
|
||||
candidates.append(("NISSE_CONFIG", self.config_file))
|
||||
offenders = []
|
||||
for label, path in candidates:
|
||||
resolved = path.expanduser().resolve()
|
||||
if resolved == workspace or workspace in resolved.parents:
|
||||
offenders.append(f"{label} ({resolved})")
|
||||
|
||||
@@ -10,8 +10,6 @@ idempotent across a crash: boot reconciliation (A3) asks Gitea whether the
|
||||
effect already happened rather than repeating it.
|
||||
"""
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
@@ -60,14 +58,6 @@ class GiteaClient:
|
||||
transport=transport,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, env: Mapping[str, str] = os.environ) -> "GiteaClient":
|
||||
base_url = env.get("NISSE_GITEA_URL")
|
||||
token = env.get("NISSE_GITEA_TOKEN")
|
||||
if not base_url or not token:
|
||||
raise GiteaError("set NISSE_GITEA_URL and NISSE_GITEA_TOKEN")
|
||||
return cls(base_url, token)
|
||||
|
||||
def _request(self, method: str, path: str, **kwargs: Any) -> Any:
|
||||
try:
|
||||
response = self._http.request(method, path, **kwargs)
|
||||
|
||||
@@ -2,18 +2,16 @@
|
||||
|
||||
One client interface, present from the first real LLM call. All traffic goes
|
||||
to the LiteLLM proxy — the VM's single LLM destination; the agent holds one
|
||||
LiteLLM token and no vendor credentials. Config comes from the environment
|
||||
(``NISSE_LLM_BASE_URL``, ``NISSE_LLM_TOKEN``), nothing hardcoded.
|
||||
LiteLLM token and no vendor credentials. The base URL and token come from the
|
||||
agent's config (nisse_dev.config), nothing hardcoded.
|
||||
|
||||
Model identity is expressed as roles (A11), resolved to concrete model strings
|
||||
from env config; the resolved mapping is pinned at task start so a mid-task
|
||||
from config; the resolved mapping is pinned at task start so a mid-task
|
||||
config change cannot split a task across two planners.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any, ClassVar, Protocol
|
||||
|
||||
@@ -49,11 +47,6 @@ RATE_LIMIT_RETRY_S = 20.0
|
||||
DEFAULT_MAX_OUTPUT_TOKENS = 16384
|
||||
|
||||
|
||||
def max_output_tokens(env: Mapping[str, str] = os.environ) -> int:
|
||||
raw = env.get("NISSE_MAX_OUTPUT_TOKENS")
|
||||
return int(raw) if raw else DEFAULT_MAX_OUTPUT_TOKENS
|
||||
|
||||
|
||||
class UpstreamUnavailable(LLMError):
|
||||
"""The proxy or provider could not be reached, or refused for reasons
|
||||
unrelated to what was asked: a network blip, a timeout, a 5xx, a rate
|
||||
@@ -113,22 +106,10 @@ class RoleMap:
|
||||
delegate: str
|
||||
consultant: str
|
||||
|
||||
_ENV_VARS: ClassVar[dict[str, str]] = {
|
||||
"executor": "NISSE_ROLE_EXECUTOR",
|
||||
"planner": "NISSE_ROLE_PLANNER",
|
||||
"delegate": "NISSE_ROLE_DELEGATE",
|
||||
"consultant": "NISSE_ROLE_CONSULTANT",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, env: Mapping[str, str] = os.environ) -> "RoleMap":
|
||||
missing = [var for var in cls._ENV_VARS.values() if not env.get(var)]
|
||||
if missing:
|
||||
raise LLMError(f"role mapping incomplete; set: {', '.join(missing)}")
|
||||
return cls(**{role: env[var] for role, var in cls._ENV_VARS.items()})
|
||||
ROLES: ClassVar[tuple[str, ...]] = ("executor", "planner", "delegate", "consultant")
|
||||
|
||||
def resolve(self, role: str) -> str:
|
||||
if role not in self._ENV_VARS:
|
||||
if role not in self.ROLES:
|
||||
raise LLMError(f"unknown role: {role!r}")
|
||||
return str(getattr(self, role))
|
||||
|
||||
@@ -156,20 +137,12 @@ class LiteLLMClient:
|
||||
)
|
||||
self._renderer = OpenAIChatRenderer()
|
||||
self._sleep = sleep
|
||||
self._max_tokens = max_tokens or max_output_tokens()
|
||||
self._max_tokens = max_tokens or DEFAULT_MAX_OUTPUT_TOKENS
|
||||
# Set after each call so a caller can meter itself (A13 caps);
|
||||
# a side channel rather than a return value, so the LLMClient
|
||||
# protocol stays the same for every implementation.
|
||||
self.last_usage: dict[str, int] = {}
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, env: Mapping[str, str] = os.environ) -> "LiteLLMClient":
|
||||
base_url = env.get("NISSE_LLM_BASE_URL")
|
||||
token = env.get("NISSE_LLM_TOKEN")
|
||||
if not base_url or not token:
|
||||
raise LLMError("set NISSE_LLM_BASE_URL and NISSE_LLM_TOKEN")
|
||||
return cls(base_url, token)
|
||||
|
||||
def complete(
|
||||
self,
|
||||
events: list[StoredEvent],
|
||||
@@ -203,7 +176,7 @@ class LiteLLMClient:
|
||||
if choice.get("finish_reason") == "length":
|
||||
raise TruncatedResponse(
|
||||
f"{requested_model} hit its output limit mid-answer; "
|
||||
"raise NISSE_MAX_OUTPUT_TOKENS"
|
||||
"raise llm.max_output_tokens"
|
||||
)
|
||||
produced_by = data.get("model") or requested_model
|
||||
usage = data.get("usage") or {}
|
||||
@@ -275,20 +248,10 @@ class AnthropicPassthroughClient:
|
||||
transport=transport,
|
||||
)
|
||||
self._renderer = AnthropicMessagesRenderer()
|
||||
self._max_tokens = max_tokens or max_output_tokens()
|
||||
self._max_tokens = max_tokens or DEFAULT_MAX_OUTPUT_TOKENS
|
||||
self._sleep = sleep
|
||||
self.last_usage: dict[str, int] = {}
|
||||
|
||||
@classmethod
|
||||
def from_env(
|
||||
cls, env: Mapping[str, str] = os.environ
|
||||
) -> "AnthropicPassthroughClient":
|
||||
base_url = env.get("NISSE_LLM_BASE_URL")
|
||||
token = env.get("NISSE_LLM_TOKEN")
|
||||
if not base_url or not token:
|
||||
raise LLMError("set NISSE_LLM_BASE_URL and NISSE_LLM_TOKEN")
|
||||
return cls(base_url, token)
|
||||
|
||||
def complete(
|
||||
self,
|
||||
events: list[StoredEvent],
|
||||
@@ -315,7 +278,7 @@ class AnthropicPassthroughClient:
|
||||
if data.get("stop_reason") == "max_tokens":
|
||||
raise TruncatedResponse(
|
||||
f"{requested_model} hit the output limit of {self._max_tokens} "
|
||||
f"tokens mid-answer; raise NISSE_MAX_OUTPUT_TOKENS"
|
||||
f"tokens mid-answer; raise llm.max_output_tokens"
|
||||
)
|
||||
content = data.get("content")
|
||||
if not isinstance(content, list):
|
||||
|
||||
@@ -20,7 +20,7 @@ from nisse_dev.envfile import load_env_file
|
||||
def main() -> int:
|
||||
load_env_file(Path(os.environ.get("NISSE_ENV", ".env")).expanduser())
|
||||
try:
|
||||
config = AgentConfig.from_env()
|
||||
config = AgentConfig.load()
|
||||
except ConfigError as exc:
|
||||
# Fail here rather than twenty minutes into a milestone on the one
|
||||
# call that needed the missing value.
|
||||
|
||||
@@ -11,12 +11,10 @@ loop backs off (capped) and retries indefinitely — an outage delays reactions
|
||||
but never fails anything.
|
||||
"""
|
||||
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import ClassVar
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -34,29 +32,6 @@ class MatrixConfig:
|
||||
token: str
|
||||
room_id: str
|
||||
|
||||
_ENV: ClassVar[tuple[str, ...]] = (
|
||||
"NISSE_MATRIX_HOMESERVER",
|
||||
"NISSE_MATRIX_USER_ID",
|
||||
"NISSE_MATRIX_TOKEN",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_env(
|
||||
cls,
|
||||
env: Mapping[str, str] = os.environ,
|
||||
room_var: str = "NISSE_MATRIX_ROOM_ID",
|
||||
) -> "MatrixConfig":
|
||||
needed = (*cls._ENV, room_var)
|
||||
missing = [var for var in needed if not env.get(var)]
|
||||
if missing:
|
||||
raise TransportError(f"matrix config incomplete; set: {', '.join(missing)}")
|
||||
return cls(
|
||||
homeserver=env["NISSE_MATRIX_HOMESERVER"].rstrip("/"),
|
||||
user_id=env["NISSE_MATRIX_USER_ID"],
|
||||
token=env["NISSE_MATRIX_TOKEN"],
|
||||
room_id=env[room_var],
|
||||
)
|
||||
|
||||
|
||||
class MatrixTransport:
|
||||
def __init__(
|
||||
|
||||
@@ -86,13 +86,6 @@ EXECUTOR_TOOLS: list[dict[str, Any]] = [
|
||||
]
|
||||
|
||||
|
||||
def _truncate(text: str) -> str:
|
||||
if len(text) <= MAX_OUTPUT_CHARS:
|
||||
return text
|
||||
dropped = len(text) - MAX_OUTPUT_CHARS
|
||||
return f"{text[:MAX_OUTPUT_CHARS]}\n[... {dropped} characters truncated]"
|
||||
|
||||
|
||||
def _child_env() -> dict[str, str]:
|
||||
"""The project's build and test code is arbitrary; it has no business
|
||||
reading the agent's LiteLLM, Matrix, or Gitea credentials."""
|
||||
@@ -106,10 +99,24 @@ class ToolOutcome:
|
||||
|
||||
|
||||
class Workspace:
|
||||
def __init__(self, root: Path, *, timeout_s: int = DEFAULT_TIMEOUT_S) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
root: Path,
|
||||
*,
|
||||
timeout_s: int = DEFAULT_TIMEOUT_S,
|
||||
max_output_chars: int = MAX_OUTPUT_CHARS,
|
||||
) -> None:
|
||||
self.root = Path(root)
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
self._timeout_s = timeout_s
|
||||
self._max_output_chars = max_output_chars
|
||||
|
||||
def _truncate(self, text: str) -> str:
|
||||
limit = self._max_output_chars
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
dropped = len(text) - limit
|
||||
return f"{text[:limit]}\n[... {dropped} characters truncated]"
|
||||
|
||||
def _resolve(self, path: str) -> Path:
|
||||
candidate = Path(path)
|
||||
@@ -142,7 +149,7 @@ class Workspace:
|
||||
def read_file(self, path: str) -> ToolOutcome:
|
||||
target = self._resolve(path)
|
||||
try:
|
||||
return ToolOutcome(_truncate(target.read_text()), success=True)
|
||||
return ToolOutcome(self._truncate(target.read_text()), success=True)
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
return ToolOutcome(f"could not read {path}: {exc}", success=False)
|
||||
|
||||
@@ -175,7 +182,9 @@ class Workspace:
|
||||
)
|
||||
except OSError as exc:
|
||||
return ToolOutcome(f"could not list {path}: {exc}", success=False)
|
||||
return ToolOutcome(_truncate("\n".join(entries) or "(empty)"), success=True)
|
||||
return ToolOutcome(
|
||||
self._truncate("\n".join(entries) or "(empty)"), success=True
|
||||
)
|
||||
|
||||
def describe(self) -> str:
|
||||
"""Told to the model, so it does not have to guess where it is."""
|
||||
@@ -212,7 +221,7 @@ class Workspace:
|
||||
f"command timed out after {self._timeout_s}s: {command}",
|
||||
success=False,
|
||||
)
|
||||
body = _truncate(
|
||||
body = self._truncate(
|
||||
f"exit code: {process.returncode}\nstdout:\n{stdout}\nstderr:\n{stderr}"
|
||||
)
|
||||
return ToolOutcome(body, success=process.returncode == 0)
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from test_config import COMPLETE
|
||||
from test_config import SECRETS, tree
|
||||
|
||||
from nisse_dev.agent import build_agent, discover_models, prepare_workspace
|
||||
from nisse_dev.catalog import ModelCatalog, UnknownModel
|
||||
@@ -34,14 +34,15 @@ TEST_CATALOG = ModelCatalog.from_payload(
|
||||
)
|
||||
|
||||
|
||||
def config_for(tmp_path: Path, **overrides: str) -> AgentConfig:
|
||||
env = {
|
||||
**COMPLETE,
|
||||
"NISSE_STATE_DB": str(tmp_path / "state.db"),
|
||||
"NISSE_WORKSPACE": str(tmp_path / "work"),
|
||||
**overrides,
|
||||
}
|
||||
return AgentConfig.from_env(env)
|
||||
def config_for(tmp_path: Path, **changes: str) -> AgentConfig:
|
||||
return AgentConfig.from_tree(
|
||||
tree(
|
||||
state_db=str(tmp_path / "state.db"),
|
||||
workspace=str(tmp_path / "work"),
|
||||
**changes,
|
||||
),
|
||||
SECRETS,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -59,7 +60,7 @@ def origin(tmp_path: Path) -> Path:
|
||||
|
||||
|
||||
def test_workspace_is_cloned_when_absent(tmp_path: Path, origin: Path) -> None:
|
||||
config = config_for(tmp_path, NISSE_GITEA_URL=f"file://{origin}")
|
||||
config = config_for(tmp_path, gitea__url=f"file://{origin}")
|
||||
# authenticated_url would mangle a file:// remote, so clone directly.
|
||||
GitRepo(tmp_path).clone(str(origin), config.workspace)
|
||||
repo = prepare_workspace(config, "nisse-dev-testing")
|
||||
@@ -206,7 +207,7 @@ def test_httpx_clients_are_not_shared_between_routes(
|
||||
def test_a_typod_role_refuses_to_boot(tmp_path: Path, monkeypatch) -> None:
|
||||
# Misconfiguration should cost a second at startup, not be discovered ten
|
||||
# minutes into a milestone.
|
||||
config = config_for(tmp_path, NISSE_ROLE_PLANNER="clause-opus-5")
|
||||
config = config_for(tmp_path, roles__planner="clause-opus-5")
|
||||
monkeypatch.setattr(
|
||||
"nisse_dev.agent.ModelCatalog.discover",
|
||||
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
||||
@@ -220,7 +221,7 @@ def test_a_role_this_key_cannot_reach_refuses_to_boot(
|
||||
) -> None:
|
||||
# /model/info is scoped per key, so the same check catches a role pointed
|
||||
# at a real model that this token has no access to.
|
||||
config = config_for(tmp_path, NISSE_ROLE_EXECUTOR="local-fast")
|
||||
config = config_for(tmp_path, roles__executor="local-fast")
|
||||
monkeypatch.setattr(
|
||||
"nisse_dev.agent.ModelCatalog.discover",
|
||||
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
||||
@@ -286,3 +287,57 @@ def test_delegation_records_what_the_alias_resolved_to(
|
||||
agent = build_offline(tmp_path, origin, monkeypatch)
|
||||
assert agent.orchestrator._delegate._backing_model == "anthropic/claude-opus-5"
|
||||
agent.stop()
|
||||
|
||||
|
||||
def test_tuning_reaches_every_component(
|
||||
tmp_path: Path, origin: Path, monkeypatch
|
||||
) -> None:
|
||||
from dataclasses import replace
|
||||
|
||||
from nisse_dev.config import (
|
||||
DelegationTuning,
|
||||
ExecutorTuning,
|
||||
ExpertTuning,
|
||||
ToolsTuning,
|
||||
Tuning,
|
||||
)
|
||||
|
||||
config = replace(
|
||||
config_for(tmp_path),
|
||||
max_output_tokens=8192,
|
||||
tuning=Tuning(
|
||||
executor=ExecutorTuning(max_turns=40, failure_limit=3),
|
||||
expert=ExpertTuning(consult_budget=5),
|
||||
delegation=DelegationTuning(
|
||||
max_turns=12, max_tokens=50_000, wall_clock_s=60
|
||||
),
|
||||
tools=ToolsTuning(command_timeout_s=300, max_output_chars=2000),
|
||||
),
|
||||
)
|
||||
GitRepo(tmp_path).clone(str(origin), config.workspace)
|
||||
monkeypatch.setattr(
|
||||
"nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"nisse_dev.agent.ModelCatalog.discover",
|
||||
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
||||
)
|
||||
monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport())
|
||||
monkeypatch.setattr(
|
||||
"nisse_dev.agent.authenticated_url",
|
||||
lambda base, repo, user, token: str(origin),
|
||||
)
|
||||
agent = build_agent(config)
|
||||
executor = agent.orchestrator._executor
|
||||
delegation = agent.orchestrator._delegate
|
||||
|
||||
assert (executor._max_turns, executor._failure_limit) == (40, 3)
|
||||
assert executor._expert._budget == 5
|
||||
assert delegation._caps.turns == 12
|
||||
assert delegation._caps.tokens == 50_000
|
||||
assert delegation._caps.wall_clock_s == 60
|
||||
assert executor._workspace._timeout_s == 300
|
||||
assert executor._workspace._max_output_chars == 2000
|
||||
assert executor._client._max_tokens == 8192
|
||||
assert delegation._client._max_tokens == 8192
|
||||
agent.stop()
|
||||
+235
-68
@@ -1,93 +1,266 @@
|
||||
import copy
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from nisse_dev.config import AgentConfig, ConfigError
|
||||
|
||||
COMPLETE = {
|
||||
"NISSE_STATE_DB": "~/demo/state.db",
|
||||
"NISSE_WORKSPACE": "~/demo/work",
|
||||
"NISSE_PROJECT": "greet",
|
||||
"NISSE_LLM_BASE_URL": "https://litellm.test",
|
||||
"NISSE_LLM_TOKEN": "sk-x",
|
||||
"NISSE_ROLE_EXECUTOR": "claude-haiku-4-5",
|
||||
"NISSE_ROLE_PLANNER": "claude-opus-5",
|
||||
"NISSE_ROLE_DELEGATE": "claude-opus-5",
|
||||
"NISSE_ROLE_CONSULTANT": "claude-haiku-4-5",
|
||||
"NISSE_MATRIX_HOMESERVER": "https://matrix.test",
|
||||
"NISSE_MATRIX_USER_ID": "@nisse-dev-testing:test",
|
||||
"NISSE_MATRIX_TOKEN": "syt_x",
|
||||
"NISSE_MATRIX_ROOM_ID": "!room:test",
|
||||
"NISSE_MATRIX_OPERATORS": "@don:test",
|
||||
"NISSE_GITEA_URL": "https://gitea.test",
|
||||
"NISSE_GITEA_TOKEN": "gt",
|
||||
"NISSE_GITEA_REPO": "don/demo",
|
||||
"NISSE_GITEA_OPERATORS": "don",
|
||||
# A complete config as the YAML file would give it, with secrets by reference.
|
||||
COMPLETE: dict[str, Any] = {
|
||||
"state_db": "~/demo/state.db",
|
||||
"workspace": "~/demo/work",
|
||||
"project": "greet",
|
||||
"llm": {"base_url": "https://litellm.test", "token": "${TEST_LLM_TOKEN}"},
|
||||
"roles": {
|
||||
"executor": "claude-haiku-4-5",
|
||||
"planner": "claude-opus-5",
|
||||
"delegate": "claude-opus-5",
|
||||
"consultant": "claude-haiku-4-5",
|
||||
},
|
||||
"matrix": {
|
||||
"homeserver": "https://matrix.test/",
|
||||
"user_id": "@nisse-dev-testing:test",
|
||||
"token": "${TEST_MATRIX_TOKEN}",
|
||||
"room_id": "!room:test",
|
||||
"operators": ["@don:test"],
|
||||
},
|
||||
"gitea": {
|
||||
"url": "https://gitea.test",
|
||||
"token": "${TEST_GITEA_TOKEN}",
|
||||
"repo": "don/demo",
|
||||
"operators": ["don"],
|
||||
},
|
||||
}
|
||||
SECRETS = {
|
||||
"TEST_LLM_TOKEN": "sk-x",
|
||||
"TEST_MATRIX_TOKEN": "syt_x",
|
||||
"TEST_GITEA_TOKEN": "gt",
|
||||
}
|
||||
|
||||
|
||||
def test_complete_environment_builds() -> None:
|
||||
config = AgentConfig.from_env(COMPLETE)
|
||||
def tree(**changes: Any) -> dict[str, Any]:
|
||||
"""COMPLETE with dotted keys changed; a value of None removes the key."""
|
||||
result = copy.deepcopy(COMPLETE)
|
||||
for dotted, value in changes.items():
|
||||
*parents, leaf = dotted.split("__")
|
||||
node = result
|
||||
for part in parents:
|
||||
node = node.setdefault(part, {})
|
||||
if value is None:
|
||||
node.pop(leaf, None)
|
||||
else:
|
||||
node[leaf] = value
|
||||
return result
|
||||
|
||||
|
||||
def build(**changes: Any) -> AgentConfig:
|
||||
return AgentConfig.from_tree(tree(**changes), SECRETS)
|
||||
|
||||
|
||||
def test_complete_config_builds_with_secrets_by_reference() -> None:
|
||||
config = build()
|
||||
assert config.project == "greet"
|
||||
assert config.llm_token == "sk-x"
|
||||
assert config.gitea_token == "gt"
|
||||
assert config.roles.planner == "claude-opus-5"
|
||||
assert config.matrix.room_id == "!room:test"
|
||||
assert config.matrix.homeserver == "https://matrix.test"
|
||||
assert config.gitea_repo == "don/demo"
|
||||
assert config.base_branch == "main"
|
||||
assert config.tuning.executor.max_turns == 24 # absent section: defaults
|
||||
|
||||
|
||||
def test_paths_expand_the_home_directory() -> None:
|
||||
config = AgentConfig.from_env(COMPLETE)
|
||||
config = build()
|
||||
assert config.workspace.is_absolute()
|
||||
assert "~" not in str(config.state_db)
|
||||
|
||||
|
||||
def test_every_missing_variable_is_reported_at_once() -> None:
|
||||
def test_every_missing_setting_is_reported_at_once() -> None:
|
||||
# Failing at first use would surface a missing token twenty minutes into
|
||||
# a milestone, on the call that mattered.
|
||||
partial = {k: v for k, v in COMPLETE.items() if k != "NISSE_MATRIX_TOKEN"}
|
||||
partial.pop("NISSE_GITEA_REPO")
|
||||
with pytest.raises(ConfigError) as excinfo:
|
||||
AgentConfig.from_env(partial)
|
||||
build(matrix__token=None, gitea__repo=None)
|
||||
message = str(excinfo.value)
|
||||
assert "NISSE_MATRIX_TOKEN" in message
|
||||
assert "NISSE_GITEA_REPO" in message
|
||||
assert "matrix.token" in message
|
||||
assert "gitea.repo" in message
|
||||
|
||||
|
||||
def test_empty_is_treated_as_missing() -> None:
|
||||
with pytest.raises(ConfigError, match="NISSE_GITEA_TOKEN"):
|
||||
AgentConfig.from_env({**COMPLETE, "NISSE_GITEA_TOKEN": " "})
|
||||
with pytest.raises(ConfigError, match="gitea.repo"):
|
||||
build(gitea__repo=" ")
|
||||
|
||||
|
||||
def test_operator_lists_are_split_and_trimmed() -> None:
|
||||
config = AgentConfig.from_env(
|
||||
{
|
||||
**COMPLETE,
|
||||
"NISSE_MATRIX_OPERATORS": " @don:test , @other:test ,",
|
||||
"NISSE_GITEA_OPERATORS": "don,",
|
||||
}
|
||||
)
|
||||
def test_a_reference_to_an_unset_variable_is_an_error() -> None:
|
||||
# Never an empty string: an empty token would pass for a value and fail
|
||||
# at the first request.
|
||||
env = {k: v for k, v in SECRETS.items() if k != "TEST_GITEA_TOKEN"}
|
||||
with pytest.raises(ConfigError, match=r"gitea.token: references \$\{TEST_GITEA"):
|
||||
AgentConfig.from_tree(tree(), env)
|
||||
|
||||
|
||||
def test_operator_lists_drop_blanks() -> None:
|
||||
config = build(matrix__operators=[" @don:test ", "@other:test", " "])
|
||||
assert config.operators == {"@don:test", "@other:test"}
|
||||
# A trailing comma must not admit an empty identity into an allowlist.
|
||||
# A blank entry must not admit an empty identity into an allowlist.
|
||||
assert "" not in config.operators
|
||||
assert config.gitea_operators == {"don"}
|
||||
|
||||
|
||||
def test_base_branch_is_overridable() -> None:
|
||||
config = AgentConfig.from_env({**COMPLETE, "NISSE_BASE_BRANCH": "trunk"})
|
||||
assert config.base_branch == "trunk"
|
||||
def test_an_operator_list_of_only_blanks_is_missing() -> None:
|
||||
with pytest.raises(ConfigError, match="gitea.operators"):
|
||||
build(gitea__operators=[" "])
|
||||
|
||||
|
||||
def test_base_branch_is_configurable() -> None:
|
||||
assert build(gitea__base_branch="trunk").base_branch == "trunk"
|
||||
|
||||
|
||||
def test_max_output_tokens_is_configurable_and_checked() -> None:
|
||||
assert build(llm__max_output_tokens=8192).max_output_tokens == 8192
|
||||
with pytest.raises(ConfigError, match="llm.max_output_tokens"):
|
||||
build(llm__max_output_tokens=0)
|
||||
|
||||
|
||||
def test_redacted_summary_carries_no_secrets() -> None:
|
||||
summary = AgentConfig.from_env(COMPLETE).redacted()
|
||||
summary = build().redacted()
|
||||
flat = " ".join(summary.values())
|
||||
for secret in ("sk-x", "syt_x", "gt"):
|
||||
for secret in SECRETS.values():
|
||||
assert secret not in flat
|
||||
assert summary["project"] == "greet"
|
||||
|
||||
|
||||
def test_state_db_and_workspace_are_separate(tmp_path: Path) -> None:
|
||||
def test_tuning_is_read() -> None:
|
||||
config = build(
|
||||
tuning={
|
||||
"executor": {"max_turns": 40, "failure_limit": 3},
|
||||
"expert": {"consult_budget": 5},
|
||||
"delegation": {"max_turns": 20, "max_tokens": 100000, "wall_clock_s": 600},
|
||||
"tools": {"command_timeout_s": 300, "max_output_chars": 2000},
|
||||
}
|
||||
)
|
||||
tuning = config.tuning
|
||||
assert (tuning.executor.max_turns, tuning.executor.failure_limit) == (40, 3)
|
||||
assert tuning.expert.consult_budget == 5
|
||||
assert tuning.delegation.max_tokens == 100_000
|
||||
assert tuning.delegation.wall_clock_s == 600.0
|
||||
assert tuning.tools.max_output_chars == 2000
|
||||
|
||||
|
||||
def test_a_partly_given_tuning_section_keeps_the_other_defaults() -> None:
|
||||
tuning = build(tuning={"executor": {"max_turns": 40}}).tuning
|
||||
assert tuning.executor.failure_limit == 2
|
||||
|
||||
|
||||
def test_every_problem_is_reported_at_once() -> None:
|
||||
# A misspelt knob that was silently ignored would look exactly like a knob
|
||||
# with no effect, so unknown keys are errors.
|
||||
bad = tree(project=None, llm_extra=1)
|
||||
bad["tuning"] = {
|
||||
"executor": {"max_turn": 40, "failure_limit": 0},
|
||||
"expert": {"consult_budget": True},
|
||||
"bogus": {"x": 1},
|
||||
}
|
||||
with pytest.raises(ConfigError) as excinfo:
|
||||
AgentConfig.from_tree(bad, SECRETS)
|
||||
message = str(excinfo.value)
|
||||
for expected in (
|
||||
"project",
|
||||
"llm_extra: unknown setting",
|
||||
"tuning.executor.max_turn: unknown setting",
|
||||
"tuning.executor.failure_limit: must be a positive number",
|
||||
"tuning.expert.consult_budget: must be a positive number",
|
||||
"tuning.bogus: unknown section",
|
||||
):
|
||||
assert expected in message
|
||||
|
||||
|
||||
def test_a_whole_number_is_required_where_one_is_meant() -> None:
|
||||
with pytest.raises(ConfigError, match="tuning.executor.max_turns"):
|
||||
build(tuning={"executor": {"max_turns": 2.5}})
|
||||
|
||||
|
||||
# --- the file --------------------------------------------------------------
|
||||
|
||||
FILE = """
|
||||
state_db: ~/demo/state.db
|
||||
workspace: ~/demo/work
|
||||
project: greet
|
||||
llm: {base_url: "https://litellm.test", token: "${TEST_LLM_TOKEN}"}
|
||||
roles:
|
||||
executor: local-fast
|
||||
planner: claude-opus-5
|
||||
delegate: claude-opus-5
|
||||
consultant: claude-haiku-4-5
|
||||
matrix:
|
||||
homeserver: https://matrix.test
|
||||
user_id: "@nisse-dev-testing:test"
|
||||
token: ${TEST_MATRIX_TOKEN}
|
||||
room_id: "!room:test"
|
||||
operators: ["@don:test"]
|
||||
gitea:
|
||||
url: https://gitea.test
|
||||
token: ${TEST_GITEA_TOKEN}
|
||||
repo: don/demo
|
||||
operators: [don]
|
||||
"""
|
||||
|
||||
|
||||
def write(directory: Path, text: str = FILE) -> Path:
|
||||
path = directory / "nisse-dev.yaml"
|
||||
path.write_text(text)
|
||||
return path
|
||||
|
||||
|
||||
def test_a_file_is_read(tmp_path: Path) -> None:
|
||||
config = AgentConfig.from_file(write(tmp_path), SECRETS)
|
||||
assert config.roles.executor == "local-fast"
|
||||
assert config.config_file == tmp_path / "nisse-dev.yaml"
|
||||
|
||||
|
||||
def test_invalid_yaml_is_reported(tmp_path: Path) -> None:
|
||||
with pytest.raises(ConfigError, match="not valid YAML"):
|
||||
AgentConfig.from_file(write(tmp_path, "roles: [unclosed"), SECRETS)
|
||||
|
||||
|
||||
def test_load_uses_nisse_config(tmp_path: Path) -> None:
|
||||
path = write(tmp_path)
|
||||
assert AgentConfig.load({**SECRETS, "NISSE_CONFIG": str(path)}).project == "greet"
|
||||
|
||||
|
||||
def test_load_defaults_to_the_working_directory(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
write(tmp_path)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
assert AgentConfig.load(SECRETS).project == "greet"
|
||||
|
||||
|
||||
def test_load_without_a_file_says_how_to_make_one(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
with pytest.raises(ConfigError, match="nisse-dev.example.yaml"):
|
||||
AgentConfig.load(SECRETS)
|
||||
|
||||
|
||||
def test_the_example_file_is_valid() -> None:
|
||||
# The example documents every knob; it must not drift out of loading.
|
||||
example = Path(__file__).resolve().parents[2] / "nisse-dev.example.yaml"
|
||||
env = {
|
||||
"NISSE_LLM_TOKEN": "sk-x",
|
||||
"NISSE_MATRIX_TOKEN": "syt_x",
|
||||
"NISSE_GITEA_TOKEN": "gt",
|
||||
}
|
||||
assert AgentConfig.from_file(example, env).tuning.executor.max_turns == 24
|
||||
|
||||
|
||||
# --- nothing that carries secrets may sit inside the clone -----------------
|
||||
|
||||
|
||||
def test_state_db_and_workspace_are_separate() -> None:
|
||||
# A2: build artefacts must not share a volume with agent state.
|
||||
config = AgentConfig.from_env(COMPLETE)
|
||||
config = build()
|
||||
assert config.state_db.parent != config.workspace
|
||||
|
||||
|
||||
@@ -96,13 +269,7 @@ def test_state_db_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
||||
# own state alongside the project.
|
||||
work = tmp_path / "project"
|
||||
with pytest.raises(ConfigError, match="inside the workspace"):
|
||||
AgentConfig.from_env(
|
||||
{
|
||||
**COMPLETE,
|
||||
"NISSE_WORKSPACE": str(work),
|
||||
"NISSE_STATE_DB": str(work / "state.db"),
|
||||
}
|
||||
)
|
||||
build(workspace=str(work), state_db=str(work / "state.db"))
|
||||
|
||||
|
||||
def test_env_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
||||
@@ -110,23 +277,23 @@ def test_env_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
||||
# as a side effect of working correctly.
|
||||
work = tmp_path / "project"
|
||||
with pytest.raises(ConfigError, match="NISSE_ENV"):
|
||||
AgentConfig.from_env(
|
||||
{
|
||||
**COMPLETE,
|
||||
"NISSE_WORKSPACE": str(work),
|
||||
"NISSE_STATE_DB": str(tmp_path / "state.db"),
|
||||
"NISSE_ENV": str(work / ".env"),
|
||||
}
|
||||
AgentConfig.from_tree(
|
||||
tree(workspace=str(work), state_db=str(tmp_path / "state.db")),
|
||||
{**SECRETS, "NISSE_ENV": str(work / ".env")},
|
||||
)
|
||||
|
||||
|
||||
def test_a_config_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
||||
work = tmp_path / "project"
|
||||
work.mkdir()
|
||||
text = FILE.replace("workspace: ~/demo/work", f"workspace: {work}")
|
||||
with pytest.raises(ConfigError, match="NISSE_CONFIG"):
|
||||
AgentConfig.from_file(write(work, text), SECRETS)
|
||||
|
||||
|
||||
def test_sibling_paths_are_fine(tmp_path: Path) -> None:
|
||||
config = AgentConfig.from_env(
|
||||
{
|
||||
**COMPLETE,
|
||||
"NISSE_WORKSPACE": str(tmp_path / "project"),
|
||||
"NISSE_STATE_DB": str(tmp_path / "state.db"),
|
||||
"NISSE_ENV": str(tmp_path / ".env"),
|
||||
}
|
||||
config = AgentConfig.from_tree(
|
||||
tree(workspace=str(tmp_path / "project"), state_db=str(tmp_path / "state.db")),
|
||||
{**SECRETS, "NISSE_ENV": str(tmp_path / ".env")},
|
||||
)
|
||||
assert config.workspace.name == "project"
|
||||
@@ -2,25 +2,23 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nisse_dev.config import REQUIRED
|
||||
from nisse_dev.main import main
|
||||
|
||||
|
||||
def test_main_refuses_to_start_without_configuration(
|
||||
def test_main_refuses_to_start_without_a_config_file(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
# Clear the real variables rather than swapping os.environ: from_env binds
|
||||
# the environment object as a default argument, so a replacement would be
|
||||
# ignored. And point NISSE_ENV at a file that does not exist, because the
|
||||
# working directory's .env holds live credentials.
|
||||
for name in REQUIRED:
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
# Run from an empty directory, and point NISSE_ENV at a file that does not
|
||||
# exist: the repository's .env holds live credentials, and a test must
|
||||
# never be one cwd away from starting a real agent.
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.delenv("NISSE_CONFIG", raising=False)
|
||||
monkeypatch.setenv("NISSE_ENV", str(tmp_path / "absent.env"))
|
||||
|
||||
assert main() == 2
|
||||
|
||||
err = capsys.readouterr().err
|
||||
assert "cannot start" in err
|
||||
assert "NISSE_LLM_TOKEN" in err
|
||||
assert "nisse-dev.example.yaml" in err
|
||||
@@ -162,11 +162,6 @@ def test_list_comments_passes_since() -> None:
|
||||
assert seen["since"] == "2026-09-07T00:00:00Z"
|
||||
|
||||
|
||||
def test_from_env_requires_config() -> None:
|
||||
with pytest.raises(GiteaError, match="NISSE_GITEA_URL"):
|
||||
GiteaClient.from_env({})
|
||||
|
||||
|
||||
# --- reconciler (A3 external truth) --------------------------------------
|
||||
|
||||
|
||||
|
||||
+11
-23
@@ -21,7 +21,6 @@ from nisse_dev.llm import (
|
||||
RoleMap,
|
||||
TruncatedResponse,
|
||||
UpstreamUnavailable,
|
||||
max_output_tokens,
|
||||
)
|
||||
|
||||
EVENTS = [StoredEvent(id=1, ts="t", event=UserMessage(text="hi", source="m:1"))]
|
||||
@@ -165,16 +164,16 @@ def test_malformed_response_raises_llm_error() -> None:
|
||||
client.complete(EVENTS, tools=None, model="local/qwen")
|
||||
|
||||
|
||||
ROLE_ENV = {
|
||||
"NISSE_ROLE_EXECUTOR": "local/qwen",
|
||||
"NISSE_ROLE_PLANNER": "anthropic/claude-opus-5",
|
||||
"NISSE_ROLE_DELEGATE": "anthropic/claude-opus-5",
|
||||
"NISSE_ROLE_CONSULTANT": "anthropic/claude-opus-5",
|
||||
}
|
||||
ROLES = RoleMap(
|
||||
executor="local/qwen",
|
||||
planner="anthropic/claude-opus-5",
|
||||
delegate="anthropic/claude-opus-5",
|
||||
consultant="anthropic/claude-opus-5",
|
||||
)
|
||||
|
||||
|
||||
def test_role_map_resolves_and_pins() -> None:
|
||||
roles = RoleMap.from_env(ROLE_ENV)
|
||||
roles = ROLES
|
||||
assert roles.resolve("executor") == "local/qwen"
|
||||
assert roles.pinned() == {
|
||||
"executor": "local/qwen",
|
||||
@@ -184,19 +183,9 @@ def test_role_map_resolves_and_pins() -> None:
|
||||
}
|
||||
|
||||
|
||||
def test_role_map_missing_env_raises() -> None:
|
||||
with pytest.raises(LLMError, match="NISSE_ROLE_PLANNER"):
|
||||
RoleMap.from_env({"NISSE_ROLE_EXECUTOR": "local/qwen"})
|
||||
|
||||
|
||||
def test_unknown_role_raises() -> None:
|
||||
with pytest.raises(LLMError, match="unknown role"):
|
||||
RoleMap.from_env(ROLE_ENV).resolve("boss")
|
||||
|
||||
|
||||
def test_client_from_env_requires_config() -> None:
|
||||
with pytest.raises(LLMError, match="NISSE_LLM_BASE_URL"):
|
||||
LiteLLMClient.from_env({})
|
||||
ROLES.resolve("boss")
|
||||
|
||||
|
||||
def test_load_env_file(tmp_path: Path) -> None:
|
||||
@@ -306,12 +295,11 @@ def test_a_complete_answer_is_not_mistaken_for_truncation() -> None:
|
||||
assert call.arguments["content"] == "a real plan"
|
||||
|
||||
|
||||
def test_output_ceiling_is_configurable_and_generous_by_default() -> None:
|
||||
def test_output_ceiling_is_generous_by_default() -> None:
|
||||
# Output is billed by what is produced, so a low ceiling buys nothing and
|
||||
# costs silent truncation.
|
||||
# costs silent truncation. The configured value is tested in test_config.
|
||||
assert DEFAULT_MAX_OUTPUT_TOKENS >= 8192
|
||||
assert max_output_tokens({}) == DEFAULT_MAX_OUTPUT_TOKENS
|
||||
assert max_output_tokens({"NISSE_MAX_OUTPUT_TOKENS": "2048"}) == 2048
|
||||
assert LiteLLMClient("http://x", "t")._max_tokens == DEFAULT_MAX_OUTPUT_TOKENS
|
||||
|
||||
|
||||
def test_truncation_is_an_llm_error_so_callers_already_handle_it() -> None:
|
||||
|
||||
@@ -143,20 +143,3 @@ def test_start_without_handler_raises() -> None:
|
||||
)
|
||||
with pytest.raises(TransportError, match="no message handler"):
|
||||
transport.start()
|
||||
|
||||
|
||||
def test_config_from_env_reports_missing() -> None:
|
||||
with pytest.raises(TransportError, match="NISSE_MATRIX_TOKEN"):
|
||||
MatrixConfig.from_env({"NISSE_MATRIX_HOMESERVER": "https://m"})
|
||||
|
||||
|
||||
def test_config_from_env_custom_room_var() -> None:
|
||||
env = {
|
||||
"NISSE_MATRIX_HOMESERVER": "https://m/",
|
||||
"NISSE_MATRIX_USER_ID": "@nisse-dev:test",
|
||||
"NISSE_MATRIX_TOKEN": "t",
|
||||
"NISSE_MATRIX_TEST_ROOM_ID": "!t:test",
|
||||
}
|
||||
config = MatrixConfig.from_env(env, room_var="NISSE_MATRIX_TEST_ROOM_ID")
|
||||
assert config.room_id == "!t:test"
|
||||
assert config.homeserver == "https://m" # trailing slash stripped
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Command-line options for the repository-level test suites.
|
||||
|
||||
Declared here rather than beside the tests that use them: pytest only reads
|
||||
options from conftest files it loads at startup, and this directory is one
|
||||
of the configured test paths.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def pytest_addoption(parser: pytest.Parser) -> None:
|
||||
parser.addoption(
|
||||
"--capability-models",
|
||||
default="",
|
||||
help=(
|
||||
"comma-separated LiteLLM aliases to measure with the capability "
|
||||
"matrix (tests/integration/test_model_capability.py). Spends real "
|
||||
"tokens on every model listed; the matrix skips when this is empty."
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Shared setup for the live integration tests.
|
||||
|
||||
They run against real services, configured by nisse-test.yaml beside this
|
||||
file: an ordinary agent config aimed at test resources, committed to the
|
||||
repository. Its secrets are ${…} references, supplied by CI as environment
|
||||
variables and locally by .env at the repository root.
|
||||
|
||||
A missing secret is a failure, not a skip. A half-configured environment must
|
||||
not pass quietly as a skip: that is how a broken CI credential hides.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nisse_dev.config import AgentConfig
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.gitea import GiteaClient
|
||||
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
TEST_CONFIG = Path(__file__).parent / "nisse-test.yaml"
|
||||
|
||||
# The secrets nisse-test.yaml refers to, for local runs. CI sets them as real
|
||||
# environment variables, which win over the file.
|
||||
load_env_file(REPO / ".env")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def live_config() -> AgentConfig:
|
||||
return AgentConfig.from_file(TEST_CONFIG)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_model(live_config: AgentConfig) -> str:
|
||||
"""The model the live tests plan, execute and delegate with."""
|
||||
return live_config.roles.planner
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def litellm(live_config: AgentConfig) -> LiteLLMClient:
|
||||
return LiteLLMClient(
|
||||
live_config.llm_base_url,
|
||||
live_config.llm_token,
|
||||
max_tokens=live_config.max_output_tokens,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def passthrough(live_config: AgentConfig) -> AnthropicPassthroughClient:
|
||||
return AnthropicPassthroughClient(
|
||||
live_config.llm_base_url,
|
||||
live_config.llm_token,
|
||||
max_tokens=live_config.max_output_tokens,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gitea(live_config: AgentConfig) -> GiteaClient:
|
||||
client = GiteaClient(live_config.gitea_url, live_config.gitea_token)
|
||||
yield client
|
||||
client.close()
|
||||
@@ -0,0 +1,40 @@
|
||||
# Agent config for the live integration tests, aimed at test resources: the
|
||||
# testing identities, the dedicated test room, and the sandbox repo. Never
|
||||
# point it at production.
|
||||
#
|
||||
# Secrets are ${…} references. CI passes them as environment variables;
|
||||
# locally they come from .env at the repository root. A missing one fails
|
||||
# the live tests that need it, rather than skipping them, so a broken
|
||||
# credential cannot hide as a skip.
|
||||
|
||||
# Unused by the live tests, which work in pytest's temporary directories;
|
||||
# present because every agent config has them.
|
||||
state_db: /tmp/nisse-test/state.db
|
||||
workspace: /tmp/nisse-test/work
|
||||
project: nisse-dev-testing
|
||||
|
||||
llm:
|
||||
base_url: https://litellm.dongar.net
|
||||
token: ${NISSE_LLM_TOKEN}
|
||||
|
||||
# The live tests exercise the gated loop and the routes over a real proxy,
|
||||
# not local-model competence (measured separately by test_model_capability),
|
||||
# so every role uses a model that holds the tool-call contract.
|
||||
roles:
|
||||
executor: claude-haiku-4-5
|
||||
planner: claude-haiku-4-5
|
||||
delegate: claude-haiku-4-5
|
||||
consultant: claude-haiku-4-5
|
||||
|
||||
matrix:
|
||||
homeserver: https://matrix.dongar.net
|
||||
user_id: "@nisse-dev-testing:dongar.net"
|
||||
token: ${NISSE_MATRIX_TOKEN}
|
||||
room_id: "!UxpXhTlQsFIeNZwkgv:dongar.net" # the test room, never production
|
||||
operators: ["@dongar:dongar.net"]
|
||||
|
||||
gitea:
|
||||
url: https://gitea.dongar.net
|
||||
token: ${NISSE_GITEA_TOKEN}
|
||||
repo: dongar/nisse-dev-testing-sandbox # the sandbox, never a real project
|
||||
operators: [dongar]
|
||||
@@ -2,30 +2,18 @@
|
||||
working code and a structured handback out."""
|
||||
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nisse_dev.delegation import Caps, Delegation
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.git import GitRepo
|
||||
from nisse_dev.llm import AnthropicPassthroughClient
|
||||
from nisse_dev.oplog import OpLog
|
||||
from nisse_dev.state import StateStore
|
||||
from nisse_dev.tools import Workspace
|
||||
|
||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
||||
|
||||
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.skipif(
|
||||
not os.environ.get(SENTINEL),
|
||||
reason=f"no live frontier route configured ({SENTINEL} unset)",
|
||||
),
|
||||
]
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
BRIEF = """Create `slugify.py` in the workspace root containing a function
|
||||
`slugify(text)` that lowercases the text and replaces every run of
|
||||
@@ -34,7 +22,9 @@ hyphen. Then create `test_slugify.py` with at least two assertions covering
|
||||
it, and run `python -m pytest test_slugify.py -q` until it passes."""
|
||||
|
||||
|
||||
def build(tmp_path: Path, caps: Caps) -> tuple[Delegation, GitRepo, StateStore]:
|
||||
def build(
|
||||
tmp_path: Path, caps: Caps, client: AnthropicPassthroughClient, model: str
|
||||
) -> tuple[Delegation, GitRepo, StateStore]:
|
||||
repo = GitRepo(tmp_path / "work")
|
||||
repo.init()
|
||||
repo.configure_identity("nisse-dev-testing", "n@t.test")
|
||||
@@ -42,18 +32,22 @@ def build(tmp_path: Path, caps: Caps) -> tuple[Delegation, GitRepo, StateStore]:
|
||||
repo.commit_all("seed")
|
||||
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
||||
delegation = Delegation(
|
||||
AnthropicPassthroughClient.from_env(),
|
||||
client,
|
||||
store,
|
||||
Workspace(repo.root, timeout_s=60),
|
||||
repo,
|
||||
model=os.environ[SENTINEL],
|
||||
model=model,
|
||||
caps=caps,
|
||||
)
|
||||
return delegation, repo, store
|
||||
|
||||
|
||||
def test_delegate_completes_a_scoped_subtask(tmp_path: Path) -> None:
|
||||
delegation, repo, _store = build(tmp_path, Caps(turns=12, wall_clock_s=300.0))
|
||||
def test_delegate_completes_a_scoped_subtask(
|
||||
tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str
|
||||
) -> None:
|
||||
delegation, repo, _store = build(
|
||||
tmp_path, Caps(turns=12, wall_clock_s=300.0), passthrough, live_model
|
||||
)
|
||||
|
||||
result = delegation.run(brief=BRIEF, branch="live/delegation-1")
|
||||
|
||||
@@ -76,10 +70,14 @@ def test_delegate_completes_a_scoped_subtask(tmp_path: Path) -> None:
|
||||
assert check.success, f"delegate's own tests do not pass: {check.result[:400]}"
|
||||
|
||||
|
||||
def test_a_cap_breach_discards_the_work(tmp_path: Path) -> None:
|
||||
def test_a_cap_breach_discards_the_work(
|
||||
tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str
|
||||
) -> None:
|
||||
# One turn cannot finish the brief, so this exercises the abort path
|
||||
# against a real model rather than a scripted one.
|
||||
delegation, repo, _ = build(tmp_path, Caps(turns=1, wall_clock_s=120.0))
|
||||
delegation, repo, _ = build(
|
||||
tmp_path, Caps(turns=1, wall_clock_s=120.0), passthrough, live_model
|
||||
)
|
||||
|
||||
result = delegation.run(brief=BRIEF, branch="live/delegation-2")
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -16,27 +17,31 @@ def test_ops_entry_point_runs_and_greets() -> None:
|
||||
assert result.stdout == "Hello from nisse-ops, the house spirit is awake.\n"
|
||||
|
||||
|
||||
def test_dev_entry_point_refuses_to_start_unconfigured() -> None:
|
||||
"""Missing configuration must fail at startup, naming everything absent.
|
||||
def test_dev_entry_point_refuses_to_start_unconfigured(tmp_path: Path) -> None:
|
||||
"""An incomplete config must fail at startup, naming everything absent.
|
||||
|
||||
Run with a deliberately empty environment and a .env path that does not
|
||||
exist: the real .env in the working directory holds live credentials, and
|
||||
a test must never be one cwd away from starting a real agent.
|
||||
Run from an empty directory, with an empty config file and a .env path
|
||||
that does not exist: the repository's .env holds live credentials, and a
|
||||
test must never be one cwd away from starting a real agent.
|
||||
"""
|
||||
config = tmp_path / "nisse-dev.yaml"
|
||||
config.write_text("")
|
||||
result = subprocess.run(
|
||||
["nisse-dev"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
cwd=tmp_path,
|
||||
env={
|
||||
"PATH": os.environ["PATH"],
|
||||
"HOME": os.environ["HOME"],
|
||||
"NISSE_ENV": "/nonexistent/.env",
|
||||
"NISSE_CONFIG": str(config),
|
||||
},
|
||||
)
|
||||
assert result.returncode == 2
|
||||
assert "cannot start" in result.stderr
|
||||
# All of them at once, not the first one it happened to reach.
|
||||
for expected in ("NISSE_STATE_DB", "NISSE_MATRIX_TOKEN", "NISSE_GITEA_REPO"):
|
||||
for expected in ("state_db", "matrix.token", "gitea.repo"):
|
||||
assert expected in result.stderr
|
||||
@@ -5,40 +5,34 @@ that parses against our local schema, since no API-level enforcement exists.
|
||||
"""
|
||||
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.evals import collect
|
||||
from nisse_dev.expert import BudgetExhausted, ExpertDesk
|
||||
from nisse_dev.llm import LiteLLMClient, RoleMap
|
||||
from nisse_dev.oplog import OpLog
|
||||
from nisse_dev.state import StateStore
|
||||
|
||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
||||
|
||||
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.skipif(
|
||||
not os.environ.get(SENTINEL),
|
||||
reason=f"no live frontier route configured ({SENTINEL} unset)",
|
||||
),
|
||||
]
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def desk(tmp_path: Path, budget: int = 2) -> tuple[ExpertDesk, StateStore]:
|
||||
model = os.environ[SENTINEL]
|
||||
@pytest.fixture
|
||||
def desk(tmp_path: Path, litellm: LiteLLMClient, live_model: str):
|
||||
def make(budget: int = 2) -> tuple[ExpertDesk, StateStore]:
|
||||
model = live_model
|
||||
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
||||
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
||||
return ExpertDesk(LiteLLMClient.from_env(), store, roles, budget=budget), store
|
||||
store = StateStore(
|
||||
tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())
|
||||
)
|
||||
return ExpertDesk(litellm, store, roles, budget=budget), store
|
||||
|
||||
return make
|
||||
|
||||
|
||||
def test_live_consult_parses_against_the_local_schema(tmp_path: Path) -> None:
|
||||
expert, store = desk(tmp_path)
|
||||
def test_live_consult_parses_against_the_local_schema(desk) -> None:
|
||||
expert, store = desk()
|
||||
|
||||
answer = expert.ask(
|
||||
"For a single-process agent storing a few thousand rows of task state, "
|
||||
@@ -53,8 +47,8 @@ def test_live_consult_parses_against_the_local_schema(tmp_path: Path) -> None:
|
||||
assert record.exit_status == "answered"
|
||||
|
||||
|
||||
def test_live_budget_stops_a_circling_caller(tmp_path: Path) -> None:
|
||||
expert, _ = desk(tmp_path, budget=1)
|
||||
def test_live_budget_stops_a_circling_caller(desk) -> None:
|
||||
expert, _ = desk(budget=1)
|
||||
expert.ask("Name one advantage of WAL mode in SQLite. Be brief.")
|
||||
with pytest.raises(BudgetExhausted):
|
||||
expert.ask("And another?")
|
||||
@@ -5,29 +5,17 @@ Cleans up after itself (closes the PR, deletes the branch) so repeated runs
|
||||
do not silt up the sandbox.
|
||||
"""
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.config import AgentConfig
|
||||
from nisse_dev.git import GitError, GitRepo, authenticated_url
|
||||
from nisse_dev.gitea import GiteaClient, pull_request_reconciler
|
||||
from nisse_dev.state import Intent
|
||||
|
||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
||||
|
||||
SENTINEL = "NISSE_GITEA_URL"
|
||||
ALSO_REQUIRED = ("NISSE_GITEA_TOKEN", "NISSE_GITEA_TEST_REPO")
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.skipif(
|
||||
not os.environ.get(SENTINEL),
|
||||
reason=f"no live Gitea configured ({SENTINEL} unset)",
|
||||
),
|
||||
]
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def intent_for(branch: str) -> Intent:
|
||||
@@ -41,16 +29,13 @@ def intent_for(branch: str) -> Intent:
|
||||
)
|
||||
|
||||
|
||||
def test_push_open_pr_and_reconcile(tmp_path: Path) -> None:
|
||||
missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)]
|
||||
assert not missing, (
|
||||
f"{SENTINEL} is set, so the live Gitea path is expected to work, "
|
||||
f"but these are missing: {', '.join(missing)}"
|
||||
)
|
||||
base_url = os.environ[SENTINEL]
|
||||
token = os.environ["NISSE_GITEA_TOKEN"]
|
||||
repo_path = os.environ["NISSE_GITEA_TEST_REPO"]
|
||||
client = GiteaClient.from_env()
|
||||
def test_push_open_pr_and_reconcile(
|
||||
tmp_path: Path, live_config: AgentConfig, gitea: GiteaClient
|
||||
) -> None:
|
||||
base_url = live_config.gitea_url
|
||||
token = live_config.gitea_token
|
||||
repo_path = live_config.gitea_repo # the sandbox
|
||||
client = gitea
|
||||
|
||||
account = client.whoami()
|
||||
assert "testing" in account, (
|
||||
|
||||
@@ -1,41 +1,17 @@
|
||||
"""Live smoke test against the real LiteLLM proxy.
|
||||
|
||||
NISSE_LLM_BASE_URL alone decides whether this runs: if it is set, the live
|
||||
path is expected to work, and anything else missing is a *failure*, not a
|
||||
skip. A half-configured environment must not pass quietly as a skip — that is
|
||||
how a broken CI credential hides.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
"""Live smoke test against the real LiteLLM proxy."""
|
||||
|
||||
import pytest
|
||||
|
||||
from nisse_dev.canonical import AssistantText, StoredEvent, UserMessage
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.config import AgentConfig
|
||||
from nisse_dev.llm import LiteLLMClient
|
||||
|
||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
||||
|
||||
SENTINEL = "NISSE_LLM_BASE_URL"
|
||||
ALSO_REQUIRED = ("NISSE_LLM_TOKEN", "NISSE_ROLE_EXECUTOR")
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.skipif(
|
||||
not os.environ.get(SENTINEL),
|
||||
reason=f"no live LLM configured ({SENTINEL} unset)",
|
||||
),
|
||||
]
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def test_executor_model_completes_via_litellm() -> None:
|
||||
missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)]
|
||||
assert not missing, (
|
||||
f"{SENTINEL} is set, so the live LLM path is expected to work, "
|
||||
f"but these are missing: {', '.join(missing)}"
|
||||
)
|
||||
client = LiteLLMClient.from_env()
|
||||
def test_executor_model_completes_via_litellm(
|
||||
litellm: LiteLLMClient, live_config: AgentConfig
|
||||
) -> None:
|
||||
events = [
|
||||
StoredEvent(
|
||||
id=1,
|
||||
@@ -45,9 +21,7 @@ def test_executor_model_completes_via_litellm() -> None:
|
||||
),
|
||||
)
|
||||
]
|
||||
result = client.complete(
|
||||
events, tools=None, model=os.environ["NISSE_ROLE_EXECUTOR"]
|
||||
)
|
||||
result = litellm.complete(events, tools=None, model=live_config.roles.executor)
|
||||
texts = [e.text for e in result if isinstance(e, AssistantText)]
|
||||
assert texts, f"no assistant text in {result!r}"
|
||||
assert "pong" in texts[0].lower()
|
||||
@@ -1,36 +1,14 @@
|
||||
"""Live smoke test against the real homeserver, bound strictly to the
|
||||
dedicated test room (never the production room).
|
||||
dedicated test room that nisse-test.yaml names (never the production room)."""
|
||||
|
||||
NISSE_MATRIX_HOMESERVER alone decides whether this runs: if it is set, the
|
||||
live path is expected to work, and anything else missing is a *failure*, not
|
||||
a skip. A half-configured environment must not pass quietly as a skip — that
|
||||
is how a broken CI credential hides."""
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.config import AgentConfig
|
||||
from nisse_dev.matrix import MatrixConfig, MatrixTransport
|
||||
|
||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
||||
|
||||
SENTINEL = "NISSE_MATRIX_HOMESERVER"
|
||||
ALSO_REQUIRED = (
|
||||
"NISSE_MATRIX_USER_ID",
|
||||
"NISSE_MATRIX_TOKEN",
|
||||
"NISSE_MATRIX_TEST_ROOM_ID",
|
||||
)
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.skipif(
|
||||
not os.environ.get(SENTINEL),
|
||||
reason=f"no live homeserver configured ({SENTINEL} unset)",
|
||||
),
|
||||
]
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _assert_testing_identity(transport: MatrixTransport, config: MatrixConfig) -> None:
|
||||
@@ -46,7 +24,7 @@ def _assert_testing_identity(transport: MatrixTransport, config: MatrixConfig) -
|
||||
whoami.raise_for_status()
|
||||
actual = whoami.json()["user_id"]
|
||||
assert actual == config.user_id, (
|
||||
f"NISSE_MATRIX_USER_ID is {config.user_id} but the token belongs to "
|
||||
f"matrix.user_id is {config.user_id} but the token belongs to "
|
||||
f"{actual}; the self-message filter would not match"
|
||||
)
|
||||
assert "testing" in actual, (
|
||||
@@ -54,13 +32,8 @@ def _assert_testing_identity(transport: MatrixTransport, config: MatrixConfig) -
|
||||
)
|
||||
|
||||
|
||||
def test_send_lands_in_test_room() -> None:
|
||||
missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)]
|
||||
assert not missing, (
|
||||
f"{SENTINEL} is set, so the live Matrix path is expected to work, "
|
||||
f"but these are missing: {', '.join(missing)}"
|
||||
)
|
||||
config = MatrixConfig.from_env(room_var="NISSE_MATRIX_TEST_ROOM_ID")
|
||||
def test_send_lands_in_test_room(live_config: AgentConfig) -> None:
|
||||
config = live_config.matrix
|
||||
marker = f"transport smoke test {uuid.uuid4().hex}"
|
||||
transport = MatrixTransport(config)
|
||||
try:
|
||||
|
||||
@@ -2,12 +2,11 @@
|
||||
approve through the gate, then let the executor role actually create and test
|
||||
a file on disk.
|
||||
|
||||
Gated on NISSE_LIVE_PLANNER_MODEL like the planning test — it needs a model
|
||||
that holds the tool-call contract on both ends.
|
||||
Uses the test config's planner model for every role: it needs a model that
|
||||
holds the tool-call contract on both ends.
|
||||
"""
|
||||
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -19,7 +18,6 @@ from nisse_dev.control import (
|
||||
EXECUTING,
|
||||
Orchestrator,
|
||||
)
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.executor import LLMExecutor
|
||||
from nisse_dev.llm import LiteLLMClient, RoleMap
|
||||
from nisse_dev.oplog import OpLog
|
||||
@@ -28,17 +26,7 @@ from nisse_dev.state import StateStore
|
||||
from nisse_dev.tools import Workspace
|
||||
from nisse_dev.transport import InMemoryTransport
|
||||
|
||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
||||
|
||||
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.skipif(
|
||||
not os.environ.get(SENTINEL),
|
||||
reason=f"no live planner route configured ({SENTINEL} unset)",
|
||||
),
|
||||
]
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _drive_to_execution(
|
||||
@@ -67,16 +55,14 @@ def _drive_to_execution(
|
||||
)
|
||||
|
||||
|
||||
def test_plan_approve_execute_produces_a_real_file(tmp_path: Path) -> None:
|
||||
assert os.environ.get("NISSE_LLM_TOKEN"), (
|
||||
f"{SENTINEL} is set, so the live path is expected to work, "
|
||||
"but NISSE_LLM_TOKEN is missing"
|
||||
)
|
||||
def test_plan_approve_execute_produces_a_real_file(
|
||||
tmp_path: Path, litellm: LiteLLMClient, live_model: str
|
||||
) -> None:
|
||||
# Both roles use the capable model here: the point is the whole gated
|
||||
# loop over a real proxy, not local-model competence (measured separately).
|
||||
model = os.environ[SENTINEL]
|
||||
model = live_model
|
||||
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
||||
client = LiteLLMClient.from_env()
|
||||
client = litellm
|
||||
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
||||
workspace = Workspace(tmp_path / "work", timeout_s=60)
|
||||
transport = InMemoryTransport()
|
||||
|
||||
@@ -9,12 +9,15 @@ What it measures is the thing that actually breaks: whether a model can hold
|
||||
the tool-call contract our loops depend on. llama3.2 could not — it returned
|
||||
`{` as an entire plan — and no amount of prompt work fixed it.
|
||||
|
||||
Opt in with NISSE_CAPABILITY_MODELS, comma separated. Skipped otherwise,
|
||||
because it spends real tokens across several models.
|
||||
"""
|
||||
Opt in by naming the models to measure, for one run:
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
uv run pytest -m integration tests/integration/test_model_capability.py \
|
||||
--capability-models=local-fast,local-coder
|
||||
|
||||
Skipped otherwise, because it spends real tokens across every model listed.
|
||||
Which models to measure is a question for a particular run, so it is an
|
||||
argument, not configuration.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -26,45 +29,56 @@ from nisse_dev.canonical import (
|
||||
UserMessage,
|
||||
)
|
||||
from nisse_dev.catalog import ModelCatalog, select_client
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.config import AgentConfig
|
||||
from nisse_dev.expert import parse_answer
|
||||
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient, LLMError
|
||||
from nisse_dev.planner import PLANNER_TOOLS
|
||||
from nisse_dev.tools import EXECUTOR_TOOLS
|
||||
|
||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
SENTINEL = "NISSE_CAPABILITY_MODELS"
|
||||
MODELS = [m.strip() for m in os.environ.get(SENTINEL, "").split(",") if m.strip()]
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.skipif(
|
||||
not MODELS, reason=f"set {SENTINEL} to a comma-separated model list"
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
|
||||
if "model" not in metafunc.fixturenames:
|
||||
return
|
||||
raw = metafunc.config.getoption("--capability-models")
|
||||
models = [m.strip() for m in raw.split(",") if m.strip()]
|
||||
if models:
|
||||
metafunc.parametrize("model", models)
|
||||
else:
|
||||
metafunc.parametrize(
|
||||
"model",
|
||||
[
|
||||
pytest.param(
|
||||
None,
|
||||
marks=pytest.mark.skip(
|
||||
reason="no models to measure; pass --capability-models"
|
||||
),
|
||||
]
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def clients() -> tuple[LiteLLMClient, AnthropicPassthroughClient]:
|
||||
return LiteLLMClient.from_env(), AnthropicPassthroughClient.from_env()
|
||||
@pytest.fixture
|
||||
def routes(
|
||||
litellm: LiteLLMClient, passthrough: AnthropicPassthroughClient
|
||||
) -> tuple[LiteLLMClient, AnthropicPassthroughClient]:
|
||||
return litellm, passthrough
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def catalog() -> ModelCatalog:
|
||||
def catalog(live_config: AgentConfig) -> ModelCatalog:
|
||||
"""The live catalog this key can see. /model/info is scoped per key, so
|
||||
the probe list and the key have to agree."""
|
||||
return ModelCatalog.discover(
|
||||
os.environ["NISSE_LLM_BASE_URL"], os.environ["NISSE_LLM_TOKEN"]
|
||||
)
|
||||
return ModelCatalog.discover(live_config.llm_base_url, live_config.llm_token)
|
||||
|
||||
|
||||
def user(text: str) -> list[StoredEvent]:
|
||||
return [StoredEvent(id=1, ts="t", event=UserMessage(text=text, source="probe"))]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_model_answers_at_all(model: str, catalog: ModelCatalog) -> None:
|
||||
normalized, passthrough = clients()
|
||||
def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None:
|
||||
normalized, passthrough = routes
|
||||
client = select_client(model, normalized, passthrough, catalog)
|
||||
produced = client.complete(
|
||||
user("Reply with exactly one word: pong"), None, model, system="Be terse."
|
||||
@@ -73,11 +87,12 @@ def test_model_answers_at_all(model: str, catalog: ModelCatalog) -> None:
|
||||
assert texts, f"{model} produced no text: {produced!r}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_model_holds_the_planner_contract(model: str, catalog: ModelCatalog) -> None:
|
||||
def test_model_holds_the_planner_contract(
|
||||
model: str, catalog: ModelCatalog, routes
|
||||
) -> None:
|
||||
"""A4's gating is structural at the tool layer: a model that answers in
|
||||
prose cannot be gated, because there is no plan to approve."""
|
||||
normalized, passthrough = clients()
|
||||
normalized, passthrough = routes
|
||||
client = select_client(model, normalized, passthrough, catalog)
|
||||
produced = client.complete(
|
||||
user("Draft an architecture plan for a command-line slug generator."),
|
||||
@@ -100,11 +115,12 @@ def test_model_holds_the_planner_contract(model: str, catalog: ModelCatalog) ->
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_model_round_trips_a_tool_result(model: str, catalog: ModelCatalog) -> None:
|
||||
def test_model_round_trips_a_tool_result(
|
||||
model: str, catalog: ModelCatalog, routes
|
||||
) -> None:
|
||||
"""The executor loop feeds results back; a model that cannot consume its
|
||||
own tool result cannot run a milestone."""
|
||||
normalized, passthrough = clients()
|
||||
normalized, passthrough = routes
|
||||
client = select_client(model, normalized, passthrough, catalog)
|
||||
events = user("List the files in the current directory using your tools.")
|
||||
produced = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.")
|
||||
@@ -128,13 +144,12 @@ def test_model_round_trips_a_tool_result(model: str, catalog: ModelCatalog) -> N
|
||||
assert follow_up, f"{model} produced nothing after its tool result"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_model_holds_the_prompt_level_json_contract(
|
||||
model: str, catalog: ModelCatalog
|
||||
model: str, catalog: ModelCatalog, routes
|
||||
) -> None:
|
||||
"""A14 contracts structured output at the prompt level because vendors
|
||||
differ on JSON enforcement. This measures whether that holds per model."""
|
||||
normalized, passthrough = clients()
|
||||
normalized, passthrough = routes
|
||||
client = select_client(model, normalized, passthrough, catalog)
|
||||
produced = client.complete(
|
||||
user("Is SQLite sufficient for a single-process agent's task state?"),
|
||||
@@ -152,12 +167,13 @@ def test_model_holds_the_prompt_level_json_contract(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_route_selection_matches_the_model(model: str, catalog: ModelCatalog) -> None:
|
||||
def test_route_selection_matches_the_model(
|
||||
model: str, catalog: ModelCatalog, routes
|
||||
) -> None:
|
||||
"""A non-Anthropic model must not be sent Anthropic-shaped bodies. This
|
||||
was hardcoded until gpt-6-astra made it observable, and name-sniffed until
|
||||
semantic aliases made the name stop meaning anything."""
|
||||
normalized, passthrough = clients()
|
||||
normalized, passthrough = routes
|
||||
facts = catalog.facts(model)
|
||||
chosen = select_client(model, normalized, passthrough, catalog)
|
||||
if facts.provider == "anthropic":
|
||||
|
||||
@@ -6,30 +6,19 @@ passthrough rather than being flattened by the proxy.
|
||||
"""
|
||||
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from nisse_dev.canonical import AssistantText, ToolCall, ToolResult, UserMessage
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.config import AgentConfig
|
||||
from nisse_dev.llm import AnthropicPassthroughClient
|
||||
from nisse_dev.oplog import OpLog
|
||||
from nisse_dev.state import StateStore
|
||||
from nisse_dev.tools import EXECUTOR_TOOLS
|
||||
|
||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
||||
|
||||
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.skipif(
|
||||
not os.environ.get(SENTINEL),
|
||||
reason=f"no live frontier route configured ({SENTINEL} unset)",
|
||||
),
|
||||
]
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def store_with(tmp_path: Path, *events) -> StateStore:
|
||||
@@ -39,18 +28,22 @@ def store_with(tmp_path: Path, *events) -> StateStore:
|
||||
return store
|
||||
|
||||
|
||||
def test_passthrough_completes_a_plain_turn(tmp_path: Path) -> None:
|
||||
def test_passthrough_completes_a_plain_turn(
|
||||
tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str
|
||||
) -> None:
|
||||
store = store_with(
|
||||
tmp_path, UserMessage(text="Reply with exactly: pong", source="live")
|
||||
)
|
||||
result = AnthropicPassthroughClient.from_env().complete(
|
||||
store.read_events(), None, os.environ[SENTINEL], system="Be terse."
|
||||
result = passthrough.complete(
|
||||
store.read_events(), None, live_model, system="Be terse."
|
||||
)
|
||||
texts = [e.text for e in result if isinstance(e, AssistantText)]
|
||||
assert texts and "pong" in texts[0].lower()
|
||||
|
||||
|
||||
def test_passthrough_round_trips_a_tool_call(tmp_path: Path) -> None:
|
||||
def test_passthrough_round_trips_a_tool_call(
|
||||
tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str
|
||||
) -> None:
|
||||
store = store_with(
|
||||
tmp_path,
|
||||
UserMessage(
|
||||
@@ -58,11 +51,11 @@ def test_passthrough_round_trips_a_tool_call(tmp_path: Path) -> None:
|
||||
source="live",
|
||||
),
|
||||
)
|
||||
client = AnthropicPassthroughClient.from_env()
|
||||
client = passthrough
|
||||
produced = client.complete(
|
||||
store.read_events(),
|
||||
EXECUTOR_TOOLS,
|
||||
os.environ[SENTINEL],
|
||||
live_model,
|
||||
system="Use a tool. Do not answer in prose.",
|
||||
)
|
||||
calls = [e for e in produced if isinstance(e, ToolCall)]
|
||||
@@ -75,18 +68,18 @@ def test_passthrough_round_trips_a_tool_call(tmp_path: Path) -> None:
|
||||
store.append_event(
|
||||
ToolResult(call_id=calls[0].call_id, result="app.py\ntests/", success=True)
|
||||
)
|
||||
follow_up = client.complete(
|
||||
store.read_events(), EXECUTOR_TOOLS, os.environ[SENTINEL]
|
||||
)
|
||||
follow_up = client.complete(store.read_events(), EXECUTOR_TOOLS, live_model)
|
||||
assert follow_up, "the vendor rejected or ignored the tool_result turn"
|
||||
|
||||
|
||||
def test_prompt_caching_survives_the_passthrough() -> None:
|
||||
def test_prompt_caching_survives_the_passthrough(
|
||||
live_config: AgentConfig, live_model: str
|
||||
) -> None:
|
||||
# A5 asks for this to be verified rather than assumed: a proxy that
|
||||
# flattened the request would silently drop cache control and turn a
|
||||
# cost optimisation into a per-call expense.
|
||||
base = os.environ["NISSE_LLM_BASE_URL"].rstrip("/")
|
||||
token = os.environ["NISSE_LLM_TOKEN"]
|
||||
base = live_config.llm_base_url.rstrip("/")
|
||||
token = live_config.llm_token
|
||||
response = httpx.post(
|
||||
f"{base}/v1/messages",
|
||||
headers={
|
||||
@@ -95,7 +88,7 @@ def test_prompt_caching_survives_the_passthrough() -> None:
|
||||
"anthropic-version": "2023-06-01",
|
||||
},
|
||||
json={
|
||||
"model": os.environ[SENTINEL],
|
||||
"model": live_model,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "Say pong."}],
|
||||
},
|
||||
|
||||
@@ -1,47 +1,30 @@
|
||||
"""Live planning turn: a real model must answer with a submit_plan or
|
||||
ask_human tool call, not prose.
|
||||
|
||||
Deliberately opt-in via NISSE_LIVE_PLANNER_MODEL rather than the usual
|
||||
NISSE_LLM_BASE_URL sentinel. Planning escalates to a frontier model
|
||||
unconditionally (A12), and the small local model does not hold the tool-call
|
||||
contract reliably — measured, see the PR — so this test needs a frontier
|
||||
route that does not exist yet. Set the variable to the model to plan with
|
||||
once one is configured; until then this skips on purpose, not by accident.
|
||||
Plans with the test config's planner role. Planning escalates to a frontier
|
||||
model unconditionally (A12), and small local models do not hold the tool-call
|
||||
contract reliably, so that role must be one that does.
|
||||
"""
|
||||
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nisse_dev.canonical import UserMessage
|
||||
from nisse_dev.control import AskHuman, SubmitPlan
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.llm import LiteLLMClient, RoleMap
|
||||
from nisse_dev.oplog import OpLog
|
||||
from nisse_dev.planner import LLMPlanningAgent
|
||||
from nisse_dev.state import StateStore
|
||||
|
||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
||||
|
||||
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.skipif(
|
||||
not os.environ.get(SENTINEL),
|
||||
reason=f"no frontier planner route configured ({SENTINEL} unset)",
|
||||
),
|
||||
]
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def test_live_planning_turn_returns_a_tool_action(tmp_path: Path) -> None:
|
||||
assert os.environ.get("NISSE_LLM_TOKEN"), (
|
||||
f"{SENTINEL} is set, so the live planning path is expected to work, "
|
||||
"but NISSE_LLM_TOKEN is missing"
|
||||
)
|
||||
model = os.environ[SENTINEL]
|
||||
def test_live_planning_turn_returns_a_tool_action(
|
||||
tmp_path: Path, litellm: LiteLLMClient, live_model: str
|
||||
) -> None:
|
||||
model = live_model
|
||||
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
||||
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
||||
store.append_event(
|
||||
@@ -53,7 +36,7 @@ def test_live_planning_turn_returns_a_tool_action(tmp_path: Path) -> None:
|
||||
source="live-test",
|
||||
)
|
||||
)
|
||||
agent = LLMPlanningAgent(LiteLLMClient.from_env(), store, roles)
|
||||
agent = LLMPlanningAgent(litellm, store, roles)
|
||||
|
||||
action = agent.plan("arch", "hello-cli")
|
||||
|
||||
|
||||
@@ -113,12 +113,14 @@ source = { editable = "nisse-dev" }
|
||||
dependencies = [
|
||||
{ name = "httpx" },
|
||||
{ name = "nisse-core" },
|
||||
{ name = "pyyaml" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "httpx", specifier = ">=0.27" },
|
||||
{ name = "nisse-core", editable = "nisse-core" },
|
||||
{ name = "pyyaml", specifier = ">=6" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -175,6 +177,52 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyyaml"
|
||||
version = "6.0.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ruff"
|
||||
version = "0.16.6"
|
||||
|
||||
Reference in new issue
Block a user