Merge pull request 'Move configuration to YAML, and make the tuning knobs configurable' (#43) from yaml-config into main
CI / checks (push) Failing after 51s
CI / checks (push) Failing after 51s
Reviewed-on: dongar/nisse#43 Reviewed-by: Don Garrett <dgarrett@acm.org>
This commit was merged in pull request #43.
This commit is contained in:
commit
31851b7396
35 files changed
+1357
-605
No files matched your search
+16
-58
@@ -1,59 +1,17 @@
|
|||||||
# Copy to .env (chmod 600). Real environment variables win over this file.
|
# Secrets only. Copy to .env (chmod 600); it is gitignored, so never commit
|
||||||
# .env is gitignored; never commit real values.
|
# real values. Real environment variables win over this file.
|
||||||
|
|
||||||
# --- LLM access (A5): the LiteLLM proxy is the single LLM destination ---
|
|
||||||
NISSE_LLM_BASE_URL=http://litellm.example:4000
|
|
||||||
NISSE_LLM_TOKEN=sk-your-litellm-virtual-key
|
|
||||||
|
|
||||||
# --- Role mapping (A11): roles resolve to concrete model strings ---
|
|
||||||
NISSE_ROLE_EXECUTOR=anthropic/claude-haiku-4-5-20251001
|
|
||||||
NISSE_ROLE_PLANNER=anthropic/claude-opus-5
|
|
||||||
NISSE_ROLE_DELEGATE=anthropic/claude-opus-5
|
|
||||||
NISSE_ROLE_CONSULTANT=anthropic/claude-opus-5
|
|
||||||
|
|
||||||
# Opt-in for the live planning test: a model the virtual key can reach that
|
|
||||||
# holds the tool-call contract. Planning escalates unconditionally (A12), and
|
|
||||||
# small local models do not hold the contract reliably — llama3.2 returns
|
|
||||||
# truncated tool arguments, Haiku-class and up are solid.
|
|
||||||
NISSE_LIVE_PLANNER_MODEL=claude-haiku-4-5
|
|
||||||
|
|
||||||
# --- Matrix transport (A1), used by nisse-dev's MatrixTransport ---
|
|
||||||
NISSE_MATRIX_HOMESERVER=https://matrix.example
|
|
||||||
NISSE_MATRIX_USER_ID=@nisse-dev:example
|
|
||||||
NISSE_MATRIX_TOKEN=syt_your_matrix_access_token
|
|
||||||
NISSE_MATRIX_ROOM_ID=!roomid:example
|
|
||||||
# Dedicated room for the live integration test; never the production room.
|
|
||||||
NISSE_MATRIX_TEST_ROOM_ID=!testroomid:example
|
|
||||||
NISSE_MATRIX_OPERATORS=@you:example
|
|
||||||
|
|
||||||
# --- Gitea (B2): service account for branches, PRs, and comment intake ---
|
|
||||||
NISSE_GITEA_URL=https://gitea.example
|
|
||||||
NISSE_GITEA_TOKEN=your-scoped-service-account-token
|
|
||||||
# Sandbox repo for live tests; never the real project repo.
|
|
||||||
NISSE_GITEA_TEST_REPO=owner/sandbox-repo
|
|
||||||
# Gitea logins allowed to drive the agent via PR comments (A4 per-channel
|
|
||||||
# allowlist; a Matrix MXID and a Gitea login are different names).
|
|
||||||
NISSE_GITEA_OPERATORS=your-gitea-login
|
|
||||||
|
|
||||||
# --- Running the agent (the demo, not the tests) ---
|
|
||||||
# The tests use NISSE_GITEA_TEST_REPO; a running agent uses these.
|
|
||||||
#
|
#
|
||||||
# Layout. NISSE_WORKSPACE is the *clone the agent works in*, not the directory
|
# Everything else lives in the YAML config (see nisse-dev.example.yaml),
|
||||||
# the agent runs from — they must not be the same place:
|
# which refers to these by name, e.g. `token: ${NISSE_LLM_TOKEN}`. The live
|
||||||
#
|
# integration tests use the same names, from tests/integration/nisse-test.yaml.
|
||||||
# ~/nisse-demo-workspace/ deployment root, not a git repo
|
|
||||||
# ├── .env this file
|
# llm.token: the LiteLLM virtual key (A5)
|
||||||
# ├── state.db the A2 state store
|
NISSE_LLM_TOKEN=replace-me
|
||||||
# └── project/ NISSE_WORKSPACE, the clone
|
# matrix.token: the Matrix access token for the agent's account (A1)
|
||||||
#
|
NISSE_MATRIX_TOKEN=replace-me
|
||||||
# The agent commits its workspace with `git add -A` and pushes it, so anything
|
# gitea.token: the Gitea service account's scoped token (B2)
|
||||||
# inside the clone reaches the project repository. A .env or state store placed
|
NISSE_GITEA_TOKEN=replace-me
|
||||||
# there would publish credentials as a side effect of the agent working
|
|
||||||
# correctly; startup refuses to proceed if either is inside the workspace.
|
# Where the config file is, if not ./nisse-dev.yaml. Where this file is, if
|
||||||
NISSE_STATE_DB=~/nisse-demo-workspace/state.db
|
# not ./.env, is NISSE_ENV, which has to be set in the real environment.
|
||||||
NISSE_WORKSPACE=~/nisse-demo-workspace/project
|
# NISSE_CONFIG=~/nisse-demo-workspace/nisse-dev.yaml
|
||||||
NISSE_PROJECT=demo
|
|
||||||
NISSE_GITEA_REPO=owner/demo-repo
|
|
||||||
# NISSE_BASE_BRANCH=main
|
|
||||||
# Path to this file, if not ./.env
|
|
||||||
# NISSE_ENV=~/nisse-demo-workspace/.env
|
|
||||||
+3
-11
@@ -34,19 +34,11 @@ jobs:
|
|||||||
run: uv run pytest -m "not integration"
|
run: uv run pytest -m "not integration"
|
||||||
|
|
||||||
- name: Integration tests
|
- name: Integration tests
|
||||||
# Live-path credentials come from repo Actions secrets; the tests
|
# Everything but the secrets is in tests/integration/nisse-test.yaml,
|
||||||
# skip-guard on these, so a missing secret or unreachable service
|
# which refers to these by name. A missing secret fails the live
|
||||||
# degrades to "skipped", never a failure on unrelated work.
|
# tests that need it; it is never quietly turned into a skip.
|
||||||
env:
|
env:
|
||||||
NISSE_LLM_BASE_URL: ${{ secrets.NISSE_LLM_BASE_URL }}
|
|
||||||
NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }}
|
NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }}
|
||||||
NISSE_ROLE_EXECUTOR: ${{ secrets.NISSE_ROLE_EXECUTOR }}
|
|
||||||
NISSE_LIVE_PLANNER_MODEL: ${{ secrets.NISSE_LIVE_PLANNER_MODEL }}
|
|
||||||
NISSE_MATRIX_HOMESERVER: ${{ secrets.NISSE_MATRIX_HOMESERVER }}
|
|
||||||
NISSE_MATRIX_USER_ID: ${{ secrets.NISSE_MATRIX_USER_ID }}
|
|
||||||
NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }}
|
NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }}
|
||||||
NISSE_MATRIX_TEST_ROOM_ID: ${{ secrets.NISSE_MATRIX_TEST_ROOM_ID }}
|
|
||||||
NISSE_GITEA_URL: ${{ secrets.NISSE_GITEA_URL }}
|
|
||||||
NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }}
|
NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }}
|
||||||
NISSE_GITEA_TEST_REPO: ${{ secrets.NISSE_GITEA_TEST_REPO }}
|
|
||||||
run: uv run pytest -m integration -rs
|
run: uv run pytest -m integration -rs
|
||||||
@@ -6,3 +6,4 @@ __pycache__/
|
|||||||
.pytest_cache/
|
.pytest_cache/
|
||||||
.ruff_cache/
|
.ruff_cache/
|
||||||
dist/
|
dist/
|
||||||
|
nisse-dev.yaml
|
||||||
@@ -389,8 +389,10 @@ Requirements:
|
|||||||
- **No vendor credentials on the agent VM.** LiteLLM holds every key, sourced from
|
- **No vendor credentials on the agent VM.** LiteLLM holds every key, sourced from
|
||||||
Bitwarden machine accounts (A7); the agent holds one LiteLLM token. Adding a vendor
|
Bitwarden machine accounts (A7); the agent holds one LiteLLM token. Adding a vendor
|
||||||
does not touch the agent VM, and a compromised agent leaks no provider keys.
|
does not touch the agent VM, and a compromised agent leaks no provider keys.
|
||||||
- Config from env/`.env`, **nothing hardcoded**: base URL, auth token, and the role
|
- Config from the agent's config file, **nothing hardcoded**: base URL, auth token,
|
||||||
mapping (A11). Changing any of it is an env change + restart, nothing more.
|
and the role mapping (A11). Secrets in it are references to environment
|
||||||
|
variables, never values. Changing any of it is a config edit + restart,
|
||||||
|
nothing more.
|
||||||
- **Budget caps and spend alerts live on the LiteLLM virtual key**, not in agent code.
|
- **Budget caps and spend alerts live on the LiteLLM virtual key**, not in agent code.
|
||||||
|
|
||||||
### A6. Browser Tool
|
### A6. Browser Tool
|
||||||
|
|||||||
@@ -0,0 +1,288 @@
|
|||||||
|
# Tuning Guide
|
||||||
|
|
||||||
|
How to fit a local model onto the inference host, and how to set the agent's
|
||||||
|
tuning knobs. Part 1 covers the ideas you need to reason about the hardware.
|
||||||
|
Part 2 covers each knob in `nisse-dev.yaml`. Part 3 gives an order to tune
|
||||||
|
in.
|
||||||
|
|
||||||
|
The knobs live under `tuning:` in the config file (see
|
||||||
|
`nisse-dev.example.yaml`). They are read at startup, so a change takes effect
|
||||||
|
on the next restart. Unknown or misspelt keys are startup errors.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Part 1 — How a local model spends VRAM
|
||||||
|
|
||||||
|
A model server holding a model in GPU memory spends it on three things:
|
||||||
|
|
||||||
|
1. **Weights**, which are fixed for a given model and quantization.
|
||||||
|
2. **The KV cache**, which grows with context length.
|
||||||
|
3. **Working memory** for activations and runtime buffers. Budget 1–2 GB of
|
||||||
|
headroom.
|
||||||
|
|
||||||
|
On a 24 GB card, the whole game is trading (1) against (2).
|
||||||
|
|
||||||
|
### Weights and quantization
|
||||||
|
|
||||||
|
A model's weights take roughly *parameters × bits per weight ÷ 8* bytes.
|
||||||
|
|
||||||
|
| | 16-bit (full) | 8-bit | ~4.5-bit (Q4_K_M, AWQ) |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 24B | 48 GB | 24 GB | ~14 GB |
|
||||||
|
| 32B | 64 GB | 32 GB | ~19 GB |
|
||||||
|
|
||||||
|
Quantization stores each weight in fewer bits. Common 4-bit formats come out
|
||||||
|
nearer 4.5–5 bits per weight once scaling factors are counted, so check the
|
||||||
|
actual file size. The quality cost is small at 5–6 bits, usually acceptable at
|
||||||
|
4, and it climbs quickly below that. For agent work the cost shows up first as
|
||||||
|
**worse tool-call discipline**, not as worse prose. The capability matrix
|
||||||
|
(Part 3) measures exactly that.
|
||||||
|
|
||||||
|
**Mixture-of-experts** models such as Qwen3-30B-A3B are different. Memory is
|
||||||
|
set by *total* parameters (30B), but speed is set by the parameters *active*
|
||||||
|
for each token (3B). An MoE model of a given size therefore generates much
|
||||||
|
faster than a dense model of the same size. It's often weaker than a dense
|
||||||
|
model of the same total size, but not always for tool use.
|
||||||
|
|
||||||
|
### The KV cache, and why context costs memory
|
||||||
|
|
||||||
|
As the model reads the prompt, every layer stores a key and a value vector for
|
||||||
|
every token, so that later tokens can attend to them without recomputing.
|
||||||
|
That store is the KV cache. Its size per token is:
|
||||||
|
|
||||||
|
2 (key and value) × layers × KV heads × head size × bytes per value
|
||||||
|
|
||||||
|
The numbers come from the model's `config.json` (`num_hidden_layers`,
|
||||||
|
`num_key_value_heads`, `head_dim`). Some worked examples at 16-bit:
|
||||||
|
|
||||||
|
| Model | Layers × KV heads × head size | Per token | 32k tokens |
|
||||||
|
|---|---|---|---|
|
||||||
|
| Qwen3-32B | 64 × 8 × 128 | 256 KB | ~8 GB |
|
||||||
|
| Mistral/Devstral Small 24B | 40 × 8 × 128 | 160 KB | ~5 GB |
|
||||||
|
| Qwen3-30B-A3B (MoE) | 48 × 4 × 128 | 96 KB | ~3 GB |
|
||||||
|
|
||||||
|
(Taken from the published configs; recheck against the model card before
|
||||||
|
relying on them.)
|
||||||
|
|
||||||
|
Two consequences:
|
||||||
|
|
||||||
|
- **Context length is a memory decision, not a free setting.** The spec's
|
||||||
|
estimate (Part H) was about 5 GB left for KV cache beside a 4-bit 32B model.
|
||||||
|
At 16-bit, that's only about 20k tokens of context for Qwen3-32B.
|
||||||
|
- **KV-cache quantization** stores keys and values in 8 bits (or 4). At 8
|
||||||
|
bits it halves the cache and is close to lossless. At 4 bits it is
|
||||||
|
noticeably worse. Quantizing the cache to 8-bit is usually the first lever
|
||||||
|
to pull, since it buys twice the context for almost nothing.
|
||||||
|
|
||||||
|
The server allocates the cache for the context length you configure. It
|
||||||
|
doesn't matter how much of it a request uses. So set the context length on the
|
||||||
|
server deliberately: it is the ceiling for *prompt plus response* on every
|
||||||
|
request.
|
||||||
|
|
||||||
|
### Prompt processing vs generation
|
||||||
|
|
||||||
|
A request has two phases, and they run at very different speeds:
|
||||||
|
|
||||||
|
- **Prefill** reads the prompt. It is parallel and fast: thousands of tokens
|
||||||
|
per second on a 4090.
|
||||||
|
- **Decode** writes the response one token at a time. Roughly 30–40 tokens per
|
||||||
|
second for a dense 32B at 4-bit, and several times that for a 3B-active MoE.
|
||||||
|
Treat these numbers as ballpark; measure your own.
|
||||||
|
|
||||||
|
An agent loop re-sends the whole transcript on every turn, which makes the
|
||||||
|
prompt long and the responses short (a tool call is often under 200 tokens).
|
||||||
|
So prefill matters more than the headline tokens-per-second suggests.
|
||||||
|
|
||||||
|
**Prefix caching** is the fix. If the start of a prompt matches the previous
|
||||||
|
request, the server reuses the KV cache for it and prefills only the new part.
|
||||||
|
llama.cpp does this per slot, and vLLM does it automatically. For this
|
||||||
|
workload it is often the single largest speed win. It works here because the
|
||||||
|
rendered transcript only grows at the end, so consecutive turns share
|
||||||
|
everything but their newest events.
|
||||||
|
|
||||||
|
### Tokens and characters
|
||||||
|
|
||||||
|
For English and code, a token is roughly **4 characters**. That's useful for
|
||||||
|
converting the character-based knobs below into context cost.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Part 2 — The knobs
|
||||||
|
|
||||||
|
For each knob: what it controls, what goes wrong at either extreme, and what
|
||||||
|
to look at when deciding.
|
||||||
|
|
||||||
|
### `llm.max_output_tokens` — default 16384
|
||||||
|
|
||||||
|
The ceiling on a single response. It is sent with every request, for every
|
||||||
|
role.
|
||||||
|
|
||||||
|
- **Why it exists:** a response that hits the ceiling is truncated
|
||||||
|
mid-sentence, or worse, mid-tool-call, where it can still parse. The agent
|
||||||
|
detects truncation and treats it as an error, but a ceiling that's too
|
||||||
|
tight still costs a failed turn. Planning with a frontier model was observed
|
||||||
|
using over 4,000 output tokens.
|
||||||
|
- **Too high, on a local model:** the server reserves room for the whole
|
||||||
|
response inside the context window. With a 32k context and a 16k ceiling,
|
||||||
|
only 16k is left for the prompt. Some servers reject any request where
|
||||||
|
prompt plus ceiling exceeds the context.
|
||||||
|
- **Too low:** truncated plans and tool calls.
|
||||||
|
- **Deciding:** executor turns are short, so 2–4k is plenty for a local
|
||||||
|
executor. Frontier planning wants the large default. This one knob covers
|
||||||
|
every role for now; a per-model setting is coming with the context budget
|
||||||
|
(see *Coming next*).
|
||||||
|
|
||||||
|
### `tuning.executor.max_turns` — default 24
|
||||||
|
|
||||||
|
How many model calls one milestone may make before the executor stops with
|
||||||
|
`capped` and asks you what to do.
|
||||||
|
|
||||||
|
- **Too low:** milestones that were making progress stop short. You'll see
|
||||||
|
`capped` outcomes on work that was going well.
|
||||||
|
- **Too high:** a confused model keeps going. Every turn also adds its tool
|
||||||
|
calls and results to the transcript, so a long run spends context as well
|
||||||
|
as time.
|
||||||
|
- **Deciding:** look at how many turns *successful* milestones take, and set
|
||||||
|
the cap comfortably above the high end, around 1.5–2× the turns of your
|
||||||
|
longer successful runs. A stronger model usually needs fewer turns, not
|
||||||
|
more.
|
||||||
|
|
||||||
|
### `tuning.executor.failure_limit` — default 2
|
||||||
|
|
||||||
|
How many consecutive failed `run_command` calls trigger escalation: the
|
||||||
|
sub-task is handed to a frontier model as a delegation (spec A12). A
|
||||||
|
successful command resets the count. Failed file reads don't count, since
|
||||||
|
reading is looking around, not doing the work.
|
||||||
|
|
||||||
|
- **Why it's a count and not a judgment:** small models are poorly calibrated
|
||||||
|
about their own competence. Asked "are you stuck?", they say no until far
|
||||||
|
too late, or yes constantly. A count of observable failures is a signal
|
||||||
|
that doesn't depend on the model's opinion of itself.
|
||||||
|
- **Too low:** normal iteration escalates. Test-first work *expects* a failing
|
||||||
|
test run before the fix, so a limit of 1 hands off routine work and spends
|
||||||
|
frontier tokens on it.
|
||||||
|
- **Too high:** a weak model circles, re-running the same failing command
|
||||||
|
with small variations, while the transcript fills up.
|
||||||
|
- **Deciding:** raise it as local models get stronger. Watch two things
|
||||||
|
together. First, how often delegations fix what the local model couldn't
|
||||||
|
(escalation is doing its job). Second, how often the local model recovers
|
||||||
|
on its own after its second failure (the limit may be too low).
|
||||||
|
|
||||||
|
### `tuning.expert.consult_budget` — default 3
|
||||||
|
|
||||||
|
How many `ask_expert` calls one milestone may make. `ask_expert` is a single
|
||||||
|
stateless question to a stronger model, with no access to the repository
|
||||||
|
(spec A14), for things like "is this API deprecated?".
|
||||||
|
|
||||||
|
- **Too low:** the executor can't get unstuck on a knowledge question it
|
||||||
|
can't answer from the code.
|
||||||
|
- **Too high:** a confused executor asks variations of the same question
|
||||||
|
instead of doing the work or failing. That's the circling A12 warns about.
|
||||||
|
- **Deciding:** count consults per milestone, and check whether the answers
|
||||||
|
changed what the executor did next. If it regularly exhausts the budget
|
||||||
|
without progress, the budget is fine and the model or the plan is the
|
||||||
|
problem.
|
||||||
|
|
||||||
|
### `tuning.delegation` — hard caps on one delegation
|
||||||
|
|
||||||
|
A delegation is a frontier model working a sub-task with full tool access
|
||||||
|
(spec A13). When a cap is hit, the delegation **fails**, and its branch and
|
||||||
|
work are discarded, because resuming on a half-edited tree isn't safe.
|
||||||
|
|
||||||
|
| Knob | Default | Caps |
|
||||||
|
|---|---|---|
|
||||||
|
| `max_turns` | 30 | model calls |
|
||||||
|
| `max_tokens` | 200000 | input + output tokens, as reported by the provider |
|
||||||
|
| `wall_clock_s` | 900 | elapsed time |
|
||||||
|
|
||||||
|
- **`max_tokens` is a scope limit, not the money limit.** Spending limits live
|
||||||
|
on the LiteLLM key (spec A5). This cap stops one delegation from growing
|
||||||
|
without bound. An unbounded one could still exhaust a daily budget and take
|
||||||
|
every frontier model away until it resets.
|
||||||
|
- **Too low:** delegations fail just short of finishing and their work is
|
||||||
|
thrown away. That is the most expensive way to be wrong, because you paid
|
||||||
|
for the work.
|
||||||
|
- **Too high:** a stuck delegation burns tokens and time before failing.
|
||||||
|
- **Deciding:** every delegation records the tokens it used and how long it
|
||||||
|
ran (the `delegation_completed` event, captured for evaluation per spec
|
||||||
|
A16). Set each cap above what successful delegations actually use, with
|
||||||
|
room to spare.
|
||||||
|
|
||||||
|
### `tuning.tools.command_timeout_s` — default 120
|
||||||
|
|
||||||
|
How long one `run_command` may run before its whole process group is killed.
|
||||||
|
|
||||||
|
- **Too low:** slow test suites, dependency installs and Docker builds get
|
||||||
|
killed and reported as failures. That also counts toward `failure_limit`.
|
||||||
|
- **Too high:** a command that never exits blocks the turn for the whole
|
||||||
|
timeout, such as a dev server started in the foreground or a test waiting on
|
||||||
|
input. `stop` waits for a running command to finish too, so this is also the
|
||||||
|
worst-case delay before a stop takes effect.
|
||||||
|
- **Deciding:** set it just above the slowest legitimate command in the
|
||||||
|
projects the agent works on.
|
||||||
|
|
||||||
|
### `tuning.tools.max_output_chars` — default 4000
|
||||||
|
|
||||||
|
How much of a tool's output (command output, file contents, directory
|
||||||
|
listings) the model sees. Anything past this is cut, with a note saying how
|
||||||
|
much was dropped.
|
||||||
|
|
||||||
|
- **This is a context knob.** Every tool result stays in the transcript and is
|
||||||
|
re-sent every turn. 4,000 characters is about 1,000 tokens, so 20 turns of
|
||||||
|
maximal output is about 20k tokens, which is already most of a local
|
||||||
|
model's context.
|
||||||
|
- **Too low:** the model doesn't see the part of the output that matters.
|
||||||
|
- **Too high:** the transcript outgrows the context window sooner.
|
||||||
|
- **A known weakness:** truncation keeps the *start* of the output, but test
|
||||||
|
runners put their failure summary at the *end*. A long, failing pytest run
|
||||||
|
can lose exactly the lines the model needed. Keeping the head and the tail
|
||||||
|
would fix this, and is a candidate for the context-budget work.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Part 3 — An order to tune in
|
||||||
|
|
||||||
|
Upstream choices change what's right downstream, so tune in this order and
|
||||||
|
change one thing at a time.
|
||||||
|
|
||||||
|
1. **Model and quantization.** Start with whether a candidate holds the
|
||||||
|
tool-call contract at all:
|
||||||
|
|
||||||
|
uv run pytest -m integration tests/integration/test_model_capability.py \
|
||||||
|
--capability-models=alias-a,alias-b
|
||||||
|
|
||||||
|
A model that falls back to prose instead of calling tools is disqualified,
|
||||||
|
however good its benchmarks look. Before comparing aliases, check what
|
||||||
|
each one actually points at in LiteLLM (`/model/info`, the
|
||||||
|
`litellm_params.model` field). Two aliases believed to be the same model
|
||||||
|
once weren't, and that cost a lot of debugging.
|
||||||
|
2. **Context length and KV-cache precision** on the server. Take the largest
|
||||||
|
context that fits beside the weights with 1–2 GB of headroom. Try an 8-bit
|
||||||
|
KV cache before giving up context.
|
||||||
|
3. **`llm.max_output_tokens`**, so the prompt keeps most of that context.
|
||||||
|
4. **Real milestones.** Run representative work and look at the evidence:
|
||||||
|
turns used, escalations, delegation outcomes, and time taken.
|
||||||
|
5. **The loop knobs**: `failure_limit`, `max_turns`, `consult_budget`, the
|
||||||
|
delegation caps, then the tool limits, one at a time.
|
||||||
|
|
||||||
|
**Expect noise.** Local model behaviour varies a lot from run to run. One
|
||||||
|
early comparison measured 12 of 20 runs holding the tool contract against 20
|
||||||
|
of 20, a real difference (p ≈ 0.003). Three runs each could easily have
|
||||||
|
pointed the other way. Before changing a knob because of a result, repeat the
|
||||||
|
run enough times that the result would survive a coin flip.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Coming next
|
||||||
|
|
||||||
|
The next changes in the 4090-readiness work add knobs, and this guide gains a
|
||||||
|
section for each:
|
||||||
|
|
||||||
|
- **A context budget.** Rendering a transcript that fits the model's context
|
||||||
|
window, with what to drop or summarize when it doesn't, and per-model output
|
||||||
|
ceilings.
|
||||||
|
- **Enforced tool calls.** An explicit `finish` tool, so the server can be told
|
||||||
|
a tool call is required and hold the model to the contract, where today it
|
||||||
|
can fall back to prose.
|
||||||
|
- **A benchmark script** that runs the capability matrix and representative
|
||||||
|
milestones across candidate models and prints a comparison table.
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
# Example configuration for nisse-dev. Copy to nisse-dev.yaml (or point
|
||||||
|
# NISSE_CONFIG at your copy) and edit.
|
||||||
|
#
|
||||||
|
# - Structure lives here. Secrets do not: write them as ${VARIABLE} references,
|
||||||
|
# resolved from the environment (or .env, see .env.example) at startup.
|
||||||
|
# - Keep this file outside the workspace clone. Startup refuses otherwise,
|
||||||
|
# because the agent commits and pushes everything in the clone.
|
||||||
|
# - Unknown keys are errors, so a misspelt knob fails at startup instead of
|
||||||
|
# silently doing nothing.
|
||||||
|
|
||||||
|
state_db: ~/nisse-demo-workspace/state.db
|
||||||
|
workspace: ~/nisse-demo-workspace/project # the clone
|
||||||
|
project: demo
|
||||||
|
|
||||||
|
llm: # A5: LiteLLM is the only LLM destination
|
||||||
|
base_url: http://litellm.example:4000
|
||||||
|
token: ${NISSE_LLM_TOKEN}
|
||||||
|
# Ceiling on each response. Must fit inside a local model's context window
|
||||||
|
# alongside the prompt; see docs/Tuning-Guide.md.
|
||||||
|
max_output_tokens: 16384
|
||||||
|
|
||||||
|
roles: # A11: role -> LiteLLM model alias
|
||||||
|
executor: local-fast
|
||||||
|
planner: claude-opus-5
|
||||||
|
delegate: claude-opus-5
|
||||||
|
consultant: claude-opus-5
|
||||||
|
|
||||||
|
matrix: # A1
|
||||||
|
homeserver: https://matrix.example
|
||||||
|
user_id: "@nisse-dev:example"
|
||||||
|
token: ${NISSE_MATRIX_TOKEN}
|
||||||
|
room_id: "!roomid:example"
|
||||||
|
operators: ["@you:example"]
|
||||||
|
|
||||||
|
gitea: # B2
|
||||||
|
url: https://gitea.example
|
||||||
|
token: ${NISSE_GITEA_TOKEN}
|
||||||
|
repo: owner/demo-repo
|
||||||
|
operators: [your-gitea-login]
|
||||||
|
base_branch: main
|
||||||
|
|
||||||
|
# What each knob does, and how to choose a value: docs/Tuning-Guide.md.
|
||||||
|
# These are the defaults; the section can be left out entirely. The file is
|
||||||
|
# read at startup, so a change takes effect on the next restart.
|
||||||
|
tuning:
|
||||||
|
executor:
|
||||||
|
max_turns: 24 # model calls per milestone before it stops as capped
|
||||||
|
failure_limit: 2 # consecutive failed commands before escalating (A12)
|
||||||
|
expert:
|
||||||
|
consult_budget: 3 # ask_expert calls per milestone (A14)
|
||||||
|
delegation: # hard caps on one delegation (A13)
|
||||||
|
max_turns: 30
|
||||||
|
max_tokens: 200000
|
||||||
|
wall_clock_s: 900
|
||||||
|
tools:
|
||||||
|
command_timeout_s: 120 # one run_command before it is killed
|
||||||
|
max_output_chars: 4000 # tool output kept for the model; the rest is cut
|
||||||
@@ -3,7 +3,11 @@ name = "nisse-dev"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
description = "Dev agent: plans and implements software projects, produces Gitea PRs"
|
description = "Dev agent: plans and implements software projects, produces Gitea PRs"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
dependencies = ["nisse-core", "httpx>=0.27"]
|
dependencies = [
|
||||||
|
"nisse-core",
|
||||||
|
"httpx>=0.27",
|
||||||
|
"pyyaml>=6",
|
||||||
|
]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
nisse-dev = "nisse_dev.main:main"
|
nisse-dev = "nisse_dev.main:main"
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from nisse_dev.cancel import Cancel
|
|||||||
from nisse_dev.catalog import ModelCatalog, select_client
|
from nisse_dev.catalog import ModelCatalog, select_client
|
||||||
from nisse_dev.config import AgentConfig
|
from nisse_dev.config import AgentConfig
|
||||||
from nisse_dev.control import Orchestrator
|
from nisse_dev.control import Orchestrator
|
||||||
from nisse_dev.delegation import Delegation
|
from nisse_dev.delegation import Caps, Delegation
|
||||||
from nisse_dev.executor import LLMExecutor
|
from nisse_dev.executor import LLMExecutor
|
||||||
from nisse_dev.expert import ExpertDesk
|
from nisse_dev.expert import ExpertDesk
|
||||||
from nisse_dev.git import GitRepo, authenticated_url
|
from nisse_dev.git import GitRepo, authenticated_url
|
||||||
@@ -149,18 +149,29 @@ def build_agent(
|
|||||||
account = gitea.whoami()
|
account = gitea.whoami()
|
||||||
|
|
||||||
git_repo = prepare_workspace(config, account)
|
git_repo = prepare_workspace(config, account)
|
||||||
workspace = Workspace(config.workspace)
|
tuning = config.tuning
|
||||||
|
workspace = Workspace(
|
||||||
|
config.workspace,
|
||||||
|
timeout_s=tuning.tools.command_timeout_s,
|
||||||
|
max_output_chars=tuning.tools.max_output_chars,
|
||||||
|
)
|
||||||
|
|
||||||
normalized = LiteLLMClient(config.llm_base_url, config.llm_token)
|
normalized = LiteLLMClient(
|
||||||
|
config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
|
||||||
|
)
|
||||||
# Delegations prefer the vendor-native route; fidelity is cheap
|
# Delegations prefer the vendor-native route; fidelity is cheap
|
||||||
# elsewhere (A5). Which route a given model gets is decided by
|
# elsewhere (A5). Which route a given model gets is decided by
|
||||||
# select_client, not by where the caller happens to sit.
|
# select_client, not by where the caller happens to sit.
|
||||||
passthrough = AnthropicPassthroughClient(config.llm_base_url, config.llm_token)
|
passthrough = AnthropicPassthroughClient(
|
||||||
|
config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
|
||||||
|
)
|
||||||
|
|
||||||
transport = MatrixTransport(config.matrix)
|
transport = MatrixTransport(config.matrix)
|
||||||
announce = notify or transport.send_message
|
announce = notify or transport.send_message
|
||||||
|
|
||||||
expert = ExpertDesk(normalized, store, config.roles)
|
expert = ExpertDesk(
|
||||||
|
normalized, store, config.roles, budget=tuning.expert.consult_budget
|
||||||
|
)
|
||||||
delegate_model = config.roles.resolve("delegate")
|
delegate_model = config.roles.resolve("delegate")
|
||||||
delegation = Delegation(
|
delegation = Delegation(
|
||||||
select_client(delegate_model, normalized, passthrough, catalog),
|
select_client(delegate_model, normalized, passthrough, catalog),
|
||||||
@@ -169,6 +180,11 @@ def build_agent(
|
|||||||
git_repo,
|
git_repo,
|
||||||
model=delegate_model,
|
model=delegate_model,
|
||||||
backing_model=catalog.facts(delegate_model).backing,
|
backing_model=catalog.facts(delegate_model).backing,
|
||||||
|
caps=Caps(
|
||||||
|
turns=tuning.delegation.max_turns,
|
||||||
|
tokens=tuning.delegation.max_tokens,
|
||||||
|
wall_clock_s=tuning.delegation.wall_clock_s,
|
||||||
|
),
|
||||||
cancel=cancel,
|
cancel=cancel,
|
||||||
)
|
)
|
||||||
executor = LLMExecutor(
|
executor = LLMExecutor(
|
||||||
@@ -180,6 +196,8 @@ def build_agent(
|
|||||||
delegate=delegation,
|
delegate=delegation,
|
||||||
notify=announce,
|
notify=announce,
|
||||||
cancel=cancel,
|
cancel=cancel,
|
||||||
|
max_turns=tuning.executor.max_turns,
|
||||||
|
failure_limit=tuning.executor.failure_limit,
|
||||||
)
|
)
|
||||||
publisher = MilestonePublisher(
|
publisher = MilestonePublisher(
|
||||||
store,
|
store,
|
||||||
|
|||||||
@@ -121,8 +121,8 @@ class ModelCatalog:
|
|||||||
|
|
||||||
Reported together rather than one per boot: an operator fixing a
|
Reported together rather than one per boot: an operator fixing a
|
||||||
config wants the whole list, not to rediscover the next typo on the
|
config wants the whole list, not to rediscover the next typo on the
|
||||||
next start (the same reason ``AgentConfig.from_env`` batches its
|
next start (the same reason ``AgentConfig`` reports every missing
|
||||||
missing variables).
|
setting at once).
|
||||||
"""
|
"""
|
||||||
unknown = {
|
unknown = {
|
||||||
role: alias for role, alias in roles.items() if alias not in self._facts
|
role: alias for role, alias in roles.items() if alias not in self._facts
|
||||||
|
|||||||
@@ -1,49 +1,198 @@
|
|||||||
"""Agent configuration (spec A5: from env/.env, nothing hardcoded).
|
"""Agent configuration (spec A5: nothing hardcoded).
|
||||||
|
|
||||||
|
Configuration is a YAML file: roles, connections, and the tuning knobs (see
|
||||||
|
docs/Tuning-Guide.md). It holds no secrets. A credential is written as a
|
||||||
|
reference, ``token: ${NISSE_LLM_TOKEN}``, and resolved from the environment,
|
||||||
|
so the file can be reviewed, shared, or committed without leaking. The
|
||||||
|
environment supplies those references and nothing else: every setting has
|
||||||
|
exactly one home.
|
||||||
|
|
||||||
Every value is collected and validated **before** anything is built, and a
|
Every value is collected and validated **before** anything is built, and a
|
||||||
single error names every missing variable at once. The alternative — failing
|
single error names every problem at once. The alternative — failing at first
|
||||||
at first use — surfaces a missing token twenty minutes into a milestone, on
|
use — surfaces a missing token twenty minutes into a milestone, on the call
|
||||||
the call that mattered.
|
that mattered. Unknown keys are errors too: a misspelt tuning knob that was
|
||||||
|
silently ignored would look exactly like a knob that has no effect.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, field, fields
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from nisse_dev.llm import RoleMap
|
import yaml
|
||||||
|
|
||||||
|
from nisse_dev.llm import DEFAULT_MAX_OUTPUT_TOKENS, RoleMap
|
||||||
from nisse_dev.matrix import MatrixConfig
|
from nisse_dev.matrix import MatrixConfig
|
||||||
|
|
||||||
REQUIRED = (
|
DEFAULT_CONFIG_FILE = "nisse-dev.yaml"
|
||||||
"NISSE_STATE_DB",
|
|
||||||
"NISSE_WORKSPACE",
|
REQUIRED_KEYS = (
|
||||||
"NISSE_PROJECT",
|
"state_db",
|
||||||
"NISSE_LLM_BASE_URL",
|
"workspace",
|
||||||
"NISSE_LLM_TOKEN",
|
"project",
|
||||||
"NISSE_ROLE_EXECUTOR",
|
"llm.base_url",
|
||||||
"NISSE_ROLE_PLANNER",
|
"llm.token",
|
||||||
"NISSE_ROLE_DELEGATE",
|
"roles.executor",
|
||||||
"NISSE_ROLE_CONSULTANT",
|
"roles.planner",
|
||||||
"NISSE_MATRIX_HOMESERVER",
|
"roles.delegate",
|
||||||
"NISSE_MATRIX_USER_ID",
|
"roles.consultant",
|
||||||
"NISSE_MATRIX_TOKEN",
|
"matrix.homeserver",
|
||||||
"NISSE_MATRIX_ROOM_ID",
|
"matrix.user_id",
|
||||||
"NISSE_MATRIX_OPERATORS",
|
"matrix.token",
|
||||||
"NISSE_GITEA_URL",
|
"matrix.room_id",
|
||||||
"NISSE_GITEA_TOKEN",
|
"matrix.operators",
|
||||||
"NISSE_GITEA_REPO",
|
"gitea.url",
|
||||||
"NISSE_GITEA_OPERATORS",
|
"gitea.token",
|
||||||
|
"gitea.repo",
|
||||||
|
"gitea.operators",
|
||||||
)
|
)
|
||||||
|
OPTIONAL_KEYS = ("llm.max_output_tokens", "gitea.base_branch")
|
||||||
|
|
||||||
|
_REFERENCE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
||||||
|
|
||||||
|
|
||||||
class ConfigError(RuntimeError):
|
class ConfigError(RuntimeError):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _people(raw: str) -> frozenset[str]:
|
# --- tuning ----------------------------------------------------------------
|
||||||
"""Operator allowlists are comma-separated; blanks are dropped so a
|
# The defaults are the values the agent ran with before they were
|
||||||
trailing comma cannot admit an empty identity."""
|
# configurable, so an absent `tuning:` section changes nothing.
|
||||||
return frozenset(part.strip() for part in raw.split(",") if part.strip())
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ExecutorTuning:
|
||||||
|
max_turns: int = 24
|
||||||
|
failure_limit: int = 2
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ExpertTuning:
|
||||||
|
consult_budget: int = 3
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DelegationTuning:
|
||||||
|
max_turns: int = 30
|
||||||
|
max_tokens: int = 200_000
|
||||||
|
wall_clock_s: float = 900.0
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ToolsTuning:
|
||||||
|
command_timeout_s: int = 120
|
||||||
|
max_output_chars: int = 4000
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Tuning:
|
||||||
|
executor: ExecutorTuning = field(default_factory=ExecutorTuning)
|
||||||
|
expert: ExpertTuning = field(default_factory=ExpertTuning)
|
||||||
|
delegation: DelegationTuning = field(default_factory=DelegationTuning)
|
||||||
|
tools: ToolsTuning = field(default_factory=ToolsTuning)
|
||||||
|
|
||||||
|
|
||||||
|
_TUNING_SECTIONS: dict[str, type] = {
|
||||||
|
f.name: f.default_factory # type: ignore[misc]
|
||||||
|
for f in fields(Tuning)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_tuning(raw: Any, errors: list[str]) -> Tuning:
|
||||||
|
if raw is None:
|
||||||
|
return Tuning()
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
errors.append("tuning: must be a mapping")
|
||||||
|
return Tuning()
|
||||||
|
sections: dict[str, Any] = {}
|
||||||
|
for name, section in raw.items():
|
||||||
|
cls = _TUNING_SECTIONS.get(name)
|
||||||
|
if cls is None:
|
||||||
|
errors.append(f"tuning.{name}: unknown section")
|
||||||
|
continue
|
||||||
|
if not isinstance(section, dict):
|
||||||
|
errors.append(f"tuning.{name}: must be a mapping")
|
||||||
|
continue
|
||||||
|
known = {f.name: f for f in fields(cls)}
|
||||||
|
values: dict[str, Any] = {}
|
||||||
|
for key, value in section.items():
|
||||||
|
spec = known.get(key)
|
||||||
|
if spec is None:
|
||||||
|
errors.append(f"tuning.{name}.{key}: unknown setting")
|
||||||
|
continue
|
||||||
|
number = _positive(value, float if spec.type in (float, "float") else int)
|
||||||
|
if number is None:
|
||||||
|
errors.append(f"tuning.{name}.{key}: must be a positive number")
|
||||||
|
continue
|
||||||
|
values[key] = number
|
||||||
|
sections[name] = cls(**values)
|
||||||
|
return Tuning(**sections)
|
||||||
|
|
||||||
|
|
||||||
|
def _positive(value: Any, kind: type) -> int | float | None:
|
||||||
|
# bool is an int in Python; `true` for a turn limit is a mistake.
|
||||||
|
if isinstance(value, bool) or not isinstance(value, int | float):
|
||||||
|
return None
|
||||||
|
if kind is int and not float(value).is_integer():
|
||||||
|
return None
|
||||||
|
if value <= 0:
|
||||||
|
return None
|
||||||
|
return kind(value)
|
||||||
|
|
||||||
|
|
||||||
|
# --- loading ---------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _people(raw: Any) -> frozenset[str]:
|
||||||
|
"""Operator allowlists: a YAML list, or comma-separated in the
|
||||||
|
environment. Blanks are dropped so a trailing comma cannot admit an empty
|
||||||
|
identity."""
|
||||||
|
parts = raw if isinstance(raw, list) else str(raw).split(",")
|
||||||
|
return frozenset(str(part).strip() for part in parts if str(part).strip())
|
||||||
|
|
||||||
|
|
||||||
|
def _flatten(tree: Mapping[str, Any], prefix: str = "") -> dict[str, Any]:
|
||||||
|
flat: dict[str, Any] = {}
|
||||||
|
for key, value in tree.items():
|
||||||
|
dotted = f"{prefix}{key}"
|
||||||
|
if isinstance(value, dict) and dotted != "tuning":
|
||||||
|
flat.update(_flatten(value, f"{dotted}."))
|
||||||
|
else:
|
||||||
|
flat[dotted] = value
|
||||||
|
return flat
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve(value: Any, key: str, env: Mapping[str, str], errors: list[str]) -> Any:
|
||||||
|
"""Replace ${VAR} references with the environment's value. A reference to
|
||||||
|
an unset variable is an error, never an empty string: an empty token would
|
||||||
|
pass for a value and fail at the first request."""
|
||||||
|
if not isinstance(value, str):
|
||||||
|
return value
|
||||||
|
|
||||||
|
def lookup(match: re.Match[str]) -> str:
|
||||||
|
name = match.group(1)
|
||||||
|
found = (env.get(name) or "").strip()
|
||||||
|
if not found:
|
||||||
|
errors.append(f"{key}: references ${{{name}}}, which is unset or empty")
|
||||||
|
return found
|
||||||
|
|
||||||
|
return _REFERENCE.sub(lookup, value)
|
||||||
|
|
||||||
|
|
||||||
|
def _read_file(path: Path, errors: list[str]) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
loaded = yaml.safe_load(path.read_text())
|
||||||
|
except yaml.YAMLError as exc:
|
||||||
|
errors.append(f"{path}: not valid YAML ({exc})")
|
||||||
|
return {}
|
||||||
|
if loaded is None:
|
||||||
|
return {}
|
||||||
|
if not isinstance(loaded, dict):
|
||||||
|
errors.append(f"{path}: must be a mapping at the top level")
|
||||||
|
return {}
|
||||||
|
return loaded
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -61,30 +210,107 @@ class AgentConfig:
|
|||||||
gitea_repo: str
|
gitea_repo: str
|
||||||
gitea_operators: frozenset[str]
|
gitea_operators: frozenset[str]
|
||||||
base_branch: str = "main"
|
base_branch: str = "main"
|
||||||
|
max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS
|
||||||
|
tuning: Tuning = field(default_factory=Tuning)
|
||||||
|
config_file: Path | None = None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_env(cls, env: Mapping[str, str] = os.environ) -> "AgentConfig":
|
def load(cls, env: Mapping[str, str] = os.environ) -> "AgentConfig":
|
||||||
|
"""The file named by NISSE_CONFIG, or ./nisse-dev.yaml."""
|
||||||
|
path = Path(env.get("NISSE_CONFIG") or DEFAULT_CONFIG_FILE).expanduser()
|
||||||
|
if not path.is_file():
|
||||||
|
raise ConfigError(
|
||||||
|
f"cannot start: no config file at {path}; create it from "
|
||||||
|
"nisse-dev.example.yaml, or point NISSE_CONFIG at one"
|
||||||
|
)
|
||||||
|
return cls.from_file(path, env)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_file(
|
||||||
|
cls, path: Path, env: Mapping[str, str] = os.environ
|
||||||
|
) -> "AgentConfig":
|
||||||
|
errors: list[str] = []
|
||||||
|
tree = _read_file(path, errors)
|
||||||
|
return cls._build(tree, env, errors, config_file=path)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_tree(
|
||||||
|
cls, tree: Mapping[str, Any], env: Mapping[str, str] = os.environ
|
||||||
|
) -> "AgentConfig":
|
||||||
|
"""From an already-parsed config, as the file would give it."""
|
||||||
|
return cls._build(tree, env, [], config_file=None)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _build(
|
||||||
|
cls,
|
||||||
|
tree: Mapping[str, Any],
|
||||||
|
env: Mapping[str, str],
|
||||||
|
errors: list[str],
|
||||||
|
*,
|
||||||
|
config_file: Path | None,
|
||||||
|
) -> "AgentConfig":
|
||||||
|
tuning = _parse_tuning(tree.get("tuning"), errors)
|
||||||
|
flat = _flatten(tree)
|
||||||
|
flat.pop("tuning", None)
|
||||||
|
|
||||||
|
known = set(REQUIRED_KEYS) | set(OPTIONAL_KEYS)
|
||||||
|
for key in sorted(set(flat) - known):
|
||||||
|
errors.append(f"{key}: unknown setting")
|
||||||
|
values = {
|
||||||
|
key: _resolve(value, key, env, errors)
|
||||||
|
for key, value in flat.items()
|
||||||
|
if key in known
|
||||||
|
}
|
||||||
|
|
||||||
# Whitespace-only counts as missing: a token of spaces would pass a
|
# Whitespace-only counts as missing: a token of spaces would pass a
|
||||||
# truthiness check and fail at the first request instead.
|
# truthiness check and fail at the first request instead.
|
||||||
missing = [name for name in REQUIRED if not (env.get(name) or "").strip()]
|
missing = [
|
||||||
|
key
|
||||||
|
for key in REQUIRED_KEYS
|
||||||
|
if values.get(key) is None
|
||||||
|
or (isinstance(values[key], str) and not values[key].strip())
|
||||||
|
or (isinstance(values[key], list) and not _people(values[key]))
|
||||||
|
]
|
||||||
if missing:
|
if missing:
|
||||||
raise ConfigError(
|
errors.append("unset or empty — " + ", ".join(missing))
|
||||||
"cannot start: these are unset or empty — " + ", ".join(missing)
|
|
||||||
)
|
max_output = values.get("llm.max_output_tokens", DEFAULT_MAX_OUTPUT_TOKENS)
|
||||||
|
try:
|
||||||
|
max_output = int(max_output)
|
||||||
|
if max_output <= 0:
|
||||||
|
raise ValueError
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
errors.append("llm.max_output_tokens: must be a positive integer")
|
||||||
|
if errors:
|
||||||
|
raise ConfigError("cannot start: " + "; ".join(errors))
|
||||||
|
|
||||||
config = cls(
|
config = cls(
|
||||||
state_db=Path(env["NISSE_STATE_DB"]).expanduser(),
|
state_db=Path(values["state_db"]).expanduser(),
|
||||||
workspace=Path(env["NISSE_WORKSPACE"]).expanduser(),
|
workspace=Path(values["workspace"]).expanduser(),
|
||||||
project=env["NISSE_PROJECT"],
|
project=str(values["project"]),
|
||||||
llm_base_url=env["NISSE_LLM_BASE_URL"],
|
llm_base_url=str(values["llm.base_url"]),
|
||||||
llm_token=env["NISSE_LLM_TOKEN"],
|
llm_token=str(values["llm.token"]),
|
||||||
roles=RoleMap.from_env(env),
|
roles=RoleMap(
|
||||||
matrix=MatrixConfig.from_env(env),
|
executor=str(values["roles.executor"]),
|
||||||
operators=_people(env["NISSE_MATRIX_OPERATORS"]),
|
planner=str(values["roles.planner"]),
|
||||||
gitea_url=env["NISSE_GITEA_URL"],
|
delegate=str(values["roles.delegate"]),
|
||||||
gitea_token=env["NISSE_GITEA_TOKEN"],
|
consultant=str(values["roles.consultant"]),
|
||||||
gitea_repo=env["NISSE_GITEA_REPO"],
|
),
|
||||||
gitea_operators=_people(env["NISSE_GITEA_OPERATORS"]),
|
matrix=MatrixConfig(
|
||||||
base_branch=env.get("NISSE_BASE_BRANCH", "main"),
|
homeserver=str(values["matrix.homeserver"]).rstrip("/"),
|
||||||
|
user_id=str(values["matrix.user_id"]),
|
||||||
|
token=str(values["matrix.token"]),
|
||||||
|
room_id=str(values["matrix.room_id"]),
|
||||||
|
),
|
||||||
|
operators=_people(values["matrix.operators"]),
|
||||||
|
gitea_url=str(values["gitea.url"]),
|
||||||
|
gitea_token=str(values["gitea.token"]),
|
||||||
|
gitea_repo=str(values["gitea.repo"]),
|
||||||
|
gitea_operators=_people(values["gitea.operators"]),
|
||||||
|
base_branch=str(values.get("gitea.base_branch") or "main"),
|
||||||
|
max_output_tokens=max_output,
|
||||||
|
tuning=tuning,
|
||||||
|
config_file=config_file,
|
||||||
)
|
)
|
||||||
config._refuse_secrets_inside_the_clone(env)
|
config._refuse_secrets_inside_the_clone(env)
|
||||||
return config
|
return config
|
||||||
@@ -93,14 +319,20 @@ class AgentConfig:
|
|||||||
"""The workspace is a git clone the agent commits with ``git add -A``
|
"""The workspace is a git clone the agent commits with ``git add -A``
|
||||||
and pushes. Anything inside it reaches the project repository, so a
|
and pushes. Anything inside it reaches the project repository, so a
|
||||||
state store or an env file placed there would publish credentials as a
|
state store or an env file placed there would publish credentials as a
|
||||||
side effect of the agent working correctly — silently, and once.
|
side effect of the agent working correctly — silently, and once. The
|
||||||
|
config file holds no secrets itself, but it names where they come
|
||||||
|
from and how the agent is deployed, which is not the project's
|
||||||
|
business either.
|
||||||
"""
|
"""
|
||||||
workspace = self.workspace.expanduser().resolve()
|
workspace = self.workspace.expanduser().resolve()
|
||||||
offenders = []
|
candidates = [
|
||||||
for label, path in (
|
("state_db", self.state_db),
|
||||||
("NISSE_STATE_DB", self.state_db),
|
|
||||||
("NISSE_ENV", Path(env.get("NISSE_ENV", ".env"))),
|
("NISSE_ENV", Path(env.get("NISSE_ENV", ".env"))),
|
||||||
):
|
]
|
||||||
|
if self.config_file is not None:
|
||||||
|
candidates.append(("NISSE_CONFIG", self.config_file))
|
||||||
|
offenders = []
|
||||||
|
for label, path in candidates:
|
||||||
resolved = path.expanduser().resolve()
|
resolved = path.expanduser().resolve()
|
||||||
if resolved == workspace or workspace in resolved.parents:
|
if resolved == workspace or workspace in resolved.parents:
|
||||||
offenders.append(f"{label} ({resolved})")
|
offenders.append(f"{label} ({resolved})")
|
||||||
|
|||||||
@@ -10,8 +10,6 @@ idempotent across a crash: boot reconciliation (A3) asks Gitea whether the
|
|||||||
effect already happened rather than repeating it.
|
effect already happened rather than repeating it.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
|
||||||
from collections.abc import Mapping
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -60,14 +58,6 @@ class GiteaClient:
|
|||||||
transport=transport,
|
transport=transport,
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_env(cls, env: Mapping[str, str] = os.environ) -> "GiteaClient":
|
|
||||||
base_url = env.get("NISSE_GITEA_URL")
|
|
||||||
token = env.get("NISSE_GITEA_TOKEN")
|
|
||||||
if not base_url or not token:
|
|
||||||
raise GiteaError("set NISSE_GITEA_URL and NISSE_GITEA_TOKEN")
|
|
||||||
return cls(base_url, token)
|
|
||||||
|
|
||||||
def _request(self, method: str, path: str, **kwargs: Any) -> Any:
|
def _request(self, method: str, path: str, **kwargs: Any) -> Any:
|
||||||
try:
|
try:
|
||||||
response = self._http.request(method, path, **kwargs)
|
response = self._http.request(method, path, **kwargs)
|
||||||
|
|||||||
@@ -2,18 +2,16 @@
|
|||||||
|
|
||||||
One client interface, present from the first real LLM call. All traffic goes
|
One client interface, present from the first real LLM call. All traffic goes
|
||||||
to the LiteLLM proxy — the VM's single LLM destination; the agent holds one
|
to the LiteLLM proxy — the VM's single LLM destination; the agent holds one
|
||||||
LiteLLM token and no vendor credentials. Config comes from the environment
|
LiteLLM token and no vendor credentials. The base URL and token come from the
|
||||||
(``NISSE_LLM_BASE_URL``, ``NISSE_LLM_TOKEN``), nothing hardcoded.
|
agent's config (nisse_dev.config), nothing hardcoded.
|
||||||
|
|
||||||
Model identity is expressed as roles (A11), resolved to concrete model strings
|
Model identity is expressed as roles (A11), resolved to concrete model strings
|
||||||
from env config; the resolved mapping is pinned at task start so a mid-task
|
from config; the resolved mapping is pinned at task start so a mid-task
|
||||||
config change cannot split a task across two planners.
|
config change cannot split a task across two planners.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
|
||||||
import time
|
import time
|
||||||
from collections.abc import Mapping
|
|
||||||
from dataclasses import asdict, dataclass
|
from dataclasses import asdict, dataclass
|
||||||
from typing import Any, ClassVar, Protocol
|
from typing import Any, ClassVar, Protocol
|
||||||
|
|
||||||
@@ -49,11 +47,6 @@ RATE_LIMIT_RETRY_S = 20.0
|
|||||||
DEFAULT_MAX_OUTPUT_TOKENS = 16384
|
DEFAULT_MAX_OUTPUT_TOKENS = 16384
|
||||||
|
|
||||||
|
|
||||||
def max_output_tokens(env: Mapping[str, str] = os.environ) -> int:
|
|
||||||
raw = env.get("NISSE_MAX_OUTPUT_TOKENS")
|
|
||||||
return int(raw) if raw else DEFAULT_MAX_OUTPUT_TOKENS
|
|
||||||
|
|
||||||
|
|
||||||
class UpstreamUnavailable(LLMError):
|
class UpstreamUnavailable(LLMError):
|
||||||
"""The proxy or provider could not be reached, or refused for reasons
|
"""The proxy or provider could not be reached, or refused for reasons
|
||||||
unrelated to what was asked: a network blip, a timeout, a 5xx, a rate
|
unrelated to what was asked: a network blip, a timeout, a 5xx, a rate
|
||||||
@@ -113,22 +106,10 @@ class RoleMap:
|
|||||||
delegate: str
|
delegate: str
|
||||||
consultant: str
|
consultant: str
|
||||||
|
|
||||||
_ENV_VARS: ClassVar[dict[str, str]] = {
|
ROLES: ClassVar[tuple[str, ...]] = ("executor", "planner", "delegate", "consultant")
|
||||||
"executor": "NISSE_ROLE_EXECUTOR",
|
|
||||||
"planner": "NISSE_ROLE_PLANNER",
|
|
||||||
"delegate": "NISSE_ROLE_DELEGATE",
|
|
||||||
"consultant": "NISSE_ROLE_CONSULTANT",
|
|
||||||
}
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_env(cls, env: Mapping[str, str] = os.environ) -> "RoleMap":
|
|
||||||
missing = [var for var in cls._ENV_VARS.values() if not env.get(var)]
|
|
||||||
if missing:
|
|
||||||
raise LLMError(f"role mapping incomplete; set: {', '.join(missing)}")
|
|
||||||
return cls(**{role: env[var] for role, var in cls._ENV_VARS.items()})
|
|
||||||
|
|
||||||
def resolve(self, role: str) -> str:
|
def resolve(self, role: str) -> str:
|
||||||
if role not in self._ENV_VARS:
|
if role not in self.ROLES:
|
||||||
raise LLMError(f"unknown role: {role!r}")
|
raise LLMError(f"unknown role: {role!r}")
|
||||||
return str(getattr(self, role))
|
return str(getattr(self, role))
|
||||||
|
|
||||||
@@ -156,20 +137,12 @@ class LiteLLMClient:
|
|||||||
)
|
)
|
||||||
self._renderer = OpenAIChatRenderer()
|
self._renderer = OpenAIChatRenderer()
|
||||||
self._sleep = sleep
|
self._sleep = sleep
|
||||||
self._max_tokens = max_tokens or max_output_tokens()
|
self._max_tokens = max_tokens or DEFAULT_MAX_OUTPUT_TOKENS
|
||||||
# Set after each call so a caller can meter itself (A13 caps);
|
# Set after each call so a caller can meter itself (A13 caps);
|
||||||
# a side channel rather than a return value, so the LLMClient
|
# a side channel rather than a return value, so the LLMClient
|
||||||
# protocol stays the same for every implementation.
|
# protocol stays the same for every implementation.
|
||||||
self.last_usage: dict[str, int] = {}
|
self.last_usage: dict[str, int] = {}
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_env(cls, env: Mapping[str, str] = os.environ) -> "LiteLLMClient":
|
|
||||||
base_url = env.get("NISSE_LLM_BASE_URL")
|
|
||||||
token = env.get("NISSE_LLM_TOKEN")
|
|
||||||
if not base_url or not token:
|
|
||||||
raise LLMError("set NISSE_LLM_BASE_URL and NISSE_LLM_TOKEN")
|
|
||||||
return cls(base_url, token)
|
|
||||||
|
|
||||||
def complete(
|
def complete(
|
||||||
self,
|
self,
|
||||||
events: list[StoredEvent],
|
events: list[StoredEvent],
|
||||||
@@ -203,7 +176,7 @@ class LiteLLMClient:
|
|||||||
if choice.get("finish_reason") == "length":
|
if choice.get("finish_reason") == "length":
|
||||||
raise TruncatedResponse(
|
raise TruncatedResponse(
|
||||||
f"{requested_model} hit its output limit mid-answer; "
|
f"{requested_model} hit its output limit mid-answer; "
|
||||||
"raise NISSE_MAX_OUTPUT_TOKENS"
|
"raise llm.max_output_tokens"
|
||||||
)
|
)
|
||||||
produced_by = data.get("model") or requested_model
|
produced_by = data.get("model") or requested_model
|
||||||
usage = data.get("usage") or {}
|
usage = data.get("usage") or {}
|
||||||
@@ -275,20 +248,10 @@ class AnthropicPassthroughClient:
|
|||||||
transport=transport,
|
transport=transport,
|
||||||
)
|
)
|
||||||
self._renderer = AnthropicMessagesRenderer()
|
self._renderer = AnthropicMessagesRenderer()
|
||||||
self._max_tokens = max_tokens or max_output_tokens()
|
self._max_tokens = max_tokens or DEFAULT_MAX_OUTPUT_TOKENS
|
||||||
self._sleep = sleep
|
self._sleep = sleep
|
||||||
self.last_usage: dict[str, int] = {}
|
self.last_usage: dict[str, int] = {}
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_env(
|
|
||||||
cls, env: Mapping[str, str] = os.environ
|
|
||||||
) -> "AnthropicPassthroughClient":
|
|
||||||
base_url = env.get("NISSE_LLM_BASE_URL")
|
|
||||||
token = env.get("NISSE_LLM_TOKEN")
|
|
||||||
if not base_url or not token:
|
|
||||||
raise LLMError("set NISSE_LLM_BASE_URL and NISSE_LLM_TOKEN")
|
|
||||||
return cls(base_url, token)
|
|
||||||
|
|
||||||
def complete(
|
def complete(
|
||||||
self,
|
self,
|
||||||
events: list[StoredEvent],
|
events: list[StoredEvent],
|
||||||
@@ -315,7 +278,7 @@ class AnthropicPassthroughClient:
|
|||||||
if data.get("stop_reason") == "max_tokens":
|
if data.get("stop_reason") == "max_tokens":
|
||||||
raise TruncatedResponse(
|
raise TruncatedResponse(
|
||||||
f"{requested_model} hit the output limit of {self._max_tokens} "
|
f"{requested_model} hit the output limit of {self._max_tokens} "
|
||||||
f"tokens mid-answer; raise NISSE_MAX_OUTPUT_TOKENS"
|
f"tokens mid-answer; raise llm.max_output_tokens"
|
||||||
)
|
)
|
||||||
content = data.get("content")
|
content = data.get("content")
|
||||||
if not isinstance(content, list):
|
if not isinstance(content, list):
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ from nisse_dev.envfile import load_env_file
|
|||||||
def main() -> int:
|
def main() -> int:
|
||||||
load_env_file(Path(os.environ.get("NISSE_ENV", ".env")).expanduser())
|
load_env_file(Path(os.environ.get("NISSE_ENV", ".env")).expanduser())
|
||||||
try:
|
try:
|
||||||
config = AgentConfig.from_env()
|
config = AgentConfig.load()
|
||||||
except ConfigError as exc:
|
except ConfigError as exc:
|
||||||
# Fail here rather than twenty minutes into a milestone on the one
|
# Fail here rather than twenty minutes into a milestone on the one
|
||||||
# call that needed the missing value.
|
# call that needed the missing value.
|
||||||
|
|||||||
@@ -11,12 +11,10 @@ loop backs off (capped) and retries indefinitely — an outage delays reactions
|
|||||||
but never fails anything.
|
but never fails anything.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
|
||||||
import threading
|
import threading
|
||||||
import uuid
|
import uuid
|
||||||
from collections.abc import Callable, Mapping
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import ClassVar
|
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
@@ -34,29 +32,6 @@ class MatrixConfig:
|
|||||||
token: str
|
token: str
|
||||||
room_id: str
|
room_id: str
|
||||||
|
|
||||||
_ENV: ClassVar[tuple[str, ...]] = (
|
|
||||||
"NISSE_MATRIX_HOMESERVER",
|
|
||||||
"NISSE_MATRIX_USER_ID",
|
|
||||||
"NISSE_MATRIX_TOKEN",
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_env(
|
|
||||||
cls,
|
|
||||||
env: Mapping[str, str] = os.environ,
|
|
||||||
room_var: str = "NISSE_MATRIX_ROOM_ID",
|
|
||||||
) -> "MatrixConfig":
|
|
||||||
needed = (*cls._ENV, room_var)
|
|
||||||
missing = [var for var in needed if not env.get(var)]
|
|
||||||
if missing:
|
|
||||||
raise TransportError(f"matrix config incomplete; set: {', '.join(missing)}")
|
|
||||||
return cls(
|
|
||||||
homeserver=env["NISSE_MATRIX_HOMESERVER"].rstrip("/"),
|
|
||||||
user_id=env["NISSE_MATRIX_USER_ID"],
|
|
||||||
token=env["NISSE_MATRIX_TOKEN"],
|
|
||||||
room_id=env[room_var],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class MatrixTransport:
|
class MatrixTransport:
|
||||||
def __init__(
|
def __init__(
|
||||||
|
|||||||
@@ -86,13 +86,6 @@ EXECUTOR_TOOLS: list[dict[str, Any]] = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def _truncate(text: str) -> str:
|
|
||||||
if len(text) <= MAX_OUTPUT_CHARS:
|
|
||||||
return text
|
|
||||||
dropped = len(text) - MAX_OUTPUT_CHARS
|
|
||||||
return f"{text[:MAX_OUTPUT_CHARS]}\n[... {dropped} characters truncated]"
|
|
||||||
|
|
||||||
|
|
||||||
def _child_env() -> dict[str, str]:
|
def _child_env() -> dict[str, str]:
|
||||||
"""The project's build and test code is arbitrary; it has no business
|
"""The project's build and test code is arbitrary; it has no business
|
||||||
reading the agent's LiteLLM, Matrix, or Gitea credentials."""
|
reading the agent's LiteLLM, Matrix, or Gitea credentials."""
|
||||||
@@ -106,10 +99,24 @@ class ToolOutcome:
|
|||||||
|
|
||||||
|
|
||||||
class Workspace:
|
class Workspace:
|
||||||
def __init__(self, root: Path, *, timeout_s: int = DEFAULT_TIMEOUT_S) -> None:
|
def __init__(
|
||||||
|
self,
|
||||||
|
root: Path,
|
||||||
|
*,
|
||||||
|
timeout_s: int = DEFAULT_TIMEOUT_S,
|
||||||
|
max_output_chars: int = MAX_OUTPUT_CHARS,
|
||||||
|
) -> None:
|
||||||
self.root = Path(root)
|
self.root = Path(root)
|
||||||
self.root.mkdir(parents=True, exist_ok=True)
|
self.root.mkdir(parents=True, exist_ok=True)
|
||||||
self._timeout_s = timeout_s
|
self._timeout_s = timeout_s
|
||||||
|
self._max_output_chars = max_output_chars
|
||||||
|
|
||||||
|
def _truncate(self, text: str) -> str:
|
||||||
|
limit = self._max_output_chars
|
||||||
|
if len(text) <= limit:
|
||||||
|
return text
|
||||||
|
dropped = len(text) - limit
|
||||||
|
return f"{text[:limit]}\n[... {dropped} characters truncated]"
|
||||||
|
|
||||||
def _resolve(self, path: str) -> Path:
|
def _resolve(self, path: str) -> Path:
|
||||||
candidate = Path(path)
|
candidate = Path(path)
|
||||||
@@ -142,7 +149,7 @@ class Workspace:
|
|||||||
def read_file(self, path: str) -> ToolOutcome:
|
def read_file(self, path: str) -> ToolOutcome:
|
||||||
target = self._resolve(path)
|
target = self._resolve(path)
|
||||||
try:
|
try:
|
||||||
return ToolOutcome(_truncate(target.read_text()), success=True)
|
return ToolOutcome(self._truncate(target.read_text()), success=True)
|
||||||
except (OSError, UnicodeDecodeError) as exc:
|
except (OSError, UnicodeDecodeError) as exc:
|
||||||
return ToolOutcome(f"could not read {path}: {exc}", success=False)
|
return ToolOutcome(f"could not read {path}: {exc}", success=False)
|
||||||
|
|
||||||
@@ -175,7 +182,9 @@ class Workspace:
|
|||||||
)
|
)
|
||||||
except OSError as exc:
|
except OSError as exc:
|
||||||
return ToolOutcome(f"could not list {path}: {exc}", success=False)
|
return ToolOutcome(f"could not list {path}: {exc}", success=False)
|
||||||
return ToolOutcome(_truncate("\n".join(entries) or "(empty)"), success=True)
|
return ToolOutcome(
|
||||||
|
self._truncate("\n".join(entries) or "(empty)"), success=True
|
||||||
|
)
|
||||||
|
|
||||||
def describe(self) -> str:
|
def describe(self) -> str:
|
||||||
"""Told to the model, so it does not have to guess where it is."""
|
"""Told to the model, so it does not have to guess where it is."""
|
||||||
@@ -212,7 +221,7 @@ class Workspace:
|
|||||||
f"command timed out after {self._timeout_s}s: {command}",
|
f"command timed out after {self._timeout_s}s: {command}",
|
||||||
success=False,
|
success=False,
|
||||||
)
|
)
|
||||||
body = _truncate(
|
body = self._truncate(
|
||||||
f"exit code: {process.returncode}\nstdout:\n{stdout}\nstderr:\n{stderr}"
|
f"exit code: {process.returncode}\nstdout:\n{stdout}\nstderr:\n{stderr}"
|
||||||
)
|
)
|
||||||
return ToolOutcome(body, success=process.returncode == 0)
|
return ToolOutcome(body, success=process.returncode == 0)
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from typing import Any
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
from test_config import COMPLETE
|
from test_config import SECRETS, tree
|
||||||
|
|
||||||
from nisse_dev.agent import build_agent, discover_models, prepare_workspace
|
from nisse_dev.agent import build_agent, discover_models, prepare_workspace
|
||||||
from nisse_dev.catalog import ModelCatalog, UnknownModel
|
from nisse_dev.catalog import ModelCatalog, UnknownModel
|
||||||
@@ -34,14 +34,15 @@ TEST_CATALOG = ModelCatalog.from_payload(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def config_for(tmp_path: Path, **overrides: str) -> AgentConfig:
|
def config_for(tmp_path: Path, **changes: str) -> AgentConfig:
|
||||||
env = {
|
return AgentConfig.from_tree(
|
||||||
**COMPLETE,
|
tree(
|
||||||
"NISSE_STATE_DB": str(tmp_path / "state.db"),
|
state_db=str(tmp_path / "state.db"),
|
||||||
"NISSE_WORKSPACE": str(tmp_path / "work"),
|
workspace=str(tmp_path / "work"),
|
||||||
**overrides,
|
**changes,
|
||||||
}
|
),
|
||||||
return AgentConfig.from_env(env)
|
SECRETS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -59,7 +60,7 @@ def origin(tmp_path: Path) -> Path:
|
|||||||
|
|
||||||
|
|
||||||
def test_workspace_is_cloned_when_absent(tmp_path: Path, origin: Path) -> None:
|
def test_workspace_is_cloned_when_absent(tmp_path: Path, origin: Path) -> None:
|
||||||
config = config_for(tmp_path, NISSE_GITEA_URL=f"file://{origin}")
|
config = config_for(tmp_path, gitea__url=f"file://{origin}")
|
||||||
# authenticated_url would mangle a file:// remote, so clone directly.
|
# authenticated_url would mangle a file:// remote, so clone directly.
|
||||||
GitRepo(tmp_path).clone(str(origin), config.workspace)
|
GitRepo(tmp_path).clone(str(origin), config.workspace)
|
||||||
repo = prepare_workspace(config, "nisse-dev-testing")
|
repo = prepare_workspace(config, "nisse-dev-testing")
|
||||||
@@ -206,7 +207,7 @@ def test_httpx_clients_are_not_shared_between_routes(
|
|||||||
def test_a_typod_role_refuses_to_boot(tmp_path: Path, monkeypatch) -> None:
|
def test_a_typod_role_refuses_to_boot(tmp_path: Path, monkeypatch) -> None:
|
||||||
# Misconfiguration should cost a second at startup, not be discovered ten
|
# Misconfiguration should cost a second at startup, not be discovered ten
|
||||||
# minutes into a milestone.
|
# minutes into a milestone.
|
||||||
config = config_for(tmp_path, NISSE_ROLE_PLANNER="clause-opus-5")
|
config = config_for(tmp_path, roles__planner="clause-opus-5")
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"nisse_dev.agent.ModelCatalog.discover",
|
"nisse_dev.agent.ModelCatalog.discover",
|
||||||
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
||||||
@@ -220,7 +221,7 @@ def test_a_role_this_key_cannot_reach_refuses_to_boot(
|
|||||||
) -> None:
|
) -> None:
|
||||||
# /model/info is scoped per key, so the same check catches a role pointed
|
# /model/info is scoped per key, so the same check catches a role pointed
|
||||||
# at a real model that this token has no access to.
|
# at a real model that this token has no access to.
|
||||||
config = config_for(tmp_path, NISSE_ROLE_EXECUTOR="local-fast")
|
config = config_for(tmp_path, roles__executor="local-fast")
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"nisse_dev.agent.ModelCatalog.discover",
|
"nisse_dev.agent.ModelCatalog.discover",
|
||||||
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
||||||
@@ -286,3 +287,57 @@ def test_delegation_records_what_the_alias_resolved_to(
|
|||||||
agent = build_offline(tmp_path, origin, monkeypatch)
|
agent = build_offline(tmp_path, origin, monkeypatch)
|
||||||
assert agent.orchestrator._delegate._backing_model == "anthropic/claude-opus-5"
|
assert agent.orchestrator._delegate._backing_model == "anthropic/claude-opus-5"
|
||||||
agent.stop()
|
agent.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_tuning_reaches_every_component(
|
||||||
|
tmp_path: Path, origin: Path, monkeypatch
|
||||||
|
) -> None:
|
||||||
|
from dataclasses import replace
|
||||||
|
|
||||||
|
from nisse_dev.config import (
|
||||||
|
DelegationTuning,
|
||||||
|
ExecutorTuning,
|
||||||
|
ExpertTuning,
|
||||||
|
ToolsTuning,
|
||||||
|
Tuning,
|
||||||
|
)
|
||||||
|
|
||||||
|
config = replace(
|
||||||
|
config_for(tmp_path),
|
||||||
|
max_output_tokens=8192,
|
||||||
|
tuning=Tuning(
|
||||||
|
executor=ExecutorTuning(max_turns=40, failure_limit=3),
|
||||||
|
expert=ExpertTuning(consult_budget=5),
|
||||||
|
delegation=DelegationTuning(
|
||||||
|
max_turns=12, max_tokens=50_000, wall_clock_s=60
|
||||||
|
),
|
||||||
|
tools=ToolsTuning(command_timeout_s=300, max_output_chars=2000),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
GitRepo(tmp_path).clone(str(origin), config.workspace)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing"
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"nisse_dev.agent.ModelCatalog.discover",
|
||||||
|
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport())
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"nisse_dev.agent.authenticated_url",
|
||||||
|
lambda base, repo, user, token: str(origin),
|
||||||
|
)
|
||||||
|
agent = build_agent(config)
|
||||||
|
executor = agent.orchestrator._executor
|
||||||
|
delegation = agent.orchestrator._delegate
|
||||||
|
|
||||||
|
assert (executor._max_turns, executor._failure_limit) == (40, 3)
|
||||||
|
assert executor._expert._budget == 5
|
||||||
|
assert delegation._caps.turns == 12
|
||||||
|
assert delegation._caps.tokens == 50_000
|
||||||
|
assert delegation._caps.wall_clock_s == 60
|
||||||
|
assert executor._workspace._timeout_s == 300
|
||||||
|
assert executor._workspace._max_output_chars == 2000
|
||||||
|
assert executor._client._max_tokens == 8192
|
||||||
|
assert delegation._client._max_tokens == 8192
|
||||||
|
agent.stop()
|
||||||
+235
-68
@@ -1,93 +1,266 @@
|
|||||||
|
import copy
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from nisse_dev.config import AgentConfig, ConfigError
|
from nisse_dev.config import AgentConfig, ConfigError
|
||||||
|
|
||||||
COMPLETE = {
|
# A complete config as the YAML file would give it, with secrets by reference.
|
||||||
"NISSE_STATE_DB": "~/demo/state.db",
|
COMPLETE: dict[str, Any] = {
|
||||||
"NISSE_WORKSPACE": "~/demo/work",
|
"state_db": "~/demo/state.db",
|
||||||
"NISSE_PROJECT": "greet",
|
"workspace": "~/demo/work",
|
||||||
"NISSE_LLM_BASE_URL": "https://litellm.test",
|
"project": "greet",
|
||||||
"NISSE_LLM_TOKEN": "sk-x",
|
"llm": {"base_url": "https://litellm.test", "token": "${TEST_LLM_TOKEN}"},
|
||||||
"NISSE_ROLE_EXECUTOR": "claude-haiku-4-5",
|
"roles": {
|
||||||
"NISSE_ROLE_PLANNER": "claude-opus-5",
|
"executor": "claude-haiku-4-5",
|
||||||
"NISSE_ROLE_DELEGATE": "claude-opus-5",
|
"planner": "claude-opus-5",
|
||||||
"NISSE_ROLE_CONSULTANT": "claude-haiku-4-5",
|
"delegate": "claude-opus-5",
|
||||||
"NISSE_MATRIX_HOMESERVER": "https://matrix.test",
|
"consultant": "claude-haiku-4-5",
|
||||||
"NISSE_MATRIX_USER_ID": "@nisse-dev-testing:test",
|
},
|
||||||
"NISSE_MATRIX_TOKEN": "syt_x",
|
"matrix": {
|
||||||
"NISSE_MATRIX_ROOM_ID": "!room:test",
|
"homeserver": "https://matrix.test/",
|
||||||
"NISSE_MATRIX_OPERATORS": "@don:test",
|
"user_id": "@nisse-dev-testing:test",
|
||||||
"NISSE_GITEA_URL": "https://gitea.test",
|
"token": "${TEST_MATRIX_TOKEN}",
|
||||||
"NISSE_GITEA_TOKEN": "gt",
|
"room_id": "!room:test",
|
||||||
"NISSE_GITEA_REPO": "don/demo",
|
"operators": ["@don:test"],
|
||||||
"NISSE_GITEA_OPERATORS": "don",
|
},
|
||||||
|
"gitea": {
|
||||||
|
"url": "https://gitea.test",
|
||||||
|
"token": "${TEST_GITEA_TOKEN}",
|
||||||
|
"repo": "don/demo",
|
||||||
|
"operators": ["don"],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
SECRETS = {
|
||||||
|
"TEST_LLM_TOKEN": "sk-x",
|
||||||
|
"TEST_MATRIX_TOKEN": "syt_x",
|
||||||
|
"TEST_GITEA_TOKEN": "gt",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_complete_environment_builds() -> None:
|
def tree(**changes: Any) -> dict[str, Any]:
|
||||||
config = AgentConfig.from_env(COMPLETE)
|
"""COMPLETE with dotted keys changed; a value of None removes the key."""
|
||||||
|
result = copy.deepcopy(COMPLETE)
|
||||||
|
for dotted, value in changes.items():
|
||||||
|
*parents, leaf = dotted.split("__")
|
||||||
|
node = result
|
||||||
|
for part in parents:
|
||||||
|
node = node.setdefault(part, {})
|
||||||
|
if value is None:
|
||||||
|
node.pop(leaf, None)
|
||||||
|
else:
|
||||||
|
node[leaf] = value
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def build(**changes: Any) -> AgentConfig:
|
||||||
|
return AgentConfig.from_tree(tree(**changes), SECRETS)
|
||||||
|
|
||||||
|
|
||||||
|
def test_complete_config_builds_with_secrets_by_reference() -> None:
|
||||||
|
config = build()
|
||||||
assert config.project == "greet"
|
assert config.project == "greet"
|
||||||
|
assert config.llm_token == "sk-x"
|
||||||
|
assert config.gitea_token == "gt"
|
||||||
assert config.roles.planner == "claude-opus-5"
|
assert config.roles.planner == "claude-opus-5"
|
||||||
assert config.matrix.room_id == "!room:test"
|
assert config.matrix.room_id == "!room:test"
|
||||||
|
assert config.matrix.homeserver == "https://matrix.test"
|
||||||
assert config.gitea_repo == "don/demo"
|
assert config.gitea_repo == "don/demo"
|
||||||
assert config.base_branch == "main"
|
assert config.base_branch == "main"
|
||||||
|
assert config.tuning.executor.max_turns == 24 # absent section: defaults
|
||||||
|
|
||||||
|
|
||||||
def test_paths_expand_the_home_directory() -> None:
|
def test_paths_expand_the_home_directory() -> None:
|
||||||
config = AgentConfig.from_env(COMPLETE)
|
config = build()
|
||||||
assert config.workspace.is_absolute()
|
assert config.workspace.is_absolute()
|
||||||
assert "~" not in str(config.state_db)
|
assert "~" not in str(config.state_db)
|
||||||
|
|
||||||
|
|
||||||
def test_every_missing_variable_is_reported_at_once() -> None:
|
def test_every_missing_setting_is_reported_at_once() -> None:
|
||||||
# Failing at first use would surface a missing token twenty minutes into
|
# Failing at first use would surface a missing token twenty minutes into
|
||||||
# a milestone, on the call that mattered.
|
# a milestone, on the call that mattered.
|
||||||
partial = {k: v for k, v in COMPLETE.items() if k != "NISSE_MATRIX_TOKEN"}
|
|
||||||
partial.pop("NISSE_GITEA_REPO")
|
|
||||||
with pytest.raises(ConfigError) as excinfo:
|
with pytest.raises(ConfigError) as excinfo:
|
||||||
AgentConfig.from_env(partial)
|
build(matrix__token=None, gitea__repo=None)
|
||||||
message = str(excinfo.value)
|
message = str(excinfo.value)
|
||||||
assert "NISSE_MATRIX_TOKEN" in message
|
assert "matrix.token" in message
|
||||||
assert "NISSE_GITEA_REPO" in message
|
assert "gitea.repo" in message
|
||||||
|
|
||||||
|
|
||||||
def test_empty_is_treated_as_missing() -> None:
|
def test_empty_is_treated_as_missing() -> None:
|
||||||
with pytest.raises(ConfigError, match="NISSE_GITEA_TOKEN"):
|
with pytest.raises(ConfigError, match="gitea.repo"):
|
||||||
AgentConfig.from_env({**COMPLETE, "NISSE_GITEA_TOKEN": " "})
|
build(gitea__repo=" ")
|
||||||
|
|
||||||
|
|
||||||
def test_operator_lists_are_split_and_trimmed() -> None:
|
def test_a_reference_to_an_unset_variable_is_an_error() -> None:
|
||||||
config = AgentConfig.from_env(
|
# Never an empty string: an empty token would pass for a value and fail
|
||||||
{
|
# at the first request.
|
||||||
**COMPLETE,
|
env = {k: v for k, v in SECRETS.items() if k != "TEST_GITEA_TOKEN"}
|
||||||
"NISSE_MATRIX_OPERATORS": " @don:test , @other:test ,",
|
with pytest.raises(ConfigError, match=r"gitea.token: references \$\{TEST_GITEA"):
|
||||||
"NISSE_GITEA_OPERATORS": "don,",
|
AgentConfig.from_tree(tree(), env)
|
||||||
}
|
|
||||||
)
|
|
||||||
|
def test_operator_lists_drop_blanks() -> None:
|
||||||
|
config = build(matrix__operators=[" @don:test ", "@other:test", " "])
|
||||||
assert config.operators == {"@don:test", "@other:test"}
|
assert config.operators == {"@don:test", "@other:test"}
|
||||||
# A trailing comma must not admit an empty identity into an allowlist.
|
# A blank entry must not admit an empty identity into an allowlist.
|
||||||
assert "" not in config.operators
|
assert "" not in config.operators
|
||||||
assert config.gitea_operators == {"don"}
|
|
||||||
|
|
||||||
|
|
||||||
def test_base_branch_is_overridable() -> None:
|
def test_an_operator_list_of_only_blanks_is_missing() -> None:
|
||||||
config = AgentConfig.from_env({**COMPLETE, "NISSE_BASE_BRANCH": "trunk"})
|
with pytest.raises(ConfigError, match="gitea.operators"):
|
||||||
assert config.base_branch == "trunk"
|
build(gitea__operators=[" "])
|
||||||
|
|
||||||
|
|
||||||
|
def test_base_branch_is_configurable() -> None:
|
||||||
|
assert build(gitea__base_branch="trunk").base_branch == "trunk"
|
||||||
|
|
||||||
|
|
||||||
|
def test_max_output_tokens_is_configurable_and_checked() -> None:
|
||||||
|
assert build(llm__max_output_tokens=8192).max_output_tokens == 8192
|
||||||
|
with pytest.raises(ConfigError, match="llm.max_output_tokens"):
|
||||||
|
build(llm__max_output_tokens=0)
|
||||||
|
|
||||||
|
|
||||||
def test_redacted_summary_carries_no_secrets() -> None:
|
def test_redacted_summary_carries_no_secrets() -> None:
|
||||||
summary = AgentConfig.from_env(COMPLETE).redacted()
|
summary = build().redacted()
|
||||||
flat = " ".join(summary.values())
|
flat = " ".join(summary.values())
|
||||||
for secret in ("sk-x", "syt_x", "gt"):
|
for secret in SECRETS.values():
|
||||||
assert secret not in flat
|
assert secret not in flat
|
||||||
assert summary["project"] == "greet"
|
assert summary["project"] == "greet"
|
||||||
|
|
||||||
|
|
||||||
def test_state_db_and_workspace_are_separate(tmp_path: Path) -> None:
|
def test_tuning_is_read() -> None:
|
||||||
|
config = build(
|
||||||
|
tuning={
|
||||||
|
"executor": {"max_turns": 40, "failure_limit": 3},
|
||||||
|
"expert": {"consult_budget": 5},
|
||||||
|
"delegation": {"max_turns": 20, "max_tokens": 100000, "wall_clock_s": 600},
|
||||||
|
"tools": {"command_timeout_s": 300, "max_output_chars": 2000},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
tuning = config.tuning
|
||||||
|
assert (tuning.executor.max_turns, tuning.executor.failure_limit) == (40, 3)
|
||||||
|
assert tuning.expert.consult_budget == 5
|
||||||
|
assert tuning.delegation.max_tokens == 100_000
|
||||||
|
assert tuning.delegation.wall_clock_s == 600.0
|
||||||
|
assert tuning.tools.max_output_chars == 2000
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_partly_given_tuning_section_keeps_the_other_defaults() -> None:
|
||||||
|
tuning = build(tuning={"executor": {"max_turns": 40}}).tuning
|
||||||
|
assert tuning.executor.failure_limit == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_problem_is_reported_at_once() -> None:
|
||||||
|
# A misspelt knob that was silently ignored would look exactly like a knob
|
||||||
|
# with no effect, so unknown keys are errors.
|
||||||
|
bad = tree(project=None, llm_extra=1)
|
||||||
|
bad["tuning"] = {
|
||||||
|
"executor": {"max_turn": 40, "failure_limit": 0},
|
||||||
|
"expert": {"consult_budget": True},
|
||||||
|
"bogus": {"x": 1},
|
||||||
|
}
|
||||||
|
with pytest.raises(ConfigError) as excinfo:
|
||||||
|
AgentConfig.from_tree(bad, SECRETS)
|
||||||
|
message = str(excinfo.value)
|
||||||
|
for expected in (
|
||||||
|
"project",
|
||||||
|
"llm_extra: unknown setting",
|
||||||
|
"tuning.executor.max_turn: unknown setting",
|
||||||
|
"tuning.executor.failure_limit: must be a positive number",
|
||||||
|
"tuning.expert.consult_budget: must be a positive number",
|
||||||
|
"tuning.bogus: unknown section",
|
||||||
|
):
|
||||||
|
assert expected in message
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_whole_number_is_required_where_one_is_meant() -> None:
|
||||||
|
with pytest.raises(ConfigError, match="tuning.executor.max_turns"):
|
||||||
|
build(tuning={"executor": {"max_turns": 2.5}})
|
||||||
|
|
||||||
|
|
||||||
|
# --- the file --------------------------------------------------------------
|
||||||
|
|
||||||
|
FILE = """
|
||||||
|
state_db: ~/demo/state.db
|
||||||
|
workspace: ~/demo/work
|
||||||
|
project: greet
|
||||||
|
llm: {base_url: "https://litellm.test", token: "${TEST_LLM_TOKEN}"}
|
||||||
|
roles:
|
||||||
|
executor: local-fast
|
||||||
|
planner: claude-opus-5
|
||||||
|
delegate: claude-opus-5
|
||||||
|
consultant: claude-haiku-4-5
|
||||||
|
matrix:
|
||||||
|
homeserver: https://matrix.test
|
||||||
|
user_id: "@nisse-dev-testing:test"
|
||||||
|
token: ${TEST_MATRIX_TOKEN}
|
||||||
|
room_id: "!room:test"
|
||||||
|
operators: ["@don:test"]
|
||||||
|
gitea:
|
||||||
|
url: https://gitea.test
|
||||||
|
token: ${TEST_GITEA_TOKEN}
|
||||||
|
repo: don/demo
|
||||||
|
operators: [don]
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def write(directory: Path, text: str = FILE) -> Path:
|
||||||
|
path = directory / "nisse-dev.yaml"
|
||||||
|
path.write_text(text)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_file_is_read(tmp_path: Path) -> None:
|
||||||
|
config = AgentConfig.from_file(write(tmp_path), SECRETS)
|
||||||
|
assert config.roles.executor == "local-fast"
|
||||||
|
assert config.config_file == tmp_path / "nisse-dev.yaml"
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_yaml_is_reported(tmp_path: Path) -> None:
|
||||||
|
with pytest.raises(ConfigError, match="not valid YAML"):
|
||||||
|
AgentConfig.from_file(write(tmp_path, "roles: [unclosed"), SECRETS)
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_uses_nisse_config(tmp_path: Path) -> None:
|
||||||
|
path = write(tmp_path)
|
||||||
|
assert AgentConfig.load({**SECRETS, "NISSE_CONFIG": str(path)}).project == "greet"
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_defaults_to_the_working_directory(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
write(tmp_path)
|
||||||
|
monkeypatch.chdir(tmp_path)
|
||||||
|
assert AgentConfig.load(SECRETS).project == "greet"
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_without_a_file_says_how_to_make_one(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.chdir(tmp_path)
|
||||||
|
with pytest.raises(ConfigError, match="nisse-dev.example.yaml"):
|
||||||
|
AgentConfig.load(SECRETS)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_example_file_is_valid() -> None:
|
||||||
|
# The example documents every knob; it must not drift out of loading.
|
||||||
|
example = Path(__file__).resolve().parents[2] / "nisse-dev.example.yaml"
|
||||||
|
env = {
|
||||||
|
"NISSE_LLM_TOKEN": "sk-x",
|
||||||
|
"NISSE_MATRIX_TOKEN": "syt_x",
|
||||||
|
"NISSE_GITEA_TOKEN": "gt",
|
||||||
|
}
|
||||||
|
assert AgentConfig.from_file(example, env).tuning.executor.max_turns == 24
|
||||||
|
|
||||||
|
|
||||||
|
# --- nothing that carries secrets may sit inside the clone -----------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_state_db_and_workspace_are_separate() -> None:
|
||||||
# A2: build artefacts must not share a volume with agent state.
|
# A2: build artefacts must not share a volume with agent state.
|
||||||
config = AgentConfig.from_env(COMPLETE)
|
config = build()
|
||||||
assert config.state_db.parent != config.workspace
|
assert config.state_db.parent != config.workspace
|
||||||
|
|
||||||
|
|
||||||
@@ -96,13 +269,7 @@ def test_state_db_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
|||||||
# own state alongside the project.
|
# own state alongside the project.
|
||||||
work = tmp_path / "project"
|
work = tmp_path / "project"
|
||||||
with pytest.raises(ConfigError, match="inside the workspace"):
|
with pytest.raises(ConfigError, match="inside the workspace"):
|
||||||
AgentConfig.from_env(
|
build(workspace=str(work), state_db=str(work / "state.db"))
|
||||||
{
|
|
||||||
**COMPLETE,
|
|
||||||
"NISSE_WORKSPACE": str(work),
|
|
||||||
"NISSE_STATE_DB": str(work / "state.db"),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_env_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
def test_env_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
||||||
@@ -110,23 +277,23 @@ def test_env_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
|||||||
# as a side effect of working correctly.
|
# as a side effect of working correctly.
|
||||||
work = tmp_path / "project"
|
work = tmp_path / "project"
|
||||||
with pytest.raises(ConfigError, match="NISSE_ENV"):
|
with pytest.raises(ConfigError, match="NISSE_ENV"):
|
||||||
AgentConfig.from_env(
|
AgentConfig.from_tree(
|
||||||
{
|
tree(workspace=str(work), state_db=str(tmp_path / "state.db")),
|
||||||
**COMPLETE,
|
{**SECRETS, "NISSE_ENV": str(work / ".env")},
|
||||||
"NISSE_WORKSPACE": str(work),
|
|
||||||
"NISSE_STATE_DB": str(tmp_path / "state.db"),
|
|
||||||
"NISSE_ENV": str(work / ".env"),
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_config_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
||||||
|
work = tmp_path / "project"
|
||||||
|
work.mkdir()
|
||||||
|
text = FILE.replace("workspace: ~/demo/work", f"workspace: {work}")
|
||||||
|
with pytest.raises(ConfigError, match="NISSE_CONFIG"):
|
||||||
|
AgentConfig.from_file(write(work, text), SECRETS)
|
||||||
|
|
||||||
|
|
||||||
def test_sibling_paths_are_fine(tmp_path: Path) -> None:
|
def test_sibling_paths_are_fine(tmp_path: Path) -> None:
|
||||||
config = AgentConfig.from_env(
|
config = AgentConfig.from_tree(
|
||||||
{
|
tree(workspace=str(tmp_path / "project"), state_db=str(tmp_path / "state.db")),
|
||||||
**COMPLETE,
|
{**SECRETS, "NISSE_ENV": str(tmp_path / ".env")},
|
||||||
"NISSE_WORKSPACE": str(tmp_path / "project"),
|
|
||||||
"NISSE_STATE_DB": str(tmp_path / "state.db"),
|
|
||||||
"NISSE_ENV": str(tmp_path / ".env"),
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
assert config.workspace.name == "project"
|
assert config.workspace.name == "project"
|
||||||
@@ -2,25 +2,23 @@ from pathlib import Path
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from nisse_dev.config import REQUIRED
|
|
||||||
from nisse_dev.main import main
|
from nisse_dev.main import main
|
||||||
|
|
||||||
|
|
||||||
def test_main_refuses_to_start_without_configuration(
|
def test_main_refuses_to_start_without_a_config_file(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
capsys: pytest.CaptureFixture[str],
|
capsys: pytest.CaptureFixture[str],
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
# Clear the real variables rather than swapping os.environ: from_env binds
|
# Run from an empty directory, and point NISSE_ENV at a file that does not
|
||||||
# the environment object as a default argument, so a replacement would be
|
# exist: the repository's .env holds live credentials, and a test must
|
||||||
# ignored. And point NISSE_ENV at a file that does not exist, because the
|
# never be one cwd away from starting a real agent.
|
||||||
# working directory's .env holds live credentials.
|
monkeypatch.chdir(tmp_path)
|
||||||
for name in REQUIRED:
|
monkeypatch.delenv("NISSE_CONFIG", raising=False)
|
||||||
monkeypatch.delenv(name, raising=False)
|
|
||||||
monkeypatch.setenv("NISSE_ENV", str(tmp_path / "absent.env"))
|
monkeypatch.setenv("NISSE_ENV", str(tmp_path / "absent.env"))
|
||||||
|
|
||||||
assert main() == 2
|
assert main() == 2
|
||||||
|
|
||||||
err = capsys.readouterr().err
|
err = capsys.readouterr().err
|
||||||
assert "cannot start" in err
|
assert "cannot start" in err
|
||||||
assert "NISSE_LLM_TOKEN" in err
|
assert "nisse-dev.example.yaml" in err
|
||||||
@@ -162,11 +162,6 @@ def test_list_comments_passes_since() -> None:
|
|||||||
assert seen["since"] == "2026-09-07T00:00:00Z"
|
assert seen["since"] == "2026-09-07T00:00:00Z"
|
||||||
|
|
||||||
|
|
||||||
def test_from_env_requires_config() -> None:
|
|
||||||
with pytest.raises(GiteaError, match="NISSE_GITEA_URL"):
|
|
||||||
GiteaClient.from_env({})
|
|
||||||
|
|
||||||
|
|
||||||
# --- reconciler (A3 external truth) --------------------------------------
|
# --- reconciler (A3 external truth) --------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+11
-23
@@ -21,7 +21,6 @@ from nisse_dev.llm import (
|
|||||||
RoleMap,
|
RoleMap,
|
||||||
TruncatedResponse,
|
TruncatedResponse,
|
||||||
UpstreamUnavailable,
|
UpstreamUnavailable,
|
||||||
max_output_tokens,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
EVENTS = [StoredEvent(id=1, ts="t", event=UserMessage(text="hi", source="m:1"))]
|
EVENTS = [StoredEvent(id=1, ts="t", event=UserMessage(text="hi", source="m:1"))]
|
||||||
@@ -165,16 +164,16 @@ def test_malformed_response_raises_llm_error() -> None:
|
|||||||
client.complete(EVENTS, tools=None, model="local/qwen")
|
client.complete(EVENTS, tools=None, model="local/qwen")
|
||||||
|
|
||||||
|
|
||||||
ROLE_ENV = {
|
ROLES = RoleMap(
|
||||||
"NISSE_ROLE_EXECUTOR": "local/qwen",
|
executor="local/qwen",
|
||||||
"NISSE_ROLE_PLANNER": "anthropic/claude-opus-5",
|
planner="anthropic/claude-opus-5",
|
||||||
"NISSE_ROLE_DELEGATE": "anthropic/claude-opus-5",
|
delegate="anthropic/claude-opus-5",
|
||||||
"NISSE_ROLE_CONSULTANT": "anthropic/claude-opus-5",
|
consultant="anthropic/claude-opus-5",
|
||||||
}
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_role_map_resolves_and_pins() -> None:
|
def test_role_map_resolves_and_pins() -> None:
|
||||||
roles = RoleMap.from_env(ROLE_ENV)
|
roles = ROLES
|
||||||
assert roles.resolve("executor") == "local/qwen"
|
assert roles.resolve("executor") == "local/qwen"
|
||||||
assert roles.pinned() == {
|
assert roles.pinned() == {
|
||||||
"executor": "local/qwen",
|
"executor": "local/qwen",
|
||||||
@@ -184,19 +183,9 @@ def test_role_map_resolves_and_pins() -> None:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_role_map_missing_env_raises() -> None:
|
|
||||||
with pytest.raises(LLMError, match="NISSE_ROLE_PLANNER"):
|
|
||||||
RoleMap.from_env({"NISSE_ROLE_EXECUTOR": "local/qwen"})
|
|
||||||
|
|
||||||
|
|
||||||
def test_unknown_role_raises() -> None:
|
def test_unknown_role_raises() -> None:
|
||||||
with pytest.raises(LLMError, match="unknown role"):
|
with pytest.raises(LLMError, match="unknown role"):
|
||||||
RoleMap.from_env(ROLE_ENV).resolve("boss")
|
ROLES.resolve("boss")
|
||||||
|
|
||||||
|
|
||||||
def test_client_from_env_requires_config() -> None:
|
|
||||||
with pytest.raises(LLMError, match="NISSE_LLM_BASE_URL"):
|
|
||||||
LiteLLMClient.from_env({})
|
|
||||||
|
|
||||||
|
|
||||||
def test_load_env_file(tmp_path: Path) -> None:
|
def test_load_env_file(tmp_path: Path) -> None:
|
||||||
@@ -306,12 +295,11 @@ def test_a_complete_answer_is_not_mistaken_for_truncation() -> None:
|
|||||||
assert call.arguments["content"] == "a real plan"
|
assert call.arguments["content"] == "a real plan"
|
||||||
|
|
||||||
|
|
||||||
def test_output_ceiling_is_configurable_and_generous_by_default() -> None:
|
def test_output_ceiling_is_generous_by_default() -> None:
|
||||||
# Output is billed by what is produced, so a low ceiling buys nothing and
|
# Output is billed by what is produced, so a low ceiling buys nothing and
|
||||||
# costs silent truncation.
|
# costs silent truncation. The configured value is tested in test_config.
|
||||||
assert DEFAULT_MAX_OUTPUT_TOKENS >= 8192
|
assert DEFAULT_MAX_OUTPUT_TOKENS >= 8192
|
||||||
assert max_output_tokens({}) == DEFAULT_MAX_OUTPUT_TOKENS
|
assert LiteLLMClient("http://x", "t")._max_tokens == DEFAULT_MAX_OUTPUT_TOKENS
|
||||||
assert max_output_tokens({"NISSE_MAX_OUTPUT_TOKENS": "2048"}) == 2048
|
|
||||||
|
|
||||||
|
|
||||||
def test_truncation_is_an_llm_error_so_callers_already_handle_it() -> None:
|
def test_truncation_is_an_llm_error_so_callers_already_handle_it() -> None:
|
||||||
|
|||||||
@@ -143,20 +143,3 @@ def test_start_without_handler_raises() -> None:
|
|||||||
)
|
)
|
||||||
with pytest.raises(TransportError, match="no message handler"):
|
with pytest.raises(TransportError, match="no message handler"):
|
||||||
transport.start()
|
transport.start()
|
||||||
|
|
||||||
|
|
||||||
def test_config_from_env_reports_missing() -> None:
|
|
||||||
with pytest.raises(TransportError, match="NISSE_MATRIX_TOKEN"):
|
|
||||||
MatrixConfig.from_env({"NISSE_MATRIX_HOMESERVER": "https://m"})
|
|
||||||
|
|
||||||
|
|
||||||
def test_config_from_env_custom_room_var() -> None:
|
|
||||||
env = {
|
|
||||||
"NISSE_MATRIX_HOMESERVER": "https://m/",
|
|
||||||
"NISSE_MATRIX_USER_ID": "@nisse-dev:test",
|
|
||||||
"NISSE_MATRIX_TOKEN": "t",
|
|
||||||
"NISSE_MATRIX_TEST_ROOM_ID": "!t:test",
|
|
||||||
}
|
|
||||||
config = MatrixConfig.from_env(env, room_var="NISSE_MATRIX_TEST_ROOM_ID")
|
|
||||||
assert config.room_id == "!t:test"
|
|
||||||
assert config.homeserver == "https://m" # trailing slash stripped
|
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
"""Command-line options for the repository-level test suites.
|
||||||
|
|
||||||
|
Declared here rather than beside the tests that use them: pytest only reads
|
||||||
|
options from conftest files it loads at startup, and this directory is one
|
||||||
|
of the configured test paths.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
def pytest_addoption(parser: pytest.Parser) -> None:
|
||||||
|
parser.addoption(
|
||||||
|
"--capability-models",
|
||||||
|
default="",
|
||||||
|
help=(
|
||||||
|
"comma-separated LiteLLM aliases to measure with the capability "
|
||||||
|
"matrix (tests/integration/test_model_capability.py). Spends real "
|
||||||
|
"tokens on every model listed; the matrix skips when this is empty."
|
||||||
|
),
|
||||||
|
)
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
"""Shared setup for the live integration tests.
|
||||||
|
|
||||||
|
They run against real services, configured by nisse-test.yaml beside this
|
||||||
|
file: an ordinary agent config aimed at test resources, committed to the
|
||||||
|
repository. Its secrets are ${…} references, supplied by CI as environment
|
||||||
|
variables and locally by .env at the repository root.
|
||||||
|
|
||||||
|
A missing secret is a failure, not a skip. A half-configured environment must
|
||||||
|
not pass quietly as a skip: that is how a broken CI credential hides.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from nisse_dev.config import AgentConfig
|
||||||
|
from nisse_dev.envfile import load_env_file
|
||||||
|
from nisse_dev.gitea import GiteaClient
|
||||||
|
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parents[2]
|
||||||
|
TEST_CONFIG = Path(__file__).parent / "nisse-test.yaml"
|
||||||
|
|
||||||
|
# The secrets nisse-test.yaml refers to, for local runs. CI sets them as real
|
||||||
|
# environment variables, which win over the file.
|
||||||
|
load_env_file(REPO / ".env")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def live_config() -> AgentConfig:
|
||||||
|
return AgentConfig.from_file(TEST_CONFIG)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def live_model(live_config: AgentConfig) -> str:
|
||||||
|
"""The model the live tests plan, execute and delegate with."""
|
||||||
|
return live_config.roles.planner
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def litellm(live_config: AgentConfig) -> LiteLLMClient:
|
||||||
|
return LiteLLMClient(
|
||||||
|
live_config.llm_base_url,
|
||||||
|
live_config.llm_token,
|
||||||
|
max_tokens=live_config.max_output_tokens,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def passthrough(live_config: AgentConfig) -> AnthropicPassthroughClient:
|
||||||
|
return AnthropicPassthroughClient(
|
||||||
|
live_config.llm_base_url,
|
||||||
|
live_config.llm_token,
|
||||||
|
max_tokens=live_config.max_output_tokens,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def gitea(live_config: AgentConfig) -> GiteaClient:
|
||||||
|
client = GiteaClient(live_config.gitea_url, live_config.gitea_token)
|
||||||
|
yield client
|
||||||
|
client.close()
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# Agent config for the live integration tests, aimed at test resources: the
|
||||||
|
# testing identities, the dedicated test room, and the sandbox repo. Never
|
||||||
|
# point it at production.
|
||||||
|
#
|
||||||
|
# Secrets are ${…} references. CI passes them as environment variables;
|
||||||
|
# locally they come from .env at the repository root. A missing one fails
|
||||||
|
# the live tests that need it, rather than skipping them, so a broken
|
||||||
|
# credential cannot hide as a skip.
|
||||||
|
|
||||||
|
# Unused by the live tests, which work in pytest's temporary directories;
|
||||||
|
# present because every agent config has them.
|
||||||
|
state_db: /tmp/nisse-test/state.db
|
||||||
|
workspace: /tmp/nisse-test/work
|
||||||
|
project: nisse-dev-testing
|
||||||
|
|
||||||
|
llm:
|
||||||
|
base_url: https://litellm.dongar.net
|
||||||
|
token: ${NISSE_LLM_TOKEN}
|
||||||
|
|
||||||
|
# The live tests exercise the gated loop and the routes over a real proxy,
|
||||||
|
# not local-model competence (measured separately by test_model_capability),
|
||||||
|
# so every role uses a model that holds the tool-call contract.
|
||||||
|
roles:
|
||||||
|
executor: claude-haiku-4-5
|
||||||
|
planner: claude-haiku-4-5
|
||||||
|
delegate: claude-haiku-4-5
|
||||||
|
consultant: claude-haiku-4-5
|
||||||
|
|
||||||
|
matrix:
|
||||||
|
homeserver: https://matrix.dongar.net
|
||||||
|
user_id: "@nisse-dev-testing:dongar.net"
|
||||||
|
token: ${NISSE_MATRIX_TOKEN}
|
||||||
|
room_id: "!UxpXhTlQsFIeNZwkgv:dongar.net" # the test room, never production
|
||||||
|
operators: ["@dongar:dongar.net"]
|
||||||
|
|
||||||
|
gitea:
|
||||||
|
url: https://gitea.dongar.net
|
||||||
|
token: ${NISSE_GITEA_TOKEN}
|
||||||
|
repo: dongar/nisse-dev-testing-sandbox # the sandbox, never a real project
|
||||||
|
operators: [dongar]
|
||||||
@@ -2,30 +2,18 @@
|
|||||||
working code and a structured handback out."""
|
working code and a structured handback out."""
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import os
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from nisse_dev.delegation import Caps, Delegation
|
from nisse_dev.delegation import Caps, Delegation
|
||||||
from nisse_dev.envfile import load_env_file
|
|
||||||
from nisse_dev.git import GitRepo
|
from nisse_dev.git import GitRepo
|
||||||
from nisse_dev.llm import AnthropicPassthroughClient
|
from nisse_dev.llm import AnthropicPassthroughClient
|
||||||
from nisse_dev.oplog import OpLog
|
from nisse_dev.oplog import OpLog
|
||||||
from nisse_dev.state import StateStore
|
from nisse_dev.state import StateStore
|
||||||
from nisse_dev.tools import Workspace
|
from nisse_dev.tools import Workspace
|
||||||
|
|
||||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
|
||||||
|
|
||||||
pytestmark = [
|
|
||||||
pytest.mark.integration,
|
|
||||||
pytest.mark.skipif(
|
|
||||||
not os.environ.get(SENTINEL),
|
|
||||||
reason=f"no live frontier route configured ({SENTINEL} unset)",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
BRIEF = """Create `slugify.py` in the workspace root containing a function
|
BRIEF = """Create `slugify.py` in the workspace root containing a function
|
||||||
`slugify(text)` that lowercases the text and replaces every run of
|
`slugify(text)` that lowercases the text and replaces every run of
|
||||||
@@ -34,7 +22,9 @@ hyphen. Then create `test_slugify.py` with at least two assertions covering
|
|||||||
it, and run `python -m pytest test_slugify.py -q` until it passes."""
|
it, and run `python -m pytest test_slugify.py -q` until it passes."""
|
||||||
|
|
||||||
|
|
||||||
def build(tmp_path: Path, caps: Caps) -> tuple[Delegation, GitRepo, StateStore]:
|
def build(
|
||||||
|
tmp_path: Path, caps: Caps, client: AnthropicPassthroughClient, model: str
|
||||||
|
) -> tuple[Delegation, GitRepo, StateStore]:
|
||||||
repo = GitRepo(tmp_path / "work")
|
repo = GitRepo(tmp_path / "work")
|
||||||
repo.init()
|
repo.init()
|
||||||
repo.configure_identity("nisse-dev-testing", "n@t.test")
|
repo.configure_identity("nisse-dev-testing", "n@t.test")
|
||||||
@@ -42,18 +32,22 @@ def build(tmp_path: Path, caps: Caps) -> tuple[Delegation, GitRepo, StateStore]:
|
|||||||
repo.commit_all("seed")
|
repo.commit_all("seed")
|
||||||
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
||||||
delegation = Delegation(
|
delegation = Delegation(
|
||||||
AnthropicPassthroughClient.from_env(),
|
client,
|
||||||
store,
|
store,
|
||||||
Workspace(repo.root, timeout_s=60),
|
Workspace(repo.root, timeout_s=60),
|
||||||
repo,
|
repo,
|
||||||
model=os.environ[SENTINEL],
|
model=model,
|
||||||
caps=caps,
|
caps=caps,
|
||||||
)
|
)
|
||||||
return delegation, repo, store
|
return delegation, repo, store
|
||||||
|
|
||||||
|
|
||||||
def test_delegate_completes_a_scoped_subtask(tmp_path: Path) -> None:
|
def test_delegate_completes_a_scoped_subtask(
|
||||||
delegation, repo, _store = build(tmp_path, Caps(turns=12, wall_clock_s=300.0))
|
tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str
|
||||||
|
) -> None:
|
||||||
|
delegation, repo, _store = build(
|
||||||
|
tmp_path, Caps(turns=12, wall_clock_s=300.0), passthrough, live_model
|
||||||
|
)
|
||||||
|
|
||||||
result = delegation.run(brief=BRIEF, branch="live/delegation-1")
|
result = delegation.run(brief=BRIEF, branch="live/delegation-1")
|
||||||
|
|
||||||
@@ -76,10 +70,14 @@ def test_delegate_completes_a_scoped_subtask(tmp_path: Path) -> None:
|
|||||||
assert check.success, f"delegate's own tests do not pass: {check.result[:400]}"
|
assert check.success, f"delegate's own tests do not pass: {check.result[:400]}"
|
||||||
|
|
||||||
|
|
||||||
def test_a_cap_breach_discards_the_work(tmp_path: Path) -> None:
|
def test_a_cap_breach_discards_the_work(
|
||||||
|
tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str
|
||||||
|
) -> None:
|
||||||
# One turn cannot finish the brief, so this exercises the abort path
|
# One turn cannot finish the brief, so this exercises the abort path
|
||||||
# against a real model rather than a scripted one.
|
# against a real model rather than a scripted one.
|
||||||
delegation, repo, _ = build(tmp_path, Caps(turns=1, wall_clock_s=120.0))
|
delegation, repo, _ = build(
|
||||||
|
tmp_path, Caps(turns=1, wall_clock_s=120.0), passthrough, live_model
|
||||||
|
)
|
||||||
|
|
||||||
result = delegation.run(brief=BRIEF, branch="live/delegation-2")
|
result = delegation.run(brief=BRIEF, branch="live/delegation-2")
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
@@ -16,27 +17,31 @@ def test_ops_entry_point_runs_and_greets() -> None:
|
|||||||
assert result.stdout == "Hello from nisse-ops, the house spirit is awake.\n"
|
assert result.stdout == "Hello from nisse-ops, the house spirit is awake.\n"
|
||||||
|
|
||||||
|
|
||||||
def test_dev_entry_point_refuses_to_start_unconfigured() -> None:
|
def test_dev_entry_point_refuses_to_start_unconfigured(tmp_path: Path) -> None:
|
||||||
"""Missing configuration must fail at startup, naming everything absent.
|
"""An incomplete config must fail at startup, naming everything absent.
|
||||||
|
|
||||||
Run with a deliberately empty environment and a .env path that does not
|
Run from an empty directory, with an empty config file and a .env path
|
||||||
exist: the real .env in the working directory holds live credentials, and
|
that does not exist: the repository's .env holds live credentials, and a
|
||||||
a test must never be one cwd away from starting a real agent.
|
test must never be one cwd away from starting a real agent.
|
||||||
"""
|
"""
|
||||||
|
config = tmp_path / "nisse-dev.yaml"
|
||||||
|
config.write_text("")
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["nisse-dev"],
|
["nisse-dev"],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
timeout=30,
|
timeout=30,
|
||||||
check=False,
|
check=False,
|
||||||
|
cwd=tmp_path,
|
||||||
env={
|
env={
|
||||||
"PATH": os.environ["PATH"],
|
"PATH": os.environ["PATH"],
|
||||||
"HOME": os.environ["HOME"],
|
"HOME": os.environ["HOME"],
|
||||||
"NISSE_ENV": "/nonexistent/.env",
|
"NISSE_ENV": "/nonexistent/.env",
|
||||||
|
"NISSE_CONFIG": str(config),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
assert result.returncode == 2
|
assert result.returncode == 2
|
||||||
assert "cannot start" in result.stderr
|
assert "cannot start" in result.stderr
|
||||||
# All of them at once, not the first one it happened to reach.
|
# All of them at once, not the first one it happened to reach.
|
||||||
for expected in ("NISSE_STATE_DB", "NISSE_MATRIX_TOKEN", "NISSE_GITEA_REPO"):
|
for expected in ("state_db", "matrix.token", "gitea.repo"):
|
||||||
assert expected in result.stderr
|
assert expected in result.stderr
|
||||||
@@ -5,40 +5,34 @@ that parses against our local schema, since no API-level enforcement exists.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import os
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from nisse_dev.envfile import load_env_file
|
|
||||||
from nisse_dev.evals import collect
|
from nisse_dev.evals import collect
|
||||||
from nisse_dev.expert import BudgetExhausted, ExpertDesk
|
from nisse_dev.expert import BudgetExhausted, ExpertDesk
|
||||||
from nisse_dev.llm import LiteLLMClient, RoleMap
|
from nisse_dev.llm import LiteLLMClient, RoleMap
|
||||||
from nisse_dev.oplog import OpLog
|
from nisse_dev.oplog import OpLog
|
||||||
from nisse_dev.state import StateStore
|
from nisse_dev.state import StateStore
|
||||||
|
|
||||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
|
||||||
|
|
||||||
pytestmark = [
|
|
||||||
pytest.mark.integration,
|
|
||||||
pytest.mark.skipif(
|
|
||||||
not os.environ.get(SENTINEL),
|
|
||||||
reason=f"no live frontier route configured ({SENTINEL} unset)",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def desk(tmp_path: Path, budget: int = 2) -> tuple[ExpertDesk, StateStore]:
|
@pytest.fixture
|
||||||
model = os.environ[SENTINEL]
|
def desk(tmp_path: Path, litellm: LiteLLMClient, live_model: str):
|
||||||
|
def make(budget: int = 2) -> tuple[ExpertDesk, StateStore]:
|
||||||
|
model = live_model
|
||||||
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
||||||
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
store = StateStore(
|
||||||
return ExpertDesk(LiteLLMClient.from_env(), store, roles, budget=budget), store
|
tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())
|
||||||
|
)
|
||||||
|
return ExpertDesk(litellm, store, roles, budget=budget), store
|
||||||
|
|
||||||
|
return make
|
||||||
|
|
||||||
|
|
||||||
def test_live_consult_parses_against_the_local_schema(tmp_path: Path) -> None:
|
def test_live_consult_parses_against_the_local_schema(desk) -> None:
|
||||||
expert, store = desk(tmp_path)
|
expert, store = desk()
|
||||||
|
|
||||||
answer = expert.ask(
|
answer = expert.ask(
|
||||||
"For a single-process agent storing a few thousand rows of task state, "
|
"For a single-process agent storing a few thousand rows of task state, "
|
||||||
@@ -53,8 +47,8 @@ def test_live_consult_parses_against_the_local_schema(tmp_path: Path) -> None:
|
|||||||
assert record.exit_status == "answered"
|
assert record.exit_status == "answered"
|
||||||
|
|
||||||
|
|
||||||
def test_live_budget_stops_a_circling_caller(tmp_path: Path) -> None:
|
def test_live_budget_stops_a_circling_caller(desk) -> None:
|
||||||
expert, _ = desk(tmp_path, budget=1)
|
expert, _ = desk(budget=1)
|
||||||
expert.ask("Name one advantage of WAL mode in SQLite. Be brief.")
|
expert.ask("Name one advantage of WAL mode in SQLite. Be brief.")
|
||||||
with pytest.raises(BudgetExhausted):
|
with pytest.raises(BudgetExhausted):
|
||||||
expert.ask("And another?")
|
expert.ask("And another?")
|
||||||
@@ -5,29 +5,17 @@ Cleans up after itself (closes the PR, deletes the branch) so repeated runs
|
|||||||
do not silt up the sandbox.
|
do not silt up the sandbox.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
|
||||||
import uuid
|
import uuid
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from nisse_dev.envfile import load_env_file
|
from nisse_dev.config import AgentConfig
|
||||||
from nisse_dev.git import GitError, GitRepo, authenticated_url
|
from nisse_dev.git import GitError, GitRepo, authenticated_url
|
||||||
from nisse_dev.gitea import GiteaClient, pull_request_reconciler
|
from nisse_dev.gitea import GiteaClient, pull_request_reconciler
|
||||||
from nisse_dev.state import Intent
|
from nisse_dev.state import Intent
|
||||||
|
|
||||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
SENTINEL = "NISSE_GITEA_URL"
|
|
||||||
ALSO_REQUIRED = ("NISSE_GITEA_TOKEN", "NISSE_GITEA_TEST_REPO")
|
|
||||||
|
|
||||||
pytestmark = [
|
|
||||||
pytest.mark.integration,
|
|
||||||
pytest.mark.skipif(
|
|
||||||
not os.environ.get(SENTINEL),
|
|
||||||
reason=f"no live Gitea configured ({SENTINEL} unset)",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def intent_for(branch: str) -> Intent:
|
def intent_for(branch: str) -> Intent:
|
||||||
@@ -41,16 +29,13 @@ def intent_for(branch: str) -> Intent:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_push_open_pr_and_reconcile(tmp_path: Path) -> None:
|
def test_push_open_pr_and_reconcile(
|
||||||
missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)]
|
tmp_path: Path, live_config: AgentConfig, gitea: GiteaClient
|
||||||
assert not missing, (
|
) -> None:
|
||||||
f"{SENTINEL} is set, so the live Gitea path is expected to work, "
|
base_url = live_config.gitea_url
|
||||||
f"but these are missing: {', '.join(missing)}"
|
token = live_config.gitea_token
|
||||||
)
|
repo_path = live_config.gitea_repo # the sandbox
|
||||||
base_url = os.environ[SENTINEL]
|
client = gitea
|
||||||
token = os.environ["NISSE_GITEA_TOKEN"]
|
|
||||||
repo_path = os.environ["NISSE_GITEA_TEST_REPO"]
|
|
||||||
client = GiteaClient.from_env()
|
|
||||||
|
|
||||||
account = client.whoami()
|
account = client.whoami()
|
||||||
assert "testing" in account, (
|
assert "testing" in account, (
|
||||||
|
|||||||
@@ -1,41 +1,17 @@
|
|||||||
"""Live smoke test against the real LiteLLM proxy.
|
"""Live smoke test against the real LiteLLM proxy."""
|
||||||
|
|
||||||
NISSE_LLM_BASE_URL alone decides whether this runs: if it is set, the live
|
|
||||||
path is expected to work, and anything else missing is a *failure*, not a
|
|
||||||
skip. A half-configured environment must not pass quietly as a skip — that is
|
|
||||||
how a broken CI credential hides.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from nisse_dev.canonical import AssistantText, StoredEvent, UserMessage
|
from nisse_dev.canonical import AssistantText, StoredEvent, UserMessage
|
||||||
from nisse_dev.envfile import load_env_file
|
from nisse_dev.config import AgentConfig
|
||||||
from nisse_dev.llm import LiteLLMClient
|
from nisse_dev.llm import LiteLLMClient
|
||||||
|
|
||||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
SENTINEL = "NISSE_LLM_BASE_URL"
|
|
||||||
ALSO_REQUIRED = ("NISSE_LLM_TOKEN", "NISSE_ROLE_EXECUTOR")
|
|
||||||
|
|
||||||
pytestmark = [
|
|
||||||
pytest.mark.integration,
|
|
||||||
pytest.mark.skipif(
|
|
||||||
not os.environ.get(SENTINEL),
|
|
||||||
reason=f"no live LLM configured ({SENTINEL} unset)",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_executor_model_completes_via_litellm() -> None:
|
def test_executor_model_completes_via_litellm(
|
||||||
missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)]
|
litellm: LiteLLMClient, live_config: AgentConfig
|
||||||
assert not missing, (
|
) -> None:
|
||||||
f"{SENTINEL} is set, so the live LLM path is expected to work, "
|
|
||||||
f"but these are missing: {', '.join(missing)}"
|
|
||||||
)
|
|
||||||
client = LiteLLMClient.from_env()
|
|
||||||
events = [
|
events = [
|
||||||
StoredEvent(
|
StoredEvent(
|
||||||
id=1,
|
id=1,
|
||||||
@@ -45,9 +21,7 @@ def test_executor_model_completes_via_litellm() -> None:
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
result = client.complete(
|
result = litellm.complete(events, tools=None, model=live_config.roles.executor)
|
||||||
events, tools=None, model=os.environ["NISSE_ROLE_EXECUTOR"]
|
|
||||||
)
|
|
||||||
texts = [e.text for e in result if isinstance(e, AssistantText)]
|
texts = [e.text for e in result if isinstance(e, AssistantText)]
|
||||||
assert texts, f"no assistant text in {result!r}"
|
assert texts, f"no assistant text in {result!r}"
|
||||||
assert "pong" in texts[0].lower()
|
assert "pong" in texts[0].lower()
|
||||||
@@ -1,36 +1,14 @@
|
|||||||
"""Live smoke test against the real homeserver, bound strictly to the
|
"""Live smoke test against the real homeserver, bound strictly to the
|
||||||
dedicated test room (never the production room).
|
dedicated test room that nisse-test.yaml names (never the production room)."""
|
||||||
|
|
||||||
NISSE_MATRIX_HOMESERVER alone decides whether this runs: if it is set, the
|
|
||||||
live path is expected to work, and anything else missing is a *failure*, not
|
|
||||||
a skip. A half-configured environment must not pass quietly as a skip — that
|
|
||||||
is how a broken CI credential hides."""
|
|
||||||
|
|
||||||
import os
|
|
||||||
import uuid
|
import uuid
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from nisse_dev.envfile import load_env_file
|
from nisse_dev.config import AgentConfig
|
||||||
from nisse_dev.matrix import MatrixConfig, MatrixTransport
|
from nisse_dev.matrix import MatrixConfig, MatrixTransport
|
||||||
|
|
||||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
SENTINEL = "NISSE_MATRIX_HOMESERVER"
|
|
||||||
ALSO_REQUIRED = (
|
|
||||||
"NISSE_MATRIX_USER_ID",
|
|
||||||
"NISSE_MATRIX_TOKEN",
|
|
||||||
"NISSE_MATRIX_TEST_ROOM_ID",
|
|
||||||
)
|
|
||||||
|
|
||||||
pytestmark = [
|
|
||||||
pytest.mark.integration,
|
|
||||||
pytest.mark.skipif(
|
|
||||||
not os.environ.get(SENTINEL),
|
|
||||||
reason=f"no live homeserver configured ({SENTINEL} unset)",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_testing_identity(transport: MatrixTransport, config: MatrixConfig) -> None:
|
def _assert_testing_identity(transport: MatrixTransport, config: MatrixConfig) -> None:
|
||||||
@@ -46,7 +24,7 @@ def _assert_testing_identity(transport: MatrixTransport, config: MatrixConfig) -
|
|||||||
whoami.raise_for_status()
|
whoami.raise_for_status()
|
||||||
actual = whoami.json()["user_id"]
|
actual = whoami.json()["user_id"]
|
||||||
assert actual == config.user_id, (
|
assert actual == config.user_id, (
|
||||||
f"NISSE_MATRIX_USER_ID is {config.user_id} but the token belongs to "
|
f"matrix.user_id is {config.user_id} but the token belongs to "
|
||||||
f"{actual}; the self-message filter would not match"
|
f"{actual}; the self-message filter would not match"
|
||||||
)
|
)
|
||||||
assert "testing" in actual, (
|
assert "testing" in actual, (
|
||||||
@@ -54,13 +32,8 @@ def _assert_testing_identity(transport: MatrixTransport, config: MatrixConfig) -
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_send_lands_in_test_room() -> None:
|
def test_send_lands_in_test_room(live_config: AgentConfig) -> None:
|
||||||
missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)]
|
config = live_config.matrix
|
||||||
assert not missing, (
|
|
||||||
f"{SENTINEL} is set, so the live Matrix path is expected to work, "
|
|
||||||
f"but these are missing: {', '.join(missing)}"
|
|
||||||
)
|
|
||||||
config = MatrixConfig.from_env(room_var="NISSE_MATRIX_TEST_ROOM_ID")
|
|
||||||
marker = f"transport smoke test {uuid.uuid4().hex}"
|
marker = f"transport smoke test {uuid.uuid4().hex}"
|
||||||
transport = MatrixTransport(config)
|
transport = MatrixTransport(config)
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -2,12 +2,11 @@
|
|||||||
approve through the gate, then let the executor role actually create and test
|
approve through the gate, then let the executor role actually create and test
|
||||||
a file on disk.
|
a file on disk.
|
||||||
|
|
||||||
Gated on NISSE_LIVE_PLANNER_MODEL like the planning test — it needs a model
|
Uses the test config's planner model for every role: it needs a model that
|
||||||
that holds the tool-call contract on both ends.
|
holds the tool-call contract on both ends.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import os
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -19,7 +18,6 @@ from nisse_dev.control import (
|
|||||||
EXECUTING,
|
EXECUTING,
|
||||||
Orchestrator,
|
Orchestrator,
|
||||||
)
|
)
|
||||||
from nisse_dev.envfile import load_env_file
|
|
||||||
from nisse_dev.executor import LLMExecutor
|
from nisse_dev.executor import LLMExecutor
|
||||||
from nisse_dev.llm import LiteLLMClient, RoleMap
|
from nisse_dev.llm import LiteLLMClient, RoleMap
|
||||||
from nisse_dev.oplog import OpLog
|
from nisse_dev.oplog import OpLog
|
||||||
@@ -28,17 +26,7 @@ from nisse_dev.state import StateStore
|
|||||||
from nisse_dev.tools import Workspace
|
from nisse_dev.tools import Workspace
|
||||||
from nisse_dev.transport import InMemoryTransport
|
from nisse_dev.transport import InMemoryTransport
|
||||||
|
|
||||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
|
||||||
|
|
||||||
pytestmark = [
|
|
||||||
pytest.mark.integration,
|
|
||||||
pytest.mark.skipif(
|
|
||||||
not os.environ.get(SENTINEL),
|
|
||||||
reason=f"no live planner route configured ({SENTINEL} unset)",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _drive_to_execution(
|
def _drive_to_execution(
|
||||||
@@ -67,16 +55,14 @@ def _drive_to_execution(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_plan_approve_execute_produces_a_real_file(tmp_path: Path) -> None:
|
def test_plan_approve_execute_produces_a_real_file(
|
||||||
assert os.environ.get("NISSE_LLM_TOKEN"), (
|
tmp_path: Path, litellm: LiteLLMClient, live_model: str
|
||||||
f"{SENTINEL} is set, so the live path is expected to work, "
|
) -> None:
|
||||||
"but NISSE_LLM_TOKEN is missing"
|
|
||||||
)
|
|
||||||
# Both roles use the capable model here: the point is the whole gated
|
# Both roles use the capable model here: the point is the whole gated
|
||||||
# loop over a real proxy, not local-model competence (measured separately).
|
# loop over a real proxy, not local-model competence (measured separately).
|
||||||
model = os.environ[SENTINEL]
|
model = live_model
|
||||||
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
||||||
client = LiteLLMClient.from_env()
|
client = litellm
|
||||||
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
||||||
workspace = Workspace(tmp_path / "work", timeout_s=60)
|
workspace = Workspace(tmp_path / "work", timeout_s=60)
|
||||||
transport = InMemoryTransport()
|
transport = InMemoryTransport()
|
||||||
|
|||||||
@@ -9,12 +9,15 @@ What it measures is the thing that actually breaks: whether a model can hold
|
|||||||
the tool-call contract our loops depend on. llama3.2 could not — it returned
|
the tool-call contract our loops depend on. llama3.2 could not — it returned
|
||||||
`{` as an entire plan — and no amount of prompt work fixed it.
|
`{` as an entire plan — and no amount of prompt work fixed it.
|
||||||
|
|
||||||
Opt in with NISSE_CAPABILITY_MODELS, comma separated. Skipped otherwise,
|
Opt in by naming the models to measure, for one run:
|
||||||
because it spends real tokens across several models.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
uv run pytest -m integration tests/integration/test_model_capability.py \
|
||||||
from pathlib import Path
|
--capability-models=local-fast,local-coder
|
||||||
|
|
||||||
|
Skipped otherwise, because it spends real tokens across every model listed.
|
||||||
|
Which models to measure is a question for a particular run, so it is an
|
||||||
|
argument, not configuration.
|
||||||
|
"""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
@@ -26,45 +29,56 @@ from nisse_dev.canonical import (
|
|||||||
UserMessage,
|
UserMessage,
|
||||||
)
|
)
|
||||||
from nisse_dev.catalog import ModelCatalog, select_client
|
from nisse_dev.catalog import ModelCatalog, select_client
|
||||||
from nisse_dev.envfile import load_env_file
|
from nisse_dev.config import AgentConfig
|
||||||
from nisse_dev.expert import parse_answer
|
from nisse_dev.expert import parse_answer
|
||||||
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient, LLMError
|
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient, LLMError
|
||||||
from nisse_dev.planner import PLANNER_TOOLS
|
from nisse_dev.planner import PLANNER_TOOLS
|
||||||
from nisse_dev.tools import EXECUTOR_TOOLS
|
from nisse_dev.tools import EXECUTOR_TOOLS
|
||||||
|
|
||||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
SENTINEL = "NISSE_CAPABILITY_MODELS"
|
|
||||||
MODELS = [m.strip() for m in os.environ.get(SENTINEL, "").split(",") if m.strip()]
|
|
||||||
|
|
||||||
pytestmark = [
|
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
|
||||||
pytest.mark.integration,
|
if "model" not in metafunc.fixturenames:
|
||||||
pytest.mark.skipif(
|
return
|
||||||
not MODELS, reason=f"set {SENTINEL} to a comma-separated model list"
|
raw = metafunc.config.getoption("--capability-models")
|
||||||
|
models = [m.strip() for m in raw.split(",") if m.strip()]
|
||||||
|
if models:
|
||||||
|
metafunc.parametrize("model", models)
|
||||||
|
else:
|
||||||
|
metafunc.parametrize(
|
||||||
|
"model",
|
||||||
|
[
|
||||||
|
pytest.param(
|
||||||
|
None,
|
||||||
|
marks=pytest.mark.skip(
|
||||||
|
reason="no models to measure; pass --capability-models"
|
||||||
),
|
),
|
||||||
]
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def clients() -> tuple[LiteLLMClient, AnthropicPassthroughClient]:
|
@pytest.fixture
|
||||||
return LiteLLMClient.from_env(), AnthropicPassthroughClient.from_env()
|
def routes(
|
||||||
|
litellm: LiteLLMClient, passthrough: AnthropicPassthroughClient
|
||||||
|
) -> tuple[LiteLLMClient, AnthropicPassthroughClient]:
|
||||||
|
return litellm, passthrough
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def catalog() -> ModelCatalog:
|
def catalog(live_config: AgentConfig) -> ModelCatalog:
|
||||||
"""The live catalog this key can see. /model/info is scoped per key, so
|
"""The live catalog this key can see. /model/info is scoped per key, so
|
||||||
the probe list and the key have to agree."""
|
the probe list and the key have to agree."""
|
||||||
return ModelCatalog.discover(
|
return ModelCatalog.discover(live_config.llm_base_url, live_config.llm_token)
|
||||||
os.environ["NISSE_LLM_BASE_URL"], os.environ["NISSE_LLM_TOKEN"]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def user(text: str) -> list[StoredEvent]:
|
def user(text: str) -> list[StoredEvent]:
|
||||||
return [StoredEvent(id=1, ts="t", event=UserMessage(text=text, source="probe"))]
|
return [StoredEvent(id=1, ts="t", event=UserMessage(text=text, source="probe"))]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("model", MODELS)
|
def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None:
|
||||||
def test_model_answers_at_all(model: str, catalog: ModelCatalog) -> None:
|
normalized, passthrough = routes
|
||||||
normalized, passthrough = clients()
|
|
||||||
client = select_client(model, normalized, passthrough, catalog)
|
client = select_client(model, normalized, passthrough, catalog)
|
||||||
produced = client.complete(
|
produced = client.complete(
|
||||||
user("Reply with exactly one word: pong"), None, model, system="Be terse."
|
user("Reply with exactly one word: pong"), None, model, system="Be terse."
|
||||||
@@ -73,11 +87,12 @@ def test_model_answers_at_all(model: str, catalog: ModelCatalog) -> None:
|
|||||||
assert texts, f"{model} produced no text: {produced!r}"
|
assert texts, f"{model} produced no text: {produced!r}"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("model", MODELS)
|
def test_model_holds_the_planner_contract(
|
||||||
def test_model_holds_the_planner_contract(model: str, catalog: ModelCatalog) -> None:
|
model: str, catalog: ModelCatalog, routes
|
||||||
|
) -> None:
|
||||||
"""A4's gating is structural at the tool layer: a model that answers in
|
"""A4's gating is structural at the tool layer: a model that answers in
|
||||||
prose cannot be gated, because there is no plan to approve."""
|
prose cannot be gated, because there is no plan to approve."""
|
||||||
normalized, passthrough = clients()
|
normalized, passthrough = routes
|
||||||
client = select_client(model, normalized, passthrough, catalog)
|
client = select_client(model, normalized, passthrough, catalog)
|
||||||
produced = client.complete(
|
produced = client.complete(
|
||||||
user("Draft an architecture plan for a command-line slug generator."),
|
user("Draft an architecture plan for a command-line slug generator."),
|
||||||
@@ -100,11 +115,12 @@ def test_model_holds_the_planner_contract(model: str, catalog: ModelCatalog) ->
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("model", MODELS)
|
def test_model_round_trips_a_tool_result(
|
||||||
def test_model_round_trips_a_tool_result(model: str, catalog: ModelCatalog) -> None:
|
model: str, catalog: ModelCatalog, routes
|
||||||
|
) -> None:
|
||||||
"""The executor loop feeds results back; a model that cannot consume its
|
"""The executor loop feeds results back; a model that cannot consume its
|
||||||
own tool result cannot run a milestone."""
|
own tool result cannot run a milestone."""
|
||||||
normalized, passthrough = clients()
|
normalized, passthrough = routes
|
||||||
client = select_client(model, normalized, passthrough, catalog)
|
client = select_client(model, normalized, passthrough, catalog)
|
||||||
events = user("List the files in the current directory using your tools.")
|
events = user("List the files in the current directory using your tools.")
|
||||||
produced = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.")
|
produced = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.")
|
||||||
@@ -128,13 +144,12 @@ def test_model_round_trips_a_tool_result(model: str, catalog: ModelCatalog) -> N
|
|||||||
assert follow_up, f"{model} produced nothing after its tool result"
|
assert follow_up, f"{model} produced nothing after its tool result"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("model", MODELS)
|
|
||||||
def test_model_holds_the_prompt_level_json_contract(
|
def test_model_holds_the_prompt_level_json_contract(
|
||||||
model: str, catalog: ModelCatalog
|
model: str, catalog: ModelCatalog, routes
|
||||||
) -> None:
|
) -> None:
|
||||||
"""A14 contracts structured output at the prompt level because vendors
|
"""A14 contracts structured output at the prompt level because vendors
|
||||||
differ on JSON enforcement. This measures whether that holds per model."""
|
differ on JSON enforcement. This measures whether that holds per model."""
|
||||||
normalized, passthrough = clients()
|
normalized, passthrough = routes
|
||||||
client = select_client(model, normalized, passthrough, catalog)
|
client = select_client(model, normalized, passthrough, catalog)
|
||||||
produced = client.complete(
|
produced = client.complete(
|
||||||
user("Is SQLite sufficient for a single-process agent's task state?"),
|
user("Is SQLite sufficient for a single-process agent's task state?"),
|
||||||
@@ -152,12 +167,13 @@ def test_model_holds_the_prompt_level_json_contract(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("model", MODELS)
|
def test_route_selection_matches_the_model(
|
||||||
def test_route_selection_matches_the_model(model: str, catalog: ModelCatalog) -> None:
|
model: str, catalog: ModelCatalog, routes
|
||||||
|
) -> None:
|
||||||
"""A non-Anthropic model must not be sent Anthropic-shaped bodies. This
|
"""A non-Anthropic model must not be sent Anthropic-shaped bodies. This
|
||||||
was hardcoded until gpt-6-astra made it observable, and name-sniffed until
|
was hardcoded until gpt-6-astra made it observable, and name-sniffed until
|
||||||
semantic aliases made the name stop meaning anything."""
|
semantic aliases made the name stop meaning anything."""
|
||||||
normalized, passthrough = clients()
|
normalized, passthrough = routes
|
||||||
facts = catalog.facts(model)
|
facts = catalog.facts(model)
|
||||||
chosen = select_client(model, normalized, passthrough, catalog)
|
chosen = select_client(model, normalized, passthrough, catalog)
|
||||||
if facts.provider == "anthropic":
|
if facts.provider == "anthropic":
|
||||||
|
|||||||
@@ -6,30 +6,19 @@ passthrough rather than being flattened by the proxy.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import os
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from nisse_dev.canonical import AssistantText, ToolCall, ToolResult, UserMessage
|
from nisse_dev.canonical import AssistantText, ToolCall, ToolResult, UserMessage
|
||||||
from nisse_dev.envfile import load_env_file
|
from nisse_dev.config import AgentConfig
|
||||||
from nisse_dev.llm import AnthropicPassthroughClient
|
from nisse_dev.llm import AnthropicPassthroughClient
|
||||||
from nisse_dev.oplog import OpLog
|
from nisse_dev.oplog import OpLog
|
||||||
from nisse_dev.state import StateStore
|
from nisse_dev.state import StateStore
|
||||||
from nisse_dev.tools import EXECUTOR_TOOLS
|
from nisse_dev.tools import EXECUTOR_TOOLS
|
||||||
|
|
||||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
|
||||||
|
|
||||||
pytestmark = [
|
|
||||||
pytest.mark.integration,
|
|
||||||
pytest.mark.skipif(
|
|
||||||
not os.environ.get(SENTINEL),
|
|
||||||
reason=f"no live frontier route configured ({SENTINEL} unset)",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def store_with(tmp_path: Path, *events) -> StateStore:
|
def store_with(tmp_path: Path, *events) -> StateStore:
|
||||||
@@ -39,18 +28,22 @@ def store_with(tmp_path: Path, *events) -> StateStore:
|
|||||||
return store
|
return store
|
||||||
|
|
||||||
|
|
||||||
def test_passthrough_completes_a_plain_turn(tmp_path: Path) -> None:
|
def test_passthrough_completes_a_plain_turn(
|
||||||
|
tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str
|
||||||
|
) -> None:
|
||||||
store = store_with(
|
store = store_with(
|
||||||
tmp_path, UserMessage(text="Reply with exactly: pong", source="live")
|
tmp_path, UserMessage(text="Reply with exactly: pong", source="live")
|
||||||
)
|
)
|
||||||
result = AnthropicPassthroughClient.from_env().complete(
|
result = passthrough.complete(
|
||||||
store.read_events(), None, os.environ[SENTINEL], system="Be terse."
|
store.read_events(), None, live_model, system="Be terse."
|
||||||
)
|
)
|
||||||
texts = [e.text for e in result if isinstance(e, AssistantText)]
|
texts = [e.text for e in result if isinstance(e, AssistantText)]
|
||||||
assert texts and "pong" in texts[0].lower()
|
assert texts and "pong" in texts[0].lower()
|
||||||
|
|
||||||
|
|
||||||
def test_passthrough_round_trips_a_tool_call(tmp_path: Path) -> None:
|
def test_passthrough_round_trips_a_tool_call(
|
||||||
|
tmp_path: Path, passthrough: AnthropicPassthroughClient, live_model: str
|
||||||
|
) -> None:
|
||||||
store = store_with(
|
store = store_with(
|
||||||
tmp_path,
|
tmp_path,
|
||||||
UserMessage(
|
UserMessage(
|
||||||
@@ -58,11 +51,11 @@ def test_passthrough_round_trips_a_tool_call(tmp_path: Path) -> None:
|
|||||||
source="live",
|
source="live",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
client = AnthropicPassthroughClient.from_env()
|
client = passthrough
|
||||||
produced = client.complete(
|
produced = client.complete(
|
||||||
store.read_events(),
|
store.read_events(),
|
||||||
EXECUTOR_TOOLS,
|
EXECUTOR_TOOLS,
|
||||||
os.environ[SENTINEL],
|
live_model,
|
||||||
system="Use a tool. Do not answer in prose.",
|
system="Use a tool. Do not answer in prose.",
|
||||||
)
|
)
|
||||||
calls = [e for e in produced if isinstance(e, ToolCall)]
|
calls = [e for e in produced if isinstance(e, ToolCall)]
|
||||||
@@ -75,18 +68,18 @@ def test_passthrough_round_trips_a_tool_call(tmp_path: Path) -> None:
|
|||||||
store.append_event(
|
store.append_event(
|
||||||
ToolResult(call_id=calls[0].call_id, result="app.py\ntests/", success=True)
|
ToolResult(call_id=calls[0].call_id, result="app.py\ntests/", success=True)
|
||||||
)
|
)
|
||||||
follow_up = client.complete(
|
follow_up = client.complete(store.read_events(), EXECUTOR_TOOLS, live_model)
|
||||||
store.read_events(), EXECUTOR_TOOLS, os.environ[SENTINEL]
|
|
||||||
)
|
|
||||||
assert follow_up, "the vendor rejected or ignored the tool_result turn"
|
assert follow_up, "the vendor rejected or ignored the tool_result turn"
|
||||||
|
|
||||||
|
|
||||||
def test_prompt_caching_survives_the_passthrough() -> None:
|
def test_prompt_caching_survives_the_passthrough(
|
||||||
|
live_config: AgentConfig, live_model: str
|
||||||
|
) -> None:
|
||||||
# A5 asks for this to be verified rather than assumed: a proxy that
|
# A5 asks for this to be verified rather than assumed: a proxy that
|
||||||
# flattened the request would silently drop cache control and turn a
|
# flattened the request would silently drop cache control and turn a
|
||||||
# cost optimisation into a per-call expense.
|
# cost optimisation into a per-call expense.
|
||||||
base = os.environ["NISSE_LLM_BASE_URL"].rstrip("/")
|
base = live_config.llm_base_url.rstrip("/")
|
||||||
token = os.environ["NISSE_LLM_TOKEN"]
|
token = live_config.llm_token
|
||||||
response = httpx.post(
|
response = httpx.post(
|
||||||
f"{base}/v1/messages",
|
f"{base}/v1/messages",
|
||||||
headers={
|
headers={
|
||||||
@@ -95,7 +88,7 @@ def test_prompt_caching_survives_the_passthrough() -> None:
|
|||||||
"anthropic-version": "2023-06-01",
|
"anthropic-version": "2023-06-01",
|
||||||
},
|
},
|
||||||
json={
|
json={
|
||||||
"model": os.environ[SENTINEL],
|
"model": live_model,
|
||||||
"max_tokens": 16,
|
"max_tokens": 16,
|
||||||
"messages": [{"role": "user", "content": "Say pong."}],
|
"messages": [{"role": "user", "content": "Say pong."}],
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,47 +1,30 @@
|
|||||||
"""Live planning turn: a real model must answer with a submit_plan or
|
"""Live planning turn: a real model must answer with a submit_plan or
|
||||||
ask_human tool call, not prose.
|
ask_human tool call, not prose.
|
||||||
|
|
||||||
Deliberately opt-in via NISSE_LIVE_PLANNER_MODEL rather than the usual
|
Plans with the test config's planner role. Planning escalates to a frontier
|
||||||
NISSE_LLM_BASE_URL sentinel. Planning escalates to a frontier model
|
model unconditionally (A12), and small local models do not hold the tool-call
|
||||||
unconditionally (A12), and the small local model does not hold the tool-call
|
contract reliably, so that role must be one that does.
|
||||||
contract reliably — measured, see the PR — so this test needs a frontier
|
|
||||||
route that does not exist yet. Set the variable to the model to plan with
|
|
||||||
once one is configured; until then this skips on purpose, not by accident.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import os
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from nisse_dev.canonical import UserMessage
|
from nisse_dev.canonical import UserMessage
|
||||||
from nisse_dev.control import AskHuman, SubmitPlan
|
from nisse_dev.control import AskHuman, SubmitPlan
|
||||||
from nisse_dev.envfile import load_env_file
|
|
||||||
from nisse_dev.llm import LiteLLMClient, RoleMap
|
from nisse_dev.llm import LiteLLMClient, RoleMap
|
||||||
from nisse_dev.oplog import OpLog
|
from nisse_dev.oplog import OpLog
|
||||||
from nisse_dev.planner import LLMPlanningAgent
|
from nisse_dev.planner import LLMPlanningAgent
|
||||||
from nisse_dev.state import StateStore
|
from nisse_dev.state import StateStore
|
||||||
|
|
||||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
|
||||||
|
|
||||||
pytestmark = [
|
|
||||||
pytest.mark.integration,
|
|
||||||
pytest.mark.skipif(
|
|
||||||
not os.environ.get(SENTINEL),
|
|
||||||
reason=f"no frontier planner route configured ({SENTINEL} unset)",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_live_planning_turn_returns_a_tool_action(tmp_path: Path) -> None:
|
def test_live_planning_turn_returns_a_tool_action(
|
||||||
assert os.environ.get("NISSE_LLM_TOKEN"), (
|
tmp_path: Path, litellm: LiteLLMClient, live_model: str
|
||||||
f"{SENTINEL} is set, so the live planning path is expected to work, "
|
) -> None:
|
||||||
"but NISSE_LLM_TOKEN is missing"
|
model = live_model
|
||||||
)
|
|
||||||
model = os.environ[SENTINEL]
|
|
||||||
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
||||||
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
||||||
store.append_event(
|
store.append_event(
|
||||||
@@ -53,7 +36,7 @@ def test_live_planning_turn_returns_a_tool_action(tmp_path: Path) -> None:
|
|||||||
source="live-test",
|
source="live-test",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
agent = LLMPlanningAgent(LiteLLMClient.from_env(), store, roles)
|
agent = LLMPlanningAgent(litellm, store, roles)
|
||||||
|
|
||||||
action = agent.plan("arch", "hello-cli")
|
action = agent.plan("arch", "hello-cli")
|
||||||
|
|
||||||
|
|||||||
@@ -113,12 +113,14 @@ source = { editable = "nisse-dev" }
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "httpx" },
|
{ name = "httpx" },
|
||||||
{ name = "nisse-core" },
|
{ name = "nisse-core" },
|
||||||
|
{ name = "pyyaml" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[package.metadata]
|
[package.metadata]
|
||||||
requires-dist = [
|
requires-dist = [
|
||||||
{ name = "httpx", specifier = ">=0.27" },
|
{ name = "httpx", specifier = ">=0.27" },
|
||||||
{ name = "nisse-core", editable = "nisse-core" },
|
{ name = "nisse-core", editable = "nisse-core" },
|
||||||
|
{ name = "pyyaml", specifier = ">=6" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -175,6 +177,52 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
|
{ url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pyyaml"
|
||||||
|
version = "6.0.3"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ruff"
|
name = "ruff"
|
||||||
version = "0.16.6"
|
version = "0.16.6"
|
||||||
|
|||||||
Reference in new issue
Block a user