From 50b46821d73c43db4296daecf6e8ad5f5d91b114 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Fri, 25 Sep 2026 00:35:24 +0000 Subject: [PATCH 1/2] Add an executor benchmark that keeps its results Choosing a local executor model, and tuning its knobs, needs measured success rates across repeated runs, comparable with every model measured before. - Four canned milestones under nisse_dev/bench/tasks: write a function, fix a bug a failing test exposes, thread an option through several files, and find one failure in output far past max_output_chars. Each is an approved plan over a seed project, judged by a hidden check the model never sees; a reference solution proves each is solvable and that its check tells the difference - Runs use the agent's own client wiring, now shared as make_llm_clients, with delegation and ask_expert off: an escalation is a failure of the model under test - Each run is stored as one JSON line in bench/results/, committed, with the model's backing name and declared facts, a note on how it was served, the task's content hash, the agent's commit and tuning, the outcome, turns, misbehaviour counts, context trims, tokens, time and LiteLLM's cost; a fallback answer can never count as a success - The report rebuilds the comparison from every stored run, with Wilson 95% intervals, marking runs against older task versions - scripts/benchmark.py runs and reports; scripts/evaluate_model.sh takes a new model through the context probe, the capability matrix and the benchmark in one command - Clients keep LiteLLM's per-call cost, and BudgetedClient counts the requests it had to trim - The tuning guide gains "Benchmarking models" Co-Authored-By: Claude Opus 5.5 (1M context) --- bench/results/.gitkeep | 0 docs/Tuning-Guide.md | 71 +++- nisse-dev/src/nisse_dev/agent.py | 77 +++-- nisse-dev/src/nisse_dev/bench/__init__.py | 17 + nisse-dev/src/nisse_dev/bench/results.py | 117 +++++++ nisse-dev/src/nisse_dev/bench/runner.py | 218 +++++++++++++ nisse-dev/src/nisse_dev/bench/tasks.py | 102 ++++++ .../hidden/original_test_inventory.py.txt | 11 + .../bench/tasks/fix-bug/hidden/test_hidden.py | 27 ++ .../bench/tasks/fix-bug/seed/inventory.py | 23 ++ .../tasks/fix-bug/seed/test_inventory.py | 11 + .../bench/tasks/fix-bug/solution/inventory.py | 23 ++ .../nisse_dev/bench/tasks/fix-bug/task.yaml | 12 + .../tasks/long-output/hidden/test_hidden.py | 11 + .../bench/tasks/long-output/seed/rates.py | 304 ++++++++++++++++++ .../tasks/long-output/seed/test_rates.py | 7 + .../bench/tasks/long-output/solution/rates.py | 304 ++++++++++++++++++ .../bench/tasks/long-output/task.yaml | 12 + .../tasks/multi-file/hidden/test_hidden.py | 33 ++ .../tasks/multi-file/seed/greeter/__init__.py | 0 .../tasks/multi-file/seed/greeter/__main__.py | 3 + .../tasks/multi-file/seed/greeter/cli.py | 11 + .../tasks/multi-file/seed/greeter/core.py | 2 + .../multi-file/seed/tests/test_greeter.py | 11 + .../tasks/multi-file/solution/greeter/cli.py | 12 + .../tasks/multi-file/solution/greeter/core.py | 3 + .../bench/tasks/multi-file/task.yaml | 16 + .../write-function/hidden/test_hidden.py | 19 ++ .../bench/tasks/write-function/seed/README.md | 1 + .../tasks/write-function/solution/slugify.py | 5 + .../bench/tasks/write-function/task.yaml | 13 + nisse-dev/src/nisse_dev/context.py | 9 + nisse-dev/src/nisse_dev/llm.py | 13 + nisse-dev/tests/test_bench.py | 265 +++++++++++++++ scripts/benchmark.py | 124 +++++++ scripts/evaluate_model.sh | 31 ++ tests/integration/test_benchmark_live.py | 31 ++ 37 files changed, 1912 insertions(+), 37 deletions(-) create mode 100644 bench/results/.gitkeep create mode 100644 nisse-dev/src/nisse_dev/bench/__init__.py create mode 100644 nisse-dev/src/nisse_dev/bench/results.py create mode 100644 nisse-dev/src/nisse_dev/bench/runner.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/original_test_inventory.py.txt create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/test_hidden.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/inventory.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/test_inventory.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/fix-bug/solution/inventory.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/long-output/hidden/test_hidden.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/rates.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/test_rates.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/long-output/solution/rates.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/multi-file/hidden/test_hidden.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__init__.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__main__.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/cli.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/core.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/tests/test_greeter.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/cli.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/core.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/write-function/hidden/test_hidden.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/write-function/seed/README.md create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/write-function/solution/slugify.py create mode 100644 nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml create mode 100644 nisse-dev/tests/test_bench.py create mode 100644 scripts/benchmark.py create mode 100755 scripts/evaluate_model.sh create mode 100644 tests/integration/test_benchmark_live.py diff --git a/bench/results/.gitkeep b/bench/results/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/Tuning-Guide.md b/docs/Tuning-Guide.md index af4c631..8977f3f 100644 --- a/docs/Tuning-Guide.md +++ b/docs/Tuning-Guide.md @@ -370,6 +370,66 @@ declaration. --- +## Benchmarking models + +The capability matrix asks whether a model *can* hold the tool contract. The +benchmark asks whether it *finishes real work*: four canned milestones, each an +approved plan over a small seed project, judged by a hidden check the model +never sees. Success means the check passes, not that the model said it was +done. + +| Task | What it exercises | +|---|---| +| `write-function` | write a small function and its tests | +| `fix-bug` | find and fix the bug a failing test exposes, without touching the test | +| `multi-file` | thread a new option through several files | +| `long-output` | find one failure in test output far longer than `max_output_chars` | + +It runs the executor exactly as the agent does (the same client wiring, +context budget and `tool_choice` handling, and your config's tuning knobs), +with **delegation and `ask_expert` turned off**. An escalation is a failure of +the model under test, not a frontier model's success. + +**Evaluating a new model** is one command, once its LiteLLM entry declares its +limits: + + scripts/evaluate_model.sh qwen3-32b-q4 "vLLM 0.29, AWQ, fp8 KV" 5 + +It runs the context probe, then the capability matrix, then five runs of +every task, and stops at the first step that fails. Or run the steps +yourself: + + uv run python scripts/benchmark.py run --models A,B --runs 5 --note "how it's served" + uv run python scripts/benchmark.py report [--models A,B] [--tasks T] + +**Results are kept.** Every run is one line in `bench/results/`, committed +with the repo, recording: +- the model's backing name and declared facts; +- your note on how it was served; +- the task's content hash; +- the agent's commit and tuning; +- the outcome, turns, failures, tokens, time and cost. + +The report rebuilds the table from every stored run. Benchmarking a new model +means running only that model and reading it next to everything before it. +Runs against an older version of a task are marked `(old)`. Runs of one model +served differently (different note) are kept apart. + +**Reading it:** +- **Compare the intervals, not the rates.** 3 of 5 (23–88%) and 5 of 5 + (57–100%) overlap: five runs don't separate them. Add runs until the + intervals stop overlapping, or accept that the models are close. +- **Look at why runs failed**, not just how many (`outcome_note` and the + counts in each result line). An early shakedown on `local-fast` failed + mostly by escalation: the model was fixing its own mistakes and hit + `failure_limit: 2` mid-recovery. That says as much about the knob as about + the model. Tuning a knob and re-running is a fair comparison, because the + tuning is recorded with every run. +- **Cost** is per run, straight from LiteLLM, so a frontier model as executor + can be compared on success rate *and* dollars. + +--- + ## Part 3 — An order to tune in Upstream choices change what's right downstream, so tune in this order and @@ -400,8 +460,8 @@ change one thing at a time. 3. **Declare the window** in LiteLLM and check it with the probe, then set the model's `max_output_tokens` there so the prompt keeps most of the window. -4. **Real milestones.** Run representative work and look at the evidence: - turns used, escalations, delegation outcomes, and time taken. +4. **The benchmark.** Run it and look at the evidence: success rate, turns, + escalations, and time taken. 5. **The loop knobs**: `failure_limit`, `max_turns`, `consult_budget`, the delegation caps, then the tool limits, one at a time. @@ -415,11 +475,10 @@ run enough times that the result would survive a coin flip. ## Coming next -The next changes in the 4090-readiness work add knobs, and this guide gains a -section for each: +Later work that will add knobs, each with its own section here: - **Compaction.** A model-written summary replacing old turns, instead of dropping them. The log already has an event for it; it needs a summarizing model and a policy for what is safe to summarize. -- **A benchmark script** that runs the capability matrix and representative - milestones across candidate models and prints a comparison table. +- **Benchmarking the other roles:** planner and delegate, with delegation + turned back on for comparisons across frontier tiers. diff --git a/nisse-dev/src/nisse_dev/agent.py b/nisse-dev/src/nisse_dev/agent.py index 4f1a04b..966f56d 100644 --- a/nisse-dev/src/nisse_dev/agent.py +++ b/nisse-dev/src/nisse_dev/agent.py @@ -113,6 +113,51 @@ class Agent: self._stopped.set() +def make_llm_clients( + config: AgentConfig, catalog: ModelCatalog +) -> tuple[BudgetedClient, BudgetedClient]: + """The two LLM routes, as every model call in the agent uses them. + + Shared with the benchmark, so that what it measures is exactly what the + agent would do: the same context budget, the same declared limits, and + tool_choice asked only of models declared to enforce it. + """ + tuning = config.tuning + + def limits_for(model: str) -> tuple[int, int | None]: + facts = catalog.facts(model) + assert facts.max_input_tokens is not None # require_limits checked + return facts.max_input_tokens, facts.max_output_tokens + + def budgeted(client: object) -> BudgetedClient: + # Every request, on either route, is fitted to its model's window. + return BudgetedClient( + client, + limits_for, + max_output_tokens=config.max_output_tokens, + keep_recent_turns=tuning.context.keep_recent_turns, + safety_margin=tuning.context.safety_margin, + enforces_tool_choice=lambda m: ( + catalog.facts(m).supports_tool_choice is True + ), + ) + + normalized = budgeted( + LiteLLMClient( + config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens + ) + ) + # Delegations prefer the vendor-native route; fidelity is cheap + # elsewhere (A5). Which route a given model gets is decided by + # select_client, not by where the caller happens to sit. + passthrough = budgeted( + AnthropicPassthroughClient( + config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens + ) + ) + return normalized, passthrough + + def prepare_workspace(config: AgentConfig, account: str) -> GitRepo: """Ensure the workspace is a clone of the project repo. @@ -161,37 +206,7 @@ def build_agent( max_output_chars=tuning.tools.max_output_chars, ) - def limits_for(model: str) -> tuple[int, int | None]: - facts = catalog.facts(model) - assert facts.max_input_tokens is not None # require_limits checked - return facts.max_input_tokens, facts.max_output_tokens - - def budgeted(client: object) -> BudgetedClient: - # Every request, on either route, is fitted to its model's window. - return BudgetedClient( - client, - limits_for, - max_output_tokens=config.max_output_tokens, - keep_recent_turns=tuning.context.keep_recent_turns, - safety_margin=tuning.context.safety_margin, - enforces_tool_choice=lambda m: ( - catalog.facts(m).supports_tool_choice is True - ), - ) - - normalized = budgeted( - LiteLLMClient( - config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens - ) - ) - # Delegations prefer the vendor-native route; fidelity is cheap - # elsewhere (A5). Which route a given model gets is decided by - # select_client, not by where the caller happens to sit. - passthrough = budgeted( - AnthropicPassthroughClient( - config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens - ) - ) + normalized, passthrough = make_llm_clients(config, catalog) transport = MatrixTransport(config.matrix) announce = notify or transport.send_message diff --git a/nisse-dev/src/nisse_dev/bench/__init__.py b/nisse-dev/src/nisse_dev/bench/__init__.py new file mode 100644 index 0000000..1e5b0a7 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/__init__.py @@ -0,0 +1,17 @@ +"""The executor benchmark: canned milestones, run repeatedly per model. + +Choosing an executor model, and later tuning its knobs, should rest on +measured success rates, not on one run or on reputation. Each task is a fixed +approved plan over a seed project, with a hidden check the model never sees; +success means the check passes, not that the model said it finished. Runs use +the agent's own client wiring (context budget, declared limits, tool_choice +only where declared), so what is measured is what the agent would do. + +Results are stored, one JSON line per run, with everything needed to compare +them later: the model's identity and declared facts, a note on how it was +served, the task's content hash, the agent's commit, and its tuning. A new +model then needs only its own runs to be compared with everything measured +before it. + +See scripts/benchmark.py, and docs/Tuning-Guide.md for how to read results. +""" diff --git a/nisse-dev/src/nisse_dev/bench/results.py b/nisse-dev/src/nisse_dev/bench/results.py new file mode 100644 index 0000000..0e3fb43 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/results.py @@ -0,0 +1,117 @@ +"""Stored benchmark results, and the report that compares them. + +Every run is one JSON line. The report is rebuilt from all of them each +time, so a model benchmarked today is read alongside every model measured +before it, as long as the task is still the same version. +""" + +import json +import math +import statistics +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from nisse_dev.bench.runner import RunRecord + + +def results_file(directory: Path, model: str, when: datetime | None = None) -> Path: + day = (when or datetime.now(UTC)).strftime("%Y-%m-%d") + return directory / f"{day}-{model}.jsonl" + + +def append(path: Path, record: RunRecord) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as out: + out.write(json.dumps(record.to_json(), sort_keys=True) + "\n") + + +def load(directory: Path) -> list[dict[str, Any]]: + records = [] + for path in sorted(directory.glob("*.jsonl")): + for line in path.read_text().splitlines(): + if line.strip(): + records.append(json.loads(line)) + return records + + +def wilson(successes: int, runs: int, z: float = 1.96) -> tuple[float, float]: + """A 95% confidence interval for a success rate, sound at small counts. + + Three successes in five runs and five in five overlap here, which is the + point: small samples of a noisy model do not separate much.""" + if runs == 0: + return 0.0, 1.0 + p = successes / runs + denominator = 1 + z * z / runs + centre = (p + z * z / (2 * runs)) / denominator + half = z * math.sqrt(p * (1 - p) / runs + z * z / (4 * runs * runs)) / denominator + return max(0.0, centre - half), min(1.0, centre + half) + + +@dataclass +class Row: + model: str + note: str + task: str + runs: int + successes: int + median_turns: float + mean_wall_s: float + cost_per_run: float + stale: bool + + @property + def interval(self) -> tuple[float, float]: + return wilson(self.successes, self.runs) + + +def summarise( + records: Iterable[dict[str, Any]], current_hashes: dict[str, str] +) -> list[Row]: + """One row per (model, how it was served, task, task version).""" + groups: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list) + for r in records: + groups[(r["backing"], r.get("note", ""), r["task"], r["task_hash"])].append(r) + rows = [] + for (backing, note, task, task_hash), runs in sorted(groups.items()): + rows.append( + Row( + model=backing, + note=note, + task=task, + runs=len(runs), + successes=sum(1 for r in runs if r["success"]), + median_turns=statistics.median(r["turns"] for r in runs), + mean_wall_s=statistics.fmean(r["wall_s"] for r in runs), + cost_per_run=statistics.fmean(r["cost"] for r in runs), + stale=current_hashes.get(task) != task_hash, + ) + ) + return rows + + +def render(rows: list[Row]) -> str: + header = ( + f"{'model':<34} {'note':<22} {'task':<15} {'success':>8} {'95% CI':>11} " + f"{'turns':>6} {'time':>7} {'$/run':>8}" + ) + lines = [header, "-" * len(header)] + for row in rows: + low, high = row.interval + task = row.task + (" (old)" if row.stale else "") + lines.append( + f"{row.model[:34]:<34} {row.note[:22]:<22} {task[:15]:<15} " + f"{row.successes:>3}/{row.runs:<4} {low:>4.0%}-{high:<5.0%} " + f"{row.median_turns:>6.0f} {row.mean_wall_s:>6.0f}s {row.cost_per_run:>8.4f}" + ) + if any(row.stale for row in rows): + lines.append("") + lines.append( + "(old) = measured against an earlier version of the task; " + "not comparable with current runs." + ) + return "\n".join(lines) diff --git a/nisse-dev/src/nisse_dev/bench/runner.py b/nisse-dev/src/nisse_dev/bench/runner.py new file mode 100644 index 0000000..ba4d02a --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/runner.py @@ -0,0 +1,218 @@ +"""Running one task once with one executor model, and recording it.""" + +import io +import subprocess +import tempfile +import time +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from nisse_dev.bench.tasks import Task +from nisse_dev.canonical import ( + CanonicalEvent, + PlanSubmitted, + StoredEvent, + ToolCall, + ToolResult, + UserMessage, +) +from nisse_dev.catalog import ModelFacts +from nisse_dev.config import Tuning +from nisse_dev.executor import LLMExecutor +from nisse_dev.llm import RoleMap +from nisse_dev.oplog import OpLog +from nisse_dev.state import StateStore +from nisse_dev.tools import Workspace + + +class MeteringClient: + """Counts what a run spends, and notices a fallback: an answer from a + model other than the one asked for is not a measurement of it.""" + + def __init__(self, inner: Any, facts: ModelFacts) -> None: + self._inner = inner + self._facts = facts + self.calls = 0 + self.prose_turns = 0 + self.tokens_in = 0 + self.tokens_out = 0 + self.cost = 0.0 + self.fallbacks: list[str] = [] + + def complete( + self, + events: list[StoredEvent], + tools: list[dict[str, Any]] | None, + model: str, + system: str | None = None, + ) -> list[CanonicalEvent]: + self.calls += 1 + produced = self._inner.complete(events, tools, model, system=system) + usage = getattr(self._inner, "last_usage", {}) or {} + self.tokens_in += int(usage.get("input", 0)) + self.tokens_out += int(usage.get("output", 0)) + self.cost += float(getattr(self._inner, "last_cost", 0.0) or 0.0) + answered = getattr(self._inner, "last_model", None) + if not self._facts.answered(answered): + self.fallbacks.append(str(answered)) + if not any(isinstance(e, ToolCall) for e in produced): + self.prose_turns += 1 + return produced + + +@dataclass +class RunRecord: + """One run, with everything needed to compare it with runs made later + against other models, other servers, or other tuning.""" + + ts: str + model: str + backing: str + provider: str + model_info: dict[str, Any] + note: str + task: str + task_hash: str + commit: str + tuning: dict[str, Any] + success: bool + outcome: str + outcome_note: str + turns: int + prose_turns: int + failed_commands: int + invented_tools: int + trimmed_requests: int + tokens_in: int + tokens_out: int + cost: float + wall_s: float + fallbacks: list[str] = field(default_factory=list) + check_output: str = "" + + def to_json(self) -> dict[str, Any]: + return asdict(self) + + +def agent_commit(repo: Path | None = None) -> str: + """The agent's commit, marked when the tree had uncommitted changes.""" + where = repo or Path(__file__).resolve().parent + try: + sha = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + cwd=where, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + dirty = subprocess.run( + ["git", "status", "--porcelain"], + cwd=where, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "unknown" + return f"{sha}+dirty" if dirty else sha + + +def _count(events: list[StoredEvent]) -> tuple[int, int]: + tools = { + e.event.call_id: e.event.tool for e in events if isinstance(e.event, ToolCall) + } + failed = invented = 0 + for stored in events: + event = stored.event + if isinstance(event, ToolResult) and not event.success: + if event.result.startswith("no such tool"): + invented += 1 + elif tools.get(event.call_id) == "run_command": + failed += 1 + return failed, invented + + +def run_task( + task: Task, + facts: ModelFacts, + client: Any, + tuning: Tuning, + *, + note: str = "", + commit: str = "", + workdir: Path | None = None, +) -> RunRecord: + """Run one task once: seed a workspace, hand the executor the approved + plan, then judge the result with the hidden check. + + Delegation and ask_expert are off: an escalation is a failure of the + model under test, not a frontier model's success.""" + with tempfile.TemporaryDirectory(dir=workdir) as tmp: + root = Path(tmp) + workspace = Workspace( + root / "work", + timeout_s=tuning.tools.command_timeout_s, + max_output_chars=tuning.tools.max_output_chars, + ) + task.seed_into(workspace.root) + store = StateStore(root / "state.db", oplog=OpLog("nisse-bench", io.StringIO())) + store.append_event(PlanSubmitted(plan=task.plan, model="benchmark")) + store.append_event(UserMessage(text="start milestone 1", source="benchmark")) + + meter = MeteringClient(client, facts) + trims_before = getattr(client, "trimmed_requests", 0) + model = facts.alias + roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model) + executor = LLMExecutor( + meter, + store, + roles, + workspace, + max_turns=tuning.executor.max_turns, + failure_limit=tuning.executor.failure_limit, + ) + started = time.monotonic() + try: + outcome = executor.run(1) + status, detail = outcome.status, outcome.note + except Exception as exc: # noqa: BLE001 -- recorded, the benchmark goes on + status, detail = "error", repr(exc) + wall = time.monotonic() - started + + success, check_output = task.run_check(workspace.root) + failed, invented = _count(store.read_events()) + store.close() + + return RunRecord( + ts=datetime.now(UTC).isoformat(timespec="seconds"), + model=model, + backing=facts.backing, + provider=facts.provider, + model_info={ + "max_input_tokens": facts.max_input_tokens, + "max_output_tokens": facts.max_output_tokens, + "supports_tool_choice": facts.supports_tool_choice, + "supports_function_calling": facts.supports_tools, + }, + note=note, + task=task.name, + task_hash=task.content_hash, + commit=commit, + tuning=asdict(tuning), + success=success and not meter.fallbacks, + outcome=status, + outcome_note=detail[:500], + turns=meter.calls, + prose_turns=meter.prose_turns, + failed_commands=failed, + invented_tools=invented, + trimmed_requests=getattr(client, "trimmed_requests", 0) - trims_before, + tokens_in=meter.tokens_in, + tokens_out=meter.tokens_out, + cost=round(meter.cost, 6), + wall_s=round(wall, 1), + fallbacks=meter.fallbacks, + check_output=check_output, + ) diff --git a/nisse-dev/src/nisse_dev/bench/tasks.py b/nisse-dev/src/nisse_dev/bench/tasks.py new file mode 100644 index 0000000..e71a4fe --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks.py @@ -0,0 +1,102 @@ +"""Benchmark tasks: a seed project, an approved plan, and a hidden check. + +Each task is a directory under ``tasks/``: + +- ``task.yaml``: ``title``, ``plan`` (the approved milestone plan, written the + way the planner is asked to write them), and ``check`` (a shell command run + in the workspace afterwards). +- ``seed/``: the project the executor starts from. +- ``hidden/``: files the check needs, copied in only after the run, into + ``.bench_check/``. The executor never sees them. +- ``solution/``: a reference solution, overlaid on the seed by the tests to + prove each task can be solved and that its check tells the difference. +""" + +import hashlib +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import yaml + +TASKS_DIR = Path(__file__).parent / "tasks" +CHECK_DIR = ".bench_check" + + +@dataclass(frozen=True) +class Task: + name: str + title: str + plan: str + check: str + path: Path + + @property + def content_hash(self) -> str: + """What the task is, so a changed task is never compared with runs of + its earlier version. Covers everything the model or the check sees; + not the reference solution.""" + digest = hashlib.sha256() + for part in ("task.yaml", "seed", "hidden"): + root = self.path / part + files = [root] if root.is_file() else sorted(root.rglob("*")) + for file in files: + if file.is_file() and "__pycache__" not in file.parts: + digest.update(str(file.relative_to(self.path)).encode()) + digest.update(file.read_bytes()) + return digest.hexdigest()[:16] + + def seed_into(self, workspace: Path) -> None: + shutil.copytree(self.path / "seed", workspace, dirs_exist_ok=True) + + def apply_solution(self, workspace: Path) -> None: + shutil.copytree(self.path / "solution", workspace, dirs_exist_ok=True) + + def run_check(self, workspace: Path, timeout_s: int = 300) -> tuple[bool, str]: + """Copy the hidden files in and run the check. Returns whether it + passed, and its output for the record.""" + target = workspace / CHECK_DIR + if target.exists(): + shutil.rmtree(target) # never trust a directory the model made + shutil.copytree(self.path / "hidden", target) + try: + result = subprocess.run( + self.check, + shell=True, + cwd=workspace, + capture_output=True, + text=True, + timeout=timeout_s, + check=False, + ) + except subprocess.TimeoutExpired: + return False, f"check timed out after {timeout_s}s" + output = (result.stdout + result.stderr)[-2000:] + return result.returncode == 0, output + + +def load_task(path: Path) -> Task: + spec = yaml.safe_load((path / "task.yaml").read_text()) + return Task( + name=path.name, + title=str(spec["title"]), + plan=str(spec["plan"]).strip(), + check=str(spec["check"]), + path=path, + ) + + +def load_tasks(names: list[str] | None = None, root: Path = TASKS_DIR) -> list[Task]: + available = { + p.name: p for p in sorted(root.iterdir()) if (p / "task.yaml").exists() + } + if names: + unknown = [n for n in names if n not in available] + if unknown: + raise ValueError( + f"unknown task(s): {', '.join(unknown)}; " + f"available: {', '.join(available)}" + ) + return [load_task(available[n]) for n in names] + return [load_task(p) for p in available.values()] diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/original_test_inventory.py.txt b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/original_test_inventory.py.txt new file mode 100644 index 0000000..8155867 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/original_test_inventory.py.txt @@ -0,0 +1,11 @@ +from inventory import Item, low_stock, total_value + + +def test_total_value_counts_every_unit() -> None: + items = [Item("bolt", 0.25, 100), Item("nut", 0.10, 40)] + assert total_value(items) == 29.0 + + +def test_low_stock_is_alphabetical() -> None: + items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100), Item("anchor", 3.0, 1)] + assert low_stock(items) == ["anchor", "washer"] diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/test_hidden.py b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/test_hidden.py new file mode 100644 index 0000000..817ca49 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/test_hidden.py @@ -0,0 +1,27 @@ +from pathlib import Path + +import pytest +from inventory import Item, low_stock, total_value + + +def test_the_visible_tests_were_not_changed() -> None: + original = (Path(__file__).parent / "original_test_inventory.py.txt").read_text() + assert Path("test_inventory.py").read_text() == original + + +@pytest.mark.parametrize( + ("items", "expected"), + [ + ([], 0.0), + ([Item("bolt", 0.25, 100), Item("nut", 0.10, 40)], 29.0), + ([Item("gear", 12.5, 3)], 37.5), + ([Item("empty", 9.99, 0)], 0.0), + ], +) +def test_total_value(items: list[Item], expected: float) -> None: + assert total_value(items) == pytest.approx(expected) + + +def test_low_stock_still_works() -> None: + items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100)] + assert low_stock(items) == ["washer"] diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/inventory.py b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/inventory.py new file mode 100644 index 0000000..052ae14 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/inventory.py @@ -0,0 +1,23 @@ +"""A tiny inventory.""" + +from dataclasses import dataclass + + +@dataclass +class Item: + name: str + unit_price: float + quantity: int + + +def total_value(items: list[Item]) -> float: + """The value of all stock: each item's unit price times its quantity.""" + total = 0.0 + for item in items: + total += item.unit_price + return total + + +def low_stock(items: list[Item], threshold: int = 5) -> list[str]: + """Names of items with fewer than `threshold` units, alphabetically.""" + return sorted(item.name for item in items if item.quantity < threshold) diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/test_inventory.py b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/test_inventory.py new file mode 100644 index 0000000..8155867 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/test_inventory.py @@ -0,0 +1,11 @@ +from inventory import Item, low_stock, total_value + + +def test_total_value_counts_every_unit() -> None: + items = [Item("bolt", 0.25, 100), Item("nut", 0.10, 40)] + assert total_value(items) == 29.0 + + +def test_low_stock_is_alphabetical() -> None: + items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100), Item("anchor", 3.0, 1)] + assert low_stock(items) == ["anchor", "washer"] diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/solution/inventory.py b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/solution/inventory.py new file mode 100644 index 0000000..44002ef --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/solution/inventory.py @@ -0,0 +1,23 @@ +"""A tiny inventory.""" + +from dataclasses import dataclass + + +@dataclass +class Item: + name: str + unit_price: float + quantity: int + + +def total_value(items: list[Item]) -> float: + """The value of all stock: each item's unit price times its quantity.""" + total = 0.0 + for item in items: + total += item.unit_price * item.quantity + return total + + +def low_stock(items: list[Item], threshold: int = 5) -> list[str]: + """Names of items with fewer than `threshold` units, alphabetically.""" + return sorted(item.name for item in items if item.quantity < threshold) diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml new file mode 100644 index 0000000..7a676d1 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml @@ -0,0 +1,12 @@ +title: Fix a bug a failing test exposes +check: python -m pytest -q .bench_check +plan: | + Milestone 1: fix the failing inventory test. + + 1. Run `python -m pytest -q` and read the failure. + 2. The failing test is correct; the bug is in `inventory.py`. Do not change + any test file. + 3. Fix `total_value` so that each item contributes its unit price times its + quantity. + 4. Run `python -m pytest -q` until every test passes. + 5. Call finish. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/long-output/hidden/test_hidden.py b/nisse-dev/src/nisse_dev/bench/tasks/long-output/hidden/test_hidden.py new file mode 100644 index 0000000..f7af64c --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/long-output/hidden/test_hidden.py @@ -0,0 +1,11 @@ +import pytest +from rates import RATES + + +def test_every_region_is_present_once() -> None: + assert sorted(RATES) == [f"R{n:03d}" for n in range(300)] + + +@pytest.mark.parametrize("n", range(300)) +def test_rate(n: int) -> None: + assert RATES[f"R{n:03d}"] == pytest.approx(n % 25 / 100) diff --git a/nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/rates.py b/nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/rates.py new file mode 100644 index 0000000..7b214fe --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/rates.py @@ -0,0 +1,304 @@ +"""Tax rate per region code.""" + +RATES = { + "R000": 0.00, + "R001": 0.01, + "R002": 0.02, + "R003": 0.03, + "R004": 0.04, + "R005": 0.05, + "R006": 0.06, + "R007": 0.07, + "R008": 0.08, + "R009": 0.09, + "R010": 0.10, + "R011": 0.11, + "R012": 0.12, + "R013": 0.13, + "R014": 0.14, + "R015": 0.15, + "R016": 0.16, + "R017": 0.17, + "R018": 0.18, + "R019": 0.19, + "R020": 0.20, + "R021": 0.21, + "R022": 0.22, + "R023": 0.23, + "R024": 0.24, + "R025": 0.00, + "R026": 0.01, + "R027": 0.02, + "R028": 0.03, + "R029": 0.04, + "R030": 0.05, + "R031": 0.06, + "R032": 0.07, + "R033": 0.08, + "R034": 0.09, + "R035": 0.10, + "R036": 0.11, + "R037": 0.12, + "R038": 0.13, + "R039": 0.14, + "R040": 0.15, + "R041": 0.16, + "R042": 0.17, + "R043": 0.18, + "R044": 0.19, + "R045": 0.20, + "R046": 0.21, + "R047": 0.22, + "R048": 0.23, + "R049": 0.24, + "R050": 0.00, + "R051": 0.01, + "R052": 0.02, + "R053": 0.03, + "R054": 0.04, + "R055": 0.05, + "R056": 0.06, + "R057": 0.07, + "R058": 0.08, + "R059": 0.09, + "R060": 0.10, + "R061": 0.11, + "R062": 0.12, + "R063": 0.13, + "R064": 0.14, + "R065": 0.15, + "R066": 0.16, + "R067": 0.17, + "R068": 0.18, + "R069": 0.19, + "R070": 0.20, + "R071": 0.21, + "R072": 0.22, + "R073": 0.23, + "R074": 0.24, + "R075": 0.00, + "R076": 0.01, + "R077": 0.02, + "R078": 0.03, + "R079": 0.04, + "R080": 0.05, + "R081": 0.06, + "R082": 0.07, + "R083": 0.08, + "R084": 0.09, + "R085": 0.10, + "R086": 0.11, + "R087": 0.12, + "R088": 0.13, + "R089": 0.14, + "R090": 0.15, + "R091": 0.16, + "R092": 0.17, + "R093": 0.18, + "R094": 0.19, + "R095": 0.20, + "R096": 0.21, + "R097": 0.22, + "R098": 0.23, + "R099": 0.24, + "R100": 0.00, + "R101": 0.01, + "R102": 0.02, + "R103": 0.03, + "R104": 0.04, + "R105": 0.05, + "R106": 0.06, + "R107": 0.07, + "R108": 0.08, + "R109": 0.09, + "R110": 0.10, + "R111": 0.11, + "R112": 0.12, + "R113": 0.13, + "R114": 0.14, + "R115": 0.15, + "R116": 0.16, + "R117": 0.17, + "R118": 0.18, + "R119": 0.19, + "R120": 0.20, + "R121": 0.21, + "R122": 0.22, + "R123": 0.23, + "R124": 0.24, + "R125": 0.00, + "R126": 0.01, + "R127": 0.02, + "R128": 0.03, + "R129": 0.04, + "R130": 0.05, + "R131": 0.06, + "R132": 0.07, + "R133": 0.08, + "R134": 0.09, + "R135": 0.10, + "R136": 0.11, + "R137": 0.12, + "R138": 0.13, + "R139": 0.14, + "R140": 0.15, + "R141": 0.16, + "R142": 0.17, + "R143": 0.18, + "R144": 0.19, + "R145": 0.20, + "R146": 0.21, + "R147": 0.22, + "R148": 0.23, + "R149": 0.24, + "R150": 0.00, + "R151": 0.01, + "R152": 0.02, + "R153": 0.03, + "R154": 0.04, + "R155": 0.05, + "R156": 0.06, + "R157": 0.07, + "R158": 0.08, + "R159": 0.09, + "R160": 0.10, + "R161": 0.11, + "R162": 0.12, + "R163": 0.13, + "R164": 0.14, + "R165": 0.15, + "R166": 0.16, + "R167": 0.17, + "R168": 0.18, + "R169": 0.19, + "R170": 0.20, + "R171": 0.21, + "R172": 0.22, + "R173": 0.23, + "R174": 0.24, + "R175": 0.00, + "R176": 0.01, + "R177": 0.02, + "R178": 0.03, + "R179": 0.04, + "R180": 0.05, + "R181": 0.06, + "R182": 0.07, + "R183": 0.08, + "R184": 0.09, + "R185": 0.10, + "R186": 0.11, + "R187": 0.12, + "R188": 0.13, + "R189": 0.14, + "R190": 0.15, + "R191": 0.16, + "R192": 0.17, + "R193": 0.18, + "R194": 0.19, + "R195": 0.20, + "R196": 0.21, + "R197": 0.22, + "R198": 0.23, + "R199": 0.24, + "R200": 0.00, + "R201": 0.01, + "R202": 0.02, + "R203": 0.03, + "R204": 0.04, + "R205": 0.05, + "R206": 0.06, + "R207": 0.07, + "R208": 0.08, + "R209": 0.09, + "R210": 0.10, + "R211": 0.11, + "R212": 0.12, + "R213": 0.13, + "R214": 0.14, + "R215": 0.15, + "R216": 0.16, + "R217": 0.17, + "R218": 0.18, + "R219": 0.19, + "R220": 0.20, + "R221": 0.21, + "R222": 0.22, + "R223": 0.23, + "R224": 0.24, + "R225": 0.00, + "R226": 0.01, + "R227": 0.02, + "R228": 0.03, + "R229": 0.04, + "R230": 0.05, + "R231": 0.06, + "R232": 0.07, + "R233": 0.08, + "R234": 0.09, + "R235": 0.10, + "R236": 0.11, + "R237": 0.21, + "R238": 0.13, + "R239": 0.14, + "R240": 0.15, + "R241": 0.16, + "R242": 0.17, + "R243": 0.18, + "R244": 0.19, + "R245": 0.20, + "R246": 0.21, + "R247": 0.22, + "R248": 0.23, + "R249": 0.24, + "R250": 0.00, + "R251": 0.01, + "R252": 0.02, + "R253": 0.03, + "R254": 0.04, + "R255": 0.05, + "R256": 0.06, + "R257": 0.07, + "R258": 0.08, + "R259": 0.09, + "R260": 0.10, + "R261": 0.11, + "R262": 0.12, + "R263": 0.13, + "R264": 0.14, + "R265": 0.15, + "R266": 0.16, + "R267": 0.17, + "R268": 0.18, + "R269": 0.19, + "R270": 0.20, + "R271": 0.21, + "R272": 0.22, + "R273": 0.23, + "R274": 0.24, + "R275": 0.00, + "R276": 0.01, + "R277": 0.02, + "R278": 0.03, + "R279": 0.04, + "R280": 0.05, + "R281": 0.06, + "R282": 0.07, + "R283": 0.08, + "R284": 0.09, + "R285": 0.10, + "R286": 0.11, + "R287": 0.12, + "R288": 0.13, + "R289": 0.14, + "R290": 0.15, + "R291": 0.16, + "R292": 0.17, + "R293": 0.18, + "R294": 0.19, + "R295": 0.20, + "R296": 0.21, + "R297": 0.22, + "R298": 0.23, + "R299": 0.24, +} diff --git a/nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/test_rates.py b/nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/test_rates.py new file mode 100644 index 0000000..a08f502 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/test_rates.py @@ -0,0 +1,7 @@ +import pytest +from rates import RATES + + +@pytest.mark.parametrize("code", [f"R{n:03d}" for n in range(300)]) +def test_rate(code: str) -> None: + assert RATES[code] == pytest.approx(int(code[1:]) % 25 / 100) diff --git a/nisse-dev/src/nisse_dev/bench/tasks/long-output/solution/rates.py b/nisse-dev/src/nisse_dev/bench/tasks/long-output/solution/rates.py new file mode 100644 index 0000000..1461336 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/long-output/solution/rates.py @@ -0,0 +1,304 @@ +"""Tax rate per region code.""" + +RATES = { + "R000": 0.00, + "R001": 0.01, + "R002": 0.02, + "R003": 0.03, + "R004": 0.04, + "R005": 0.05, + "R006": 0.06, + "R007": 0.07, + "R008": 0.08, + "R009": 0.09, + "R010": 0.10, + "R011": 0.11, + "R012": 0.12, + "R013": 0.13, + "R014": 0.14, + "R015": 0.15, + "R016": 0.16, + "R017": 0.17, + "R018": 0.18, + "R019": 0.19, + "R020": 0.20, + "R021": 0.21, + "R022": 0.22, + "R023": 0.23, + "R024": 0.24, + "R025": 0.00, + "R026": 0.01, + "R027": 0.02, + "R028": 0.03, + "R029": 0.04, + "R030": 0.05, + "R031": 0.06, + "R032": 0.07, + "R033": 0.08, + "R034": 0.09, + "R035": 0.10, + "R036": 0.11, + "R037": 0.12, + "R038": 0.13, + "R039": 0.14, + "R040": 0.15, + "R041": 0.16, + "R042": 0.17, + "R043": 0.18, + "R044": 0.19, + "R045": 0.20, + "R046": 0.21, + "R047": 0.22, + "R048": 0.23, + "R049": 0.24, + "R050": 0.00, + "R051": 0.01, + "R052": 0.02, + "R053": 0.03, + "R054": 0.04, + "R055": 0.05, + "R056": 0.06, + "R057": 0.07, + "R058": 0.08, + "R059": 0.09, + "R060": 0.10, + "R061": 0.11, + "R062": 0.12, + "R063": 0.13, + "R064": 0.14, + "R065": 0.15, + "R066": 0.16, + "R067": 0.17, + "R068": 0.18, + "R069": 0.19, + "R070": 0.20, + "R071": 0.21, + "R072": 0.22, + "R073": 0.23, + "R074": 0.24, + "R075": 0.00, + "R076": 0.01, + "R077": 0.02, + "R078": 0.03, + "R079": 0.04, + "R080": 0.05, + "R081": 0.06, + "R082": 0.07, + "R083": 0.08, + "R084": 0.09, + "R085": 0.10, + "R086": 0.11, + "R087": 0.12, + "R088": 0.13, + "R089": 0.14, + "R090": 0.15, + "R091": 0.16, + "R092": 0.17, + "R093": 0.18, + "R094": 0.19, + "R095": 0.20, + "R096": 0.21, + "R097": 0.22, + "R098": 0.23, + "R099": 0.24, + "R100": 0.00, + "R101": 0.01, + "R102": 0.02, + "R103": 0.03, + "R104": 0.04, + "R105": 0.05, + "R106": 0.06, + "R107": 0.07, + "R108": 0.08, + "R109": 0.09, + "R110": 0.10, + "R111": 0.11, + "R112": 0.12, + "R113": 0.13, + "R114": 0.14, + "R115": 0.15, + "R116": 0.16, + "R117": 0.17, + "R118": 0.18, + "R119": 0.19, + "R120": 0.20, + "R121": 0.21, + "R122": 0.22, + "R123": 0.23, + "R124": 0.24, + "R125": 0.00, + "R126": 0.01, + "R127": 0.02, + "R128": 0.03, + "R129": 0.04, + "R130": 0.05, + "R131": 0.06, + "R132": 0.07, + "R133": 0.08, + "R134": 0.09, + "R135": 0.10, + "R136": 0.11, + "R137": 0.12, + "R138": 0.13, + "R139": 0.14, + "R140": 0.15, + "R141": 0.16, + "R142": 0.17, + "R143": 0.18, + "R144": 0.19, + "R145": 0.20, + "R146": 0.21, + "R147": 0.22, + "R148": 0.23, + "R149": 0.24, + "R150": 0.00, + "R151": 0.01, + "R152": 0.02, + "R153": 0.03, + "R154": 0.04, + "R155": 0.05, + "R156": 0.06, + "R157": 0.07, + "R158": 0.08, + "R159": 0.09, + "R160": 0.10, + "R161": 0.11, + "R162": 0.12, + "R163": 0.13, + "R164": 0.14, + "R165": 0.15, + "R166": 0.16, + "R167": 0.17, + "R168": 0.18, + "R169": 0.19, + "R170": 0.20, + "R171": 0.21, + "R172": 0.22, + "R173": 0.23, + "R174": 0.24, + "R175": 0.00, + "R176": 0.01, + "R177": 0.02, + "R178": 0.03, + "R179": 0.04, + "R180": 0.05, + "R181": 0.06, + "R182": 0.07, + "R183": 0.08, + "R184": 0.09, + "R185": 0.10, + "R186": 0.11, + "R187": 0.12, + "R188": 0.13, + "R189": 0.14, + "R190": 0.15, + "R191": 0.16, + "R192": 0.17, + "R193": 0.18, + "R194": 0.19, + "R195": 0.20, + "R196": 0.21, + "R197": 0.22, + "R198": 0.23, + "R199": 0.24, + "R200": 0.00, + "R201": 0.01, + "R202": 0.02, + "R203": 0.03, + "R204": 0.04, + "R205": 0.05, + "R206": 0.06, + "R207": 0.07, + "R208": 0.08, + "R209": 0.09, + "R210": 0.10, + "R211": 0.11, + "R212": 0.12, + "R213": 0.13, + "R214": 0.14, + "R215": 0.15, + "R216": 0.16, + "R217": 0.17, + "R218": 0.18, + "R219": 0.19, + "R220": 0.20, + "R221": 0.21, + "R222": 0.22, + "R223": 0.23, + "R224": 0.24, + "R225": 0.00, + "R226": 0.01, + "R227": 0.02, + "R228": 0.03, + "R229": 0.04, + "R230": 0.05, + "R231": 0.06, + "R232": 0.07, + "R233": 0.08, + "R234": 0.09, + "R235": 0.10, + "R236": 0.11, + "R237": 0.12, + "R238": 0.13, + "R239": 0.14, + "R240": 0.15, + "R241": 0.16, + "R242": 0.17, + "R243": 0.18, + "R244": 0.19, + "R245": 0.20, + "R246": 0.21, + "R247": 0.22, + "R248": 0.23, + "R249": 0.24, + "R250": 0.00, + "R251": 0.01, + "R252": 0.02, + "R253": 0.03, + "R254": 0.04, + "R255": 0.05, + "R256": 0.06, + "R257": 0.07, + "R258": 0.08, + "R259": 0.09, + "R260": 0.10, + "R261": 0.11, + "R262": 0.12, + "R263": 0.13, + "R264": 0.14, + "R265": 0.15, + "R266": 0.16, + "R267": 0.17, + "R268": 0.18, + "R269": 0.19, + "R270": 0.20, + "R271": 0.21, + "R272": 0.22, + "R273": 0.23, + "R274": 0.24, + "R275": 0.00, + "R276": 0.01, + "R277": 0.02, + "R278": 0.03, + "R279": 0.04, + "R280": 0.05, + "R281": 0.06, + "R282": 0.07, + "R283": 0.08, + "R284": 0.09, + "R285": 0.10, + "R286": 0.11, + "R287": 0.12, + "R288": 0.13, + "R289": 0.14, + "R290": 0.15, + "R291": 0.16, + "R292": 0.17, + "R293": 0.18, + "R294": 0.19, + "R295": 0.20, + "R296": 0.21, + "R297": 0.22, + "R298": 0.23, + "R299": 0.24, +} diff --git a/nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml new file mode 100644 index 0000000..3ac8ae3 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml @@ -0,0 +1,12 @@ +title: Find one failure in long test output +check: python -m pytest -q .bench_check +plan: | + Milestone 1: fix the one wrong entry in the rate table. + + 1. Run `python -m pytest -v`. Exactly one test fails, and its output names a + region code. + 2. The tests are correct. In `rates.py`, the rate for region code RNNN must be + (NNN mod 25) / 100. Correct only the failing region's entry; leave every + other line as it is. + 3. Run `python -m pytest -q` until every test passes. + 4. Call finish. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/hidden/test_hidden.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/hidden/test_hidden.py new file mode 100644 index 0000000..61187c4 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/hidden/test_hidden.py @@ -0,0 +1,33 @@ +import subprocess +import sys + +from greeter.cli import main +from greeter.core import greet + + +def test_default_is_unchanged() -> None: + assert greet("Bob") == "Hello, Bob!" + + +def test_shout() -> None: + assert greet("Bob", shout=True) == "HELLO, BOB!" + + +def test_cli_default(capsys) -> None: + assert main(["Ada"]) == 0 + assert capsys.readouterr().out == "Hello, Ada!\n" + + +def test_cli_shout(capsys) -> None: + assert main(["--shout", "Ada"]) == 0 + assert capsys.readouterr().out == "HELLO, ADA!\n" + + +def test_module_entry_point() -> None: + result = subprocess.run( + [sys.executable, "-m", "greeter", "--shout", "Bob"], + capture_output=True, + text=True, + check=True, + ) + assert result.stdout == "HELLO, BOB!\n" diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__init__.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__main__.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__main__.py new file mode 100644 index 0000000..5664529 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__main__.py @@ -0,0 +1,3 @@ +from greeter.cli import main + +raise SystemExit(main()) diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/cli.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/cli.py new file mode 100644 index 0000000..a38c398 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/cli.py @@ -0,0 +1,11 @@ +import argparse + +from greeter.core import greet + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="greeter") + parser.add_argument("name") + args = parser.parse_args(argv) + print(greet(args.name)) + return 0 diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/core.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/core.py new file mode 100644 index 0000000..1e21eeb --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/core.py @@ -0,0 +1,2 @@ +def greet(name: str) -> str: + return f"Hello, {name}!" diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/tests/test_greeter.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/tests/test_greeter.py new file mode 100644 index 0000000..54c92e2 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/tests/test_greeter.py @@ -0,0 +1,11 @@ +from greeter.cli import main +from greeter.core import greet + + +def test_greet() -> None: + assert greet("Bob") == "Hello, Bob!" + + +def test_cli_prints_the_greeting(capsys) -> None: + assert main(["Bob"]) == 0 + assert capsys.readouterr().out == "Hello, Bob!\n" diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/cli.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/cli.py new file mode 100644 index 0000000..82f856b --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/cli.py @@ -0,0 +1,12 @@ +import argparse + +from greeter.core import greet + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="greeter") + parser.add_argument("name") + parser.add_argument("--shout", action="store_true") + args = parser.parse_args(argv) + print(greet(args.name, shout=args.shout)) + return 0 diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/core.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/core.py new file mode 100644 index 0000000..819f836 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/core.py @@ -0,0 +1,3 @@ +def greet(name: str, shout: bool = False) -> str: + greeting = f"Hello, {name}!" + return greeting.upper() if shout else greeting diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml new file mode 100644 index 0000000..44f6902 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml @@ -0,0 +1,16 @@ +title: Thread a new option through several files +check: python -m pytest -q .bench_check +plan: | + Milestone 1: add a --shout option to the greeter. + + 1. In `greeter/core.py`, give `greet` a keyword argument `shout: bool = False`. + When it is true, return the whole greeting in upper case: + greet("Bob", shout=True) == "HELLO, BOB!". The default behaviour is unchanged. + 2. In `greeter/cli.py`, add a `--shout` flag (action="store_true") and pass it + to `greet`. + 3. Add two tests to `tests/test_greeter.py`: one for greet with shout=True, and + one that calls `main(["--shout", "Bob"])` and checks the printed output + using pytest's capsys fixture. + 4. Run `python -m pytest -q` until every test passes, and check that + `python -m greeter --shout Bob` prints `HELLO, BOB!`. + 5. Call finish. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/write-function/hidden/test_hidden.py b/nisse-dev/src/nisse_dev/bench/tasks/write-function/hidden/test_hidden.py new file mode 100644 index 0000000..cf925f6 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/write-function/hidden/test_hidden.py @@ -0,0 +1,19 @@ +import pytest +from slugify import slugify + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("Hello, World!", "hello-world"), + (" Nisse Agent ", "nisse-agent"), + ("a--b", "a-b"), + ("", ""), + ("Python 3.12 Release", "python-3-12-release"), + ("--leading and trailing--", "leading-and-trailing"), + ("MiXeD CaSe", "mixed-case"), + ("!!!", ""), + ], +) +def test_slugify(text: str, expected: str) -> None: + assert slugify(text) == expected diff --git a/nisse-dev/src/nisse_dev/bench/tasks/write-function/seed/README.md b/nisse-dev/src/nisse_dev/bench/tasks/write-function/seed/README.md new file mode 100644 index 0000000..e09c91f --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/write-function/seed/README.md @@ -0,0 +1 @@ +A small utilities project. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/write-function/solution/slugify.py b/nisse-dev/src/nisse_dev/bench/tasks/write-function/solution/slugify.py new file mode 100644 index 0000000..c4d3700 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/write-function/solution/slugify.py @@ -0,0 +1,5 @@ +import re + + +def slugify(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") diff --git a/nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml new file mode 100644 index 0000000..9955265 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml @@ -0,0 +1,13 @@ +title: Write a small function and its tests +check: python -m pytest -q .bench_check +plan: | + Milestone 1: add a slugify function. + + 1. Create `slugify.py` in the workspace root, defining `slugify(text: str) -> str`. + 2. It lowercases the text, replaces every run of characters that are not a-z or + 0-9 with a single hyphen, and strips hyphens from both ends. Examples: + "Hello, World!" -> "hello-world"; " Nisse Agent " -> "nisse-agent"; + "a--b" -> "a-b"; "" -> "". + 3. Create `test_slugify.py` with pytest tests covering the four examples. + 4. Run `python -m pytest -q test_slugify.py`, and fix anything until it passes. + 5. Call finish. diff --git a/nisse-dev/src/nisse_dev/context.py b/nisse-dev/src/nisse_dev/context.py index d5c162b..d77eb61 100644 --- a/nisse-dev/src/nisse_dev/context.py +++ b/nisse-dev/src/nisse_dev/context.py @@ -230,6 +230,7 @@ class BudgetedClient: # Per model: how much the character estimate undercounted, as learned # from reported usage. Starts at 1 and only grows. self._factors: dict[str, float] = {} + self.trimmed_requests = 0 @property def last_usage(self) -> dict[str, int]: @@ -239,6 +240,10 @@ class BudgetedClient: def last_model(self) -> str | None: return getattr(self._inner, "last_model", None) + @property + def last_cost(self) -> float: + return float(getattr(self._inner, "last_cost", 0.0) or 0.0) + def budget_for(self, model: str) -> Budget: window, model_max_output = self._limits_for(model) ceiling = self._max_output_tokens @@ -261,6 +266,10 @@ class BudgetedClient: budget = self.budget_for(model) factor = self._factors.get(model, 1.0) fitted = fit(events, system, tools, budget, factor) + # Counted so the benchmark can report how often a model's window was + # the constraint: any request that had to be cut down or shortened. + if fitted != list(events): + self.trimmed_requests += 1 produced = self._inner.complete( fitted, tools, diff --git a/nisse-dev/src/nisse_dev/llm.py b/nisse-dev/src/nisse_dev/llm.py index 10c8877..1579a60 100644 --- a/nisse-dev/src/nisse_dev/llm.py +++ b/nisse-dev/src/nisse_dev/llm.py @@ -143,6 +143,7 @@ class LiteLLMClient: # protocol stays the same for every implementation. self.last_usage: dict[str, int] = {} self.last_model: str | None = None + self.last_cost: float = 0.0 def complete( self, @@ -171,6 +172,8 @@ class LiteLLMClient: raise LLMError( f"LLM request returned {response.status_code}: {response.text[:500]}" ) + # LiteLLM prices every call; kept so a caller can meter spending. + self.last_cost = _response_cost(response) return self._parse(response.json(), requested_model=model) def _parse( @@ -263,6 +266,7 @@ class AnthropicPassthroughClient: self._sleep = sleep self.last_usage: dict[str, int] = {} self.last_model: str | None = None + self.last_cost: float = 0.0 def complete( self, @@ -291,6 +295,8 @@ class AnthropicPassthroughClient: raise LLMError( f"passthrough returned {response.status_code}: {response.text[:500]}" ) + # LiteLLM prices every call; kept so a caller can meter spending. + self.last_cost = _response_cost(response) return self._parse(response.json(), requested_model=model) def _parse( @@ -358,6 +364,13 @@ def _post_with_rate_limit_retry( # Model families we have a native renderer adapter for. A13: delegations take # the vendor's passthrough route because a normalising middlebox is lossy # exactly where a long-running delegation lives. +def _response_cost(response: httpx.Response) -> float: + try: + return float(response.headers.get("x-litellm-response-cost") or 0.0) + except ValueError: + return 0.0 + + def _raise_for_upstream(response: httpx.Response) -> None: """Classify a non-200 as upstream trouble where it plainly is. diff --git a/nisse-dev/tests/test_bench.py b/nisse-dev/tests/test_bench.py new file mode 100644 index 0000000..a48eff1 --- /dev/null +++ b/nisse-dev/tests/test_bench.py @@ -0,0 +1,265 @@ +"""The executor benchmark.""" + +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from nisse_dev.bench.results import ( + append, + load, + render, + results_file, + summarise, + wilson, +) +from nisse_dev.bench.runner import MeteringClient, run_task +from nisse_dev.bench.tasks import CHECK_DIR, load_tasks +from nisse_dev.canonical import AssistantText, ToolCall +from nisse_dev.catalog import ModelFacts +from nisse_dev.config import ExecutorTuning, Tuning + +TASKS = load_tasks() +FACTS = ModelFacts( + alias="local-fast", + backing="openai/qwen3-8b-awq", + provider="openai", + supports_tools=True, + max_input_tokens=32768, + max_output_tokens=8192, + supports_tool_choice=True, +) + + +# --- the tasks themselves --------------------------------------------------- + + +def test_the_bundled_tasks_are_the_four_planned() -> None: + assert [t.name for t in TASKS] == [ + "fix-bug", + "long-output", + "multi-file", + "write-function", + ] + + +@pytest.mark.parametrize("task", TASKS, ids=lambda t: t.name) +def test_the_seed_fails_its_check_and_the_solution_passes(task, tmp_path: Path) -> None: + # A check that passes on the untouched seed measures nothing; a task its + # own reference solution can't pass is unfair to every model. + seed = tmp_path / "seed" + task.seed_into(seed) + assert not task.run_check(seed)[0], f"{task.name}: the seed already passes" + + solved = tmp_path / "solved" + task.seed_into(solved) + task.apply_solution(solved) + passed, output = task.run_check(solved) + assert passed, f"{task.name}: the reference solution fails:\n{output}" + + +@pytest.mark.parametrize("task", TASKS, ids=lambda t: t.name) +def test_the_hidden_check_is_not_in_the_seed(task, tmp_path: Path) -> None: + task.seed_into(tmp_path) + assert not (tmp_path / CHECK_DIR).exists() + assert "Call finish" in task.plan # plans end the way the executor must + + +def test_long_output_really_is_long(tmp_path: Path) -> None: + """The point of the task is output far past tools.max_output_chars.""" + import subprocess + import sys + + [task] = load_tasks(["long-output"]) + task.seed_into(tmp_path) + run = subprocess.run( + [sys.executable, "-m", "pytest", "-v"], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + assert len(run.stdout) > 4 * Tuning().tools.max_output_chars + assert "R237" in run.stdout[-3000:] # the failure is named at the end + + +def test_a_changed_task_gets_a_new_hash(tmp_path: Path) -> None: + [task] = load_tasks(["write-function"]) + shutil.copytree(task.path, tmp_path / "write-function") + [copy] = load_tasks(root=tmp_path) + assert copy.content_hash == task.content_hash + (copy.path / "seed" / "README.md").write_text("changed\n") + assert copy.content_hash != task.content_hash + + +def test_an_unknown_task_is_refused() -> None: + with pytest.raises(ValueError, match="unknown task"): + load_tasks(["nope"]) + + +# --- running a task ---------------------------------------------------------- + + +class Scripted: + """A model that plays back fixed turns, reporting itself as `answered`.""" + + def __init__(self, turns: list[list[Any]], answered: str = "local-fast") -> None: + self.turns = turns + self.last_usage = {"input": 1000, "output": 50} + self.last_cost = 0.0 + self.last_model = answered + self.trimmed_requests = 0 + + def complete(self, events, tools, model, system=None): + if not self.turns: + return [AssistantText(text="nothing left to do", model=model)] + return self.turns.pop(0) + + +def call(tool: str, cid: str, **arguments: Any) -> ToolCall: + return ToolCall(call_id=cid, tool=tool, arguments=arguments) + + +SOLVE_SLUGIFY = [ + [ + call( + "write_file", + "w", + path="slugify.py", + content=( + "import re\n\n\ndef slugify(text):\n" + " return re.sub(r'[^a-z0-9]+', '-', text.lower()).strip('-')\n" + ), + ) + ], + [call("run_command", "r", command="python -c 'import slugify'")], + [call("finish", "f", summary="added slugify")], +] + + +def test_a_solved_task_is_a_success(tmp_path: Path) -> None: + [task] = load_tasks(["write-function"]) + record = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), note="n") + + assert record.success + assert record.outcome == "completed" + assert record.turns == 3 + assert record.tokens_in == 3000 + assert record.task_hash == task.content_hash + assert record.model_info["supports_tool_choice"] is True + assert record.tuning["executor"]["max_turns"] == 24 + + +def test_saying_finished_is_not_success(tmp_path: Path) -> None: + # Success is the hidden check passing, never the model's own claim. + [task] = load_tasks(["write-function"]) + wrong = [ + [ + call( + "write_file", + "w", + path="slugify.py", + content="def slugify(t):\n return t\n", + ) + ], + [call("finish", "f", summary="done")], + ] + record = run_task(task, FACTS, Scripted(wrong), Tuning()) + assert record.outcome == "completed" + assert not record.success + + +def test_misbehaviour_is_counted(tmp_path: Path) -> None: + [task] = load_tasks(["write-function"]) + turns = [ + [call("chat", "x1")], + [call("run_command", "r1", command="exit 1")], + [AssistantText(text="let me explain", model="m")], + ] + # A higher failure limit, so the invented tool and the failed command + # don't escalate before the prose turn is reached. + tuning = Tuning(executor=ExecutorTuning(failure_limit=5)) + record = run_task(task, FACTS, Scripted(turns), tuning) + assert record.invented_tools == 1 + assert record.failed_commands == 1 + assert record.prose_turns >= 1 + assert not record.success + + +def test_a_fallback_answer_can_never_be_a_success(tmp_path: Path) -> None: + [task] = load_tasks(["write-function"]) + client = Scripted(list(SOLVE_SLUGIFY), answered="claude-sonnet-5") + record = run_task(task, FACTS, client, Tuning()) + assert record.fallbacks == ["claude-sonnet-5"] * 3 + assert not record.success # the check passed, but not for this model + + +# --- storing and reporting --------------------------------------------------- + + +def test_wilson_intervals_behave() -> None: + low, high = wilson(5, 5) + assert high == 1.0 and 0.5 < low < 0.6 + # 3 of 5 and 5 of 5 overlap: five runs do not separate them. + assert wilson(3, 5)[1] > wilson(5, 5)[0] + assert wilson(0, 0) == (0.0, 1.0) + + +def record_json(**overrides: Any) -> dict[str, Any]: + [task] = load_tasks(["write-function"]) + base = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning()).to_json() + base.update(overrides) + return base + + +def test_results_round_trip_and_report(tmp_path: Path) -> None: + from nisse_dev.bench.runner import RunRecord + + rec = RunRecord(**record_json()) + path = results_file(tmp_path, "local-fast") + append(path, rec) + append(path, RunRecord(**record_json(success=False))) + stored = load(tmp_path) + assert len(stored) == 2 + assert json.loads(path.read_text().splitlines()[0])["model"] == "local-fast" + + current = {t.name: t.content_hash for t in TASKS} + [row] = summarise(stored, current) + assert (row.successes, row.runs) == (1, 2) + assert not row.stale + table = render([row]) + assert "openai/qwen3-8b-awq" in table + assert "1/2" in table + + +def test_runs_of_an_older_task_version_are_flagged() -> None: + current = {t.name: t.content_hash for t in TASKS} + rows = summarise([record_json(task_hash="0123456789abcdef")], current) + assert rows[0].stale + assert "(old)" in render(rows) + + +def test_models_served_differently_are_kept_apart() -> None: + current = {t.name: t.content_hash for t in TASKS} + rows = summarise( + [record_json(note="vLLM, AWQ"), record_json(note="llama.cpp, Q4_K_M")], current + ) + assert len(rows) == 2 + + +def test_metering_counts_prose_and_cost() -> None: + class Inner: + def __init__(self) -> None: + self.last_usage = {"input": 10, "output": 2} + self.last_cost = 0.5 + self.last_model = "local-fast" + + def complete(self, events, tools, model, system=None): + return [AssistantText(text="hi", model=model)] + + meter = MeteringClient(Inner(), FACTS) + meter.complete([], None, "local-fast") + meter.complete([], None, "local-fast") + assert (meter.calls, meter.prose_turns, meter.cost) == (2, 2, 1.0) diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100644 index 0000000..c694a80 --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,124 @@ +"""Benchmark executor models on canned milestones, and compare the results. + + uv run python scripts/benchmark.py run --models A,B [--runs 5] [--tasks T,...] + [--note "vLLM 0.29, AWQ, fp8 KV"] [--config PATH] + uv run python scripts/benchmark.py report [--models A,B] [--tasks T,...] + +`run` appends one JSON line per run to bench/results/, which is committed: +the report compares every run ever stored, so benchmarking a new model means +running only that model. See docs/Tuning-Guide.md, "Benchmarking models". +""" + +import argparse +import os +import sys +from pathlib import Path + +from nisse_dev.agent import make_llm_clients +from nisse_dev.bench.results import append, load, render, results_file, summarise +from nisse_dev.bench.runner import agent_commit, run_task +from nisse_dev.bench.tasks import load_tasks +from nisse_dev.catalog import ModelCatalog +from nisse_dev.config import AgentConfig, ConfigError +from nisse_dev.envfile import load_env_file + +REPO = Path(__file__).resolve().parents[1] +RESULTS = REPO / "bench" / "results" + + +def _names(raw: str | None) -> list[str] | None: + return [n.strip() for n in raw.split(",") if n.strip()] if raw else None + + +def run(args: argparse.Namespace) -> int: + load_env_file(Path(os.environ.get("NISSE_ENV", ".env")).expanduser()) + try: + config = ( + AgentConfig.from_file(Path(args.config)) + if args.config + else AgentConfig.load() + ) + except ConfigError as exc: + print(exc, file=sys.stderr) + return 2 + models = _names(args.models) or [] + tasks = load_tasks(_names(args.tasks)) + catalog = ModelCatalog.discover(config.llm_base_url, config.llm_token) + catalog.require_limits({m: m for m in models}) + normalized, _passthrough = make_llm_clients(config, catalog) + commit = agent_commit(REPO) + + for model in models: + facts = catalog.facts(model) + out = results_file(args.results, model) + for task in tasks: + for n in range(args.runs): + # The executor always takes the normalized route in the agent, + # so it does here too. + record = run_task( + task, + facts, + normalized, + config.tuning, + note=args.note, + commit=commit, + ) + append(out, record) + verdict = "pass" if record.success else f"FAIL ({record.outcome})" + extra = ( + f" fallback to {record.fallbacks[0]}!" if record.fallbacks else "" + ) + print( + f"{model} {task.name} run {n + 1}/{args.runs}: {verdict}, " + f"{record.turns} turns, {record.wall_s:.0f}s, " + f"${record.cost:.4f}{extra}", + flush=True, + ) + print() + return report(args) + + +def report(args: argparse.Namespace) -> int: + records = load(args.results) + models = _names(getattr(args, "models", None)) + tasks = _names(getattr(args, "tasks", None)) + if models: + records = [r for r in records if r["model"] in models or r["backing"] in models] + if tasks: + records = [r for r in records if r["task"] in tasks] + if not records: + print("no stored results match", file=sys.stderr) + return 1 + current = {task.name: task.content_hash for task in load_tasks()} + print(render(summarise(records, current))) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--results", type=Path, default=RESULTS, help=argparse.SUPPRESS) + sub = parser.add_subparsers(dest="command", required=True) + + run_p = sub.add_parser("run", help="benchmark models, then report") + run_p.add_argument( + "--models", required=True, help="LiteLLM aliases, comma separated" + ) + run_p.add_argument("--runs", type=int, default=5, help="runs per task (default 5)") + run_p.add_argument("--tasks", help="task names, comma separated (default: all)") + run_p.add_argument( + "--note", default="", help='how the model was served, e.g. "vLLM 0.29, AWQ"' + ) + run_p.add_argument("--config", help="agent config (default: NISSE_CONFIG)") + run_p.set_defaults(handler=run) + + report_p = sub.add_parser("report", help="compare stored results") + report_p.add_argument("--models", help="aliases or backing models to include") + report_p.add_argument("--tasks", help="tasks to include") + report_p.set_defaults(handler=report) + + args = parser.parse_args() + return args.handler(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/evaluate_model.sh b/scripts/evaluate_model.sh new file mode 100755 index 0000000..81dcf22 --- /dev/null +++ b/scripts/evaluate_model.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Evaluate a model for the executor role, from a declared LiteLLM alias to a +# benchmark report next to every model measured before it. +# +# scripts/evaluate_model.sh ALIAS "how it is served" [RUNS] [--config PATH] +# +# 1. The context probe: does the server's window match what LiteLLM declares? +# 2. The capability matrix: does it hold the tool contract, and is its +# supports_tool_choice declaration true? +# 3. The benchmark: canned milestones, RUNS times each (default 5). +# Stops at the first step that fails; there is no point benchmarking a model +# that can't hold the tool contract. +set -euo pipefail +cd "$(dirname "$0")/.." + +alias="${1:?usage: evaluate_model.sh ALIAS NOTE [RUNS] [--config PATH]}" +note="${2:?give a note on how the model is served, e.g. \"vLLM 0.29, AWQ\"}" +runs="${3:-5}" +shift $(( $# < 3 ? $# : 3 )) +config=("$@") + +echo "== 1/3 context probe" +uv run python scripts/probe_context.py "$alias" "${config[@]}" + +echo "== 2/3 capability matrix" +uv run pytest -m integration -q -rs tests/integration/test_model_capability.py \ + --capability-models="$alias" + +echo "== 3/3 benchmark ($runs runs per task)" +uv run python scripts/benchmark.py run --models "$alias" --runs "$runs" \ + --note "$note" "${config[@]}" diff --git a/tests/integration/test_benchmark_live.py b/tests/integration/test_benchmark_live.py new file mode 100644 index 0000000..38eeda6 --- /dev/null +++ b/tests/integration/test_benchmark_live.py @@ -0,0 +1,31 @@ +"""One live benchmark run: the simplest task, through the agent's own +client wiring, with the test config's executor.""" + +from pathlib import Path + +import pytest + +from nisse_dev.agent import make_llm_clients +from nisse_dev.bench.runner import run_task +from nisse_dev.bench.tasks import load_tasks +from nisse_dev.catalog import ModelCatalog +from nisse_dev.config import AgentConfig + +pytestmark = pytest.mark.integration + + +def test_the_simplest_task_passes_live( + live_config: AgentConfig, tmp_path: Path +) -> None: + catalog = ModelCatalog.discover(live_config.llm_base_url, live_config.llm_token) + normalized, _ = make_llm_clients(live_config, catalog) + facts = catalog.facts(live_config.roles.executor) + [task] = load_tasks(["write-function"]) + + record = run_task(task, facts, normalized, live_config.tuning, workdir=tmp_path) + + assert record.fallbacks == [] + assert record.success, ( + f"{record.outcome}: {record.outcome_note}\n{record.check_output}" + ) + assert record.cost >= 0 -- 2.54.0 From 0cccfe1e9484470bfbda8c685a6804e309c43370 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Fri, 25 Sep 2026 01:25:13 +0000 Subject: [PATCH 2/2] Add the local baseline: qwen3-8b-awq and qwen3-14b, 5 runs per task The 4070's two local models with default tuning, as the baseline the 4090's candidates will be compared with. Every one of the 22 failures is an escalation from failure_limit: 2; there were no prose turns, invented tools, context trims, or finished-but-wrong results. Co-Authored-By: Claude Opus 5.5 (1M context) --- bench/results/2026-09-25-local-coder.jsonl | 20 ++++++++++++++++++++ bench/results/2026-09-25-local-fast.jsonl | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 bench/results/2026-09-25-local-coder.jsonl create mode 100644 bench/results/2026-09-25-local-fast.jsonl diff --git a/bench/results/2026-09-25-local-coder.jsonl b/bench/results/2026-09-25-local-coder.jsonl new file mode 100644 index 0000000..0ae8fc4 --- /dev/null +++ b/bench/results/2026-09-25-local-coder.jsonl @@ -0,0 +1,20 @@ +{"backing": "openai/qwen3-14b", "check_output": "...... [100%]\n6 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Fixed total_value to multiply unit price by quantity. All tests now pass.", "prose_turns": 0, "provider": "openai", "success": true, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 5328, "tokens_out": 1492, "trimmed_requests": 0, "ts": "2026-09-25T01:03:24+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 46.4} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\nImportError while importing test module '/tmp/tmpr1q1pe0r/work/.bench_check/test_hidden.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n.bench_check/test_hidden.py:4: in \n from inventory import Item, low_stock, total_value\nE ImportError: cannot import name 'Item' from 'inventory' (/tmp/tmpr1q1pe0r/work/inventory.py)\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.10s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n______________________ ERROR collecting test_inventory.py ______________________\nImportError while importing test module '/tmp/tmpr1q1pe0r/work/test_inventory.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], packa", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 2560, "tokens_out": 1136, "trimmed_requests": 0, "ts": "2026-09-25T01:03:49+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 3, "wall_s": 24.3} +{"backing": "openai/qwen3-14b", "check_output": "...... [100%]\n6 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Fixed the total_value function to multiply unit price by quantity, passing all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 5318, "tokens_out": 1623, "trimmed_requests": 0, "ts": "2026-09-25T01:04:23+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 34.5} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n.bench_check/test_hidden.py:4: in \n from inventory import Item, low_stock, total_value\ninventory.py:1: in \n def total_value(items: list[Item]) -> float:\n ^^^^\nE NameError: name 'Item' is not defined\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py - NameError: name 'Item' is not defined\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.11s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n______________________ ERROR collecting test_inventory.py ______________________\ntest_inventory.py:1: in \n from inventory import Item, low_stock, total_value\ninventory.py:1: in \n def total_value(items: list[Item]) -> float:\n ^^^^\nE NameError: name 'Item' is not defined\n=========================== short ", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 2565, "tokens_out": 958, "trimmed_requests": 0, "ts": "2026-09-25T01:04:45+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 3, "wall_s": 20.6} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n.bench_check/test_hidden.py:4: in \n from inventory import Item, low_stock, total_value\ninventory.py:1: in \n def total_value(items: list[Item]) -> float:\n ^^^^\nE NameError: name 'Item' is not defined\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py - NameError: name 'Item' is not defined\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.09s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n______________________ ERROR collecting test_inventory.py ______________________\ntest_inventory.py:1: in \n from inventory import Item, low_stock, total_value\ninventory.py:1: in \n def total_value(items: list[Item]) -> float:\n ^^^^\nE NameError: name 'Item' is not defined\n=========================== short ", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 2565, "tokens_out": 1051, "trimmed_requests": 0, "ts": "2026-09-25T01:05:07+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 3, "wall_s": 22.6} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmpq8iz1j8z/work/rates.py\", line 56\nE [... 1445 characters omitted ...]\nE ^^^^^^^^\nE SyntaxError: invalid syntax. Perhaps you forgot a comma?\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 13616, "tokens_out": 3186, "trimmed_requests": 0, "ts": "2026-09-25T01:06:21+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 72.8} +{"backing": "openai/qwen3-14b", "check_output": ": 'R271'\nFAILED .bench_check/test_hidden.py::test_rate[272] - KeyError: 'R272'\nFAILED .bench_check/test_hidden.py::test_rate[273] - KeyError: 'R273'\nFAILED .bench_check/test_hidden.py::test_rate[274] - KeyError: 'R274'\nFAILED .bench_check/test_hidden.py::test_rate[275] - KeyError: 'R275'\nFAILED .bench_check/test_hidden.py::test_rate[276] - KeyError: 'R276'\nFAILED .bench_check/test_hidden.py::test_rate[277] - KeyError: 'R277'\nFAILED .bench_check/test_hidden.py::test_rate[278] - KeyError: 'R278'\nFAILED .bench_check/test_hidden.py::test_rate[279] - KeyError: 'R279'\nFAILED .bench_check/test_hidden.py::test_rate[280] - KeyError: 'R280'\nFAILED .bench_check/test_hidden.py::test_rate[281] - KeyError: 'R281'\nFAILED .bench_check/test_hidden.py::test_rate[282] - KeyError: 'R282'\nFAILED .bench_check/test_hidden.py::test_rate[283] - KeyError: 'R283'\nFAILED .bench_check/test_hidden.py::test_rate[284] - KeyError: 'R284'\nFAILED .bench_check/test_hidden.py::test_rate[285] - KeyError: 'R285'\nFAILED .bench_check/test_hidden.py::test_rate[286] - KeyError: 'R286'\nFAILED .bench_check/test_hidden.py::test_rate[287] - KeyError: 'R287'\nFAILED .bench_check/test_hidden.py::test_rate[288] - KeyError: 'R288'\nFAILED .bench_check/test_hidden.py::test_rate[289] - KeyError: 'R289'\nFAILED .bench_check/test_hidden.py::test_rate[290] - KeyError: 'R290'\nFAILED .bench_check/test_hidden.py::test_rate[291] - KeyError: 'R291'\nFAILED .bench_check/test_hidden.py::test_rate[292] - KeyError: 'R292'\nFAILED .bench_check/test_hidden.py::test_rate[293] - KeyError: 'R293'\nFAILED .bench_check/test_hidden.py::test_rate[294] - KeyError: 'R294'\nFAILED .bench_check/test_hidden.py::test_rate[295] - KeyError: 'R295'\nFAILED .bench_check/test_hidden.py::test_rate[296] - KeyError: 'R296'\nFAILED .bench_check/test_hidden.py::test_rate[297] - KeyError: 'R297'\nFAILED .bench_check/test_hidden.py::test_rate[298] - KeyError: 'R298'\nFAILED .bench_check/test_hidden.py::test_rate[299] - KeyError: 'R299'\n247 failed, 54 passed in 0.49s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n.....................................................FFFFFFFFFFFFFFFFFFF [ 24%]\nFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF [ 48%]\nFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF [ 72%]\nFFFFFFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF [ 96%]\nFFFFFFFFFFFF [100%]\n=================================== FAILU", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 12726, "tokens_out": 2366, "trimmed_requests": 0, "ts": "2026-09-25T01:07:16+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 54.4} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmphac71897/work/rates.py\", line 56\nE [... 1445 characters omitted ...]\nE ^^^^^^^^\nE SyntaxError: invalid syntax. Perhaps you forgot a comma?\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.15s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 13614, "tokens_out": 3000, "trimmed_requests": 0, "ts": "2026-09-25T01:08:25+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 69.2} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmp0vs0w4kr/work/rates.py\", line 56\nE [... 1445 characters omitted ...]\nE ^^^^^^^^\nE SyntaxError: invalid syntax. Perhaps you forgot a comma?\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 13607, "tokens_out": 3223, "trimmed_requests": 0, "ts": "2026-09-25T01:09:40+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 73.8} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmp_pxm0ldr/work/rates.py\", line 56\nE [... 1445 characters omitted ...]\nE ^^^^^^^^\nE SyntaxError: invalid syntax. Perhaps you forgot a comma?\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.15s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 13609, "tokens_out": 3180, "trimmed_requests": 0, "ts": "2026-09-25T01:10:53+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 73.0} +{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x777eb60e3e90>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 13753, "tokens_out": 5755, "trimmed_requests": 0, "ts": "2026-09-25T01:12:55+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 121.2} +{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added --shout option to greeter, updated core and cli code, added tests, and verified with pytest.", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 21373, "tokens_out": 9952, "trimmed_requests": 0, "ts": "2026-09-25T01:16:25+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 13, "wall_s": 210.0} +{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x7a2788fdbe30>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9432, "tokens_out": 6691, "trimmed_requests": 0, "ts": "2026-09-25T01:18:47+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 142.1} +{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added --shout option to greeter, updated core and cli code, added tests, and verified with pytest.", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9104, "tokens_out": 3626, "trimmed_requests": 0, "ts": "2026-09-25T01:20:03+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 75.8} +{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x76eac82e3da0>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9382, "tokens_out": 7755, "trimmed_requests": 0, "ts": "2026-09-25T01:22:48+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 164.5} +{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function and passed all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 1769, "tokens_out": 1300, "trimmed_requests": 0, "ts": "2026-09-25T01:23:16+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 27.7} +{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function with tests; all tests passed.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 1782, "tokens_out": 1239, "trimmed_requests": 0, "ts": "2026-09-25T01:23:43+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 26.2} +{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function and passed all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3627, "tokens_out": 1332, "trimmed_requests": 0, "ts": "2026-09-25T01:24:11+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 28.1} +{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function and passed all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 734, "tokens_out": 653, "trimmed_requests": 0, "ts": "2026-09-25T01:24:25+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 1, "wall_s": 13.9} +{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function and passed all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 733, "tokens_out": 523, "trimmed_requests": 0, "ts": "2026-09-25T01:24:37+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 1, "wall_s": 11.3} diff --git a/bench/results/2026-09-25-local-fast.jsonl b/bench/results/2026-09-25-local-fast.jsonl new file mode 100644 index 0000000..1cd5ac9 --- /dev/null +++ b/bench/results/2026-09-25-local-fast.jsonl @@ -0,0 +1,20 @@ +{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1578, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:41+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 63.2} +{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1576, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:43+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4} +{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1576, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:44+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4} +{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1574, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:46+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4} +{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1578, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:48+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4} +{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmpl4mdto5g/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11000, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:00:52+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2} +{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmpsqvx_94y/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11000, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:00:57+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2} +{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmpe3eakwxj/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11013, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:01:01+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2} +{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmprlm2camm/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 10987, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:01:06+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2} +{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmpvzw7f5tn/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11000, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:01:11+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2} +{"backing": "openai/qwen3-8b-awq", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n...F [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x713c88ce3c20>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 13039, "tokens_out": 566, "trimmed_requests": 0, "ts": "2026-09-25T01:01:19+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 8.4} +{"backing": "openai/qwen3-8b-awq", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n________________________________ test_cli_shout ________________________________\n\n def test_cli_shout() -> None:\n> from pytest import capsys\nE ImportError: cannot import name 'capsys' from 'pytest' (/root/nisse/.venv/lib/python3.12/site-packages/pytest/__init__.py)", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 11243, "tokens_out": 715, "trimmed_requests": 0, "ts": "2026-09-25T01:01:29+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 9.9} +{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:4: in \n from greeter.cli import main\nE File \"/tmp/tmp3f09u7jk/work/greeter/cli.py\", line 11\nE print(greet(args.name, shout=args.--shout))\nE ^\nE SyntaxError: invalid syntax\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n\nstderr:\nTraceback (most recent call last):\n File \"\", line 198, in _run_module_as_main\n File \"\", line 88, in _run_code\n File \"/tmp/tmp3f09u7jk/work/greeter/__main__.py\", line 1, in \n from greeter.cli import main\n File \"/tmp/tmp3f09u7jk/work/greeter/cli.py\", line 11\n print(greet(args.name, shout=args.--shout))\n ^\nSyntaxError: invalid syntax\n", "prose_turns": 0, "provider": "openai", "success": false, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9358, "tokens_out": 450, "trimmed_requests": 0, "ts": "2026-09-25T01:01:56+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 26.5} +{"backing": "openai/qwen3-8b-awq", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "completed", "outcome_note": "Added --shout option to greeter. Tests pass.", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 10784, "tokens_out": 549, "trimmed_requests": 0, "ts": "2026-09-25T01:02:04+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 7.5} +{"backing": "openai/qwen3-8b-awq", "check_output": " stderr arguments may not be used '\n 'with capture_output.')\n kwargs['stdout'] = PIPE\n kwargs['stderr'] = PIPE\n \n with Popen(*popenargs, **kwargs) as process:\n try:\n stdout, stderr = process.communicate(input, timeout=timeout)\n except TimeoutExpired as exc:\n process.kill()\n if _mswindows:\n # Windows accumulates the output in a single blocking\n # read() call run on child threads, with the timeout\n # being done in a join() on those threads. communicate()\n # _after_ kill() is required to collect that and add it\n # to the exception.\n exc.stdout, exc.stderr = process.communicate()\n else:\n # POSIX _communicate already populated the output so\n # far into the TimeoutExpired exception.\n process.wait()\n raise\n except: # Including KeyboardInterrupt, communicate handled that.\n process.kill()\n # We don't call process.wait() as .__exit__ does that for us.\n raise\n retcode = process.poll()\n if check and retcode:\n> raise CalledProcessError(retcode, process.args,\n output=stdout, stderr=stderr)\nE subprocess.CalledProcessError: Command '['/root/nisse/.venv/bin/python', '-m', 'greeter', '--shout', 'Bob']' returned non-zero exit status 2.\n\n/usr/lib/python3.12/subprocess.py:571: CalledProcessError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_cli_default - SystemExit: 2\nFAILED .bench_check/test_hidden.py::test_cli_shout - SystemExit: 2\nFAILED .bench_check/test_hidden.py::test_module_entry_point - subprocess.Call...\n3 failed, 2 passed in 0.19s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..F [100%]\n=================================== FAILURES ===================================\n________________________________ test_cli_shout ________________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x763f352e3980>\n\n def test_cli_shout(capsys) -> None:\n> assert main([\"--shout\", \"Bob\"]) == 0\n ^^^^^^^^^^^^^^^^^^^^^^^^\n\ntest", "prose_turns": 0, "provider": "openai", "success": false, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 13124, "tokens_out": 589, "trimmed_requests": 0, "ts": "2026-09-25T01:02:13+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 8.4} +{"backing": "openai/qwen3-8b-awq", "check_output": " (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'leading-and-trailing'\nE \nE - leading-and-trailing\n\n.bench_check/test_hidden.py:19: AssertionError\n_____________________ test_slugify[MiXeD CaSe-mixed-case] ______________________\n\ntext = 'MiXeD CaSe', expected = 'mixed-case'\n\n @pytest.mark.parametrize(\n (\"text\", \"expected\"),\n [\n (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'mixed-case'\nE \nE - mixed-case\n\n.bench_check/test_hidden.py:19: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_slugify[Hello, World!-hello-world]\nFAILED .bench_check/test_hidden.py::test_slugify[ Nisse Agent -nisse-agent]\nFAILED .bench_check/test_hidden.py::test_slugify[a--b-a-b] - AssertionError: ...\nFAILED .bench_check/test_hidden.py::test_slugify[Python 3.12 Release-python-3-12-release]\nFAILED .bench_check/test_hidden.py::test_slugify[--leading and trailing---leading-and-trailing]\nFAILED .bench_check/test_hidden.py::test_slugify[MiXeD CaSe-mixed-case] - Ass...\n6 failed, 2 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\nE AssertionError: assert '' == 'hello-world'\nE \nE - hello-world\n\ntest_slugify.py:6: AssertionEr", "prose_turns": 0, "provider": "openai", "success": false, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 4942, "tokens_out": 377, "trimmed_requests": 0, "ts": "2026-09-25T01:02:19+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 5.5} +{"backing": "openai/qwen3-8b-awq", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\n ^^^^^^^\nE NameError: name 'slugify' is not defined\n\ntest_slugify.py:4: NameError\n===============", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3688, "tokens_out": 255, "trimmed_requests": 0, "ts": "2026-09-25T01:02:23+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 3.9} +{"backing": "openai/qwen3-8b-awq", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\n ^^^^^^^\nE NameError: name 'slugify' is not defined\n\ntest_slugify.py:4: NameError\n===============", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3696, "tokens_out": 255, "trimmed_requests": 0, "ts": "2026-09-25T01:02:27+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 3.9} +{"backing": "openai/qwen3-8b-awq", "check_output": " (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'leading-and-trailing'\nE \nE - leading-and-trailing\n\n.bench_check/test_hidden.py:19: AssertionError\n_____________________ test_slugify[MiXeD CaSe-mixed-case] ______________________\n\ntext = 'MiXeD CaSe', expected = 'mixed-case'\n\n @pytest.mark.parametrize(\n (\"text\", \"expected\"),\n [\n (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'mixed-case'\nE \nE - mixed-case\n\n.bench_check/test_hidden.py:19: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_slugify[Hello, World!-hello-world]\nFAILED .bench_check/test_hidden.py::test_slugify[ Nisse Agent -nisse-agent]\nFAILED .bench_check/test_hidden.py::test_slugify[a--b-a-b] - AssertionError: ...\nFAILED .bench_check/test_hidden.py::test_slugify[Python 3.12 Release-python-3-12-release]\nFAILED .bench_check/test_hidden.py::test_slugify[--leading and trailing---leading-and-trailing]\nFAILED .bench_check/test_hidden.py::test_slugify[MiXeD CaSe-mixed-case] - Ass...\n6 failed, 2 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\nE AssertionError: assert '' == 'hello-world'\nE \nE - hello-world\n\ntest_slugify.py:6: AssertionEr", "prose_turns": 0, "provider": "openai", "success": false, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 4936, "tokens_out": 377, "trimmed_requests": 0, "ts": "2026-09-25T01:02:33+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 5.5} +{"backing": "openai/qwen3-8b-awq", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\n ^^^^^^^\nE NameError: name 'slugify' is not defined\n\ntest_slugify.py:4: NameError\n===============", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3686, "tokens_out": 251, "trimmed_requests": 0, "ts": "2026-09-25T01:02:37+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 3.9} -- 2.54.0