Add an executor benchmark that keeps its results
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m11s
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m11s
Choosing a local executor model, and tuning its knobs, needs measured success rates across repeated runs, comparable with every model measured before. - Four canned milestones under nisse_dev/bench/tasks: write a function, fix a bug a failing test exposes, thread an option through several files, and find one failure in output far past max_output_chars. Each is an approved plan over a seed project, judged by a hidden check the model never sees; a reference solution proves each is solvable and that its check tells the difference - Runs use the agent's own client wiring, now shared as make_llm_clients, with delegation and ask_expert off: an escalation is a failure of the model under test - Each run is stored as one JSON line in bench/results/, committed, with the model's backing name and declared facts, a note on how it was served, the task's content hash, the agent's commit and tuning, the outcome, turns, misbehaviour counts, context trims, tokens, time and LiteLLM's cost; a fallback answer can never count as a success - The report rebuilds the comparison from every stored run, with Wilson 95% intervals, marking runs against older task versions - scripts/benchmark.py runs and reports; scripts/evaluate_model.sh takes a new model through the context probe, the capability matrix and the benchmark in one command - Clients keep LiteLLM's per-call cost, and BudgetedClient counts the requests it had to trim - The tuning guide gains "Benchmarking models" Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
dongarandClaude Opus 5.5 committed
1 parent
b2bbf70b7b
commit
50b46821d7
37 files changed
+1912
-37
No files matched your search
@@ -113,6 +113,51 @@ class Agent:
|
||||
self._stopped.set()
|
||||
|
||||
|
||||
def make_llm_clients(
|
||||
config: AgentConfig, catalog: ModelCatalog
|
||||
) -> tuple[BudgetedClient, BudgetedClient]:
|
||||
"""The two LLM routes, as every model call in the agent uses them.
|
||||
|
||||
Shared with the benchmark, so that what it measures is exactly what the
|
||||
agent would do: the same context budget, the same declared limits, and
|
||||
tool_choice asked only of models declared to enforce it.
|
||||
"""
|
||||
tuning = config.tuning
|
||||
|
||||
def limits_for(model: str) -> tuple[int, int | None]:
|
||||
facts = catalog.facts(model)
|
||||
assert facts.max_input_tokens is not None # require_limits checked
|
||||
return facts.max_input_tokens, facts.max_output_tokens
|
||||
|
||||
def budgeted(client: object) -> BudgetedClient:
|
||||
# Every request, on either route, is fitted to its model's window.
|
||||
return BudgetedClient(
|
||||
client,
|
||||
limits_for,
|
||||
max_output_tokens=config.max_output_tokens,
|
||||
keep_recent_turns=tuning.context.keep_recent_turns,
|
||||
safety_margin=tuning.context.safety_margin,
|
||||
enforces_tool_choice=lambda m: (
|
||||
catalog.facts(m).supports_tool_choice is True
|
||||
),
|
||||
)
|
||||
|
||||
normalized = budgeted(
|
||||
LiteLLMClient(
|
||||
config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
|
||||
)
|
||||
)
|
||||
# Delegations prefer the vendor-native route; fidelity is cheap
|
||||
# elsewhere (A5). Which route a given model gets is decided by
|
||||
# select_client, not by where the caller happens to sit.
|
||||
passthrough = budgeted(
|
||||
AnthropicPassthroughClient(
|
||||
config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
|
||||
)
|
||||
)
|
||||
return normalized, passthrough
|
||||
|
||||
|
||||
def prepare_workspace(config: AgentConfig, account: str) -> GitRepo:
|
||||
"""Ensure the workspace is a clone of the project repo.
|
||||
|
||||
@@ -161,37 +206,7 @@ def build_agent(
|
||||
max_output_chars=tuning.tools.max_output_chars,
|
||||
)
|
||||
|
||||
def limits_for(model: str) -> tuple[int, int | None]:
|
||||
facts = catalog.facts(model)
|
||||
assert facts.max_input_tokens is not None # require_limits checked
|
||||
return facts.max_input_tokens, facts.max_output_tokens
|
||||
|
||||
def budgeted(client: object) -> BudgetedClient:
|
||||
# Every request, on either route, is fitted to its model's window.
|
||||
return BudgetedClient(
|
||||
client,
|
||||
limits_for,
|
||||
max_output_tokens=config.max_output_tokens,
|
||||
keep_recent_turns=tuning.context.keep_recent_turns,
|
||||
safety_margin=tuning.context.safety_margin,
|
||||
enforces_tool_choice=lambda m: (
|
||||
catalog.facts(m).supports_tool_choice is True
|
||||
),
|
||||
)
|
||||
|
||||
normalized = budgeted(
|
||||
LiteLLMClient(
|
||||
config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
|
||||
)
|
||||
)
|
||||
# Delegations prefer the vendor-native route; fidelity is cheap
|
||||
# elsewhere (A5). Which route a given model gets is decided by
|
||||
# select_client, not by where the caller happens to sit.
|
||||
passthrough = budgeted(
|
||||
AnthropicPassthroughClient(
|
||||
config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
|
||||
)
|
||||
)
|
||||
normalized, passthrough = make_llm_clients(config, catalog)
|
||||
|
||||
transport = MatrixTransport(config.matrix)
|
||||
announce = notify or transport.send_message
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""The executor benchmark: canned milestones, run repeatedly per model.
|
||||
|
||||
Choosing an executor model, and later tuning its knobs, should rest on
|
||||
measured success rates, not on one run or on reputation. Each task is a fixed
|
||||
approved plan over a seed project, with a hidden check the model never sees;
|
||||
success means the check passes, not that the model said it finished. Runs use
|
||||
the agent's own client wiring (context budget, declared limits, tool_choice
|
||||
only where declared), so what is measured is what the agent would do.
|
||||
|
||||
Results are stored, one JSON line per run, with everything needed to compare
|
||||
them later: the model's identity and declared facts, a note on how it was
|
||||
served, the task's content hash, the agent's commit, and its tuning. A new
|
||||
model then needs only its own runs to be compared with everything measured
|
||||
before it.
|
||||
|
||||
See scripts/benchmark.py, and docs/Tuning-Guide.md for how to read results.
|
||||
"""
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Stored benchmark results, and the report that compares them.
|
||||
|
||||
Every run is one JSON line. The report is rebuilt from all of them each
|
||||
time, so a model benchmarked today is read alongside every model measured
|
||||
before it, as long as the task is still the same version.
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from nisse_dev.bench.runner import RunRecord
|
||||
|
||||
|
||||
def results_file(directory: Path, model: str, when: datetime | None = None) -> Path:
|
||||
day = (when or datetime.now(UTC)).strftime("%Y-%m-%d")
|
||||
return directory / f"{day}-{model}.jsonl"
|
||||
|
||||
|
||||
def append(path: Path, record: RunRecord) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("a") as out:
|
||||
out.write(json.dumps(record.to_json(), sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def load(directory: Path) -> list[dict[str, Any]]:
|
||||
records = []
|
||||
for path in sorted(directory.glob("*.jsonl")):
|
||||
for line in path.read_text().splitlines():
|
||||
if line.strip():
|
||||
records.append(json.loads(line))
|
||||
return records
|
||||
|
||||
|
||||
def wilson(successes: int, runs: int, z: float = 1.96) -> tuple[float, float]:
|
||||
"""A 95% confidence interval for a success rate, sound at small counts.
|
||||
|
||||
Three successes in five runs and five in five overlap here, which is the
|
||||
point: small samples of a noisy model do not separate much."""
|
||||
if runs == 0:
|
||||
return 0.0, 1.0
|
||||
p = successes / runs
|
||||
denominator = 1 + z * z / runs
|
||||
centre = (p + z * z / (2 * runs)) / denominator
|
||||
half = z * math.sqrt(p * (1 - p) / runs + z * z / (4 * runs * runs)) / denominator
|
||||
return max(0.0, centre - half), min(1.0, centre + half)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Row:
|
||||
model: str
|
||||
note: str
|
||||
task: str
|
||||
runs: int
|
||||
successes: int
|
||||
median_turns: float
|
||||
mean_wall_s: float
|
||||
cost_per_run: float
|
||||
stale: bool
|
||||
|
||||
@property
|
||||
def interval(self) -> tuple[float, float]:
|
||||
return wilson(self.successes, self.runs)
|
||||
|
||||
|
||||
def summarise(
|
||||
records: Iterable[dict[str, Any]], current_hashes: dict[str, str]
|
||||
) -> list[Row]:
|
||||
"""One row per (model, how it was served, task, task version)."""
|
||||
groups: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
|
||||
for r in records:
|
||||
groups[(r["backing"], r.get("note", ""), r["task"], r["task_hash"])].append(r)
|
||||
rows = []
|
||||
for (backing, note, task, task_hash), runs in sorted(groups.items()):
|
||||
rows.append(
|
||||
Row(
|
||||
model=backing,
|
||||
note=note,
|
||||
task=task,
|
||||
runs=len(runs),
|
||||
successes=sum(1 for r in runs if r["success"]),
|
||||
median_turns=statistics.median(r["turns"] for r in runs),
|
||||
mean_wall_s=statistics.fmean(r["wall_s"] for r in runs),
|
||||
cost_per_run=statistics.fmean(r["cost"] for r in runs),
|
||||
stale=current_hashes.get(task) != task_hash,
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def render(rows: list[Row]) -> str:
|
||||
header = (
|
||||
f"{'model':<34} {'note':<22} {'task':<15} {'success':>8} {'95% CI':>11} "
|
||||
f"{'turns':>6} {'time':>7} {'$/run':>8}"
|
||||
)
|
||||
lines = [header, "-" * len(header)]
|
||||
for row in rows:
|
||||
low, high = row.interval
|
||||
task = row.task + (" (old)" if row.stale else "")
|
||||
lines.append(
|
||||
f"{row.model[:34]:<34} {row.note[:22]:<22} {task[:15]:<15} "
|
||||
f"{row.successes:>3}/{row.runs:<4} {low:>4.0%}-{high:<5.0%} "
|
||||
f"{row.median_turns:>6.0f} {row.mean_wall_s:>6.0f}s {row.cost_per_run:>8.4f}"
|
||||
)
|
||||
if any(row.stale for row in rows):
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"(old) = measured against an earlier version of the task; "
|
||||
"not comparable with current runs."
|
||||
)
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Running one task once with one executor model, and recording it."""
|
||||
|
||||
import io
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from nisse_dev.bench.tasks import Task
|
||||
from nisse_dev.canonical import (
|
||||
CanonicalEvent,
|
||||
PlanSubmitted,
|
||||
StoredEvent,
|
||||
ToolCall,
|
||||
ToolResult,
|
||||
UserMessage,
|
||||
)
|
||||
from nisse_dev.catalog import ModelFacts
|
||||
from nisse_dev.config import Tuning
|
||||
from nisse_dev.executor import LLMExecutor
|
||||
from nisse_dev.llm import RoleMap
|
||||
from nisse_dev.oplog import OpLog
|
||||
from nisse_dev.state import StateStore
|
||||
from nisse_dev.tools import Workspace
|
||||
|
||||
|
||||
class MeteringClient:
|
||||
"""Counts what a run spends, and notices a fallback: an answer from a
|
||||
model other than the one asked for is not a measurement of it."""
|
||||
|
||||
def __init__(self, inner: Any, facts: ModelFacts) -> None:
|
||||
self._inner = inner
|
||||
self._facts = facts
|
||||
self.calls = 0
|
||||
self.prose_turns = 0
|
||||
self.tokens_in = 0
|
||||
self.tokens_out = 0
|
||||
self.cost = 0.0
|
||||
self.fallbacks: list[str] = []
|
||||
|
||||
def complete(
|
||||
self,
|
||||
events: list[StoredEvent],
|
||||
tools: list[dict[str, Any]] | None,
|
||||
model: str,
|
||||
system: str | None = None,
|
||||
) -> list[CanonicalEvent]:
|
||||
self.calls += 1
|
||||
produced = self._inner.complete(events, tools, model, system=system)
|
||||
usage = getattr(self._inner, "last_usage", {}) or {}
|
||||
self.tokens_in += int(usage.get("input", 0))
|
||||
self.tokens_out += int(usage.get("output", 0))
|
||||
self.cost += float(getattr(self._inner, "last_cost", 0.0) or 0.0)
|
||||
answered = getattr(self._inner, "last_model", None)
|
||||
if not self._facts.answered(answered):
|
||||
self.fallbacks.append(str(answered))
|
||||
if not any(isinstance(e, ToolCall) for e in produced):
|
||||
self.prose_turns += 1
|
||||
return produced
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunRecord:
|
||||
"""One run, with everything needed to compare it with runs made later
|
||||
against other models, other servers, or other tuning."""
|
||||
|
||||
ts: str
|
||||
model: str
|
||||
backing: str
|
||||
provider: str
|
||||
model_info: dict[str, Any]
|
||||
note: str
|
||||
task: str
|
||||
task_hash: str
|
||||
commit: str
|
||||
tuning: dict[str, Any]
|
||||
success: bool
|
||||
outcome: str
|
||||
outcome_note: str
|
||||
turns: int
|
||||
prose_turns: int
|
||||
failed_commands: int
|
||||
invented_tools: int
|
||||
trimmed_requests: int
|
||||
tokens_in: int
|
||||
tokens_out: int
|
||||
cost: float
|
||||
wall_s: float
|
||||
fallbacks: list[str] = field(default_factory=list)
|
||||
check_output: str = ""
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def agent_commit(repo: Path | None = None) -> str:
|
||||
"""The agent's commit, marked when the tree had uncommitted changes."""
|
||||
where = repo or Path(__file__).resolve().parent
|
||||
try:
|
||||
sha = subprocess.run(
|
||||
["git", "rev-parse", "--short", "HEAD"],
|
||||
cwd=where,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
dirty = subprocess.run(
|
||||
["git", "status", "--porcelain"],
|
||||
cwd=where,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return "unknown"
|
||||
return f"{sha}+dirty" if dirty else sha
|
||||
|
||||
|
||||
def _count(events: list[StoredEvent]) -> tuple[int, int]:
|
||||
tools = {
|
||||
e.event.call_id: e.event.tool for e in events if isinstance(e.event, ToolCall)
|
||||
}
|
||||
failed = invented = 0
|
||||
for stored in events:
|
||||
event = stored.event
|
||||
if isinstance(event, ToolResult) and not event.success:
|
||||
if event.result.startswith("no such tool"):
|
||||
invented += 1
|
||||
elif tools.get(event.call_id) == "run_command":
|
||||
failed += 1
|
||||
return failed, invented
|
||||
|
||||
|
||||
def run_task(
|
||||
task: Task,
|
||||
facts: ModelFacts,
|
||||
client: Any,
|
||||
tuning: Tuning,
|
||||
*,
|
||||
note: str = "",
|
||||
commit: str = "",
|
||||
workdir: Path | None = None,
|
||||
) -> RunRecord:
|
||||
"""Run one task once: seed a workspace, hand the executor the approved
|
||||
plan, then judge the result with the hidden check.
|
||||
|
||||
Delegation and ask_expert are off: an escalation is a failure of the
|
||||
model under test, not a frontier model's success."""
|
||||
with tempfile.TemporaryDirectory(dir=workdir) as tmp:
|
||||
root = Path(tmp)
|
||||
workspace = Workspace(
|
||||
root / "work",
|
||||
timeout_s=tuning.tools.command_timeout_s,
|
||||
max_output_chars=tuning.tools.max_output_chars,
|
||||
)
|
||||
task.seed_into(workspace.root)
|
||||
store = StateStore(root / "state.db", oplog=OpLog("nisse-bench", io.StringIO()))
|
||||
store.append_event(PlanSubmitted(plan=task.plan, model="benchmark"))
|
||||
store.append_event(UserMessage(text="start milestone 1", source="benchmark"))
|
||||
|
||||
meter = MeteringClient(client, facts)
|
||||
trims_before = getattr(client, "trimmed_requests", 0)
|
||||
model = facts.alias
|
||||
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
||||
executor = LLMExecutor(
|
||||
meter,
|
||||
store,
|
||||
roles,
|
||||
workspace,
|
||||
max_turns=tuning.executor.max_turns,
|
||||
failure_limit=tuning.executor.failure_limit,
|
||||
)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
outcome = executor.run(1)
|
||||
status, detail = outcome.status, outcome.note
|
||||
except Exception as exc: # noqa: BLE001 -- recorded, the benchmark goes on
|
||||
status, detail = "error", repr(exc)
|
||||
wall = time.monotonic() - started
|
||||
|
||||
success, check_output = task.run_check(workspace.root)
|
||||
failed, invented = _count(store.read_events())
|
||||
store.close()
|
||||
|
||||
return RunRecord(
|
||||
ts=datetime.now(UTC).isoformat(timespec="seconds"),
|
||||
model=model,
|
||||
backing=facts.backing,
|
||||
provider=facts.provider,
|
||||
model_info={
|
||||
"max_input_tokens": facts.max_input_tokens,
|
||||
"max_output_tokens": facts.max_output_tokens,
|
||||
"supports_tool_choice": facts.supports_tool_choice,
|
||||
"supports_function_calling": facts.supports_tools,
|
||||
},
|
||||
note=note,
|
||||
task=task.name,
|
||||
task_hash=task.content_hash,
|
||||
commit=commit,
|
||||
tuning=asdict(tuning),
|
||||
success=success and not meter.fallbacks,
|
||||
outcome=status,
|
||||
outcome_note=detail[:500],
|
||||
turns=meter.calls,
|
||||
prose_turns=meter.prose_turns,
|
||||
failed_commands=failed,
|
||||
invented_tools=invented,
|
||||
trimmed_requests=getattr(client, "trimmed_requests", 0) - trims_before,
|
||||
tokens_in=meter.tokens_in,
|
||||
tokens_out=meter.tokens_out,
|
||||
cost=round(meter.cost, 6),
|
||||
wall_s=round(wall, 1),
|
||||
fallbacks=meter.fallbacks,
|
||||
check_output=check_output,
|
||||
)
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Benchmark tasks: a seed project, an approved plan, and a hidden check.
|
||||
|
||||
Each task is a directory under ``tasks/``:
|
||||
|
||||
- ``task.yaml``: ``title``, ``plan`` (the approved milestone plan, written the
|
||||
way the planner is asked to write them), and ``check`` (a shell command run
|
||||
in the workspace afterwards).
|
||||
- ``seed/``: the project the executor starts from.
|
||||
- ``hidden/``: files the check needs, copied in only after the run, into
|
||||
``.bench_check/``. The executor never sees them.
|
||||
- ``solution/``: a reference solution, overlaid on the seed by the tests to
|
||||
prove each task can be solved and that its check tells the difference.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import shutil
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
TASKS_DIR = Path(__file__).parent / "tasks"
|
||||
CHECK_DIR = ".bench_check"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Task:
|
||||
name: str
|
||||
title: str
|
||||
plan: str
|
||||
check: str
|
||||
path: Path
|
||||
|
||||
@property
|
||||
def content_hash(self) -> str:
|
||||
"""What the task is, so a changed task is never compared with runs of
|
||||
its earlier version. Covers everything the model or the check sees;
|
||||
not the reference solution."""
|
||||
digest = hashlib.sha256()
|
||||
for part in ("task.yaml", "seed", "hidden"):
|
||||
root = self.path / part
|
||||
files = [root] if root.is_file() else sorted(root.rglob("*"))
|
||||
for file in files:
|
||||
if file.is_file() and "__pycache__" not in file.parts:
|
||||
digest.update(str(file.relative_to(self.path)).encode())
|
||||
digest.update(file.read_bytes())
|
||||
return digest.hexdigest()[:16]
|
||||
|
||||
def seed_into(self, workspace: Path) -> None:
|
||||
shutil.copytree(self.path / "seed", workspace, dirs_exist_ok=True)
|
||||
|
||||
def apply_solution(self, workspace: Path) -> None:
|
||||
shutil.copytree(self.path / "solution", workspace, dirs_exist_ok=True)
|
||||
|
||||
def run_check(self, workspace: Path, timeout_s: int = 300) -> tuple[bool, str]:
|
||||
"""Copy the hidden files in and run the check. Returns whether it
|
||||
passed, and its output for the record."""
|
||||
target = workspace / CHECK_DIR
|
||||
if target.exists():
|
||||
shutil.rmtree(target) # never trust a directory the model made
|
||||
shutil.copytree(self.path / "hidden", target)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
self.check,
|
||||
shell=True,
|
||||
cwd=workspace,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout_s,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, f"check timed out after {timeout_s}s"
|
||||
output = (result.stdout + result.stderr)[-2000:]
|
||||
return result.returncode == 0, output
|
||||
|
||||
|
||||
def load_task(path: Path) -> Task:
|
||||
spec = yaml.safe_load((path / "task.yaml").read_text())
|
||||
return Task(
|
||||
name=path.name,
|
||||
title=str(spec["title"]),
|
||||
plan=str(spec["plan"]).strip(),
|
||||
check=str(spec["check"]),
|
||||
path=path,
|
||||
)
|
||||
|
||||
|
||||
def load_tasks(names: list[str] | None = None, root: Path = TASKS_DIR) -> list[Task]:
|
||||
available = {
|
||||
p.name: p for p in sorted(root.iterdir()) if (p / "task.yaml").exists()
|
||||
}
|
||||
if names:
|
||||
unknown = [n for n in names if n not in available]
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"unknown task(s): {', '.join(unknown)}; "
|
||||
f"available: {', '.join(available)}"
|
||||
)
|
||||
return [load_task(available[n]) for n in names]
|
||||
return [load_task(p) for p in available.values()]
|
||||
@@ -0,0 +1,11 @@
|
||||
from inventory import Item, low_stock, total_value
|
||||
|
||||
|
||||
def test_total_value_counts_every_unit() -> None:
|
||||
items = [Item("bolt", 0.25, 100), Item("nut", 0.10, 40)]
|
||||
assert total_value(items) == 29.0
|
||||
|
||||
|
||||
def test_low_stock_is_alphabetical() -> None:
|
||||
items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100), Item("anchor", 3.0, 1)]
|
||||
assert low_stock(items) == ["anchor", "washer"]
|
||||
@@ -0,0 +1,27 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from inventory import Item, low_stock, total_value
|
||||
|
||||
|
||||
def test_the_visible_tests_were_not_changed() -> None:
|
||||
original = (Path(__file__).parent / "original_test_inventory.py.txt").read_text()
|
||||
assert Path("test_inventory.py").read_text() == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("items", "expected"),
|
||||
[
|
||||
([], 0.0),
|
||||
([Item("bolt", 0.25, 100), Item("nut", 0.10, 40)], 29.0),
|
||||
([Item("gear", 12.5, 3)], 37.5),
|
||||
([Item("empty", 9.99, 0)], 0.0),
|
||||
],
|
||||
)
|
||||
def test_total_value(items: list[Item], expected: float) -> None:
|
||||
assert total_value(items) == pytest.approx(expected)
|
||||
|
||||
|
||||
def test_low_stock_still_works() -> None:
|
||||
items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100)]
|
||||
assert low_stock(items) == ["washer"]
|
||||
@@ -0,0 +1,23 @@
|
||||
"""A tiny inventory."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class Item:
|
||||
name: str
|
||||
unit_price: float
|
||||
quantity: int
|
||||
|
||||
|
||||
def total_value(items: list[Item]) -> float:
|
||||
"""The value of all stock: each item's unit price times its quantity."""
|
||||
total = 0.0
|
||||
for item in items:
|
||||
total += item.unit_price
|
||||
return total
|
||||
|
||||
|
||||
def low_stock(items: list[Item], threshold: int = 5) -> list[str]:
|
||||
"""Names of items with fewer than `threshold` units, alphabetically."""
|
||||
return sorted(item.name for item in items if item.quantity < threshold)
|
||||
@@ -0,0 +1,11 @@
|
||||
from inventory import Item, low_stock, total_value
|
||||
|
||||
|
||||
def test_total_value_counts_every_unit() -> None:
|
||||
items = [Item("bolt", 0.25, 100), Item("nut", 0.10, 40)]
|
||||
assert total_value(items) == 29.0
|
||||
|
||||
|
||||
def test_low_stock_is_alphabetical() -> None:
|
||||
items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100), Item("anchor", 3.0, 1)]
|
||||
assert low_stock(items) == ["anchor", "washer"]
|
||||
@@ -0,0 +1,23 @@
|
||||
"""A tiny inventory."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class Item:
|
||||
name: str
|
||||
unit_price: float
|
||||
quantity: int
|
||||
|
||||
|
||||
def total_value(items: list[Item]) -> float:
|
||||
"""The value of all stock: each item's unit price times its quantity."""
|
||||
total = 0.0
|
||||
for item in items:
|
||||
total += item.unit_price * item.quantity
|
||||
return total
|
||||
|
||||
|
||||
def low_stock(items: list[Item], threshold: int = 5) -> list[str]:
|
||||
"""Names of items with fewer than `threshold` units, alphabetically."""
|
||||
return sorted(item.name for item in items if item.quantity < threshold)
|
||||
@@ -0,0 +1,12 @@
|
||||
title: Fix a bug a failing test exposes
|
||||
check: python -m pytest -q .bench_check
|
||||
plan: |
|
||||
Milestone 1: fix the failing inventory test.
|
||||
|
||||
1. Run `python -m pytest -q` and read the failure.
|
||||
2. The failing test is correct; the bug is in `inventory.py`. Do not change
|
||||
any test file.
|
||||
3. Fix `total_value` so that each item contributes its unit price times its
|
||||
quantity.
|
||||
4. Run `python -m pytest -q` until every test passes.
|
||||
5. Call finish.
|
||||
@@ -0,0 +1,11 @@
|
||||
import pytest
|
||||
from rates import RATES
|
||||
|
||||
|
||||
def test_every_region_is_present_once() -> None:
|
||||
assert sorted(RATES) == [f"R{n:03d}" for n in range(300)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", range(300))
|
||||
def test_rate(n: int) -> None:
|
||||
assert RATES[f"R{n:03d}"] == pytest.approx(n % 25 / 100)
|
||||
@@ -0,0 +1,304 @@
|
||||
"""Tax rate per region code."""
|
||||
|
||||
RATES = {
|
||||
"R000": 0.00,
|
||||
"R001": 0.01,
|
||||
"R002": 0.02,
|
||||
"R003": 0.03,
|
||||
"R004": 0.04,
|
||||
"R005": 0.05,
|
||||
"R006": 0.06,
|
||||
"R007": 0.07,
|
||||
"R008": 0.08,
|
||||
"R009": 0.09,
|
||||
"R010": 0.10,
|
||||
"R011": 0.11,
|
||||
"R012": 0.12,
|
||||
"R013": 0.13,
|
||||
"R014": 0.14,
|
||||
"R015": 0.15,
|
||||
"R016": 0.16,
|
||||
"R017": 0.17,
|
||||
"R018": 0.18,
|
||||
"R019": 0.19,
|
||||
"R020": 0.20,
|
||||
"R021": 0.21,
|
||||
"R022": 0.22,
|
||||
"R023": 0.23,
|
||||
"R024": 0.24,
|
||||
"R025": 0.00,
|
||||
"R026": 0.01,
|
||||
"R027": 0.02,
|
||||
"R028": 0.03,
|
||||
"R029": 0.04,
|
||||
"R030": 0.05,
|
||||
"R031": 0.06,
|
||||
"R032": 0.07,
|
||||
"R033": 0.08,
|
||||
"R034": 0.09,
|
||||
"R035": 0.10,
|
||||
"R036": 0.11,
|
||||
"R037": 0.12,
|
||||
"R038": 0.13,
|
||||
"R039": 0.14,
|
||||
"R040": 0.15,
|
||||
"R041": 0.16,
|
||||
"R042": 0.17,
|
||||
"R043": 0.18,
|
||||
"R044": 0.19,
|
||||
"R045": 0.20,
|
||||
"R046": 0.21,
|
||||
"R047": 0.22,
|
||||
"R048": 0.23,
|
||||
"R049": 0.24,
|
||||
"R050": 0.00,
|
||||
"R051": 0.01,
|
||||
"R052": 0.02,
|
||||
"R053": 0.03,
|
||||
"R054": 0.04,
|
||||
"R055": 0.05,
|
||||
"R056": 0.06,
|
||||
"R057": 0.07,
|
||||
"R058": 0.08,
|
||||
"R059": 0.09,
|
||||
"R060": 0.10,
|
||||
"R061": 0.11,
|
||||
"R062": 0.12,
|
||||
"R063": 0.13,
|
||||
"R064": 0.14,
|
||||
"R065": 0.15,
|
||||
"R066": 0.16,
|
||||
"R067": 0.17,
|
||||
"R068": 0.18,
|
||||
"R069": 0.19,
|
||||
"R070": 0.20,
|
||||
"R071": 0.21,
|
||||
"R072": 0.22,
|
||||
"R073": 0.23,
|
||||
"R074": 0.24,
|
||||
"R075": 0.00,
|
||||
"R076": 0.01,
|
||||
"R077": 0.02,
|
||||
"R078": 0.03,
|
||||
"R079": 0.04,
|
||||
"R080": 0.05,
|
||||
"R081": 0.06,
|
||||
"R082": 0.07,
|
||||
"R083": 0.08,
|
||||
"R084": 0.09,
|
||||
"R085": 0.10,
|
||||
"R086": 0.11,
|
||||
"R087": 0.12,
|
||||
"R088": 0.13,
|
||||
"R089": 0.14,
|
||||
"R090": 0.15,
|
||||
"R091": 0.16,
|
||||
"R092": 0.17,
|
||||
"R093": 0.18,
|
||||
"R094": 0.19,
|
||||
"R095": 0.20,
|
||||
"R096": 0.21,
|
||||
"R097": 0.22,
|
||||
"R098": 0.23,
|
||||
"R099": 0.24,
|
||||
"R100": 0.00,
|
||||
"R101": 0.01,
|
||||
"R102": 0.02,
|
||||
"R103": 0.03,
|
||||
"R104": 0.04,
|
||||
"R105": 0.05,
|
||||
"R106": 0.06,
|
||||
"R107": 0.07,
|
||||
"R108": 0.08,
|
||||
"R109": 0.09,
|
||||
"R110": 0.10,
|
||||
"R111": 0.11,
|
||||
"R112": 0.12,
|
||||
"R113": 0.13,
|
||||
"R114": 0.14,
|
||||
"R115": 0.15,
|
||||
"R116": 0.16,
|
||||
"R117": 0.17,
|
||||
"R118": 0.18,
|
||||
"R119": 0.19,
|
||||
"R120": 0.20,
|
||||
"R121": 0.21,
|
||||
"R122": 0.22,
|
||||
"R123": 0.23,
|
||||
"R124": 0.24,
|
||||
"R125": 0.00,
|
||||
"R126": 0.01,
|
||||
"R127": 0.02,
|
||||
"R128": 0.03,
|
||||
"R129": 0.04,
|
||||
"R130": 0.05,
|
||||
"R131": 0.06,
|
||||
"R132": 0.07,
|
||||
"R133": 0.08,
|
||||
"R134": 0.09,
|
||||
"R135": 0.10,
|
||||
"R136": 0.11,
|
||||
"R137": 0.12,
|
||||
"R138": 0.13,
|
||||
"R139": 0.14,
|
||||
"R140": 0.15,
|
||||
"R141": 0.16,
|
||||
"R142": 0.17,
|
||||
"R143": 0.18,
|
||||
"R144": 0.19,
|
||||
"R145": 0.20,
|
||||
"R146": 0.21,
|
||||
"R147": 0.22,
|
||||
"R148": 0.23,
|
||||
"R149": 0.24,
|
||||
"R150": 0.00,
|
||||
"R151": 0.01,
|
||||
"R152": 0.02,
|
||||
"R153": 0.03,
|
||||
"R154": 0.04,
|
||||
"R155": 0.05,
|
||||
"R156": 0.06,
|
||||
"R157": 0.07,
|
||||
"R158": 0.08,
|
||||
"R159": 0.09,
|
||||
"R160": 0.10,
|
||||
"R161": 0.11,
|
||||
"R162": 0.12,
|
||||
"R163": 0.13,
|
||||
"R164": 0.14,
|
||||
"R165": 0.15,
|
||||
"R166": 0.16,
|
||||
"R167": 0.17,
|
||||
"R168": 0.18,
|
||||
"R169": 0.19,
|
||||
"R170": 0.20,
|
||||
"R171": 0.21,
|
||||
"R172": 0.22,
|
||||
"R173": 0.23,
|
||||
"R174": 0.24,
|
||||
"R175": 0.00,
|
||||
"R176": 0.01,
|
||||
"R177": 0.02,
|
||||
"R178": 0.03,
|
||||
"R179": 0.04,
|
||||
"R180": 0.05,
|
||||
"R181": 0.06,
|
||||
"R182": 0.07,
|
||||
"R183": 0.08,
|
||||
"R184": 0.09,
|
||||
"R185": 0.10,
|
||||
"R186": 0.11,
|
||||
"R187": 0.12,
|
||||
"R188": 0.13,
|
||||
"R189": 0.14,
|
||||
"R190": 0.15,
|
||||
"R191": 0.16,
|
||||
"R192": 0.17,
|
||||
"R193": 0.18,
|
||||
"R194": 0.19,
|
||||
"R195": 0.20,
|
||||
"R196": 0.21,
|
||||
"R197": 0.22,
|
||||
"R198": 0.23,
|
||||
"R199": 0.24,
|
||||
"R200": 0.00,
|
||||
"R201": 0.01,
|
||||
"R202": 0.02,
|
||||
"R203": 0.03,
|
||||
"R204": 0.04,
|
||||
"R205": 0.05,
|
||||
"R206": 0.06,
|
||||
"R207": 0.07,
|
||||
"R208": 0.08,
|
||||
"R209": 0.09,
|
||||
"R210": 0.10,
|
||||
"R211": 0.11,
|
||||
"R212": 0.12,
|
||||
"R213": 0.13,
|
||||
"R214": 0.14,
|
||||
"R215": 0.15,
|
||||
"R216": 0.16,
|
||||
"R217": 0.17,
|
||||
"R218": 0.18,
|
||||
"R219": 0.19,
|
||||
"R220": 0.20,
|
||||
"R221": 0.21,
|
||||
"R222": 0.22,
|
||||
"R223": 0.23,
|
||||
"R224": 0.24,
|
||||
"R225": 0.00,
|
||||
"R226": 0.01,
|
||||
"R227": 0.02,
|
||||
"R228": 0.03,
|
||||
"R229": 0.04,
|
||||
"R230": 0.05,
|
||||
"R231": 0.06,
|
||||
"R232": 0.07,
|
||||
"R233": 0.08,
|
||||
"R234": 0.09,
|
||||
"R235": 0.10,
|
||||
"R236": 0.11,
|
||||
"R237": 0.21,
|
||||
"R238": 0.13,
|
||||
"R239": 0.14,
|
||||
"R240": 0.15,
|
||||
"R241": 0.16,
|
||||
"R242": 0.17,
|
||||
"R243": 0.18,
|
||||
"R244": 0.19,
|
||||
"R245": 0.20,
|
||||
"R246": 0.21,
|
||||
"R247": 0.22,
|
||||
"R248": 0.23,
|
||||
"R249": 0.24,
|
||||
"R250": 0.00,
|
||||
"R251": 0.01,
|
||||
"R252": 0.02,
|
||||
"R253": 0.03,
|
||||
"R254": 0.04,
|
||||
"R255": 0.05,
|
||||
"R256": 0.06,
|
||||
"R257": 0.07,
|
||||
"R258": 0.08,
|
||||
"R259": 0.09,
|
||||
"R260": 0.10,
|
||||
"R261": 0.11,
|
||||
"R262": 0.12,
|
||||
"R263": 0.13,
|
||||
"R264": 0.14,
|
||||
"R265": 0.15,
|
||||
"R266": 0.16,
|
||||
"R267": 0.17,
|
||||
"R268": 0.18,
|
||||
"R269": 0.19,
|
||||
"R270": 0.20,
|
||||
"R271": 0.21,
|
||||
"R272": 0.22,
|
||||
"R273": 0.23,
|
||||
"R274": 0.24,
|
||||
"R275": 0.00,
|
||||
"R276": 0.01,
|
||||
"R277": 0.02,
|
||||
"R278": 0.03,
|
||||
"R279": 0.04,
|
||||
"R280": 0.05,
|
||||
"R281": 0.06,
|
||||
"R282": 0.07,
|
||||
"R283": 0.08,
|
||||
"R284": 0.09,
|
||||
"R285": 0.10,
|
||||
"R286": 0.11,
|
||||
"R287": 0.12,
|
||||
"R288": 0.13,
|
||||
"R289": 0.14,
|
||||
"R290": 0.15,
|
||||
"R291": 0.16,
|
||||
"R292": 0.17,
|
||||
"R293": 0.18,
|
||||
"R294": 0.19,
|
||||
"R295": 0.20,
|
||||
"R296": 0.21,
|
||||
"R297": 0.22,
|
||||
"R298": 0.23,
|
||||
"R299": 0.24,
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import pytest
|
||||
from rates import RATES
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", [f"R{n:03d}" for n in range(300)])
|
||||
def test_rate(code: str) -> None:
|
||||
assert RATES[code] == pytest.approx(int(code[1:]) % 25 / 100)
|
||||
@@ -0,0 +1,304 @@
|
||||
"""Tax rate per region code."""
|
||||
|
||||
RATES = {
|
||||
"R000": 0.00,
|
||||
"R001": 0.01,
|
||||
"R002": 0.02,
|
||||
"R003": 0.03,
|
||||
"R004": 0.04,
|
||||
"R005": 0.05,
|
||||
"R006": 0.06,
|
||||
"R007": 0.07,
|
||||
"R008": 0.08,
|
||||
"R009": 0.09,
|
||||
"R010": 0.10,
|
||||
"R011": 0.11,
|
||||
"R012": 0.12,
|
||||
"R013": 0.13,
|
||||
"R014": 0.14,
|
||||
"R015": 0.15,
|
||||
"R016": 0.16,
|
||||
"R017": 0.17,
|
||||
"R018": 0.18,
|
||||
"R019": 0.19,
|
||||
"R020": 0.20,
|
||||
"R021": 0.21,
|
||||
"R022": 0.22,
|
||||
"R023": 0.23,
|
||||
"R024": 0.24,
|
||||
"R025": 0.00,
|
||||
"R026": 0.01,
|
||||
"R027": 0.02,
|
||||
"R028": 0.03,
|
||||
"R029": 0.04,
|
||||
"R030": 0.05,
|
||||
"R031": 0.06,
|
||||
"R032": 0.07,
|
||||
"R033": 0.08,
|
||||
"R034": 0.09,
|
||||
"R035": 0.10,
|
||||
"R036": 0.11,
|
||||
"R037": 0.12,
|
||||
"R038": 0.13,
|
||||
"R039": 0.14,
|
||||
"R040": 0.15,
|
||||
"R041": 0.16,
|
||||
"R042": 0.17,
|
||||
"R043": 0.18,
|
||||
"R044": 0.19,
|
||||
"R045": 0.20,
|
||||
"R046": 0.21,
|
||||
"R047": 0.22,
|
||||
"R048": 0.23,
|
||||
"R049": 0.24,
|
||||
"R050": 0.00,
|
||||
"R051": 0.01,
|
||||
"R052": 0.02,
|
||||
"R053": 0.03,
|
||||
"R054": 0.04,
|
||||
"R055": 0.05,
|
||||
"R056": 0.06,
|
||||
"R057": 0.07,
|
||||
"R058": 0.08,
|
||||
"R059": 0.09,
|
||||
"R060": 0.10,
|
||||
"R061": 0.11,
|
||||
"R062": 0.12,
|
||||
"R063": 0.13,
|
||||
"R064": 0.14,
|
||||
"R065": 0.15,
|
||||
"R066": 0.16,
|
||||
"R067": 0.17,
|
||||
"R068": 0.18,
|
||||
"R069": 0.19,
|
||||
"R070": 0.20,
|
||||
"R071": 0.21,
|
||||
"R072": 0.22,
|
||||
"R073": 0.23,
|
||||
"R074": 0.24,
|
||||
"R075": 0.00,
|
||||
"R076": 0.01,
|
||||
"R077": 0.02,
|
||||
"R078": 0.03,
|
||||
"R079": 0.04,
|
||||
"R080": 0.05,
|
||||
"R081": 0.06,
|
||||
"R082": 0.07,
|
||||
"R083": 0.08,
|
||||
"R084": 0.09,
|
||||
"R085": 0.10,
|
||||
"R086": 0.11,
|
||||
"R087": 0.12,
|
||||
"R088": 0.13,
|
||||
"R089": 0.14,
|
||||
"R090": 0.15,
|
||||
"R091": 0.16,
|
||||
"R092": 0.17,
|
||||
"R093": 0.18,
|
||||
"R094": 0.19,
|
||||
"R095": 0.20,
|
||||
"R096": 0.21,
|
||||
"R097": 0.22,
|
||||
"R098": 0.23,
|
||||
"R099": 0.24,
|
||||
"R100": 0.00,
|
||||
"R101": 0.01,
|
||||
"R102": 0.02,
|
||||
"R103": 0.03,
|
||||
"R104": 0.04,
|
||||
"R105": 0.05,
|
||||
"R106": 0.06,
|
||||
"R107": 0.07,
|
||||
"R108": 0.08,
|
||||
"R109": 0.09,
|
||||
"R110": 0.10,
|
||||
"R111": 0.11,
|
||||
"R112": 0.12,
|
||||
"R113": 0.13,
|
||||
"R114": 0.14,
|
||||
"R115": 0.15,
|
||||
"R116": 0.16,
|
||||
"R117": 0.17,
|
||||
"R118": 0.18,
|
||||
"R119": 0.19,
|
||||
"R120": 0.20,
|
||||
"R121": 0.21,
|
||||
"R122": 0.22,
|
||||
"R123": 0.23,
|
||||
"R124": 0.24,
|
||||
"R125": 0.00,
|
||||
"R126": 0.01,
|
||||
"R127": 0.02,
|
||||
"R128": 0.03,
|
||||
"R129": 0.04,
|
||||
"R130": 0.05,
|
||||
"R131": 0.06,
|
||||
"R132": 0.07,
|
||||
"R133": 0.08,
|
||||
"R134": 0.09,
|
||||
"R135": 0.10,
|
||||
"R136": 0.11,
|
||||
"R137": 0.12,
|
||||
"R138": 0.13,
|
||||
"R139": 0.14,
|
||||
"R140": 0.15,
|
||||
"R141": 0.16,
|
||||
"R142": 0.17,
|
||||
"R143": 0.18,
|
||||
"R144": 0.19,
|
||||
"R145": 0.20,
|
||||
"R146": 0.21,
|
||||
"R147": 0.22,
|
||||
"R148": 0.23,
|
||||
"R149": 0.24,
|
||||
"R150": 0.00,
|
||||
"R151": 0.01,
|
||||
"R152": 0.02,
|
||||
"R153": 0.03,
|
||||
"R154": 0.04,
|
||||
"R155": 0.05,
|
||||
"R156": 0.06,
|
||||
"R157": 0.07,
|
||||
"R158": 0.08,
|
||||
"R159": 0.09,
|
||||
"R160": 0.10,
|
||||
"R161": 0.11,
|
||||
"R162": 0.12,
|
||||
"R163": 0.13,
|
||||
"R164": 0.14,
|
||||
"R165": 0.15,
|
||||
"R166": 0.16,
|
||||
"R167": 0.17,
|
||||
"R168": 0.18,
|
||||
"R169": 0.19,
|
||||
"R170": 0.20,
|
||||
"R171": 0.21,
|
||||
"R172": 0.22,
|
||||
"R173": 0.23,
|
||||
"R174": 0.24,
|
||||
"R175": 0.00,
|
||||
"R176": 0.01,
|
||||
"R177": 0.02,
|
||||
"R178": 0.03,
|
||||
"R179": 0.04,
|
||||
"R180": 0.05,
|
||||
"R181": 0.06,
|
||||
"R182": 0.07,
|
||||
"R183": 0.08,
|
||||
"R184": 0.09,
|
||||
"R185": 0.10,
|
||||
"R186": 0.11,
|
||||
"R187": 0.12,
|
||||
"R188": 0.13,
|
||||
"R189": 0.14,
|
||||
"R190": 0.15,
|
||||
"R191": 0.16,
|
||||
"R192": 0.17,
|
||||
"R193": 0.18,
|
||||
"R194": 0.19,
|
||||
"R195": 0.20,
|
||||
"R196": 0.21,
|
||||
"R197": 0.22,
|
||||
"R198": 0.23,
|
||||
"R199": 0.24,
|
||||
"R200": 0.00,
|
||||
"R201": 0.01,
|
||||
"R202": 0.02,
|
||||
"R203": 0.03,
|
||||
"R204": 0.04,
|
||||
"R205": 0.05,
|
||||
"R206": 0.06,
|
||||
"R207": 0.07,
|
||||
"R208": 0.08,
|
||||
"R209": 0.09,
|
||||
"R210": 0.10,
|
||||
"R211": 0.11,
|
||||
"R212": 0.12,
|
||||
"R213": 0.13,
|
||||
"R214": 0.14,
|
||||
"R215": 0.15,
|
||||
"R216": 0.16,
|
||||
"R217": 0.17,
|
||||
"R218": 0.18,
|
||||
"R219": 0.19,
|
||||
"R220": 0.20,
|
||||
"R221": 0.21,
|
||||
"R222": 0.22,
|
||||
"R223": 0.23,
|
||||
"R224": 0.24,
|
||||
"R225": 0.00,
|
||||
"R226": 0.01,
|
||||
"R227": 0.02,
|
||||
"R228": 0.03,
|
||||
"R229": 0.04,
|
||||
"R230": 0.05,
|
||||
"R231": 0.06,
|
||||
"R232": 0.07,
|
||||
"R233": 0.08,
|
||||
"R234": 0.09,
|
||||
"R235": 0.10,
|
||||
"R236": 0.11,
|
||||
"R237": 0.12,
|
||||
"R238": 0.13,
|
||||
"R239": 0.14,
|
||||
"R240": 0.15,
|
||||
"R241": 0.16,
|
||||
"R242": 0.17,
|
||||
"R243": 0.18,
|
||||
"R244": 0.19,
|
||||
"R245": 0.20,
|
||||
"R246": 0.21,
|
||||
"R247": 0.22,
|
||||
"R248": 0.23,
|
||||
"R249": 0.24,
|
||||
"R250": 0.00,
|
||||
"R251": 0.01,
|
||||
"R252": 0.02,
|
||||
"R253": 0.03,
|
||||
"R254": 0.04,
|
||||
"R255": 0.05,
|
||||
"R256": 0.06,
|
||||
"R257": 0.07,
|
||||
"R258": 0.08,
|
||||
"R259": 0.09,
|
||||
"R260": 0.10,
|
||||
"R261": 0.11,
|
||||
"R262": 0.12,
|
||||
"R263": 0.13,
|
||||
"R264": 0.14,
|
||||
"R265": 0.15,
|
||||
"R266": 0.16,
|
||||
"R267": 0.17,
|
||||
"R268": 0.18,
|
||||
"R269": 0.19,
|
||||
"R270": 0.20,
|
||||
"R271": 0.21,
|
||||
"R272": 0.22,
|
||||
"R273": 0.23,
|
||||
"R274": 0.24,
|
||||
"R275": 0.00,
|
||||
"R276": 0.01,
|
||||
"R277": 0.02,
|
||||
"R278": 0.03,
|
||||
"R279": 0.04,
|
||||
"R280": 0.05,
|
||||
"R281": 0.06,
|
||||
"R282": 0.07,
|
||||
"R283": 0.08,
|
||||
"R284": 0.09,
|
||||
"R285": 0.10,
|
||||
"R286": 0.11,
|
||||
"R287": 0.12,
|
||||
"R288": 0.13,
|
||||
"R289": 0.14,
|
||||
"R290": 0.15,
|
||||
"R291": 0.16,
|
||||
"R292": 0.17,
|
||||
"R293": 0.18,
|
||||
"R294": 0.19,
|
||||
"R295": 0.20,
|
||||
"R296": 0.21,
|
||||
"R297": 0.22,
|
||||
"R298": 0.23,
|
||||
"R299": 0.24,
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
title: Find one failure in long test output
|
||||
check: python -m pytest -q .bench_check
|
||||
plan: |
|
||||
Milestone 1: fix the one wrong entry in the rate table.
|
||||
|
||||
1. Run `python -m pytest -v`. Exactly one test fails, and its output names a
|
||||
region code.
|
||||
2. The tests are correct. In `rates.py`, the rate for region code RNNN must be
|
||||
(NNN mod 25) / 100. Correct only the failing region's entry; leave every
|
||||
other line as it is.
|
||||
3. Run `python -m pytest -q` until every test passes.
|
||||
4. Call finish.
|
||||
@@ -0,0 +1,33 @@
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from greeter.cli import main
|
||||
from greeter.core import greet
|
||||
|
||||
|
||||
def test_default_is_unchanged() -> None:
|
||||
assert greet("Bob") == "Hello, Bob!"
|
||||
|
||||
|
||||
def test_shout() -> None:
|
||||
assert greet("Bob", shout=True) == "HELLO, BOB!"
|
||||
|
||||
|
||||
def test_cli_default(capsys) -> None:
|
||||
assert main(["Ada"]) == 0
|
||||
assert capsys.readouterr().out == "Hello, Ada!\n"
|
||||
|
||||
|
||||
def test_cli_shout(capsys) -> None:
|
||||
assert main(["--shout", "Ada"]) == 0
|
||||
assert capsys.readouterr().out == "HELLO, ADA!\n"
|
||||
|
||||
|
||||
def test_module_entry_point() -> None:
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "greeter", "--shout", "Bob"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
assert result.stdout == "HELLO, BOB!\n"
|
||||
Whitespace-only changes.
@@ -0,0 +1,3 @@
|
||||
from greeter.cli import main
|
||||
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,11 @@
|
||||
import argparse
|
||||
|
||||
from greeter.core import greet
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="greeter")
|
||||
parser.add_argument("name")
|
||||
args = parser.parse_args(argv)
|
||||
print(greet(args.name))
|
||||
return 0
|
||||
@@ -0,0 +1,2 @@
|
||||
def greet(name: str) -> str:
|
||||
return f"Hello, {name}!"
|
||||
@@ -0,0 +1,11 @@
|
||||
from greeter.cli import main
|
||||
from greeter.core import greet
|
||||
|
||||
|
||||
def test_greet() -> None:
|
||||
assert greet("Bob") == "Hello, Bob!"
|
||||
|
||||
|
||||
def test_cli_prints_the_greeting(capsys) -> None:
|
||||
assert main(["Bob"]) == 0
|
||||
assert capsys.readouterr().out == "Hello, Bob!\n"
|
||||
@@ -0,0 +1,12 @@
|
||||
import argparse
|
||||
|
||||
from greeter.core import greet
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="greeter")
|
||||
parser.add_argument("name")
|
||||
parser.add_argument("--shout", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
print(greet(args.name, shout=args.shout))
|
||||
return 0
|
||||
@@ -0,0 +1,3 @@
|
||||
def greet(name: str, shout: bool = False) -> str:
|
||||
greeting = f"Hello, {name}!"
|
||||
return greeting.upper() if shout else greeting
|
||||
@@ -0,0 +1,16 @@
|
||||
title: Thread a new option through several files
|
||||
check: python -m pytest -q .bench_check
|
||||
plan: |
|
||||
Milestone 1: add a --shout option to the greeter.
|
||||
|
||||
1. In `greeter/core.py`, give `greet` a keyword argument `shout: bool = False`.
|
||||
When it is true, return the whole greeting in upper case:
|
||||
greet("Bob", shout=True) == "HELLO, BOB!". The default behaviour is unchanged.
|
||||
2. In `greeter/cli.py`, add a `--shout` flag (action="store_true") and pass it
|
||||
to `greet`.
|
||||
3. Add two tests to `tests/test_greeter.py`: one for greet with shout=True, and
|
||||
one that calls `main(["--shout", "Bob"])` and checks the printed output
|
||||
using pytest's capsys fixture.
|
||||
4. Run `python -m pytest -q` until every test passes, and check that
|
||||
`python -m greeter --shout Bob` prints `HELLO, BOB!`.
|
||||
5. Call finish.
|
||||
@@ -0,0 +1,19 @@
|
||||
import pytest
|
||||
from slugify import slugify
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
("Hello, World!", "hello-world"),
|
||||
(" Nisse Agent ", "nisse-agent"),
|
||||
("a--b", "a-b"),
|
||||
("", ""),
|
||||
("Python 3.12 Release", "python-3-12-release"),
|
||||
("--leading and trailing--", "leading-and-trailing"),
|
||||
("MiXeD CaSe", "mixed-case"),
|
||||
("!!!", ""),
|
||||
],
|
||||
)
|
||||
def test_slugify(text: str, expected: str) -> None:
|
||||
assert slugify(text) == expected
|
||||
@@ -0,0 +1 @@
|
||||
A small utilities project.
|
||||
@@ -0,0 +1,5 @@
|
||||
import re
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
|
||||
@@ -0,0 +1,13 @@
|
||||
title: Write a small function and its tests
|
||||
check: python -m pytest -q .bench_check
|
||||
plan: |
|
||||
Milestone 1: add a slugify function.
|
||||
|
||||
1. Create `slugify.py` in the workspace root, defining `slugify(text: str) -> str`.
|
||||
2. It lowercases the text, replaces every run of characters that are not a-z or
|
||||
0-9 with a single hyphen, and strips hyphens from both ends. Examples:
|
||||
"Hello, World!" -> "hello-world"; " Nisse Agent " -> "nisse-agent";
|
||||
"a--b" -> "a-b"; "" -> "".
|
||||
3. Create `test_slugify.py` with pytest tests covering the four examples.
|
||||
4. Run `python -m pytest -q test_slugify.py`, and fix anything until it passes.
|
||||
5. Call finish.
|
||||
@@ -230,6 +230,7 @@ class BudgetedClient:
|
||||
# Per model: how much the character estimate undercounted, as learned
|
||||
# from reported usage. Starts at 1 and only grows.
|
||||
self._factors: dict[str, float] = {}
|
||||
self.trimmed_requests = 0
|
||||
|
||||
@property
|
||||
def last_usage(self) -> dict[str, int]:
|
||||
@@ -239,6 +240,10 @@ class BudgetedClient:
|
||||
def last_model(self) -> str | None:
|
||||
return getattr(self._inner, "last_model", None)
|
||||
|
||||
@property
|
||||
def last_cost(self) -> float:
|
||||
return float(getattr(self._inner, "last_cost", 0.0) or 0.0)
|
||||
|
||||
def budget_for(self, model: str) -> Budget:
|
||||
window, model_max_output = self._limits_for(model)
|
||||
ceiling = self._max_output_tokens
|
||||
@@ -261,6 +266,10 @@ class BudgetedClient:
|
||||
budget = self.budget_for(model)
|
||||
factor = self._factors.get(model, 1.0)
|
||||
fitted = fit(events, system, tools, budget, factor)
|
||||
# Counted so the benchmark can report how often a model's window was
|
||||
# the constraint: any request that had to be cut down or shortened.
|
||||
if fitted != list(events):
|
||||
self.trimmed_requests += 1
|
||||
produced = self._inner.complete(
|
||||
fitted,
|
||||
tools,
|
||||
|
||||
@@ -143,6 +143,7 @@ class LiteLLMClient:
|
||||
# protocol stays the same for every implementation.
|
||||
self.last_usage: dict[str, int] = {}
|
||||
self.last_model: str | None = None
|
||||
self.last_cost: float = 0.0
|
||||
|
||||
def complete(
|
||||
self,
|
||||
@@ -171,6 +172,8 @@ class LiteLLMClient:
|
||||
raise LLMError(
|
||||
f"LLM request returned {response.status_code}: {response.text[:500]}"
|
||||
)
|
||||
# LiteLLM prices every call; kept so a caller can meter spending.
|
||||
self.last_cost = _response_cost(response)
|
||||
return self._parse(response.json(), requested_model=model)
|
||||
|
||||
def _parse(
|
||||
@@ -263,6 +266,7 @@ class AnthropicPassthroughClient:
|
||||
self._sleep = sleep
|
||||
self.last_usage: dict[str, int] = {}
|
||||
self.last_model: str | None = None
|
||||
self.last_cost: float = 0.0
|
||||
|
||||
def complete(
|
||||
self,
|
||||
@@ -291,6 +295,8 @@ class AnthropicPassthroughClient:
|
||||
raise LLMError(
|
||||
f"passthrough returned {response.status_code}: {response.text[:500]}"
|
||||
)
|
||||
# LiteLLM prices every call; kept so a caller can meter spending.
|
||||
self.last_cost = _response_cost(response)
|
||||
return self._parse(response.json(), requested_model=model)
|
||||
|
||||
def _parse(
|
||||
@@ -358,6 +364,13 @@ def _post_with_rate_limit_retry(
|
||||
# Model families we have a native renderer adapter for. A13: delegations take
|
||||
# the vendor's passthrough route because a normalising middlebox is lossy
|
||||
# exactly where a long-running delegation lives.
|
||||
def _response_cost(response: httpx.Response) -> float:
|
||||
try:
|
||||
return float(response.headers.get("x-litellm-response-cost") or 0.0)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _raise_for_upstream(response: httpx.Response) -> None:
|
||||
"""Classify a non-200 as upstream trouble where it plainly is.
|
||||
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
"""The executor benchmark."""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from nisse_dev.bench.results import (
|
||||
append,
|
||||
load,
|
||||
render,
|
||||
results_file,
|
||||
summarise,
|
||||
wilson,
|
||||
)
|
||||
from nisse_dev.bench.runner import MeteringClient, run_task
|
||||
from nisse_dev.bench.tasks import CHECK_DIR, load_tasks
|
||||
from nisse_dev.canonical import AssistantText, ToolCall
|
||||
from nisse_dev.catalog import ModelFacts
|
||||
from nisse_dev.config import ExecutorTuning, Tuning
|
||||
|
||||
TASKS = load_tasks()
|
||||
FACTS = ModelFacts(
|
||||
alias="local-fast",
|
||||
backing="openai/qwen3-8b-awq",
|
||||
provider="openai",
|
||||
supports_tools=True,
|
||||
max_input_tokens=32768,
|
||||
max_output_tokens=8192,
|
||||
supports_tool_choice=True,
|
||||
)
|
||||
|
||||
|
||||
# --- the tasks themselves ---------------------------------------------------
|
||||
|
||||
|
||||
def test_the_bundled_tasks_are_the_four_planned() -> None:
|
||||
assert [t.name for t in TASKS] == [
|
||||
"fix-bug",
|
||||
"long-output",
|
||||
"multi-file",
|
||||
"write-function",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("task", TASKS, ids=lambda t: t.name)
|
||||
def test_the_seed_fails_its_check_and_the_solution_passes(task, tmp_path: Path) -> None:
|
||||
# A check that passes on the untouched seed measures nothing; a task its
|
||||
# own reference solution can't pass is unfair to every model.
|
||||
seed = tmp_path / "seed"
|
||||
task.seed_into(seed)
|
||||
assert not task.run_check(seed)[0], f"{task.name}: the seed already passes"
|
||||
|
||||
solved = tmp_path / "solved"
|
||||
task.seed_into(solved)
|
||||
task.apply_solution(solved)
|
||||
passed, output = task.run_check(solved)
|
||||
assert passed, f"{task.name}: the reference solution fails:\n{output}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("task", TASKS, ids=lambda t: t.name)
|
||||
def test_the_hidden_check_is_not_in_the_seed(task, tmp_path: Path) -> None:
|
||||
task.seed_into(tmp_path)
|
||||
assert not (tmp_path / CHECK_DIR).exists()
|
||||
assert "Call finish" in task.plan # plans end the way the executor must
|
||||
|
||||
|
||||
def test_long_output_really_is_long(tmp_path: Path) -> None:
|
||||
"""The point of the task is output far past tools.max_output_chars."""
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
[task] = load_tasks(["long-output"])
|
||||
task.seed_into(tmp_path)
|
||||
run = subprocess.run(
|
||||
[sys.executable, "-m", "pytest", "-v"],
|
||||
cwd=tmp_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert len(run.stdout) > 4 * Tuning().tools.max_output_chars
|
||||
assert "R237" in run.stdout[-3000:] # the failure is named at the end
|
||||
|
||||
|
||||
def test_a_changed_task_gets_a_new_hash(tmp_path: Path) -> None:
|
||||
[task] = load_tasks(["write-function"])
|
||||
shutil.copytree(task.path, tmp_path / "write-function")
|
||||
[copy] = load_tasks(root=tmp_path)
|
||||
assert copy.content_hash == task.content_hash
|
||||
(copy.path / "seed" / "README.md").write_text("changed\n")
|
||||
assert copy.content_hash != task.content_hash
|
||||
|
||||
|
||||
def test_an_unknown_task_is_refused() -> None:
|
||||
with pytest.raises(ValueError, match="unknown task"):
|
||||
load_tasks(["nope"])
|
||||
|
||||
|
||||
# --- running a task ----------------------------------------------------------
|
||||
|
||||
|
||||
class Scripted:
|
||||
"""A model that plays back fixed turns, reporting itself as `answered`."""
|
||||
|
||||
def __init__(self, turns: list[list[Any]], answered: str = "local-fast") -> None:
|
||||
self.turns = turns
|
||||
self.last_usage = {"input": 1000, "output": 50}
|
||||
self.last_cost = 0.0
|
||||
self.last_model = answered
|
||||
self.trimmed_requests = 0
|
||||
|
||||
def complete(self, events, tools, model, system=None):
|
||||
if not self.turns:
|
||||
return [AssistantText(text="nothing left to do", model=model)]
|
||||
return self.turns.pop(0)
|
||||
|
||||
|
||||
def call(tool: str, cid: str, **arguments: Any) -> ToolCall:
|
||||
return ToolCall(call_id=cid, tool=tool, arguments=arguments)
|
||||
|
||||
|
||||
SOLVE_SLUGIFY = [
|
||||
[
|
||||
call(
|
||||
"write_file",
|
||||
"w",
|
||||
path="slugify.py",
|
||||
content=(
|
||||
"import re\n\n\ndef slugify(text):\n"
|
||||
" return re.sub(r'[^a-z0-9]+', '-', text.lower()).strip('-')\n"
|
||||
),
|
||||
)
|
||||
],
|
||||
[call("run_command", "r", command="python -c 'import slugify'")],
|
||||
[call("finish", "f", summary="added slugify")],
|
||||
]
|
||||
|
||||
|
||||
def test_a_solved_task_is_a_success(tmp_path: Path) -> None:
|
||||
[task] = load_tasks(["write-function"])
|
||||
record = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), note="n")
|
||||
|
||||
assert record.success
|
||||
assert record.outcome == "completed"
|
||||
assert record.turns == 3
|
||||
assert record.tokens_in == 3000
|
||||
assert record.task_hash == task.content_hash
|
||||
assert record.model_info["supports_tool_choice"] is True
|
||||
assert record.tuning["executor"]["max_turns"] == 24
|
||||
|
||||
|
||||
def test_saying_finished_is_not_success(tmp_path: Path) -> None:
|
||||
# Success is the hidden check passing, never the model's own claim.
|
||||
[task] = load_tasks(["write-function"])
|
||||
wrong = [
|
||||
[
|
||||
call(
|
||||
"write_file",
|
||||
"w",
|
||||
path="slugify.py",
|
||||
content="def slugify(t):\n return t\n",
|
||||
)
|
||||
],
|
||||
[call("finish", "f", summary="done")],
|
||||
]
|
||||
record = run_task(task, FACTS, Scripted(wrong), Tuning())
|
||||
assert record.outcome == "completed"
|
||||
assert not record.success
|
||||
|
||||
|
||||
def test_misbehaviour_is_counted(tmp_path: Path) -> None:
|
||||
[task] = load_tasks(["write-function"])
|
||||
turns = [
|
||||
[call("chat", "x1")],
|
||||
[call("run_command", "r1", command="exit 1")],
|
||||
[AssistantText(text="let me explain", model="m")],
|
||||
]
|
||||
# A higher failure limit, so the invented tool and the failed command
|
||||
# don't escalate before the prose turn is reached.
|
||||
tuning = Tuning(executor=ExecutorTuning(failure_limit=5))
|
||||
record = run_task(task, FACTS, Scripted(turns), tuning)
|
||||
assert record.invented_tools == 1
|
||||
assert record.failed_commands == 1
|
||||
assert record.prose_turns >= 1
|
||||
assert not record.success
|
||||
|
||||
|
||||
def test_a_fallback_answer_can_never_be_a_success(tmp_path: Path) -> None:
|
||||
[task] = load_tasks(["write-function"])
|
||||
client = Scripted(list(SOLVE_SLUGIFY), answered="claude-sonnet-5")
|
||||
record = run_task(task, FACTS, client, Tuning())
|
||||
assert record.fallbacks == ["claude-sonnet-5"] * 3
|
||||
assert not record.success # the check passed, but not for this model
|
||||
|
||||
|
||||
# --- storing and reporting ---------------------------------------------------
|
||||
|
||||
|
||||
def test_wilson_intervals_behave() -> None:
|
||||
low, high = wilson(5, 5)
|
||||
assert high == 1.0 and 0.5 < low < 0.6
|
||||
# 3 of 5 and 5 of 5 overlap: five runs do not separate them.
|
||||
assert wilson(3, 5)[1] > wilson(5, 5)[0]
|
||||
assert wilson(0, 0) == (0.0, 1.0)
|
||||
|
||||
|
||||
def record_json(**overrides: Any) -> dict[str, Any]:
|
||||
[task] = load_tasks(["write-function"])
|
||||
base = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning()).to_json()
|
||||
base.update(overrides)
|
||||
return base
|
||||
|
||||
|
||||
def test_results_round_trip_and_report(tmp_path: Path) -> None:
|
||||
from nisse_dev.bench.runner import RunRecord
|
||||
|
||||
rec = RunRecord(**record_json())
|
||||
path = results_file(tmp_path, "local-fast")
|
||||
append(path, rec)
|
||||
append(path, RunRecord(**record_json(success=False)))
|
||||
stored = load(tmp_path)
|
||||
assert len(stored) == 2
|
||||
assert json.loads(path.read_text().splitlines()[0])["model"] == "local-fast"
|
||||
|
||||
current = {t.name: t.content_hash for t in TASKS}
|
||||
[row] = summarise(stored, current)
|
||||
assert (row.successes, row.runs) == (1, 2)
|
||||
assert not row.stale
|
||||
table = render([row])
|
||||
assert "openai/qwen3-8b-awq" in table
|
||||
assert "1/2" in table
|
||||
|
||||
|
||||
def test_runs_of_an_older_task_version_are_flagged() -> None:
|
||||
current = {t.name: t.content_hash for t in TASKS}
|
||||
rows = summarise([record_json(task_hash="0123456789abcdef")], current)
|
||||
assert rows[0].stale
|
||||
assert "(old)" in render(rows)
|
||||
|
||||
|
||||
def test_models_served_differently_are_kept_apart() -> None:
|
||||
current = {t.name: t.content_hash for t in TASKS}
|
||||
rows = summarise(
|
||||
[record_json(note="vLLM, AWQ"), record_json(note="llama.cpp, Q4_K_M")], current
|
||||
)
|
||||
assert len(rows) == 2
|
||||
|
||||
|
||||
def test_metering_counts_prose_and_cost() -> None:
|
||||
class Inner:
|
||||
def __init__(self) -> None:
|
||||
self.last_usage = {"input": 10, "output": 2}
|
||||
self.last_cost = 0.5
|
||||
self.last_model = "local-fast"
|
||||
|
||||
def complete(self, events, tools, model, system=None):
|
||||
return [AssistantText(text="hi", model=model)]
|
||||
|
||||
meter = MeteringClient(Inner(), FACTS)
|
||||
meter.complete([], None, "local-fast")
|
||||
meter.complete([], None, "local-fast")
|
||||
assert (meter.calls, meter.prose_turns, meter.cost) == (2, 2, 1.0)
|
||||
Reference in new issue
Block a user