dev/nisse
2
1 Fork 0
Code Issues 23 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request 'Add an executor benchmark that keeps its results' (#47) from benchmark into main
CI / checks (push) Successful in 1m15s
Details

Browse Source 
Reviewed-on: dongar/nisse#47
Reviewed-by: Don Garrett <dgarrett@acm.org>
This commit was merged in pull request #47.
This commit is contained in:
dongar committed 2026-09-25 02:23:57 +00:00
2 parents b2bbf70b7b 0cccfe1e94
commit 8a8d8dc27f
39 files changed
+1952 -37
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

bench/results/.gitkeep
View File
Whitespace-only changes.
bench/results/2026-09-25-local-coder.jsonl
+20
View File
@@ -0,0 +1,20 @@
{"backing": "openai/qwen3-14b", "check_output": "...... [100%]\n6 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Fixed total_value to multiply unit price by quantity. All tests now pass.", "prose_turns": 0, "provider": "openai", "success": true, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 5328, "tokens_out": 1492, "trimmed_requests": 0, "ts": "2026-09-25T01:03:24+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 46.4}
{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\nImportError while importing test module '/tmp/tmpr1q1pe0r/work/.bench_check/test_hidden.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n.bench_check/test_hidden.py:4: in <module>\n from inventory import Item, low_stock, total_value\nE ImportError: cannot import name 'Item' from 'inventory' (/tmp/tmpr1q1pe0r/work/inventory.py)\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.10s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n______________________ ERROR collecting test_inventory.py ______________________\nImportError while importing test module '/tmp/tmpr1q1pe0r/work/test_inventory.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], packa", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 2560, "tokens_out": 1136, "trimmed_requests": 0, "ts": "2026-09-25T01:03:49+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 3, "wall_s": 24.3}
{"backing": "openai/qwen3-14b", "check_output": "...... [100%]\n6 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Fixed the total_value function to multiply unit price by quantity, passing all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 5318, "tokens_out": 1623, "trimmed_requests": 0, "ts": "2026-09-25T01:04:23+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 34.5}
{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n.bench_check/test_hidden.py:4: in <module>\n from inventory import Item, low_stock, total_value\ninventory.py:1: in <module>\n def total_value(items: list[Item]) -> float:\n ^^^^\nE NameError: name 'Item' is not defined\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py - NameError: name 'Item' is not defined\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.11s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n______________________ ERROR collecting test_inventory.py ______________________\ntest_inventory.py:1: in <module>\n from inventory import Item, low_stock, total_value\ninventory.py:1: in <module>\n def total_value(items: list[Item]) -> float:\n ^^^^\nE NameError: name 'Item' is not defined\n=========================== short ", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 2565, "tokens_out": 958, "trimmed_requests": 0, "ts": "2026-09-25T01:04:45+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 3, "wall_s": 20.6}
{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n.bench_check/test_hidden.py:4: in <module>\n from inventory import Item, low_stock, total_value\ninventory.py:1: in <module>\n def total_value(items: list[Item]) -> float:\n ^^^^\nE NameError: name 'Item' is not defined\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py - NameError: name 'Item' is not defined\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.09s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n______________________ ERROR collecting test_inventory.py ______________________\ntest_inventory.py:1: in <module>\n from inventory import Item, low_stock, total_value\ninventory.py:1: in <module>\n def total_value(items: list[Item]) -> float:\n ^^^^\nE NameError: name 'Item' is not defined\n=========================== short ", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 2565, "tokens_out": 1051, "trimmed_requests": 0, "ts": "2026-09-25T01:05:07+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 3, "wall_s": 22.6}
{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmpq8iz1j8z/work/rates.py\", line 56\nE [... 1445 characters omitted ...]\nE ^^^^^^^^\nE SyntaxError: invalid syntax. Perhaps you forgot a comma?\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 13616, "tokens_out": 3186, "trimmed_requests": 0, "ts": "2026-09-25T01:06:21+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 72.8}
{"backing": "openai/qwen3-14b", "check_output": ": 'R271'\nFAILED .bench_check/test_hidden.py::test_rate[272] - KeyError: 'R272'\nFAILED .bench_check/test_hidden.py::test_rate[273] - KeyError: 'R273'\nFAILED .bench_check/test_hidden.py::test_rate[274] - KeyError: 'R274'\nFAILED .bench_check/test_hidden.py::test_rate[275] - KeyError: 'R275'\nFAILED .bench_check/test_hidden.py::test_rate[276] - KeyError: 'R276'\nFAILED .bench_check/test_hidden.py::test_rate[277] - KeyError: 'R277'\nFAILED .bench_check/test_hidden.py::test_rate[278] - KeyError: 'R278'\nFAILED .bench_check/test_hidden.py::test_rate[279] - KeyError: 'R279'\nFAILED .bench_check/test_hidden.py::test_rate[280] - KeyError: 'R280'\nFAILED .bench_check/test_hidden.py::test_rate[281] - KeyError: 'R281'\nFAILED .bench_check/test_hidden.py::test_rate[282] - KeyError: 'R282'\nFAILED .bench_check/test_hidden.py::test_rate[283] - KeyError: 'R283'\nFAILED .bench_check/test_hidden.py::test_rate[284] - KeyError: 'R284'\nFAILED .bench_check/test_hidden.py::test_rate[285] - KeyError: 'R285'\nFAILED .bench_check/test_hidden.py::test_rate[286] - KeyError: 'R286'\nFAILED .bench_check/test_hidden.py::test_rate[287] - KeyError: 'R287'\nFAILED .bench_check/test_hidden.py::test_rate[288] - KeyError: 'R288'\nFAILED .bench_check/test_hidden.py::test_rate[289] - KeyError: 'R289'\nFAILED .bench_check/test_hidden.py::test_rate[290] - KeyError: 'R290'\nFAILED .bench_check/test_hidden.py::test_rate[291] - KeyError: 'R291'\nFAILED .bench_check/test_hidden.py::test_rate[292] - KeyError: 'R292'\nFAILED .bench_check/test_hidden.py::test_rate[293] - KeyError: 'R293'\nFAILED .bench_check/test_hidden.py::test_rate[294] - KeyError: 'R294'\nFAILED .bench_check/test_hidden.py::test_rate[295] - KeyError: 'R295'\nFAILED .bench_check/test_hidden.py::test_rate[296] - KeyError: 'R296'\nFAILED .bench_check/test_hidden.py::test_rate[297] - KeyError: 'R297'\nFAILED .bench_check/test_hidden.py::test_rate[298] - KeyError: 'R298'\nFAILED .bench_check/test_hidden.py::test_rate[299] - KeyError: 'R299'\n247 failed, 54 passed in 0.49s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n.....................................................FFFFFFFFFFFFFFFFFFF [ 24%]\nFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF [ 48%]\nFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF [ 72%]\nFFFFFFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF [ 96%]\nFFFFFFFFFFFF [100%]\n=================================== FAILU", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 12726, "tokens_out": 2366, "trimmed_requests": 0, "ts": "2026-09-25T01:07:16+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 54.4}
{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmphac71897/work/rates.py\", line 56\nE [... 1445 characters omitted ...]\nE ^^^^^^^^\nE SyntaxError: invalid syntax. Perhaps you forgot a comma?\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.15s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 13614, "tokens_out": 3000, "trimmed_requests": 0, "ts": "2026-09-25T01:08:25+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 69.2}
{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmp0vs0w4kr/work/rates.py\", line 56\nE [... 1445 characters omitted ...]\nE ^^^^^^^^\nE SyntaxError: invalid syntax. Perhaps you forgot a comma?\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 13607, "tokens_out": 3223, "trimmed_requests": 0, "ts": "2026-09-25T01:09:40+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 73.8}
{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmp_pxm0ldr/work/rates.py\", line 56\nE [... 1445 characters omitted ...]\nE ^^^^^^^^\nE SyntaxError: invalid syntax. Perhaps you forgot a comma?\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.15s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 13609, "tokens_out": 3180, "trimmed_requests": 0, "ts": "2026-09-25T01:10:53+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 73.0}
{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x777eb60e3e90>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 13753, "tokens_out": 5755, "trimmed_requests": 0, "ts": "2026-09-25T01:12:55+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 121.2}
{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added --shout option to greeter, updated core and cli code, added tests, and verified with pytest.", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 21373, "tokens_out": 9952, "trimmed_requests": 0, "ts": "2026-09-25T01:16:25+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 13, "wall_s": 210.0}
{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x7a2788fdbe30>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9432, "tokens_out": 6691, "trimmed_requests": 0, "ts": "2026-09-25T01:18:47+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 142.1}
{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added --shout option to greeter, updated core and cli code, added tests, and verified with pytest.", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9104, "tokens_out": 3626, "trimmed_requests": 0, "ts": "2026-09-25T01:20:03+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 75.8}
{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x76eac82e3da0>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9382, "tokens_out": 7755, "trimmed_requests": 0, "ts": "2026-09-25T01:22:48+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 164.5}
{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function and passed all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 1769, "tokens_out": 1300, "trimmed_requests": 0, "ts": "2026-09-25T01:23:16+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 27.7}
{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function with tests; all tests passed.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 1782, "tokens_out": 1239, "trimmed_requests": 0, "ts": "2026-09-25T01:23:43+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 26.2}
{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function and passed all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3627, "tokens_out": 1332, "trimmed_requests": 0, "ts": "2026-09-25T01:24:11+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 28.1}
{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function and passed all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 734, "tokens_out": 653, "trimmed_requests": 0, "ts": "2026-09-25T01:24:25+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 1, "wall_s": 13.9}
{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function and passed all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 733, "tokens_out": 523, "trimmed_requests": 0, "ts": "2026-09-25T01:24:37+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 1, "wall_s": 11.3}
bench/results/2026-09-25-local-fast.jsonl
+20
View File
@@ -0,0 +1,20 @@
{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1578, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:41+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 63.2}
{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1576, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:43+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4}
{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1576, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:44+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4}
{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1574, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:46+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4}
{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1578, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:48+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4}
{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmpl4mdto5g/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11000, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:00:52+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2}
{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmpsqvx_94y/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11000, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:00:57+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2}
{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmpe3eakwxj/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11013, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:01:01+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2}
{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmprlm2camm/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 10987, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:01:06+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2}
{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmpvzw7f5tn/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11000, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:01:11+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2}
{"backing": "openai/qwen3-8b-awq", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n...F [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x713c88ce3c20>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 13039, "tokens_out": 566, "trimmed_requests": 0, "ts": "2026-09-25T01:01:19+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 8.4}
{"backing": "openai/qwen3-8b-awq", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n________________________________ test_cli_shout ________________________________\n\n def test_cli_shout() -> None:\n> from pytest import capsys\nE ImportError: cannot import name 'capsys' from 'pytest' (/root/nisse/.venv/lib/python3.12/site-packages/pytest/__init__.py)", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 11243, "tokens_out": 715, "trimmed_requests": 0, "ts": "2026-09-25T01:01:29+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 9.9}
{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:4: in <module>\n from greeter.cli import main\nE File \"/tmp/tmp3f09u7jk/work/greeter/cli.py\", line 11\nE print(greet(args.name, shout=args.--shout))\nE ^\nE SyntaxError: invalid syntax\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n\nstderr:\nTraceback (most recent call last):\n File \"<frozen runpy>\", line 198, in _run_module_as_main\n File \"<frozen runpy>\", line 88, in _run_code\n File \"/tmp/tmp3f09u7jk/work/greeter/__main__.py\", line 1, in <module>\n from greeter.cli import main\n File \"/tmp/tmp3f09u7jk/work/greeter/cli.py\", line 11\n print(greet(args.name, shout=args.--shout))\n ^\nSyntaxError: invalid syntax\n", "prose_turns": 0, "provider": "openai", "success": false, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9358, "tokens_out": 450, "trimmed_requests": 0, "ts": "2026-09-25T01:01:56+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 26.5}
{"backing": "openai/qwen3-8b-awq", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "completed", "outcome_note": "Added --shout option to greeter. Tests pass.", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 10784, "tokens_out": 549, "trimmed_requests": 0, "ts": "2026-09-25T01:02:04+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 7.5}
{"backing": "openai/qwen3-8b-awq", "check_output": " stderr arguments may not be used '\n 'with capture_output.')\n kwargs['stdout'] = PIPE\n kwargs['stderr'] = PIPE\n \n with Popen(*popenargs, **kwargs) as process:\n try:\n stdout, stderr = process.communicate(input, timeout=timeout)\n except TimeoutExpired as exc:\n process.kill()\n if _mswindows:\n # Windows accumulates the output in a single blocking\n # read() call run on child threads, with the timeout\n # being done in a join() on those threads. communicate()\n # _after_ kill() is required to collect that and add it\n # to the exception.\n exc.stdout, exc.stderr = process.communicate()\n else:\n # POSIX _communicate already populated the output so\n # far into the TimeoutExpired exception.\n process.wait()\n raise\n except: # Including KeyboardInterrupt, communicate handled that.\n process.kill()\n # We don't call process.wait() as .__exit__ does that for us.\n raise\n retcode = process.poll()\n if check and retcode:\n> raise CalledProcessError(retcode, process.args,\n output=stdout, stderr=stderr)\nE subprocess.CalledProcessError: Command '['/root/nisse/.venv/bin/python', '-m', 'greeter', '--shout', 'Bob']' returned non-zero exit status 2.\n\n/usr/lib/python3.12/subprocess.py:571: CalledProcessError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_cli_default - SystemExit: 2\nFAILED .bench_check/test_hidden.py::test_cli_shout - SystemExit: 2\nFAILED .bench_check/test_hidden.py::test_module_entry_point - subprocess.Call...\n3 failed, 2 passed in 0.19s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..F [100%]\n=================================== FAILURES ===================================\n________________________________ test_cli_shout ________________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x763f352e3980>\n\n def test_cli_shout(capsys) -> None:\n> assert main([\"--shout\", \"Bob\"]) == 0\n ^^^^^^^^^^^^^^^^^^^^^^^^\n\ntest", "prose_turns": 0, "provider": "openai", "success": false, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 13124, "tokens_out": 589, "trimmed_requests": 0, "ts": "2026-09-25T01:02:13+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 8.4}
{"backing": "openai/qwen3-8b-awq", "check_output": " (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'leading-and-trailing'\nE \nE - leading-and-trailing\n\n.bench_check/test_hidden.py:19: AssertionError\n_____________________ test_slugify[MiXeD CaSe-mixed-case] ______________________\n\ntext = 'MiXeD CaSe', expected = 'mixed-case'\n\n @pytest.mark.parametrize(\n (\"text\", \"expected\"),\n [\n (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'mixed-case'\nE \nE - mixed-case\n\n.bench_check/test_hidden.py:19: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_slugify[Hello, World!-hello-world]\nFAILED .bench_check/test_hidden.py::test_slugify[ Nisse Agent -nisse-agent]\nFAILED .bench_check/test_hidden.py::test_slugify[a--b-a-b] - AssertionError: ...\nFAILED .bench_check/test_hidden.py::test_slugify[Python 3.12 Release-python-3-12-release]\nFAILED .bench_check/test_hidden.py::test_slugify[--leading and trailing---leading-and-trailing]\nFAILED .bench_check/test_hidden.py::test_slugify[MiXeD CaSe-mixed-case] - Ass...\n6 failed, 2 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\nE AssertionError: assert '' == 'hello-world'\nE \nE - hello-world\n\ntest_slugify.py:6: AssertionEr", "prose_turns": 0, "provider": "openai", "success": false, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 4942, "tokens_out": 377, "trimmed_requests": 0, "ts": "2026-09-25T01:02:19+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 5.5}
{"backing": "openai/qwen3-8b-awq", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\n ^^^^^^^\nE NameError: name 'slugify' is not defined\n\ntest_slugify.py:4: NameError\n===============", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3688, "tokens_out": 255, "trimmed_requests": 0, "ts": "2026-09-25T01:02:23+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 3.9}
{"backing": "openai/qwen3-8b-awq", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\n ^^^^^^^\nE NameError: name 'slugify' is not defined\n\ntest_slugify.py:4: NameError\n===============", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3696, "tokens_out": 255, "trimmed_requests": 0, "ts": "2026-09-25T01:02:27+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 3.9}
{"backing": "openai/qwen3-8b-awq", "check_output": " (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'leading-and-trailing'\nE \nE - leading-and-trailing\n\n.bench_check/test_hidden.py:19: AssertionError\n_____________________ test_slugify[MiXeD CaSe-mixed-case] ______________________\n\ntext = 'MiXeD CaSe', expected = 'mixed-case'\n\n @pytest.mark.parametrize(\n (\"text\", \"expected\"),\n [\n (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'mixed-case'\nE \nE - mixed-case\n\n.bench_check/test_hidden.py:19: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_slugify[Hello, World!-hello-world]\nFAILED .bench_check/test_hidden.py::test_slugify[ Nisse Agent -nisse-agent]\nFAILED .bench_check/test_hidden.py::test_slugify[a--b-a-b] - AssertionError: ...\nFAILED .bench_check/test_hidden.py::test_slugify[Python 3.12 Release-python-3-12-release]\nFAILED .bench_check/test_hidden.py::test_slugify[--leading and trailing---leading-and-trailing]\nFAILED .bench_check/test_hidden.py::test_slugify[MiXeD CaSe-mixed-case] - Ass...\n6 failed, 2 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\nE AssertionError: assert '' == 'hello-world'\nE \nE - hello-world\n\ntest_slugify.py:6: AssertionEr", "prose_turns": 0, "provider": "openai", "success": false, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 4936, "tokens_out": 377, "trimmed_requests": 0, "ts": "2026-09-25T01:02:33+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 5.5}
{"backing": "openai/qwen3-8b-awq", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\n ^^^^^^^\nE NameError: name 'slugify' is not defined\n\ntest_slugify.py:4: NameError\n===============", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3686, "tokens_out": 251, "trimmed_requests": 0, "ts": "2026-09-25T01:02:37+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 3.9}
docs/Tuning-Guide.md
+65 -6
View File
@@ -370,6 +370,66 @@ declaration.
---
## Benchmarking models
The capability matrix asks whether a model *can* hold the tool contract. The
benchmark asks whether it *finishes real work*: four canned milestones, each an
approved plan over a small seed project, judged by a hidden check the model
never sees. Success means the check passes, not that the model said it was
done.
| Task | What it exercises |
|---|---|
| `write-function` | write a small function and its tests |
| `fix-bug` | find and fix the bug a failing test exposes, without touching the test |
| `multi-file` | thread a new option through several files |
| `long-output` | find one failure in test output far longer than `max_output_chars` |
It runs the executor exactly as the agent does (the same client wiring,
context budget and `tool_choice` handling, and your config's tuning knobs),
with **delegation and `ask_expert` turned off**. An escalation is a failure of
the model under test, not a frontier model's success.
**Evaluating a new model** is one command, once its LiteLLM entry declares its
limits:
scripts/evaluate_model.sh qwen3-32b-q4 "vLLM 0.29, AWQ, fp8 KV" 5
It runs the context probe, then the capability matrix, then five runs of
every task, and stops at the first step that fails. Or run the steps
yourself:
uv run python scripts/benchmark.py run --models A,B --runs 5 --note "how it's served"
uv run python scripts/benchmark.py report [--models A,B] [--tasks T]
**Results are kept.** Every run is one line in `bench/results/`, committed
with the repo, recording:
- the model's backing name and declared facts;
- your note on how it was served;
- the task's content hash;
- the agent's commit and tuning;
- the outcome, turns, failures, tokens, time and cost.
The report rebuilds the table from every stored run. Benchmarking a new model
means running only that model and reading it next to everything before it.
Runs against an older version of a task are marked `(old)`. Runs of one model
served differently (different note) are kept apart.
**Reading it:**
- **Compare the intervals, not the rates.** 3 of 5 (23–88%) and 5 of 5
(57–100%) overlap: five runs don't separate them. Add runs until the
intervals stop overlapping, or accept that the models are close.
- **Look at why runs failed**, not just how many (`outcome_note` and the
counts in each result line). An early shakedown on `local-fast` failed
mostly by escalation: the model was fixing its own mistakes and hit
`failure_limit: 2` mid-recovery. That says as much about the knob as about
the model. Tuning a knob and re-running is a fair comparison, because the
tuning is recorded with every run.
- **Cost** is per run, straight from LiteLLM, so a frontier model as executor
can be compared on success rate *and* dollars.
---
## Part 3 — An order to tune in
Upstream choices change what's right downstream, so tune in this order and
@@ -400,8 +460,8 @@ change one thing at a time.
3. **Declare the window** in LiteLLM and check it with the probe, then set
the model's `max_output_tokens` there so the prompt keeps most of the
window.
4. **Real milestones.** Run representative work and look at the evidence:
turns used, escalations, delegation outcomes, and time taken.
4. **The benchmark.** Run it and look at the evidence: success rate, turns,
escalations, and time taken.
5. **The loop knobs**: `failure_limit`, `max_turns`, `consult_budget`, the
delegation caps, then the tool limits, one at a time.
@@ -415,11 +475,10 @@ run enough times that the result would survive a coin flip.
## Coming next
The next changes in the 4090-readiness work add knobs, and this guide gains a
section for each:
Later work that will add knobs, each with its own section here:
- **Compaction.** A model-written summary replacing old turns, instead of
dropping them. The log already has an event for it; it needs a summarizing
model and a policy for what is safe to summarize.
- **A benchmark script** that runs the capability matrix and representative
milestones across candidate models and prints a comparison table.
- **Benchmarking the other roles:** planner and delegate, with delegation
turned back on for comparisons across frontier tiers.
nisse-dev/src/nisse_dev/agent.py
+46 -31
View File
@@ -113,6 +113,51 @@ class Agent:
self._stopped.set()
def make_llm_clients(
config: AgentConfig, catalog: ModelCatalog
) -> tuple[BudgetedClient, BudgetedClient]:
"""The two LLM routes, as every model call in the agent uses them.
Shared with the benchmark, so that what it measures is exactly what the
agent would do: the same context budget, the same declared limits, and
tool_choice asked only of models declared to enforce it.
"""
tuning = config.tuning
def limits_for(model: str) -> tuple[int, int | None]:
facts = catalog.facts(model)
assert facts.max_input_tokens is not None # require_limits checked
return facts.max_input_tokens, facts.max_output_tokens
def budgeted(client: object) -> BudgetedClient:
# Every request, on either route, is fitted to its model's window.
return BudgetedClient(
client,
limits_for,
max_output_tokens=config.max_output_tokens,
keep_recent_turns=tuning.context.keep_recent_turns,
safety_margin=tuning.context.safety_margin,
enforces_tool_choice=lambda m: (
catalog.facts(m).supports_tool_choice is True
),
)
normalized = budgeted(
LiteLLMClient(
config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
)
)
# Delegations prefer the vendor-native route; fidelity is cheap
# elsewhere (A5). Which route a given model gets is decided by
# select_client, not by where the caller happens to sit.
passthrough = budgeted(
AnthropicPassthroughClient(
config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
)
)
return normalized, passthrough
def prepare_workspace(config: AgentConfig, account: str) -> GitRepo:
"""Ensure the workspace is a clone of the project repo.
@@ -161,37 +206,7 @@ def build_agent(
max_output_chars=tuning.tools.max_output_chars,
)
def limits_for(model: str) -> tuple[int, int | None]:
facts = catalog.facts(model)
assert facts.max_input_tokens is not None # require_limits checked
return facts.max_input_tokens, facts.max_output_tokens
def budgeted(client: object) -> BudgetedClient:
# Every request, on either route, is fitted to its model's window.
return BudgetedClient(
client,
limits_for,
max_output_tokens=config.max_output_tokens,
keep_recent_turns=tuning.context.keep_recent_turns,
safety_margin=tuning.context.safety_margin,
enforces_tool_choice=lambda m: (
catalog.facts(m).supports_tool_choice is True
),
)
normalized = budgeted(
LiteLLMClient(
config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
)
)
# Delegations prefer the vendor-native route; fidelity is cheap
# elsewhere (A5). Which route a given model gets is decided by
# select_client, not by where the caller happens to sit.
passthrough = budgeted(
AnthropicPassthroughClient(
config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
)
)
normalized, passthrough = make_llm_clients(config, catalog)
transport = MatrixTransport(config.matrix)
announce = notify or transport.send_message
nisse-dev/src/nisse_dev/bench/__init__.py
+17
View File
@@ -0,0 +1,17 @@
"""The executor benchmark: canned milestones, run repeatedly per model.
Choosing an executor model, and later tuning its knobs, should rest on
measured success rates, not on one run or on reputation. Each task is a fixed
approved plan over a seed project, with a hidden check the model never sees;
success means the check passes, not that the model said it finished. Runs use
the agent's own client wiring (context budget, declared limits, tool_choice
only where declared), so what is measured is what the agent would do.
Results are stored, one JSON line per run, with everything needed to compare
them later: the model's identity and declared facts, a note on how it was
served, the task's content hash, the agent's commit, and its tuning. A new
model then needs only its own runs to be compared with everything measured
before it.
See scripts/benchmark.py, and docs/Tuning-Guide.md for how to read results.
"""
nisse-dev/src/nisse_dev/bench/results.py
+117
View File
@@ -0,0 +1,117 @@
"""Stored benchmark results, and the report that compares them.
Every run is one JSON line. The report is rebuilt from all of them each
time, so a model benchmarked today is read alongside every model measured
before it, as long as the task is still the same version.
"""
import json
import math
import statistics
from collections import defaultdict
from collections.abc import Iterable
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from nisse_dev.bench.runner import RunRecord
def results_file(directory: Path, model: str, when: datetime | None = None) -> Path:
day = (when or datetime.now(UTC)).strftime("%Y-%m-%d")
return directory / f"{day}-{model}.jsonl"
def append(path: Path, record: RunRecord) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("a") as out:
out.write(json.dumps(record.to_json(), sort_keys=True) + "\n")
def load(directory: Path) -> list[dict[str, Any]]:
records = []
for path in sorted(directory.glob("*.jsonl")):
for line in path.read_text().splitlines():
if line.strip():
records.append(json.loads(line))
return records
def wilson(successes: int, runs: int, z: float = 1.96) -> tuple[float, float]:
"""A 95% confidence interval for a success rate, sound at small counts.
Three successes in five runs and five in five overlap here, which is the
point: small samples of a noisy model do not separate much."""
if runs == 0:
return 0.0, 1.0
p = successes / runs
denominator = 1 + z * z / runs
centre = (p + z * z / (2 * runs)) / denominator
half = z * math.sqrt(p * (1 - p) / runs + z * z / (4 * runs * runs)) / denominator
return max(0.0, centre - half), min(1.0, centre + half)
@dataclass
class Row:
model: str
note: str
task: str
runs: int
successes: int
median_turns: float
mean_wall_s: float
cost_per_run: float
stale: bool
@property
def interval(self) -> tuple[float, float]:
return wilson(self.successes, self.runs)
def summarise(
records: Iterable[dict[str, Any]], current_hashes: dict[str, str]
) -> list[Row]:
"""One row per (model, how it was served, task, task version)."""
groups: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
for r in records:
groups[(r["backing"], r.get("note", ""), r["task"], r["task_hash"])].append(r)
rows = []
for (backing, note, task, task_hash), runs in sorted(groups.items()):
rows.append(
Row(
model=backing,
note=note,
task=task,
runs=len(runs),
successes=sum(1 for r in runs if r["success"]),
median_turns=statistics.median(r["turns"] for r in runs),
mean_wall_s=statistics.fmean(r["wall_s"] for r in runs),
cost_per_run=statistics.fmean(r["cost"] for r in runs),
stale=current_hashes.get(task) != task_hash,
)
)
return rows
def render(rows: list[Row]) -> str:
header = (
f"{'model':<34} {'note':<22} {'task':<15} {'success':>8} {'95% CI':>11} "
f"{'turns':>6} {'time':>7} {'$/run':>8}"
)
lines = [header, "-" * len(header)]
for row in rows:
low, high = row.interval
task = row.task + (" (old)" if row.stale else "")
lines.append(
f"{row.model[:34]:<34} {row.note[:22]:<22} {task[:15]:<15} "
f"{row.successes:>3}/{row.runs:<4} {low:>4.0%}-{high:<5.0%} "
f"{row.median_turns:>6.0f} {row.mean_wall_s:>6.0f}s {row.cost_per_run:>8.4f}"
)
if any(row.stale for row in rows):
lines.append("")
lines.append(
"(old) = measured against an earlier version of the task; "
"not comparable with current runs."
)
return "\n".join(lines)
nisse-dev/src/nisse_dev/bench/runner.py
+218
View File
@@ -0,0 +1,218 @@
"""Running one task once with one executor model, and recording it."""
import io
import subprocess
import tempfile
import time
from dataclasses import asdict, dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from nisse_dev.bench.tasks import Task
from nisse_dev.canonical import (
CanonicalEvent,
PlanSubmitted,
StoredEvent,
ToolCall,
ToolResult,
UserMessage,
)
from nisse_dev.catalog import ModelFacts
from nisse_dev.config import Tuning
from nisse_dev.executor import LLMExecutor
from nisse_dev.llm import RoleMap
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore
from nisse_dev.tools import Workspace
class MeteringClient:
"""Counts what a run spends, and notices a fallback: an answer from a
model other than the one asked for is not a measurement of it."""
def __init__(self, inner: Any, facts: ModelFacts) -> None:
self._inner = inner
self._facts = facts
self.calls = 0
self.prose_turns = 0
self.tokens_in = 0
self.tokens_out = 0
self.cost = 0.0
self.fallbacks: list[str] = []
def complete(
self,
events: list[StoredEvent],
tools: list[dict[str, Any]] | None,
model: str,
system: str | None = None,
) -> list[CanonicalEvent]:
self.calls += 1
produced = self._inner.complete(events, tools, model, system=system)
usage = getattr(self._inner, "last_usage", {}) or {}
self.tokens_in += int(usage.get("input", 0))
self.tokens_out += int(usage.get("output", 0))
self.cost += float(getattr(self._inner, "last_cost", 0.0) or 0.0)
answered = getattr(self._inner, "last_model", None)
if not self._facts.answered(answered):
self.fallbacks.append(str(answered))
if not any(isinstance(e, ToolCall) for e in produced):
self.prose_turns += 1
return produced
@dataclass
class RunRecord:
"""One run, with everything needed to compare it with runs made later
against other models, other servers, or other tuning."""
ts: str
model: str
backing: str
provider: str
model_info: dict[str, Any]
note: str
task: str
task_hash: str
commit: str
tuning: dict[str, Any]
success: bool
outcome: str
outcome_note: str
turns: int
prose_turns: int
failed_commands: int
invented_tools: int
trimmed_requests: int
tokens_in: int
tokens_out: int
cost: float
wall_s: float
fallbacks: list[str] = field(default_factory=list)
check_output: str = ""
def to_json(self) -> dict[str, Any]:
return asdict(self)
def agent_commit(repo: Path | None = None) -> str:
"""The agent's commit, marked when the tree had uncommitted changes."""
where = repo or Path(__file__).resolve().parent
try:
sha = subprocess.run(
["git", "rev-parse", "--short", "HEAD"],
cwd=where,
capture_output=True,
text=True,
check=True,
).stdout.strip()
dirty = subprocess.run(
["git", "status", "--porcelain"],
cwd=where,
capture_output=True,
text=True,
check=True,
).stdout.strip()
except (OSError, subprocess.CalledProcessError):
return "unknown"
return f"{sha}+dirty" if dirty else sha
def _count(events: list[StoredEvent]) -> tuple[int, int]:
tools = {
e.event.call_id: e.event.tool for e in events if isinstance(e.event, ToolCall)
}
failed = invented = 0
for stored in events:
event = stored.event
if isinstance(event, ToolResult) and not event.success:
if event.result.startswith("no such tool"):
invented += 1
elif tools.get(event.call_id) == "run_command":
failed += 1
return failed, invented
def run_task(
task: Task,
facts: ModelFacts,
client: Any,
tuning: Tuning,
*,
note: str = "",
commit: str = "",
workdir: Path | None = None,
) -> RunRecord:
"""Run one task once: seed a workspace, hand the executor the approved
plan, then judge the result with the hidden check.
Delegation and ask_expert are off: an escalation is a failure of the
model under test, not a frontier model's success."""
with tempfile.TemporaryDirectory(dir=workdir) as tmp:
root = Path(tmp)
workspace = Workspace(
root / "work",
timeout_s=tuning.tools.command_timeout_s,
max_output_chars=tuning.tools.max_output_chars,
)
task.seed_into(workspace.root)
store = StateStore(root / "state.db", oplog=OpLog("nisse-bench", io.StringIO()))
store.append_event(PlanSubmitted(plan=task.plan, model="benchmark"))
store.append_event(UserMessage(text="start milestone 1", source="benchmark"))
meter = MeteringClient(client, facts)
trims_before = getattr(client, "trimmed_requests", 0)
model = facts.alias
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
executor = LLMExecutor(
meter,
store,
roles,
workspace,
max_turns=tuning.executor.max_turns,
failure_limit=tuning.executor.failure_limit,
)
started = time.monotonic()
try:
outcome = executor.run(1)
status, detail = outcome.status, outcome.note
except Exception as exc: # noqa: BLE001 -- recorded, the benchmark goes on
status, detail = "error", repr(exc)
wall = time.monotonic() - started
success, check_output = task.run_check(workspace.root)
failed, invented = _count(store.read_events())
store.close()
return RunRecord(
ts=datetime.now(UTC).isoformat(timespec="seconds"),
model=model,
backing=facts.backing,
provider=facts.provider,
model_info={
"max_input_tokens": facts.max_input_tokens,
"max_output_tokens": facts.max_output_tokens,
"supports_tool_choice": facts.supports_tool_choice,
"supports_function_calling": facts.supports_tools,
},
note=note,
task=task.name,
task_hash=task.content_hash,
commit=commit,
tuning=asdict(tuning),
success=success and not meter.fallbacks,
outcome=status,
outcome_note=detail[:500],
turns=meter.calls,
prose_turns=meter.prose_turns,
failed_commands=failed,
invented_tools=invented,
trimmed_requests=getattr(client, "trimmed_requests", 0) - trims_before,
tokens_in=meter.tokens_in,
tokens_out=meter.tokens_out,
cost=round(meter.cost, 6),
wall_s=round(wall, 1),
fallbacks=meter.fallbacks,
check_output=check_output,
)
nisse-dev/src/nisse_dev/bench/tasks.py
+102
View File
@@ -0,0 +1,102 @@
"""Benchmark tasks: a seed project, an approved plan, and a hidden check.
Each task is a directory under ``tasks/``:
- ``task.yaml``: ``title``, ``plan`` (the approved milestone plan, written the
way the planner is asked to write them), and ``check`` (a shell command run
in the workspace afterwards).
- ``seed/``: the project the executor starts from.
- ``hidden/``: files the check needs, copied in only after the run, into
``.bench_check/``. The executor never sees them.
- ``solution/``: a reference solution, overlaid on the seed by the tests to
prove each task can be solved and that its check tells the difference.
"""
import hashlib
import shutil
import subprocess
from dataclasses import dataclass
from pathlib import Path
import yaml
TASKS_DIR = Path(__file__).parent / "tasks"
CHECK_DIR = ".bench_check"
@dataclass(frozen=True)
class Task:
name: str
title: str
plan: str
check: str
path: Path
@property
def content_hash(self) -> str:
"""What the task is, so a changed task is never compared with runs of
its earlier version. Covers everything the model or the check sees;
not the reference solution."""
digest = hashlib.sha256()
for part in ("task.yaml", "seed", "hidden"):
root = self.path / part
files = [root] if root.is_file() else sorted(root.rglob("*"))
for file in files:
if file.is_file() and "__pycache__" not in file.parts:
digest.update(str(file.relative_to(self.path)).encode())
digest.update(file.read_bytes())
return digest.hexdigest()[:16]
def seed_into(self, workspace: Path) -> None:
shutil.copytree(self.path / "seed", workspace, dirs_exist_ok=True)
def apply_solution(self, workspace: Path) -> None:
shutil.copytree(self.path / "solution", workspace, dirs_exist_ok=True)
def run_check(self, workspace: Path, timeout_s: int = 300) -> tuple[bool, str]:
"""Copy the hidden files in and run the check. Returns whether it
passed, and its output for the record."""
target = workspace / CHECK_DIR
if target.exists():
shutil.rmtree(target) # never trust a directory the model made
shutil.copytree(self.path / "hidden", target)
try:
result = subprocess.run(
self.check,
shell=True,
cwd=workspace,
capture_output=True,
text=True,
timeout=timeout_s,
check=False,
)
except subprocess.TimeoutExpired:
return False, f"check timed out after {timeout_s}s"
output = (result.stdout + result.stderr)[-2000:]
return result.returncode == 0, output
def load_task(path: Path) -> Task:
spec = yaml.safe_load((path / "task.yaml").read_text())
return Task(
name=path.name,
title=str(spec["title"]),
plan=str(spec["plan"]).strip(),
check=str(spec["check"]),
path=path,
)
def load_tasks(names: list[str] | None = None, root: Path = TASKS_DIR) -> list[Task]:
available = {
p.name: p for p in sorted(root.iterdir()) if (p / "task.yaml").exists()
}
if names:
unknown = [n for n in names if n not in available]
if unknown:
raise ValueError(
f"unknown task(s): {', '.join(unknown)}; "
f"available: {', '.join(available)}"
)
return [load_task(available[n]) for n in names]
return [load_task(p) for p in available.values()]
nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/original_test_inventory.py.txt
+11
View File
@@ -0,0 +1,11 @@
from inventory import Item, low_stock, total_value
def test_total_value_counts_every_unit() -> None:
items = [Item("bolt", 0.25, 100), Item("nut", 0.10, 40)]
assert total_value(items) == 29.0
def test_low_stock_is_alphabetical() -> None:
items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100), Item("anchor", 3.0, 1)]
assert low_stock(items) == ["anchor", "washer"]
nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/test_hidden.py
+27
View File
@@ -0,0 +1,27 @@
from pathlib import Path
import pytest
from inventory import Item, low_stock, total_value
def test_the_visible_tests_were_not_changed() -> None:
original = (Path(__file__).parent / "original_test_inventory.py.txt").read_text()
assert Path("test_inventory.py").read_text() == original
@pytest.mark.parametrize(
("items", "expected"),
[
([], 0.0),
([Item("bolt", 0.25, 100), Item("nut", 0.10, 40)], 29.0),
([Item("gear", 12.5, 3)], 37.5),
([Item("empty", 9.99, 0)], 0.0),
],
)
def test_total_value(items: list[Item], expected: float) -> None:
assert total_value(items) == pytest.approx(expected)
def test_low_stock_still_works() -> None:
items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100)]
assert low_stock(items) == ["washer"]
nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/inventory.py
+23
View File
@@ -0,0 +1,23 @@
"""A tiny inventory."""
from dataclasses import dataclass
@dataclass
class Item:
name: str
unit_price: float
quantity: int
def total_value(items: list[Item]) -> float:
"""The value of all stock: each item's unit price times its quantity."""
total = 0.0
for item in items:
total += item.unit_price
return total
def low_stock(items: list[Item], threshold: int = 5) -> list[str]:
"""Names of items with fewer than `threshold` units, alphabetically."""
return sorted(item.name for item in items if item.quantity < threshold)
nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/test_inventory.py
+11
View File
@@ -0,0 +1,11 @@
from inventory import Item, low_stock, total_value
def test_total_value_counts_every_unit() -> None:
items = [Item("bolt", 0.25, 100), Item("nut", 0.10, 40)]
assert total_value(items) == 29.0
def test_low_stock_is_alphabetical() -> None:
items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100), Item("anchor", 3.0, 1)]
assert low_stock(items) == ["anchor", "washer"]
nisse-dev/src/nisse_dev/bench/tasks/fix-bug/solution/inventory.py
+23
View File
@@ -0,0 +1,23 @@
"""A tiny inventory."""
from dataclasses import dataclass
@dataclass
class Item:
name: str
unit_price: float
quantity: int
def total_value(items: list[Item]) -> float:
"""The value of all stock: each item's unit price times its quantity."""
total = 0.0
for item in items:
total += item.unit_price * item.quantity
return total
def low_stock(items: list[Item], threshold: int = 5) -> list[str]:
"""Names of items with fewer than `threshold` units, alphabetically."""
return sorted(item.name for item in items if item.quantity < threshold)
nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml
+12
View File
@@ -0,0 +1,12 @@
title: Fix a bug a failing test exposes
check: python -m pytest -q .bench_check
plan: |
Milestone 1: fix the failing inventory test.
1. Run `python -m pytest -q` and read the failure.
2. The failing test is correct; the bug is in `inventory.py`. Do not change
any test file.
3. Fix `total_value` so that each item contributes its unit price times its
quantity.
4. Run `python -m pytest -q` until every test passes.
5. Call finish.
nisse-dev/src/nisse_dev/bench/tasks/long-output/hidden/test_hidden.py
+11
View File
@@ -0,0 +1,11 @@
import pytest
from rates import RATES
def test_every_region_is_present_once() -> None:
assert sorted(RATES) == [f"R{n:03d}" for n in range(300)]
@pytest.mark.parametrize("n", range(300))
def test_rate(n: int) -> None:
assert RATES[f"R{n:03d}"] == pytest.approx(n % 25 / 100)
nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/rates.py
+304
View File
@@ -0,0 +1,304 @@
"""Tax rate per region code."""
RATES = {
"R000": 0.00,
"R001": 0.01,
"R002": 0.02,
"R003": 0.03,
"R004": 0.04,
"R005": 0.05,
"R006": 0.06,
"R007": 0.07,
"R008": 0.08,
"R009": 0.09,
"R010": 0.10,
"R011": 0.11,
"R012": 0.12,
"R013": 0.13,
"R014": 0.14,
"R015": 0.15,
"R016": 0.16,
"R017": 0.17,
"R018": 0.18,
"R019": 0.19,
"R020": 0.20,
"R021": 0.21,
"R022": 0.22,
"R023": 0.23,
"R024": 0.24,
"R025": 0.00,
"R026": 0.01,
"R027": 0.02,
"R028": 0.03,
"R029": 0.04,
"R030": 0.05,
"R031": 0.06,
"R032": 0.07,
"R033": 0.08,
"R034": 0.09,
"R035": 0.10,
"R036": 0.11,
"R037": 0.12,
"R038": 0.13,
"R039": 0.14,
"R040": 0.15,
"R041": 0.16,
"R042": 0.17,
"R043": 0.18,
"R044": 0.19,
"R045": 0.20,
"R046": 0.21,
"R047": 0.22,
"R048": 0.23,
"R049": 0.24,
"R050": 0.00,
"R051": 0.01,
"R052": 0.02,
"R053": 0.03,
"R054": 0.04,
"R055": 0.05,
"R056": 0.06,
"R057": 0.07,
"R058": 0.08,
"R059": 0.09,
"R060": 0.10,
"R061": 0.11,
"R062": 0.12,
"R063": 0.13,
"R064": 0.14,
"R065": 0.15,
"R066": 0.16,
"R067": 0.17,
"R068": 0.18,
"R069": 0.19,
"R070": 0.20,
"R071": 0.21,
"R072": 0.22,
"R073": 0.23,
"R074": 0.24,
"R075": 0.00,
"R076": 0.01,
"R077": 0.02,
"R078": 0.03,
"R079": 0.04,
"R080": 0.05,
"R081": 0.06,
"R082": 0.07,
"R083": 0.08,
"R084": 0.09,
"R085": 0.10,
"R086": 0.11,
"R087": 0.12,
"R088": 0.13,
"R089": 0.14,
"R090": 0.15,
"R091": 0.16,
"R092": 0.17,
"R093": 0.18,
"R094": 0.19,
"R095": 0.20,
"R096": 0.21,
"R097": 0.22,
"R098": 0.23,
"R099": 0.24,
"R100": 0.00,
"R101": 0.01,
"R102": 0.02,
"R103": 0.03,
"R104": 0.04,
"R105": 0.05,
"R106": 0.06,
"R107": 0.07,
"R108": 0.08,
"R109": 0.09,
"R110": 0.10,
"R111": 0.11,
"R112": 0.12,
"R113": 0.13,
"R114": 0.14,
"R115": 0.15,
"R116": 0.16,
"R117": 0.17,
"R118": 0.18,
"R119": 0.19,
"R120": 0.20,
"R121": 0.21,
"R122": 0.22,
"R123": 0.23,
"R124": 0.24,
"R125": 0.00,
"R126": 0.01,
"R127": 0.02,
"R128": 0.03,
"R129": 0.04,
"R130": 0.05,
"R131": 0.06,
"R132": 0.07,
"R133": 0.08,
"R134": 0.09,
"R135": 0.10,
"R136": 0.11,
"R137": 0.12,
"R138": 0.13,
"R139": 0.14,
"R140": 0.15,
"R141": 0.16,
"R142": 0.17,
"R143": 0.18,
"R144": 0.19,
"R145": 0.20,
"R146": 0.21,
"R147": 0.22,
"R148": 0.23,
"R149": 0.24,
"R150": 0.00,
"R151": 0.01,
"R152": 0.02,
"R153": 0.03,
"R154": 0.04,
"R155": 0.05,
"R156": 0.06,
"R157": 0.07,
"R158": 0.08,
"R159": 0.09,
"R160": 0.10,
"R161": 0.11,
"R162": 0.12,
"R163": 0.13,
"R164": 0.14,
"R165": 0.15,
"R166": 0.16,
"R167": 0.17,
"R168": 0.18,
"R169": 0.19,
"R170": 0.20,
"R171": 0.21,
"R172": 0.22,
"R173": 0.23,
"R174": 0.24,
"R175": 0.00,
"R176": 0.01,
"R177": 0.02,
"R178": 0.03,
"R179": 0.04,
"R180": 0.05,
"R181": 0.06,
"R182": 0.07,
"R183": 0.08,
"R184": 0.09,
"R185": 0.10,
"R186": 0.11,
"R187": 0.12,
"R188": 0.13,
"R189": 0.14,
"R190": 0.15,
"R191": 0.16,
"R192": 0.17,
"R193": 0.18,
"R194": 0.19,
"R195": 0.20,
"R196": 0.21,
"R197": 0.22,
"R198": 0.23,
"R199": 0.24,
"R200": 0.00,
"R201": 0.01,
"R202": 0.02,
"R203": 0.03,
"R204": 0.04,
"R205": 0.05,
"R206": 0.06,
"R207": 0.07,
"R208": 0.08,
"R209": 0.09,
"R210": 0.10,
"R211": 0.11,
"R212": 0.12,
"R213": 0.13,
"R214": 0.14,
"R215": 0.15,
"R216": 0.16,
"R217": 0.17,
"R218": 0.18,
"R219": 0.19,
"R220": 0.20,
"R221": 0.21,
"R222": 0.22,
"R223": 0.23,
"R224": 0.24,
"R225": 0.00,
"R226": 0.01,
"R227": 0.02,
"R228": 0.03,
"R229": 0.04,
"R230": 0.05,
"R231": 0.06,
"R232": 0.07,
"R233": 0.08,
"R234": 0.09,
"R235": 0.10,
"R236": 0.11,
"R237": 0.21,
"R238": 0.13,
"R239": 0.14,
"R240": 0.15,
"R241": 0.16,
"R242": 0.17,
"R243": 0.18,
"R244": 0.19,
"R245": 0.20,
"R246": 0.21,
"R247": 0.22,
"R248": 0.23,
"R249": 0.24,
"R250": 0.00,
"R251": 0.01,
"R252": 0.02,
"R253": 0.03,
"R254": 0.04,
"R255": 0.05,
"R256": 0.06,
"R257": 0.07,
"R258": 0.08,
"R259": 0.09,
"R260": 0.10,
"R261": 0.11,
"R262": 0.12,
"R263": 0.13,
"R264": 0.14,
"R265": 0.15,
"R266": 0.16,
"R267": 0.17,
"R268": 0.18,
"R269": 0.19,
"R270": 0.20,
"R271": 0.21,
"R272": 0.22,
"R273": 0.23,
"R274": 0.24,
"R275": 0.00,
"R276": 0.01,
"R277": 0.02,
"R278": 0.03,
"R279": 0.04,
"R280": 0.05,
"R281": 0.06,
"R282": 0.07,
"R283": 0.08,
"R284": 0.09,
"R285": 0.10,
"R286": 0.11,
"R287": 0.12,
"R288": 0.13,
"R289": 0.14,
"R290": 0.15,
"R291": 0.16,
"R292": 0.17,
"R293": 0.18,
"R294": 0.19,
"R295": 0.20,
"R296": 0.21,
"R297": 0.22,
"R298": 0.23,
"R299": 0.24,
}
nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/test_rates.py
+7
View File
@@ -0,0 +1,7 @@
import pytest
from rates import RATES
@pytest.mark.parametrize("code", [f"R{n:03d}" for n in range(300)])
def test_rate(code: str) -> None:
assert RATES[code] == pytest.approx(int(code[1:]) % 25 / 100)
nisse-dev/src/nisse_dev/bench/tasks/long-output/solution/rates.py
+304
View File
@@ -0,0 +1,304 @@
"""Tax rate per region code."""
RATES = {
"R000": 0.00,
"R001": 0.01,
"R002": 0.02,
"R003": 0.03,
"R004": 0.04,
"R005": 0.05,
"R006": 0.06,
"R007": 0.07,
"R008": 0.08,
"R009": 0.09,
"R010": 0.10,
"R011": 0.11,
"R012": 0.12,
"R013": 0.13,
"R014": 0.14,
"R015": 0.15,
"R016": 0.16,
"R017": 0.17,
"R018": 0.18,
"R019": 0.19,
"R020": 0.20,
"R021": 0.21,
"R022": 0.22,
"R023": 0.23,
"R024": 0.24,
"R025": 0.00,
"R026": 0.01,
"R027": 0.02,
"R028": 0.03,
"R029": 0.04,
"R030": 0.05,
"R031": 0.06,
"R032": 0.07,
"R033": 0.08,
"R034": 0.09,
"R035": 0.10,
"R036": 0.11,
"R037": 0.12,
"R038": 0.13,
"R039": 0.14,
"R040": 0.15,
"R041": 0.16,
"R042": 0.17,
"R043": 0.18,
"R044": 0.19,
"R045": 0.20,
"R046": 0.21,
"R047": 0.22,
"R048": 0.23,
"R049": 0.24,
"R050": 0.00,
"R051": 0.01,
"R052": 0.02,
"R053": 0.03,
"R054": 0.04,
"R055": 0.05,
"R056": 0.06,
"R057": 0.07,
"R058": 0.08,
"R059": 0.09,
"R060": 0.10,
"R061": 0.11,
"R062": 0.12,
"R063": 0.13,
"R064": 0.14,
"R065": 0.15,
"R066": 0.16,
"R067": 0.17,
"R068": 0.18,
"R069": 0.19,
"R070": 0.20,
"R071": 0.21,
"R072": 0.22,
"R073": 0.23,
"R074": 0.24,
"R075": 0.00,
"R076": 0.01,
"R077": 0.02,
"R078": 0.03,
"R079": 0.04,
"R080": 0.05,
"R081": 0.06,
"R082": 0.07,
"R083": 0.08,
"R084": 0.09,
"R085": 0.10,
"R086": 0.11,
"R087": 0.12,
"R088": 0.13,
"R089": 0.14,
"R090": 0.15,
"R091": 0.16,
"R092": 0.17,
"R093": 0.18,
"R094": 0.19,
"R095": 0.20,
"R096": 0.21,
"R097": 0.22,
"R098": 0.23,
"R099": 0.24,
"R100": 0.00,
"R101": 0.01,
"R102": 0.02,
"R103": 0.03,
"R104": 0.04,
"R105": 0.05,
"R106": 0.06,
"R107": 0.07,
"R108": 0.08,
"R109": 0.09,
"R110": 0.10,
"R111": 0.11,
"R112": 0.12,
"R113": 0.13,
"R114": 0.14,
"R115": 0.15,
"R116": 0.16,
"R117": 0.17,
"R118": 0.18,
"R119": 0.19,
"R120": 0.20,
"R121": 0.21,
"R122": 0.22,
"R123": 0.23,
"R124": 0.24,
"R125": 0.00,
"R126": 0.01,
"R127": 0.02,
"R128": 0.03,
"R129": 0.04,
"R130": 0.05,
"R131": 0.06,
"R132": 0.07,
"R133": 0.08,
"R134": 0.09,
"R135": 0.10,
"R136": 0.11,
"R137": 0.12,
"R138": 0.13,
"R139": 0.14,
"R140": 0.15,
"R141": 0.16,
"R142": 0.17,
"R143": 0.18,
"R144": 0.19,
"R145": 0.20,
"R146": 0.21,
"R147": 0.22,
"R148": 0.23,
"R149": 0.24,
"R150": 0.00,
"R151": 0.01,
"R152": 0.02,
"R153": 0.03,
"R154": 0.04,
"R155": 0.05,
"R156": 0.06,
"R157": 0.07,
"R158": 0.08,
"R159": 0.09,
"R160": 0.10,
"R161": 0.11,
"R162": 0.12,
"R163": 0.13,
"R164": 0.14,
"R165": 0.15,
"R166": 0.16,
"R167": 0.17,
"R168": 0.18,
"R169": 0.19,
"R170": 0.20,
"R171": 0.21,
"R172": 0.22,
"R173": 0.23,
"R174": 0.24,
"R175": 0.00,
"R176": 0.01,
"R177": 0.02,
"R178": 0.03,
"R179": 0.04,
"R180": 0.05,
"R181": 0.06,
"R182": 0.07,
"R183": 0.08,
"R184": 0.09,
"R185": 0.10,
"R186": 0.11,
"R187": 0.12,
"R188": 0.13,
"R189": 0.14,
"R190": 0.15,
"R191": 0.16,
"R192": 0.17,
"R193": 0.18,
"R194": 0.19,
"R195": 0.20,
"R196": 0.21,
"R197": 0.22,
"R198": 0.23,
"R199": 0.24,
"R200": 0.00,
"R201": 0.01,
"R202": 0.02,
"R203": 0.03,
"R204": 0.04,
"R205": 0.05,
"R206": 0.06,
"R207": 0.07,
"R208": 0.08,
"R209": 0.09,
"R210": 0.10,
"R211": 0.11,
"R212": 0.12,
"R213": 0.13,
"R214": 0.14,
"R215": 0.15,
"R216": 0.16,
"R217": 0.17,
"R218": 0.18,
"R219": 0.19,
"R220": 0.20,
"R221": 0.21,
"R222": 0.22,
"R223": 0.23,
"R224": 0.24,
"R225": 0.00,
"R226": 0.01,
"R227": 0.02,
"R228": 0.03,
"R229": 0.04,
"R230": 0.05,
"R231": 0.06,
"R232": 0.07,
"R233": 0.08,
"R234": 0.09,
"R235": 0.10,
"R236": 0.11,
"R237": 0.12,
"R238": 0.13,
"R239": 0.14,
"R240": 0.15,
"R241": 0.16,
"R242": 0.17,
"R243": 0.18,
"R244": 0.19,
"R245": 0.20,
"R246": 0.21,
"R247": 0.22,
"R248": 0.23,
"R249": 0.24,
"R250": 0.00,
"R251": 0.01,
"R252": 0.02,
"R253": 0.03,
"R254": 0.04,
"R255": 0.05,
"R256": 0.06,
"R257": 0.07,
"R258": 0.08,
"R259": 0.09,
"R260": 0.10,
"R261": 0.11,
"R262": 0.12,
"R263": 0.13,
"R264": 0.14,
"R265": 0.15,
"R266": 0.16,
"R267": 0.17,
"R268": 0.18,
"R269": 0.19,
"R270": 0.20,
"R271": 0.21,
"R272": 0.22,
"R273": 0.23,
"R274": 0.24,
"R275": 0.00,
"R276": 0.01,
"R277": 0.02,
"R278": 0.03,
"R279": 0.04,
"R280": 0.05,
"R281": 0.06,
"R282": 0.07,
"R283": 0.08,
"R284": 0.09,
"R285": 0.10,
"R286": 0.11,
"R287": 0.12,
"R288": 0.13,
"R289": 0.14,
"R290": 0.15,
"R291": 0.16,
"R292": 0.17,
"R293": 0.18,
"R294": 0.19,
"R295": 0.20,
"R296": 0.21,
"R297": 0.22,
"R298": 0.23,
"R299": 0.24,
}
nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml
+12
View File
@@ -0,0 +1,12 @@
title: Find one failure in long test output
check: python -m pytest -q .bench_check
plan: |
Milestone 1: fix the one wrong entry in the rate table.
1. Run `python -m pytest -v`. Exactly one test fails, and its output names a
region code.
2. The tests are correct. In `rates.py`, the rate for region code RNNN must be
(NNN mod 25) / 100. Correct only the failing region's entry; leave every
other line as it is.
3. Run `python -m pytest -q` until every test passes.
4. Call finish.
nisse-dev/src/nisse_dev/bench/tasks/multi-file/hidden/test_hidden.py
+33
View File
@@ -0,0 +1,33 @@
import subprocess
import sys
from greeter.cli import main
from greeter.core import greet
def test_default_is_unchanged() -> None:
assert greet("Bob") == "Hello, Bob!"
def test_shout() -> None:
assert greet("Bob", shout=True) == "HELLO, BOB!"
def test_cli_default(capsys) -> None:
assert main(["Ada"]) == 0
assert capsys.readouterr().out == "Hello, Ada!\n"
def test_cli_shout(capsys) -> None:
assert main(["--shout", "Ada"]) == 0
assert capsys.readouterr().out == "HELLO, ADA!\n"
def test_module_entry_point() -> None:
result = subprocess.run(
[sys.executable, "-m", "greeter", "--shout", "Bob"],
capture_output=True,
text=True,
check=True,
)
assert result.stdout == "HELLO, BOB!\n"
nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__init__.py
View File
Whitespace-only changes.
nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__main__.py
+3
View File
@@ -0,0 +1,3 @@
from greeter.cli import main
raise SystemExit(main())
nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/cli.py
+11
View File
@@ -0,0 +1,11 @@
import argparse
from greeter.core import greet
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="greeter")
parser.add_argument("name")
args = parser.parse_args(argv)
print(greet(args.name))
return 0
nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/core.py
+2
View File
@@ -0,0 +1,2 @@
def greet(name: str) -> str:
return f"Hello, {name}!"
nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/tests/test_greeter.py
+11
View File
@@ -0,0 +1,11 @@
from greeter.cli import main
from greeter.core import greet
def test_greet() -> None:
assert greet("Bob") == "Hello, Bob!"
def test_cli_prints_the_greeting(capsys) -> None:
assert main(["Bob"]) == 0
assert capsys.readouterr().out == "Hello, Bob!\n"
nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/cli.py
+12
View File
@@ -0,0 +1,12 @@
import argparse
from greeter.core import greet
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="greeter")
parser.add_argument("name")
parser.add_argument("--shout", action="store_true")
args = parser.parse_args(argv)
print(greet(args.name, shout=args.shout))
return 0
nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/core.py
+3
View File
@@ -0,0 +1,3 @@
def greet(name: str, shout: bool = False) -> str:
greeting = f"Hello, {name}!"
return greeting.upper() if shout else greeting
nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml
+16
View File
@@ -0,0 +1,16 @@
title: Thread a new option through several files
check: python -m pytest -q .bench_check
plan: |
Milestone 1: add a --shout option to the greeter.
1. In `greeter/core.py`, give `greet` a keyword argument `shout: bool = False`.
When it is true, return the whole greeting in upper case:
greet("Bob", shout=True) == "HELLO, BOB!". The default behaviour is unchanged.
2. In `greeter/cli.py`, add a `--shout` flag (action="store_true") and pass it
to `greet`.
3. Add two tests to `tests/test_greeter.py`: one for greet with shout=True, and
one that calls `main(["--shout", "Bob"])` and checks the printed output
using pytest's capsys fixture.
4. Run `python -m pytest -q` until every test passes, and check that
`python -m greeter --shout Bob` prints `HELLO, BOB!`.
5. Call finish.
nisse-dev/src/nisse_dev/bench/tasks/write-function/hidden/test_hidden.py
+19
View File
@@ -0,0 +1,19 @@
import pytest
from slugify import slugify
@pytest.mark.parametrize(
("text", "expected"),
[
("Hello, World!", "hello-world"),
(" Nisse Agent ", "nisse-agent"),
("a--b", "a-b"),
("", ""),
("Python 3.12 Release", "python-3-12-release"),
("--leading and trailing--", "leading-and-trailing"),
("MiXeD CaSe", "mixed-case"),
("!!!", ""),
],
)
def test_slugify(text: str, expected: str) -> None:
assert slugify(text) == expected
nisse-dev/src/nisse_dev/bench/tasks/write-function/seed/README.md
+1
View File
@@ -0,0 +1 @@
A small utilities project.
nisse-dev/src/nisse_dev/bench/tasks/write-function/solution/slugify.py
+5
View File
@@ -0,0 +1,5 @@
import re
def slugify(text: str) -> str:
return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml
+13
View File
@@ -0,0 +1,13 @@
title: Write a small function and its tests
check: python -m pytest -q .bench_check
plan: |
Milestone 1: add a slugify function.
1. Create `slugify.py` in the workspace root, defining `slugify(text: str) -> str`.
2. It lowercases the text, replaces every run of characters that are not a-z or
0-9 with a single hyphen, and strips hyphens from both ends. Examples:
"Hello, World!" -> "hello-world"; " Nisse Agent " -> "nisse-agent";
"a--b" -> "a-b"; "" -> "".
3. Create `test_slugify.py` with pytest tests covering the four examples.
4. Run `python -m pytest -q test_slugify.py`, and fix anything until it passes.
5. Call finish.
nisse-dev/src/nisse_dev/context.py
+9
View File
@@ -230,6 +230,7 @@ class BudgetedClient:
# Per model: how much the character estimate undercounted, as learned
# from reported usage. Starts at 1 and only grows.
self._factors: dict[str, float] = {}
self.trimmed_requests = 0
@property
def last_usage(self) -> dict[str, int]:
@@ -239,6 +240,10 @@ class BudgetedClient:
def last_model(self) -> str | None:
return getattr(self._inner, "last_model", None)
@property
def last_cost(self) -> float:
return float(getattr(self._inner, "last_cost", 0.0) or 0.0)
def budget_for(self, model: str) -> Budget:
window, model_max_output = self._limits_for(model)
ceiling = self._max_output_tokens
@@ -261,6 +266,10 @@ class BudgetedClient:
budget = self.budget_for(model)
factor = self._factors.get(model, 1.0)
fitted = fit(events, system, tools, budget, factor)
# Counted so the benchmark can report how often a model's window was
# the constraint: any request that had to be cut down or shortened.
if fitted != list(events):
self.trimmed_requests += 1
produced = self._inner.complete(
fitted,
tools,
nisse-dev/src/nisse_dev/llm.py
+13
View File
@@ -143,6 +143,7 @@ class LiteLLMClient:
# protocol stays the same for every implementation.
self.last_usage: dict[str, int] = {}
self.last_model: str | None = None
self.last_cost: float = 0.0
def complete(
self,
@@ -171,6 +172,8 @@ class LiteLLMClient:
raise LLMError(
f"LLM request returned {response.status_code}: {response.text[:500]}"
)
# LiteLLM prices every call; kept so a caller can meter spending.
self.last_cost = _response_cost(response)
return self._parse(response.json(), requested_model=model)
def _parse(
@@ -263,6 +266,7 @@ class AnthropicPassthroughClient:
self._sleep = sleep
self.last_usage: dict[str, int] = {}
self.last_model: str | None = None
self.last_cost: float = 0.0
def complete(
self,
@@ -291,6 +295,8 @@ class AnthropicPassthroughClient:
raise LLMError(
f"passthrough returned {response.status_code}: {response.text[:500]}"
)
# LiteLLM prices every call; kept so a caller can meter spending.
self.last_cost = _response_cost(response)
return self._parse(response.json(), requested_model=model)
def _parse(
@@ -358,6 +364,13 @@ def _post_with_rate_limit_retry(
# Model families we have a native renderer adapter for. A13: delegations take
# the vendor's passthrough route because a normalising middlebox is lossy
# exactly where a long-running delegation lives.
def _response_cost(response: httpx.Response) -> float:
try:
return float(response.headers.get("x-litellm-response-cost") or 0.0)
except ValueError:
return 0.0
def _raise_for_upstream(response: httpx.Response) -> None:
"""Classify a non-200 as upstream trouble where it plainly is.
nisse-dev/tests/test_bench.py
+265
View File
@@ -0,0 +1,265 @@
"""The executor benchmark."""
import json
import shutil
from pathlib import Path
from typing import Any
import pytest
from nisse_dev.bench.results import (
append,
load,
render,
results_file,
summarise,
wilson,
)
from nisse_dev.bench.runner import MeteringClient, run_task
from nisse_dev.bench.tasks import CHECK_DIR, load_tasks
from nisse_dev.canonical import AssistantText, ToolCall
from nisse_dev.catalog import ModelFacts
from nisse_dev.config import ExecutorTuning, Tuning
TASKS = load_tasks()
FACTS = ModelFacts(
alias="local-fast",
backing="openai/qwen3-8b-awq",
provider="openai",
supports_tools=True,
max_input_tokens=32768,
max_output_tokens=8192,
supports_tool_choice=True,
)
# --- the tasks themselves ---------------------------------------------------
def test_the_bundled_tasks_are_the_four_planned() -> None:
assert [t.name for t in TASKS] == [
"fix-bug",
"long-output",
"multi-file",
"write-function",
]
@pytest.mark.parametrize("task", TASKS, ids=lambda t: t.name)
def test_the_seed_fails_its_check_and_the_solution_passes(task, tmp_path: Path) -> None:
# A check that passes on the untouched seed measures nothing; a task its
# own reference solution can't pass is unfair to every model.
seed = tmp_path / "seed"
task.seed_into(seed)
assert not task.run_check(seed)[0], f"{task.name}: the seed already passes"
solved = tmp_path / "solved"
task.seed_into(solved)
task.apply_solution(solved)
passed, output = task.run_check(solved)
assert passed, f"{task.name}: the reference solution fails:\n{output}"
@pytest.mark.parametrize("task", TASKS, ids=lambda t: t.name)
def test_the_hidden_check_is_not_in_the_seed(task, tmp_path: Path) -> None:
task.seed_into(tmp_path)
assert not (tmp_path / CHECK_DIR).exists()
assert "Call finish" in task.plan # plans end the way the executor must
def test_long_output_really_is_long(tmp_path: Path) -> None:
"""The point of the task is output far past tools.max_output_chars."""
import subprocess
import sys
[task] = load_tasks(["long-output"])
task.seed_into(tmp_path)
run = subprocess.run(
[sys.executable, "-m", "pytest", "-v"],
cwd=tmp_path,
capture_output=True,
text=True,
check=False,
)
assert len(run.stdout) > 4 * Tuning().tools.max_output_chars
assert "R237" in run.stdout[-3000:] # the failure is named at the end
def test_a_changed_task_gets_a_new_hash(tmp_path: Path) -> None:
[task] = load_tasks(["write-function"])
shutil.copytree(task.path, tmp_path / "write-function")
[copy] = load_tasks(root=tmp_path)
assert copy.content_hash == task.content_hash
(copy.path / "seed" / "README.md").write_text("changed\n")
assert copy.content_hash != task.content_hash
def test_an_unknown_task_is_refused() -> None:
with pytest.raises(ValueError, match="unknown task"):
load_tasks(["nope"])
# --- running a task ----------------------------------------------------------
class Scripted:
"""A model that plays back fixed turns, reporting itself as `answered`."""
def __init__(self, turns: list[list[Any]], answered: str = "local-fast") -> None:
self.turns = turns
self.last_usage = {"input": 1000, "output": 50}
self.last_cost = 0.0
self.last_model = answered
self.trimmed_requests = 0
def complete(self, events, tools, model, system=None):
if not self.turns:
return [AssistantText(text="nothing left to do", model=model)]
return self.turns.pop(0)
def call(tool: str, cid: str, **arguments: Any) -> ToolCall:
return ToolCall(call_id=cid, tool=tool, arguments=arguments)
SOLVE_SLUGIFY = [
[
call(
"write_file",
"w",
path="slugify.py",
content=(
"import re\n\n\ndef slugify(text):\n"
" return re.sub(r'[^a-z0-9]+', '-', text.lower()).strip('-')\n"
),
)
],
[call("run_command", "r", command="python -c 'import slugify'")],
[call("finish", "f", summary="added slugify")],
]
def test_a_solved_task_is_a_success(tmp_path: Path) -> None:
[task] = load_tasks(["write-function"])
record = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), note="n")
assert record.success
assert record.outcome == "completed"
assert record.turns == 3
assert record.tokens_in == 3000
assert record.task_hash == task.content_hash
assert record.model_info["supports_tool_choice"] is True
assert record.tuning["executor"]["max_turns"] == 24
def test_saying_finished_is_not_success(tmp_path: Path) -> None:
# Success is the hidden check passing, never the model's own claim.
[task] = load_tasks(["write-function"])
wrong = [
[
call(
"write_file",
"w",
path="slugify.py",
content="def slugify(t):\n return t\n",
)
],
[call("finish", "f", summary="done")],
]
record = run_task(task, FACTS, Scripted(wrong), Tuning())
assert record.outcome == "completed"
assert not record.success
def test_misbehaviour_is_counted(tmp_path: Path) -> None:
[task] = load_tasks(["write-function"])
turns = [
[call("chat", "x1")],
[call("run_command", "r1", command="exit 1")],
[AssistantText(text="let me explain", model="m")],
]
# A higher failure limit, so the invented tool and the failed command
# don't escalate before the prose turn is reached.
tuning = Tuning(executor=ExecutorTuning(failure_limit=5))
record = run_task(task, FACTS, Scripted(turns), tuning)
assert record.invented_tools == 1
assert record.failed_commands == 1
assert record.prose_turns >= 1
assert not record.success
def test_a_fallback_answer_can_never_be_a_success(tmp_path: Path) -> None:
[task] = load_tasks(["write-function"])
client = Scripted(list(SOLVE_SLUGIFY), answered="claude-sonnet-5")
record = run_task(task, FACTS, client, Tuning())
assert record.fallbacks == ["claude-sonnet-5"] * 3
assert not record.success # the check passed, but not for this model
# --- storing and reporting ---------------------------------------------------
def test_wilson_intervals_behave() -> None:
low, high = wilson(5, 5)
assert high == 1.0 and 0.5 < low < 0.6
# 3 of 5 and 5 of 5 overlap: five runs do not separate them.
assert wilson(3, 5)[1] > wilson(5, 5)[0]
assert wilson(0, 0) == (0.0, 1.0)
def record_json(**overrides: Any) -> dict[str, Any]:
[task] = load_tasks(["write-function"])
base = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning()).to_json()
base.update(overrides)
return base
def test_results_round_trip_and_report(tmp_path: Path) -> None:
from nisse_dev.bench.runner import RunRecord
rec = RunRecord(**record_json())
path = results_file(tmp_path, "local-fast")
append(path, rec)
append(path, RunRecord(**record_json(success=False)))
stored = load(tmp_path)
assert len(stored) == 2
assert json.loads(path.read_text().splitlines()[0])["model"] == "local-fast"
current = {t.name: t.content_hash for t in TASKS}
[row] = summarise(stored, current)
assert (row.successes, row.runs) == (1, 2)
assert not row.stale
table = render([row])
assert "openai/qwen3-8b-awq" in table
assert "1/2" in table
def test_runs_of_an_older_task_version_are_flagged() -> None:
current = {t.name: t.content_hash for t in TASKS}
rows = summarise([record_json(task_hash="0123456789abcdef")], current)
assert rows[0].stale
assert "(old)" in render(rows)
def test_models_served_differently_are_kept_apart() -> None:
current = {t.name: t.content_hash for t in TASKS}
rows = summarise(
[record_json(note="vLLM, AWQ"), record_json(note="llama.cpp, Q4_K_M")], current
)
assert len(rows) == 2
def test_metering_counts_prose_and_cost() -> None:
class Inner:
def __init__(self) -> None:
self.last_usage = {"input": 10, "output": 2}
self.last_cost = 0.5
self.last_model = "local-fast"
def complete(self, events, tools, model, system=None):
return [AssistantText(text="hi", model=model)]
meter = MeteringClient(Inner(), FACTS)
meter.complete([], None, "local-fast")
meter.complete([], None, "local-fast")
assert (meter.calls, meter.prose_turns, meter.cost) == (2, 2, 1.0)
scripts/benchmark.py
+124
View File
@@ -0,0 +1,124 @@
"""Benchmark executor models on canned milestones, and compare the results.
uv run python scripts/benchmark.py run --models A,B [--runs 5] [--tasks T,...]
[--note "vLLM 0.29, AWQ, fp8 KV"] [--config PATH]
uv run python scripts/benchmark.py report [--models A,B] [--tasks T,...]
`run` appends one JSON line per run to bench/results/, which is committed:
the report compares every run ever stored, so benchmarking a new model means
running only that model. See docs/Tuning-Guide.md, "Benchmarking models".
"""
import argparse
import os
import sys
from pathlib import Path
from nisse_dev.agent import make_llm_clients
from nisse_dev.bench.results import append, load, render, results_file, summarise
from nisse_dev.bench.runner import agent_commit, run_task
from nisse_dev.bench.tasks import load_tasks
from nisse_dev.catalog import ModelCatalog
from nisse_dev.config import AgentConfig, ConfigError
from nisse_dev.envfile import load_env_file
REPO = Path(__file__).resolve().parents[1]
RESULTS = REPO / "bench" / "results"
def _names(raw: str | None) -> list[str] | None:
return [n.strip() for n in raw.split(",") if n.strip()] if raw else None
def run(args: argparse.Namespace) -> int:
load_env_file(Path(os.environ.get("NISSE_ENV", ".env")).expanduser())
try:
config = (
AgentConfig.from_file(Path(args.config))
if args.config
else AgentConfig.load()
)
except ConfigError as exc:
print(exc, file=sys.stderr)
return 2
models = _names(args.models) or []
tasks = load_tasks(_names(args.tasks))
catalog = ModelCatalog.discover(config.llm_base_url, config.llm_token)
catalog.require_limits({m: m for m in models})
normalized, _passthrough = make_llm_clients(config, catalog)
commit = agent_commit(REPO)
for model in models:
facts = catalog.facts(model)
out = results_file(args.results, model)
for task in tasks:
for n in range(args.runs):
# The executor always takes the normalized route in the agent,
# so it does here too.
record = run_task(
task,
facts,
normalized,
config.tuning,
note=args.note,
commit=commit,
)
append(out, record)
verdict = "pass" if record.success else f"FAIL ({record.outcome})"
extra = (
f" fallback to {record.fallbacks[0]}!" if record.fallbacks else ""
)
print(
f"{model} {task.name} run {n + 1}/{args.runs}: {verdict}, "
f"{record.turns} turns, {record.wall_s:.0f}s, "
f"${record.cost:.4f}{extra}",
flush=True,
)
print()
return report(args)
def report(args: argparse.Namespace) -> int:
records = load(args.results)
models = _names(getattr(args, "models", None))
tasks = _names(getattr(args, "tasks", None))
if models:
records = [r for r in records if r["model"] in models or r["backing"] in models]
if tasks:
records = [r for r in records if r["task"] in tasks]
if not records:
print("no stored results match", file=sys.stderr)
return 1
current = {task.name: task.content_hash for task in load_tasks()}
print(render(summarise(records, current)))
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--results", type=Path, default=RESULTS, help=argparse.SUPPRESS)
sub = parser.add_subparsers(dest="command", required=True)
run_p = sub.add_parser("run", help="benchmark models, then report")
run_p.add_argument(
"--models", required=True, help="LiteLLM aliases, comma separated"
)
run_p.add_argument("--runs", type=int, default=5, help="runs per task (default 5)")
run_p.add_argument("--tasks", help="task names, comma separated (default: all)")
run_p.add_argument(
"--note", default="", help='how the model was served, e.g. "vLLM 0.29, AWQ"'
)
run_p.add_argument("--config", help="agent config (default: NISSE_CONFIG)")
run_p.set_defaults(handler=run)
report_p = sub.add_parser("report", help="compare stored results")
report_p.add_argument("--models", help="aliases or backing models to include")
report_p.add_argument("--tasks", help="tasks to include")
report_p.set_defaults(handler=report)
args = parser.parse_args()
return args.handler(args)
if __name__ == "__main__":
raise SystemExit(main())
scripts/evaluate_model.sh
Executable
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env bash
# Evaluate a model for the executor role, from a declared LiteLLM alias to a
# benchmark report next to every model measured before it.
#
# scripts/evaluate_model.sh ALIAS "how it is served" [RUNS] [--config PATH]
#
# 1. The context probe: does the server's window match what LiteLLM declares?
# 2. The capability matrix: does it hold the tool contract, and is its
# supports_tool_choice declaration true?
# 3. The benchmark: canned milestones, RUNS times each (default 5).
# Stops at the first step that fails; there is no point benchmarking a model
# that can't hold the tool contract.
set -euo pipefail
cd "$(dirname "$0")/.."
alias="${1:?usage: evaluate_model.sh ALIAS NOTE [RUNS] [--config PATH]}"
note="${2:?give a note on how the model is served, e.g. \"vLLM 0.29, AWQ\"}"
runs="${3:-5}"
shift $(( $# < 3 ? $# : 3 ))
config=("$@")
echo "== 1/3 context probe"
uv run python scripts/probe_context.py "$alias" "${config[@]}"
echo "== 2/3 capability matrix"
uv run pytest -m integration -q -rs tests/integration/test_model_capability.py \
--capability-models="$alias"
echo "== 3/3 benchmark ($runs runs per task)"
uv run python scripts/benchmark.py run --models "$alias" --runs "$runs" \
--note "$note" "${config[@]}"
tests/integration/test_benchmark_live.py
+31
View File
@@ -0,0 +1,31 @@
"""One live benchmark run: the simplest task, through the agent's own
client wiring, with the test config's executor."""
from pathlib import Path
import pytest
from nisse_dev.agent import make_llm_clients
from nisse_dev.bench.runner import run_task
from nisse_dev.bench.tasks import load_tasks
from nisse_dev.catalog import ModelCatalog
from nisse_dev.config import AgentConfig
pytestmark = pytest.mark.integration
def test_the_simplest_task_passes_live(
live_config: AgentConfig, tmp_path: Path
) -> None:
catalog = ModelCatalog.discover(live_config.llm_base_url, live_config.llm_token)
normalized, _ = make_llm_clients(live_config, catalog)
facts = catalog.facts(live_config.roles.executor)
[task] = load_tasks(["write-function"])
record = run_task(task, facts, normalized, live_config.tuning, workdir=tmp_path)
assert record.fallbacks == []
assert record.success, (
f"{record.outcome}: {record.outcome_note}\n{record.check_output}"
)
assert record.cost >= 0