diff --git a/bench/results/.gitkeep b/bench/results/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/bench/results/2026-09-25-local-coder.jsonl b/bench/results/2026-09-25-local-coder.jsonl new file mode 100644 index 0000000..0ae8fc4 --- /dev/null +++ b/bench/results/2026-09-25-local-coder.jsonl @@ -0,0 +1,20 @@ +{"backing": "openai/qwen3-14b", "check_output": "...... [100%]\n6 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Fixed total_value to multiply unit price by quantity. All tests now pass.", "prose_turns": 0, "provider": "openai", "success": true, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 5328, "tokens_out": 1492, "trimmed_requests": 0, "ts": "2026-09-25T01:03:24+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 46.4} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\nImportError while importing test module '/tmp/tmpr1q1pe0r/work/.bench_check/test_hidden.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n.bench_check/test_hidden.py:4: in \n from inventory import Item, low_stock, total_value\nE ImportError: cannot import name 'Item' from 'inventory' (/tmp/tmpr1q1pe0r/work/inventory.py)\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.10s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n______________________ ERROR collecting test_inventory.py ______________________\nImportError while importing test module '/tmp/tmpr1q1pe0r/work/test_inventory.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], packa", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 2560, "tokens_out": 1136, "trimmed_requests": 0, "ts": "2026-09-25T01:03:49+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 3, "wall_s": 24.3} +{"backing": "openai/qwen3-14b", "check_output": "...... [100%]\n6 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Fixed the total_value function to multiply unit price by quantity, passing all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 5318, "tokens_out": 1623, "trimmed_requests": 0, "ts": "2026-09-25T01:04:23+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 34.5} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n.bench_check/test_hidden.py:4: in \n from inventory import Item, low_stock, total_value\ninventory.py:1: in \n def total_value(items: list[Item]) -> float:\n ^^^^\nE NameError: name 'Item' is not defined\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py - NameError: name 'Item' is not defined\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.11s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n______________________ ERROR collecting test_inventory.py ______________________\ntest_inventory.py:1: in \n from inventory import Item, low_stock, total_value\ninventory.py:1: in \n def total_value(items: list[Item]) -> float:\n ^^^^\nE NameError: name 'Item' is not defined\n=========================== short ", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 2565, "tokens_out": 958, "trimmed_requests": 0, "ts": "2026-09-25T01:04:45+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 3, "wall_s": 20.6} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n.bench_check/test_hidden.py:4: in \n from inventory import Item, low_stock, total_value\ninventory.py:1: in \n def total_value(items: list[Item]) -> float:\n ^^^^\nE NameError: name 'Item' is not defined\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py - NameError: name 'Item' is not defined\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.09s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n______________________ ERROR collecting test_inventory.py ______________________\ntest_inventory.py:1: in \n from inventory import Item, low_stock, total_value\ninventory.py:1: in \n def total_value(items: list[Item]) -> float:\n ^^^^\nE NameError: name 'Item' is not defined\n=========================== short ", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 2565, "tokens_out": 1051, "trimmed_requests": 0, "ts": "2026-09-25T01:05:07+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 3, "wall_s": 22.6} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmpq8iz1j8z/work/rates.py\", line 56\nE [... 1445 characters omitted ...]\nE ^^^^^^^^\nE SyntaxError: invalid syntax. Perhaps you forgot a comma?\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 13616, "tokens_out": 3186, "trimmed_requests": 0, "ts": "2026-09-25T01:06:21+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 72.8} +{"backing": "openai/qwen3-14b", "check_output": ": 'R271'\nFAILED .bench_check/test_hidden.py::test_rate[272] - KeyError: 'R272'\nFAILED .bench_check/test_hidden.py::test_rate[273] - KeyError: 'R273'\nFAILED .bench_check/test_hidden.py::test_rate[274] - KeyError: 'R274'\nFAILED .bench_check/test_hidden.py::test_rate[275] - KeyError: 'R275'\nFAILED .bench_check/test_hidden.py::test_rate[276] - KeyError: 'R276'\nFAILED .bench_check/test_hidden.py::test_rate[277] - KeyError: 'R277'\nFAILED .bench_check/test_hidden.py::test_rate[278] - KeyError: 'R278'\nFAILED .bench_check/test_hidden.py::test_rate[279] - KeyError: 'R279'\nFAILED .bench_check/test_hidden.py::test_rate[280] - KeyError: 'R280'\nFAILED .bench_check/test_hidden.py::test_rate[281] - KeyError: 'R281'\nFAILED .bench_check/test_hidden.py::test_rate[282] - KeyError: 'R282'\nFAILED .bench_check/test_hidden.py::test_rate[283] - KeyError: 'R283'\nFAILED .bench_check/test_hidden.py::test_rate[284] - KeyError: 'R284'\nFAILED .bench_check/test_hidden.py::test_rate[285] - KeyError: 'R285'\nFAILED .bench_check/test_hidden.py::test_rate[286] - KeyError: 'R286'\nFAILED .bench_check/test_hidden.py::test_rate[287] - KeyError: 'R287'\nFAILED .bench_check/test_hidden.py::test_rate[288] - KeyError: 'R288'\nFAILED .bench_check/test_hidden.py::test_rate[289] - KeyError: 'R289'\nFAILED .bench_check/test_hidden.py::test_rate[290] - KeyError: 'R290'\nFAILED .bench_check/test_hidden.py::test_rate[291] - KeyError: 'R291'\nFAILED .bench_check/test_hidden.py::test_rate[292] - KeyError: 'R292'\nFAILED .bench_check/test_hidden.py::test_rate[293] - KeyError: 'R293'\nFAILED .bench_check/test_hidden.py::test_rate[294] - KeyError: 'R294'\nFAILED .bench_check/test_hidden.py::test_rate[295] - KeyError: 'R295'\nFAILED .bench_check/test_hidden.py::test_rate[296] - KeyError: 'R296'\nFAILED .bench_check/test_hidden.py::test_rate[297] - KeyError: 'R297'\nFAILED .bench_check/test_hidden.py::test_rate[298] - KeyError: 'R298'\nFAILED .bench_check/test_hidden.py::test_rate[299] - KeyError: 'R299'\n247 failed, 54 passed in 0.49s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n.....................................................FFFFFFFFFFFFFFFFFFF [ 24%]\nFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF [ 48%]\nFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF [ 72%]\nFFFFFFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF [ 96%]\nFFFFFFFFFFFF [100%]\n=================================== FAILU", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 12726, "tokens_out": 2366, "trimmed_requests": 0, "ts": "2026-09-25T01:07:16+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 54.4} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmphac71897/work/rates.py\", line 56\nE [... 1445 characters omitted ...]\nE ^^^^^^^^\nE SyntaxError: invalid syntax. Perhaps you forgot a comma?\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.15s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 13614, "tokens_out": 3000, "trimmed_requests": 0, "ts": "2026-09-25T01:08:25+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 69.2} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmp0vs0w4kr/work/rates.py\", line 56\nE [... 1445 characters omitted ...]\nE ^^^^^^^^\nE SyntaxError: invalid syntax. Perhaps you forgot a comma?\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 13607, "tokens_out": 3223, "trimmed_requests": 0, "ts": "2026-09-25T01:09:40+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 73.8} +{"backing": "openai/qwen3-14b", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmp_pxm0ldr/work/rates.py\", line 56\nE [... 1445 characters omitted ...]\nE ^^^^^^^^\nE SyntaxError: invalid syntax. Perhaps you forgot a comma?\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.15s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 13609, "tokens_out": 3180, "trimmed_requests": 0, "ts": "2026-09-25T01:10:53+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 73.0} +{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x777eb60e3e90>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 13753, "tokens_out": 5755, "trimmed_requests": 0, "ts": "2026-09-25T01:12:55+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 121.2} +{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added --shout option to greeter, updated core and cli code, added tests, and verified with pytest.", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 21373, "tokens_out": 9952, "trimmed_requests": 0, "ts": "2026-09-25T01:16:25+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 13, "wall_s": 210.0} +{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x7a2788fdbe30>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9432, "tokens_out": 6691, "trimmed_requests": 0, "ts": "2026-09-25T01:18:47+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 142.1} +{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added --shout option to greeter, updated core and cli code, added tests, and verified with pytest.", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9104, "tokens_out": 3626, "trimmed_requests": 0, "ts": "2026-09-25T01:20:03+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 75.8} +{"backing": "openai/qwen3-14b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x76eac82e3da0>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9382, "tokens_out": 7755, "trimmed_requests": 0, "ts": "2026-09-25T01:22:48+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 164.5} +{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function and passed all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 1769, "tokens_out": 1300, "trimmed_requests": 0, "ts": "2026-09-25T01:23:16+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 27.7} +{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function with tests; all tests passed.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 1782, "tokens_out": 1239, "trimmed_requests": 0, "ts": "2026-09-25T01:23:43+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 26.2} +{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function and passed all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3627, "tokens_out": 1332, "trimmed_requests": 0, "ts": "2026-09-25T01:24:11+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 28.1} +{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function and passed all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 734, "tokens_out": 653, "trimmed_requests": 0, "ts": "2026-09-25T01:24:25+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 1, "wall_s": 13.9} +{"backing": "openai/qwen3-14b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682+dirty", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-coder", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": false}, "note": "llama.cpp b11028, Qwen3-14B Q4_K_M, q8 KV, 32k", "outcome": "completed", "outcome_note": "Added slugify function and passed all tests.", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 733, "tokens_out": 523, "trimmed_requests": 0, "ts": "2026-09-25T01:24:37+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 1, "wall_s": 11.3} diff --git a/bench/results/2026-09-25-local-fast.jsonl b/bench/results/2026-09-25-local-fast.jsonl new file mode 100644 index 0000000..1cd5ac9 --- /dev/null +++ b/bench/results/2026-09-25-local-fast.jsonl @@ -0,0 +1,20 @@ +{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1578, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:41+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 63.2} +{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1576, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:43+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4} +{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1576, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:44+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4} +{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1574, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:46+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4} +{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1578, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:48+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4} +{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmpl4mdto5g/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11000, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:00:52+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2} +{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmpsqvx_94y/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11000, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:00:57+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2} +{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmpe3eakwxj/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11013, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:01:01+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2} +{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmprlm2camm/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 10987, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:01:06+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2} +{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in \n from rates import RATES\nE File \"/tmp/tmpvzw7f5tn/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11000, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:01:11+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2} +{"backing": "openai/qwen3-8b-awq", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n...F [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x713c88ce3c20>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 13039, "tokens_out": 566, "trimmed_requests": 0, "ts": "2026-09-25T01:01:19+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 8.4} +{"backing": "openai/qwen3-8b-awq", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n________________________________ test_cli_shout ________________________________\n\n def test_cli_shout() -> None:\n> from pytest import capsys\nE ImportError: cannot import name 'capsys' from 'pytest' (/root/nisse/.venv/lib/python3.12/site-packages/pytest/__init__.py)", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 11243, "tokens_out": 715, "trimmed_requests": 0, "ts": "2026-09-25T01:01:29+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 9.9} +{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:4: in \n from greeter.cli import main\nE File \"/tmp/tmp3f09u7jk/work/greeter/cli.py\", line 11\nE print(greet(args.name, shout=args.--shout))\nE ^\nE SyntaxError: invalid syntax\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n\nstderr:\nTraceback (most recent call last):\n File \"\", line 198, in _run_module_as_main\n File \"\", line 88, in _run_code\n File \"/tmp/tmp3f09u7jk/work/greeter/__main__.py\", line 1, in \n from greeter.cli import main\n File \"/tmp/tmp3f09u7jk/work/greeter/cli.py\", line 11\n print(greet(args.name, shout=args.--shout))\n ^\nSyntaxError: invalid syntax\n", "prose_turns": 0, "provider": "openai", "success": false, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9358, "tokens_out": 450, "trimmed_requests": 0, "ts": "2026-09-25T01:01:56+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 26.5} +{"backing": "openai/qwen3-8b-awq", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "completed", "outcome_note": "Added --shout option to greeter. Tests pass.", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 10784, "tokens_out": 549, "trimmed_requests": 0, "ts": "2026-09-25T01:02:04+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 7.5} +{"backing": "openai/qwen3-8b-awq", "check_output": " stderr arguments may not be used '\n 'with capture_output.')\n kwargs['stdout'] = PIPE\n kwargs['stderr'] = PIPE\n \n with Popen(*popenargs, **kwargs) as process:\n try:\n stdout, stderr = process.communicate(input, timeout=timeout)\n except TimeoutExpired as exc:\n process.kill()\n if _mswindows:\n # Windows accumulates the output in a single blocking\n # read() call run on child threads, with the timeout\n # being done in a join() on those threads. communicate()\n # _after_ kill() is required to collect that and add it\n # to the exception.\n exc.stdout, exc.stderr = process.communicate()\n else:\n # POSIX _communicate already populated the output so\n # far into the TimeoutExpired exception.\n process.wait()\n raise\n except: # Including KeyboardInterrupt, communicate handled that.\n process.kill()\n # We don't call process.wait() as .__exit__ does that for us.\n raise\n retcode = process.poll()\n if check and retcode:\n> raise CalledProcessError(retcode, process.args,\n output=stdout, stderr=stderr)\nE subprocess.CalledProcessError: Command '['/root/nisse/.venv/bin/python', '-m', 'greeter', '--shout', 'Bob']' returned non-zero exit status 2.\n\n/usr/lib/python3.12/subprocess.py:571: CalledProcessError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_cli_default - SystemExit: 2\nFAILED .bench_check/test_hidden.py::test_cli_shout - SystemExit: 2\nFAILED .bench_check/test_hidden.py::test_module_entry_point - subprocess.Call...\n3 failed, 2 passed in 0.19s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..F [100%]\n=================================== FAILURES ===================================\n________________________________ test_cli_shout ________________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x763f352e3980>\n\n def test_cli_shout(capsys) -> None:\n> assert main([\"--shout\", \"Bob\"]) == 0\n ^^^^^^^^^^^^^^^^^^^^^^^^\n\ntest", "prose_turns": 0, "provider": "openai", "success": false, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 13124, "tokens_out": 589, "trimmed_requests": 0, "ts": "2026-09-25T01:02:13+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 8.4} +{"backing": "openai/qwen3-8b-awq", "check_output": " (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'leading-and-trailing'\nE \nE - leading-and-trailing\n\n.bench_check/test_hidden.py:19: AssertionError\n_____________________ test_slugify[MiXeD CaSe-mixed-case] ______________________\n\ntext = 'MiXeD CaSe', expected = 'mixed-case'\n\n @pytest.mark.parametrize(\n (\"text\", \"expected\"),\n [\n (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'mixed-case'\nE \nE - mixed-case\n\n.bench_check/test_hidden.py:19: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_slugify[Hello, World!-hello-world]\nFAILED .bench_check/test_hidden.py::test_slugify[ Nisse Agent -nisse-agent]\nFAILED .bench_check/test_hidden.py::test_slugify[a--b-a-b] - AssertionError: ...\nFAILED .bench_check/test_hidden.py::test_slugify[Python 3.12 Release-python-3-12-release]\nFAILED .bench_check/test_hidden.py::test_slugify[--leading and trailing---leading-and-trailing]\nFAILED .bench_check/test_hidden.py::test_slugify[MiXeD CaSe-mixed-case] - Ass...\n6 failed, 2 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\nE AssertionError: assert '' == 'hello-world'\nE \nE - hello-world\n\ntest_slugify.py:6: AssertionEr", "prose_turns": 0, "provider": "openai", "success": false, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 4942, "tokens_out": 377, "trimmed_requests": 0, "ts": "2026-09-25T01:02:19+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 5.5} +{"backing": "openai/qwen3-8b-awq", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\n ^^^^^^^\nE NameError: name 'slugify' is not defined\n\ntest_slugify.py:4: NameError\n===============", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3688, "tokens_out": 255, "trimmed_requests": 0, "ts": "2026-09-25T01:02:23+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 3.9} +{"backing": "openai/qwen3-8b-awq", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\n ^^^^^^^\nE NameError: name 'slugify' is not defined\n\ntest_slugify.py:4: NameError\n===============", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3696, "tokens_out": 255, "trimmed_requests": 0, "ts": "2026-09-25T01:02:27+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 3.9} +{"backing": "openai/qwen3-8b-awq", "check_output": " (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'leading-and-trailing'\nE \nE - leading-and-trailing\n\n.bench_check/test_hidden.py:19: AssertionError\n_____________________ test_slugify[MiXeD CaSe-mixed-case] ______________________\n\ntext = 'MiXeD CaSe', expected = 'mixed-case'\n\n @pytest.mark.parametrize(\n (\"text\", \"expected\"),\n [\n (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'mixed-case'\nE \nE - mixed-case\n\n.bench_check/test_hidden.py:19: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_slugify[Hello, World!-hello-world]\nFAILED .bench_check/test_hidden.py::test_slugify[ Nisse Agent -nisse-agent]\nFAILED .bench_check/test_hidden.py::test_slugify[a--b-a-b] - AssertionError: ...\nFAILED .bench_check/test_hidden.py::test_slugify[Python 3.12 Release-python-3-12-release]\nFAILED .bench_check/test_hidden.py::test_slugify[--leading and trailing---leading-and-trailing]\nFAILED .bench_check/test_hidden.py::test_slugify[MiXeD CaSe-mixed-case] - Ass...\n6 failed, 2 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\nE AssertionError: assert '' == 'hello-world'\nE \nE - hello-world\n\ntest_slugify.py:6: AssertionEr", "prose_turns": 0, "provider": "openai", "success": false, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 4936, "tokens_out": 377, "trimmed_requests": 0, "ts": "2026-09-25T01:02:33+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 5.5} +{"backing": "openai/qwen3-8b-awq", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\n ^^^^^^^\nE NameError: name 'slugify' is not defined\n\ntest_slugify.py:4: NameError\n===============", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3686, "tokens_out": 251, "trimmed_requests": 0, "ts": "2026-09-25T01:02:37+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 3.9} diff --git a/docs/Tuning-Guide.md b/docs/Tuning-Guide.md index af4c631..8977f3f 100644 --- a/docs/Tuning-Guide.md +++ b/docs/Tuning-Guide.md @@ -370,6 +370,66 @@ declaration. --- +## Benchmarking models + +The capability matrix asks whether a model *can* hold the tool contract. The +benchmark asks whether it *finishes real work*: four canned milestones, each an +approved plan over a small seed project, judged by a hidden check the model +never sees. Success means the check passes, not that the model said it was +done. + +| Task | What it exercises | +|---|---| +| `write-function` | write a small function and its tests | +| `fix-bug` | find and fix the bug a failing test exposes, without touching the test | +| `multi-file` | thread a new option through several files | +| `long-output` | find one failure in test output far longer than `max_output_chars` | + +It runs the executor exactly as the agent does (the same client wiring, +context budget and `tool_choice` handling, and your config's tuning knobs), +with **delegation and `ask_expert` turned off**. An escalation is a failure of +the model under test, not a frontier model's success. + +**Evaluating a new model** is one command, once its LiteLLM entry declares its +limits: + + scripts/evaluate_model.sh qwen3-32b-q4 "vLLM 0.29, AWQ, fp8 KV" 5 + +It runs the context probe, then the capability matrix, then five runs of +every task, and stops at the first step that fails. Or run the steps +yourself: + + uv run python scripts/benchmark.py run --models A,B --runs 5 --note "how it's served" + uv run python scripts/benchmark.py report [--models A,B] [--tasks T] + +**Results are kept.** Every run is one line in `bench/results/`, committed +with the repo, recording: +- the model's backing name and declared facts; +- your note on how it was served; +- the task's content hash; +- the agent's commit and tuning; +- the outcome, turns, failures, tokens, time and cost. + +The report rebuilds the table from every stored run. Benchmarking a new model +means running only that model and reading it next to everything before it. +Runs against an older version of a task are marked `(old)`. Runs of one model +served differently (different note) are kept apart. + +**Reading it:** +- **Compare the intervals, not the rates.** 3 of 5 (23–88%) and 5 of 5 + (57–100%) overlap: five runs don't separate them. Add runs until the + intervals stop overlapping, or accept that the models are close. +- **Look at why runs failed**, not just how many (`outcome_note` and the + counts in each result line). An early shakedown on `local-fast` failed + mostly by escalation: the model was fixing its own mistakes and hit + `failure_limit: 2` mid-recovery. That says as much about the knob as about + the model. Tuning a knob and re-running is a fair comparison, because the + tuning is recorded with every run. +- **Cost** is per run, straight from LiteLLM, so a frontier model as executor + can be compared on success rate *and* dollars. + +--- + ## Part 3 — An order to tune in Upstream choices change what's right downstream, so tune in this order and @@ -400,8 +460,8 @@ change one thing at a time. 3. **Declare the window** in LiteLLM and check it with the probe, then set the model's `max_output_tokens` there so the prompt keeps most of the window. -4. **Real milestones.** Run representative work and look at the evidence: - turns used, escalations, delegation outcomes, and time taken. +4. **The benchmark.** Run it and look at the evidence: success rate, turns, + escalations, and time taken. 5. **The loop knobs**: `failure_limit`, `max_turns`, `consult_budget`, the delegation caps, then the tool limits, one at a time. @@ -415,11 +475,10 @@ run enough times that the result would survive a coin flip. ## Coming next -The next changes in the 4090-readiness work add knobs, and this guide gains a -section for each: +Later work that will add knobs, each with its own section here: - **Compaction.** A model-written summary replacing old turns, instead of dropping them. The log already has an event for it; it needs a summarizing model and a policy for what is safe to summarize. -- **A benchmark script** that runs the capability matrix and representative - milestones across candidate models and prints a comparison table. +- **Benchmarking the other roles:** planner and delegate, with delegation + turned back on for comparisons across frontier tiers. diff --git a/nisse-dev/src/nisse_dev/agent.py b/nisse-dev/src/nisse_dev/agent.py index 4f1a04b..966f56d 100644 --- a/nisse-dev/src/nisse_dev/agent.py +++ b/nisse-dev/src/nisse_dev/agent.py @@ -113,6 +113,51 @@ class Agent: self._stopped.set() +def make_llm_clients( + config: AgentConfig, catalog: ModelCatalog +) -> tuple[BudgetedClient, BudgetedClient]: + """The two LLM routes, as every model call in the agent uses them. + + Shared with the benchmark, so that what it measures is exactly what the + agent would do: the same context budget, the same declared limits, and + tool_choice asked only of models declared to enforce it. + """ + tuning = config.tuning + + def limits_for(model: str) -> tuple[int, int | None]: + facts = catalog.facts(model) + assert facts.max_input_tokens is not None # require_limits checked + return facts.max_input_tokens, facts.max_output_tokens + + def budgeted(client: object) -> BudgetedClient: + # Every request, on either route, is fitted to its model's window. + return BudgetedClient( + client, + limits_for, + max_output_tokens=config.max_output_tokens, + keep_recent_turns=tuning.context.keep_recent_turns, + safety_margin=tuning.context.safety_margin, + enforces_tool_choice=lambda m: ( + catalog.facts(m).supports_tool_choice is True + ), + ) + + normalized = budgeted( + LiteLLMClient( + config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens + ) + ) + # Delegations prefer the vendor-native route; fidelity is cheap + # elsewhere (A5). Which route a given model gets is decided by + # select_client, not by where the caller happens to sit. + passthrough = budgeted( + AnthropicPassthroughClient( + config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens + ) + ) + return normalized, passthrough + + def prepare_workspace(config: AgentConfig, account: str) -> GitRepo: """Ensure the workspace is a clone of the project repo. @@ -161,37 +206,7 @@ def build_agent( max_output_chars=tuning.tools.max_output_chars, ) - def limits_for(model: str) -> tuple[int, int | None]: - facts = catalog.facts(model) - assert facts.max_input_tokens is not None # require_limits checked - return facts.max_input_tokens, facts.max_output_tokens - - def budgeted(client: object) -> BudgetedClient: - # Every request, on either route, is fitted to its model's window. - return BudgetedClient( - client, - limits_for, - max_output_tokens=config.max_output_tokens, - keep_recent_turns=tuning.context.keep_recent_turns, - safety_margin=tuning.context.safety_margin, - enforces_tool_choice=lambda m: ( - catalog.facts(m).supports_tool_choice is True - ), - ) - - normalized = budgeted( - LiteLLMClient( - config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens - ) - ) - # Delegations prefer the vendor-native route; fidelity is cheap - # elsewhere (A5). Which route a given model gets is decided by - # select_client, not by where the caller happens to sit. - passthrough = budgeted( - AnthropicPassthroughClient( - config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens - ) - ) + normalized, passthrough = make_llm_clients(config, catalog) transport = MatrixTransport(config.matrix) announce = notify or transport.send_message diff --git a/nisse-dev/src/nisse_dev/bench/__init__.py b/nisse-dev/src/nisse_dev/bench/__init__.py new file mode 100644 index 0000000..1e5b0a7 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/__init__.py @@ -0,0 +1,17 @@ +"""The executor benchmark: canned milestones, run repeatedly per model. + +Choosing an executor model, and later tuning its knobs, should rest on +measured success rates, not on one run or on reputation. Each task is a fixed +approved plan over a seed project, with a hidden check the model never sees; +success means the check passes, not that the model said it finished. Runs use +the agent's own client wiring (context budget, declared limits, tool_choice +only where declared), so what is measured is what the agent would do. + +Results are stored, one JSON line per run, with everything needed to compare +them later: the model's identity and declared facts, a note on how it was +served, the task's content hash, the agent's commit, and its tuning. A new +model then needs only its own runs to be compared with everything measured +before it. + +See scripts/benchmark.py, and docs/Tuning-Guide.md for how to read results. +""" diff --git a/nisse-dev/src/nisse_dev/bench/results.py b/nisse-dev/src/nisse_dev/bench/results.py new file mode 100644 index 0000000..0e3fb43 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/results.py @@ -0,0 +1,117 @@ +"""Stored benchmark results, and the report that compares them. + +Every run is one JSON line. The report is rebuilt from all of them each +time, so a model benchmarked today is read alongside every model measured +before it, as long as the task is still the same version. +""" + +import json +import math +import statistics +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from nisse_dev.bench.runner import RunRecord + + +def results_file(directory: Path, model: str, when: datetime | None = None) -> Path: + day = (when or datetime.now(UTC)).strftime("%Y-%m-%d") + return directory / f"{day}-{model}.jsonl" + + +def append(path: Path, record: RunRecord) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as out: + out.write(json.dumps(record.to_json(), sort_keys=True) + "\n") + + +def load(directory: Path) -> list[dict[str, Any]]: + records = [] + for path in sorted(directory.glob("*.jsonl")): + for line in path.read_text().splitlines(): + if line.strip(): + records.append(json.loads(line)) + return records + + +def wilson(successes: int, runs: int, z: float = 1.96) -> tuple[float, float]: + """A 95% confidence interval for a success rate, sound at small counts. + + Three successes in five runs and five in five overlap here, which is the + point: small samples of a noisy model do not separate much.""" + if runs == 0: + return 0.0, 1.0 + p = successes / runs + denominator = 1 + z * z / runs + centre = (p + z * z / (2 * runs)) / denominator + half = z * math.sqrt(p * (1 - p) / runs + z * z / (4 * runs * runs)) / denominator + return max(0.0, centre - half), min(1.0, centre + half) + + +@dataclass +class Row: + model: str + note: str + task: str + runs: int + successes: int + median_turns: float + mean_wall_s: float + cost_per_run: float + stale: bool + + @property + def interval(self) -> tuple[float, float]: + return wilson(self.successes, self.runs) + + +def summarise( + records: Iterable[dict[str, Any]], current_hashes: dict[str, str] +) -> list[Row]: + """One row per (model, how it was served, task, task version).""" + groups: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list) + for r in records: + groups[(r["backing"], r.get("note", ""), r["task"], r["task_hash"])].append(r) + rows = [] + for (backing, note, task, task_hash), runs in sorted(groups.items()): + rows.append( + Row( + model=backing, + note=note, + task=task, + runs=len(runs), + successes=sum(1 for r in runs if r["success"]), + median_turns=statistics.median(r["turns"] for r in runs), + mean_wall_s=statistics.fmean(r["wall_s"] for r in runs), + cost_per_run=statistics.fmean(r["cost"] for r in runs), + stale=current_hashes.get(task) != task_hash, + ) + ) + return rows + + +def render(rows: list[Row]) -> str: + header = ( + f"{'model':<34} {'note':<22} {'task':<15} {'success':>8} {'95% CI':>11} " + f"{'turns':>6} {'time':>7} {'$/run':>8}" + ) + lines = [header, "-" * len(header)] + for row in rows: + low, high = row.interval + task = row.task + (" (old)" if row.stale else "") + lines.append( + f"{row.model[:34]:<34} {row.note[:22]:<22} {task[:15]:<15} " + f"{row.successes:>3}/{row.runs:<4} {low:>4.0%}-{high:<5.0%} " + f"{row.median_turns:>6.0f} {row.mean_wall_s:>6.0f}s {row.cost_per_run:>8.4f}" + ) + if any(row.stale for row in rows): + lines.append("") + lines.append( + "(old) = measured against an earlier version of the task; " + "not comparable with current runs." + ) + return "\n".join(lines) diff --git a/nisse-dev/src/nisse_dev/bench/runner.py b/nisse-dev/src/nisse_dev/bench/runner.py new file mode 100644 index 0000000..ba4d02a --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/runner.py @@ -0,0 +1,218 @@ +"""Running one task once with one executor model, and recording it.""" + +import io +import subprocess +import tempfile +import time +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from nisse_dev.bench.tasks import Task +from nisse_dev.canonical import ( + CanonicalEvent, + PlanSubmitted, + StoredEvent, + ToolCall, + ToolResult, + UserMessage, +) +from nisse_dev.catalog import ModelFacts +from nisse_dev.config import Tuning +from nisse_dev.executor import LLMExecutor +from nisse_dev.llm import RoleMap +from nisse_dev.oplog import OpLog +from nisse_dev.state import StateStore +from nisse_dev.tools import Workspace + + +class MeteringClient: + """Counts what a run spends, and notices a fallback: an answer from a + model other than the one asked for is not a measurement of it.""" + + def __init__(self, inner: Any, facts: ModelFacts) -> None: + self._inner = inner + self._facts = facts + self.calls = 0 + self.prose_turns = 0 + self.tokens_in = 0 + self.tokens_out = 0 + self.cost = 0.0 + self.fallbacks: list[str] = [] + + def complete( + self, + events: list[StoredEvent], + tools: list[dict[str, Any]] | None, + model: str, + system: str | None = None, + ) -> list[CanonicalEvent]: + self.calls += 1 + produced = self._inner.complete(events, tools, model, system=system) + usage = getattr(self._inner, "last_usage", {}) or {} + self.tokens_in += int(usage.get("input", 0)) + self.tokens_out += int(usage.get("output", 0)) + self.cost += float(getattr(self._inner, "last_cost", 0.0) or 0.0) + answered = getattr(self._inner, "last_model", None) + if not self._facts.answered(answered): + self.fallbacks.append(str(answered)) + if not any(isinstance(e, ToolCall) for e in produced): + self.prose_turns += 1 + return produced + + +@dataclass +class RunRecord: + """One run, with everything needed to compare it with runs made later + against other models, other servers, or other tuning.""" + + ts: str + model: str + backing: str + provider: str + model_info: dict[str, Any] + note: str + task: str + task_hash: str + commit: str + tuning: dict[str, Any] + success: bool + outcome: str + outcome_note: str + turns: int + prose_turns: int + failed_commands: int + invented_tools: int + trimmed_requests: int + tokens_in: int + tokens_out: int + cost: float + wall_s: float + fallbacks: list[str] = field(default_factory=list) + check_output: str = "" + + def to_json(self) -> dict[str, Any]: + return asdict(self) + + +def agent_commit(repo: Path | None = None) -> str: + """The agent's commit, marked when the tree had uncommitted changes.""" + where = repo or Path(__file__).resolve().parent + try: + sha = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + cwd=where, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + dirty = subprocess.run( + ["git", "status", "--porcelain"], + cwd=where, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "unknown" + return f"{sha}+dirty" if dirty else sha + + +def _count(events: list[StoredEvent]) -> tuple[int, int]: + tools = { + e.event.call_id: e.event.tool for e in events if isinstance(e.event, ToolCall) + } + failed = invented = 0 + for stored in events: + event = stored.event + if isinstance(event, ToolResult) and not event.success: + if event.result.startswith("no such tool"): + invented += 1 + elif tools.get(event.call_id) == "run_command": + failed += 1 + return failed, invented + + +def run_task( + task: Task, + facts: ModelFacts, + client: Any, + tuning: Tuning, + *, + note: str = "", + commit: str = "", + workdir: Path | None = None, +) -> RunRecord: + """Run one task once: seed a workspace, hand the executor the approved + plan, then judge the result with the hidden check. + + Delegation and ask_expert are off: an escalation is a failure of the + model under test, not a frontier model's success.""" + with tempfile.TemporaryDirectory(dir=workdir) as tmp: + root = Path(tmp) + workspace = Workspace( + root / "work", + timeout_s=tuning.tools.command_timeout_s, + max_output_chars=tuning.tools.max_output_chars, + ) + task.seed_into(workspace.root) + store = StateStore(root / "state.db", oplog=OpLog("nisse-bench", io.StringIO())) + store.append_event(PlanSubmitted(plan=task.plan, model="benchmark")) + store.append_event(UserMessage(text="start milestone 1", source="benchmark")) + + meter = MeteringClient(client, facts) + trims_before = getattr(client, "trimmed_requests", 0) + model = facts.alias + roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model) + executor = LLMExecutor( + meter, + store, + roles, + workspace, + max_turns=tuning.executor.max_turns, + failure_limit=tuning.executor.failure_limit, + ) + started = time.monotonic() + try: + outcome = executor.run(1) + status, detail = outcome.status, outcome.note + except Exception as exc: # noqa: BLE001 -- recorded, the benchmark goes on + status, detail = "error", repr(exc) + wall = time.monotonic() - started + + success, check_output = task.run_check(workspace.root) + failed, invented = _count(store.read_events()) + store.close() + + return RunRecord( + ts=datetime.now(UTC).isoformat(timespec="seconds"), + model=model, + backing=facts.backing, + provider=facts.provider, + model_info={ + "max_input_tokens": facts.max_input_tokens, + "max_output_tokens": facts.max_output_tokens, + "supports_tool_choice": facts.supports_tool_choice, + "supports_function_calling": facts.supports_tools, + }, + note=note, + task=task.name, + task_hash=task.content_hash, + commit=commit, + tuning=asdict(tuning), + success=success and not meter.fallbacks, + outcome=status, + outcome_note=detail[:500], + turns=meter.calls, + prose_turns=meter.prose_turns, + failed_commands=failed, + invented_tools=invented, + trimmed_requests=getattr(client, "trimmed_requests", 0) - trims_before, + tokens_in=meter.tokens_in, + tokens_out=meter.tokens_out, + cost=round(meter.cost, 6), + wall_s=round(wall, 1), + fallbacks=meter.fallbacks, + check_output=check_output, + ) diff --git a/nisse-dev/src/nisse_dev/bench/tasks.py b/nisse-dev/src/nisse_dev/bench/tasks.py new file mode 100644 index 0000000..e71a4fe --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks.py @@ -0,0 +1,102 @@ +"""Benchmark tasks: a seed project, an approved plan, and a hidden check. + +Each task is a directory under ``tasks/``: + +- ``task.yaml``: ``title``, ``plan`` (the approved milestone plan, written the + way the planner is asked to write them), and ``check`` (a shell command run + in the workspace afterwards). +- ``seed/``: the project the executor starts from. +- ``hidden/``: files the check needs, copied in only after the run, into + ``.bench_check/``. The executor never sees them. +- ``solution/``: a reference solution, overlaid on the seed by the tests to + prove each task can be solved and that its check tells the difference. +""" + +import hashlib +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import yaml + +TASKS_DIR = Path(__file__).parent / "tasks" +CHECK_DIR = ".bench_check" + + +@dataclass(frozen=True) +class Task: + name: str + title: str + plan: str + check: str + path: Path + + @property + def content_hash(self) -> str: + """What the task is, so a changed task is never compared with runs of + its earlier version. Covers everything the model or the check sees; + not the reference solution.""" + digest = hashlib.sha256() + for part in ("task.yaml", "seed", "hidden"): + root = self.path / part + files = [root] if root.is_file() else sorted(root.rglob("*")) + for file in files: + if file.is_file() and "__pycache__" not in file.parts: + digest.update(str(file.relative_to(self.path)).encode()) + digest.update(file.read_bytes()) + return digest.hexdigest()[:16] + + def seed_into(self, workspace: Path) -> None: + shutil.copytree(self.path / "seed", workspace, dirs_exist_ok=True) + + def apply_solution(self, workspace: Path) -> None: + shutil.copytree(self.path / "solution", workspace, dirs_exist_ok=True) + + def run_check(self, workspace: Path, timeout_s: int = 300) -> tuple[bool, str]: + """Copy the hidden files in and run the check. Returns whether it + passed, and its output for the record.""" + target = workspace / CHECK_DIR + if target.exists(): + shutil.rmtree(target) # never trust a directory the model made + shutil.copytree(self.path / "hidden", target) + try: + result = subprocess.run( + self.check, + shell=True, + cwd=workspace, + capture_output=True, + text=True, + timeout=timeout_s, + check=False, + ) + except subprocess.TimeoutExpired: + return False, f"check timed out after {timeout_s}s" + output = (result.stdout + result.stderr)[-2000:] + return result.returncode == 0, output + + +def load_task(path: Path) -> Task: + spec = yaml.safe_load((path / "task.yaml").read_text()) + return Task( + name=path.name, + title=str(spec["title"]), + plan=str(spec["plan"]).strip(), + check=str(spec["check"]), + path=path, + ) + + +def load_tasks(names: list[str] | None = None, root: Path = TASKS_DIR) -> list[Task]: + available = { + p.name: p for p in sorted(root.iterdir()) if (p / "task.yaml").exists() + } + if names: + unknown = [n for n in names if n not in available] + if unknown: + raise ValueError( + f"unknown task(s): {', '.join(unknown)}; " + f"available: {', '.join(available)}" + ) + return [load_task(available[n]) for n in names] + return [load_task(p) for p in available.values()] diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/original_test_inventory.py.txt b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/original_test_inventory.py.txt new file mode 100644 index 0000000..8155867 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/original_test_inventory.py.txt @@ -0,0 +1,11 @@ +from inventory import Item, low_stock, total_value + + +def test_total_value_counts_every_unit() -> None: + items = [Item("bolt", 0.25, 100), Item("nut", 0.10, 40)] + assert total_value(items) == 29.0 + + +def test_low_stock_is_alphabetical() -> None: + items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100), Item("anchor", 3.0, 1)] + assert low_stock(items) == ["anchor", "washer"] diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/test_hidden.py b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/test_hidden.py new file mode 100644 index 0000000..817ca49 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/hidden/test_hidden.py @@ -0,0 +1,27 @@ +from pathlib import Path + +import pytest +from inventory import Item, low_stock, total_value + + +def test_the_visible_tests_were_not_changed() -> None: + original = (Path(__file__).parent / "original_test_inventory.py.txt").read_text() + assert Path("test_inventory.py").read_text() == original + + +@pytest.mark.parametrize( + ("items", "expected"), + [ + ([], 0.0), + ([Item("bolt", 0.25, 100), Item("nut", 0.10, 40)], 29.0), + ([Item("gear", 12.5, 3)], 37.5), + ([Item("empty", 9.99, 0)], 0.0), + ], +) +def test_total_value(items: list[Item], expected: float) -> None: + assert total_value(items) == pytest.approx(expected) + + +def test_low_stock_still_works() -> None: + items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100)] + assert low_stock(items) == ["washer"] diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/inventory.py b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/inventory.py new file mode 100644 index 0000000..052ae14 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/inventory.py @@ -0,0 +1,23 @@ +"""A tiny inventory.""" + +from dataclasses import dataclass + + +@dataclass +class Item: + name: str + unit_price: float + quantity: int + + +def total_value(items: list[Item]) -> float: + """The value of all stock: each item's unit price times its quantity.""" + total = 0.0 + for item in items: + total += item.unit_price + return total + + +def low_stock(items: list[Item], threshold: int = 5) -> list[str]: + """Names of items with fewer than `threshold` units, alphabetically.""" + return sorted(item.name for item in items if item.quantity < threshold) diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/test_inventory.py b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/test_inventory.py new file mode 100644 index 0000000..8155867 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/seed/test_inventory.py @@ -0,0 +1,11 @@ +from inventory import Item, low_stock, total_value + + +def test_total_value_counts_every_unit() -> None: + items = [Item("bolt", 0.25, 100), Item("nut", 0.10, 40)] + assert total_value(items) == 29.0 + + +def test_low_stock_is_alphabetical() -> None: + items = [Item("washer", 0.05, 2), Item("bolt", 0.25, 100), Item("anchor", 3.0, 1)] + assert low_stock(items) == ["anchor", "washer"] diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/solution/inventory.py b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/solution/inventory.py new file mode 100644 index 0000000..44002ef --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/solution/inventory.py @@ -0,0 +1,23 @@ +"""A tiny inventory.""" + +from dataclasses import dataclass + + +@dataclass +class Item: + name: str + unit_price: float + quantity: int + + +def total_value(items: list[Item]) -> float: + """The value of all stock: each item's unit price times its quantity.""" + total = 0.0 + for item in items: + total += item.unit_price * item.quantity + return total + + +def low_stock(items: list[Item], threshold: int = 5) -> list[str]: + """Names of items with fewer than `threshold` units, alphabetically.""" + return sorted(item.name for item in items if item.quantity < threshold) diff --git a/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml new file mode 100644 index 0000000..7a676d1 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml @@ -0,0 +1,12 @@ +title: Fix a bug a failing test exposes +check: python -m pytest -q .bench_check +plan: | + Milestone 1: fix the failing inventory test. + + 1. Run `python -m pytest -q` and read the failure. + 2. The failing test is correct; the bug is in `inventory.py`. Do not change + any test file. + 3. Fix `total_value` so that each item contributes its unit price times its + quantity. + 4. Run `python -m pytest -q` until every test passes. + 5. Call finish. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/long-output/hidden/test_hidden.py b/nisse-dev/src/nisse_dev/bench/tasks/long-output/hidden/test_hidden.py new file mode 100644 index 0000000..f7af64c --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/long-output/hidden/test_hidden.py @@ -0,0 +1,11 @@ +import pytest +from rates import RATES + + +def test_every_region_is_present_once() -> None: + assert sorted(RATES) == [f"R{n:03d}" for n in range(300)] + + +@pytest.mark.parametrize("n", range(300)) +def test_rate(n: int) -> None: + assert RATES[f"R{n:03d}"] == pytest.approx(n % 25 / 100) diff --git a/nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/rates.py b/nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/rates.py new file mode 100644 index 0000000..7b214fe --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/rates.py @@ -0,0 +1,304 @@ +"""Tax rate per region code.""" + +RATES = { + "R000": 0.00, + "R001": 0.01, + "R002": 0.02, + "R003": 0.03, + "R004": 0.04, + "R005": 0.05, + "R006": 0.06, + "R007": 0.07, + "R008": 0.08, + "R009": 0.09, + "R010": 0.10, + "R011": 0.11, + "R012": 0.12, + "R013": 0.13, + "R014": 0.14, + "R015": 0.15, + "R016": 0.16, + "R017": 0.17, + "R018": 0.18, + "R019": 0.19, + "R020": 0.20, + "R021": 0.21, + "R022": 0.22, + "R023": 0.23, + "R024": 0.24, + "R025": 0.00, + "R026": 0.01, + "R027": 0.02, + "R028": 0.03, + "R029": 0.04, + "R030": 0.05, + "R031": 0.06, + "R032": 0.07, + "R033": 0.08, + "R034": 0.09, + "R035": 0.10, + "R036": 0.11, + "R037": 0.12, + "R038": 0.13, + "R039": 0.14, + "R040": 0.15, + "R041": 0.16, + "R042": 0.17, + "R043": 0.18, + "R044": 0.19, + "R045": 0.20, + "R046": 0.21, + "R047": 0.22, + "R048": 0.23, + "R049": 0.24, + "R050": 0.00, + "R051": 0.01, + "R052": 0.02, + "R053": 0.03, + "R054": 0.04, + "R055": 0.05, + "R056": 0.06, + "R057": 0.07, + "R058": 0.08, + "R059": 0.09, + "R060": 0.10, + "R061": 0.11, + "R062": 0.12, + "R063": 0.13, + "R064": 0.14, + "R065": 0.15, + "R066": 0.16, + "R067": 0.17, + "R068": 0.18, + "R069": 0.19, + "R070": 0.20, + "R071": 0.21, + "R072": 0.22, + "R073": 0.23, + "R074": 0.24, + "R075": 0.00, + "R076": 0.01, + "R077": 0.02, + "R078": 0.03, + "R079": 0.04, + "R080": 0.05, + "R081": 0.06, + "R082": 0.07, + "R083": 0.08, + "R084": 0.09, + "R085": 0.10, + "R086": 0.11, + "R087": 0.12, + "R088": 0.13, + "R089": 0.14, + "R090": 0.15, + "R091": 0.16, + "R092": 0.17, + "R093": 0.18, + "R094": 0.19, + "R095": 0.20, + "R096": 0.21, + "R097": 0.22, + "R098": 0.23, + "R099": 0.24, + "R100": 0.00, + "R101": 0.01, + "R102": 0.02, + "R103": 0.03, + "R104": 0.04, + "R105": 0.05, + "R106": 0.06, + "R107": 0.07, + "R108": 0.08, + "R109": 0.09, + "R110": 0.10, + "R111": 0.11, + "R112": 0.12, + "R113": 0.13, + "R114": 0.14, + "R115": 0.15, + "R116": 0.16, + "R117": 0.17, + "R118": 0.18, + "R119": 0.19, + "R120": 0.20, + "R121": 0.21, + "R122": 0.22, + "R123": 0.23, + "R124": 0.24, + "R125": 0.00, + "R126": 0.01, + "R127": 0.02, + "R128": 0.03, + "R129": 0.04, + "R130": 0.05, + "R131": 0.06, + "R132": 0.07, + "R133": 0.08, + "R134": 0.09, + "R135": 0.10, + "R136": 0.11, + "R137": 0.12, + "R138": 0.13, + "R139": 0.14, + "R140": 0.15, + "R141": 0.16, + "R142": 0.17, + "R143": 0.18, + "R144": 0.19, + "R145": 0.20, + "R146": 0.21, + "R147": 0.22, + "R148": 0.23, + "R149": 0.24, + "R150": 0.00, + "R151": 0.01, + "R152": 0.02, + "R153": 0.03, + "R154": 0.04, + "R155": 0.05, + "R156": 0.06, + "R157": 0.07, + "R158": 0.08, + "R159": 0.09, + "R160": 0.10, + "R161": 0.11, + "R162": 0.12, + "R163": 0.13, + "R164": 0.14, + "R165": 0.15, + "R166": 0.16, + "R167": 0.17, + "R168": 0.18, + "R169": 0.19, + "R170": 0.20, + "R171": 0.21, + "R172": 0.22, + "R173": 0.23, + "R174": 0.24, + "R175": 0.00, + "R176": 0.01, + "R177": 0.02, + "R178": 0.03, + "R179": 0.04, + "R180": 0.05, + "R181": 0.06, + "R182": 0.07, + "R183": 0.08, + "R184": 0.09, + "R185": 0.10, + "R186": 0.11, + "R187": 0.12, + "R188": 0.13, + "R189": 0.14, + "R190": 0.15, + "R191": 0.16, + "R192": 0.17, + "R193": 0.18, + "R194": 0.19, + "R195": 0.20, + "R196": 0.21, + "R197": 0.22, + "R198": 0.23, + "R199": 0.24, + "R200": 0.00, + "R201": 0.01, + "R202": 0.02, + "R203": 0.03, + "R204": 0.04, + "R205": 0.05, + "R206": 0.06, + "R207": 0.07, + "R208": 0.08, + "R209": 0.09, + "R210": 0.10, + "R211": 0.11, + "R212": 0.12, + "R213": 0.13, + "R214": 0.14, + "R215": 0.15, + "R216": 0.16, + "R217": 0.17, + "R218": 0.18, + "R219": 0.19, + "R220": 0.20, + "R221": 0.21, + "R222": 0.22, + "R223": 0.23, + "R224": 0.24, + "R225": 0.00, + "R226": 0.01, + "R227": 0.02, + "R228": 0.03, + "R229": 0.04, + "R230": 0.05, + "R231": 0.06, + "R232": 0.07, + "R233": 0.08, + "R234": 0.09, + "R235": 0.10, + "R236": 0.11, + "R237": 0.21, + "R238": 0.13, + "R239": 0.14, + "R240": 0.15, + "R241": 0.16, + "R242": 0.17, + "R243": 0.18, + "R244": 0.19, + "R245": 0.20, + "R246": 0.21, + "R247": 0.22, + "R248": 0.23, + "R249": 0.24, + "R250": 0.00, + "R251": 0.01, + "R252": 0.02, + "R253": 0.03, + "R254": 0.04, + "R255": 0.05, + "R256": 0.06, + "R257": 0.07, + "R258": 0.08, + "R259": 0.09, + "R260": 0.10, + "R261": 0.11, + "R262": 0.12, + "R263": 0.13, + "R264": 0.14, + "R265": 0.15, + "R266": 0.16, + "R267": 0.17, + "R268": 0.18, + "R269": 0.19, + "R270": 0.20, + "R271": 0.21, + "R272": 0.22, + "R273": 0.23, + "R274": 0.24, + "R275": 0.00, + "R276": 0.01, + "R277": 0.02, + "R278": 0.03, + "R279": 0.04, + "R280": 0.05, + "R281": 0.06, + "R282": 0.07, + "R283": 0.08, + "R284": 0.09, + "R285": 0.10, + "R286": 0.11, + "R287": 0.12, + "R288": 0.13, + "R289": 0.14, + "R290": 0.15, + "R291": 0.16, + "R292": 0.17, + "R293": 0.18, + "R294": 0.19, + "R295": 0.20, + "R296": 0.21, + "R297": 0.22, + "R298": 0.23, + "R299": 0.24, +} diff --git a/nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/test_rates.py b/nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/test_rates.py new file mode 100644 index 0000000..a08f502 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/long-output/seed/test_rates.py @@ -0,0 +1,7 @@ +import pytest +from rates import RATES + + +@pytest.mark.parametrize("code", [f"R{n:03d}" for n in range(300)]) +def test_rate(code: str) -> None: + assert RATES[code] == pytest.approx(int(code[1:]) % 25 / 100) diff --git a/nisse-dev/src/nisse_dev/bench/tasks/long-output/solution/rates.py b/nisse-dev/src/nisse_dev/bench/tasks/long-output/solution/rates.py new file mode 100644 index 0000000..1461336 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/long-output/solution/rates.py @@ -0,0 +1,304 @@ +"""Tax rate per region code.""" + +RATES = { + "R000": 0.00, + "R001": 0.01, + "R002": 0.02, + "R003": 0.03, + "R004": 0.04, + "R005": 0.05, + "R006": 0.06, + "R007": 0.07, + "R008": 0.08, + "R009": 0.09, + "R010": 0.10, + "R011": 0.11, + "R012": 0.12, + "R013": 0.13, + "R014": 0.14, + "R015": 0.15, + "R016": 0.16, + "R017": 0.17, + "R018": 0.18, + "R019": 0.19, + "R020": 0.20, + "R021": 0.21, + "R022": 0.22, + "R023": 0.23, + "R024": 0.24, + "R025": 0.00, + "R026": 0.01, + "R027": 0.02, + "R028": 0.03, + "R029": 0.04, + "R030": 0.05, + "R031": 0.06, + "R032": 0.07, + "R033": 0.08, + "R034": 0.09, + "R035": 0.10, + "R036": 0.11, + "R037": 0.12, + "R038": 0.13, + "R039": 0.14, + "R040": 0.15, + "R041": 0.16, + "R042": 0.17, + "R043": 0.18, + "R044": 0.19, + "R045": 0.20, + "R046": 0.21, + "R047": 0.22, + "R048": 0.23, + "R049": 0.24, + "R050": 0.00, + "R051": 0.01, + "R052": 0.02, + "R053": 0.03, + "R054": 0.04, + "R055": 0.05, + "R056": 0.06, + "R057": 0.07, + "R058": 0.08, + "R059": 0.09, + "R060": 0.10, + "R061": 0.11, + "R062": 0.12, + "R063": 0.13, + "R064": 0.14, + "R065": 0.15, + "R066": 0.16, + "R067": 0.17, + "R068": 0.18, + "R069": 0.19, + "R070": 0.20, + "R071": 0.21, + "R072": 0.22, + "R073": 0.23, + "R074": 0.24, + "R075": 0.00, + "R076": 0.01, + "R077": 0.02, + "R078": 0.03, + "R079": 0.04, + "R080": 0.05, + "R081": 0.06, + "R082": 0.07, + "R083": 0.08, + "R084": 0.09, + "R085": 0.10, + "R086": 0.11, + "R087": 0.12, + "R088": 0.13, + "R089": 0.14, + "R090": 0.15, + "R091": 0.16, + "R092": 0.17, + "R093": 0.18, + "R094": 0.19, + "R095": 0.20, + "R096": 0.21, + "R097": 0.22, + "R098": 0.23, + "R099": 0.24, + "R100": 0.00, + "R101": 0.01, + "R102": 0.02, + "R103": 0.03, + "R104": 0.04, + "R105": 0.05, + "R106": 0.06, + "R107": 0.07, + "R108": 0.08, + "R109": 0.09, + "R110": 0.10, + "R111": 0.11, + "R112": 0.12, + "R113": 0.13, + "R114": 0.14, + "R115": 0.15, + "R116": 0.16, + "R117": 0.17, + "R118": 0.18, + "R119": 0.19, + "R120": 0.20, + "R121": 0.21, + "R122": 0.22, + "R123": 0.23, + "R124": 0.24, + "R125": 0.00, + "R126": 0.01, + "R127": 0.02, + "R128": 0.03, + "R129": 0.04, + "R130": 0.05, + "R131": 0.06, + "R132": 0.07, + "R133": 0.08, + "R134": 0.09, + "R135": 0.10, + "R136": 0.11, + "R137": 0.12, + "R138": 0.13, + "R139": 0.14, + "R140": 0.15, + "R141": 0.16, + "R142": 0.17, + "R143": 0.18, + "R144": 0.19, + "R145": 0.20, + "R146": 0.21, + "R147": 0.22, + "R148": 0.23, + "R149": 0.24, + "R150": 0.00, + "R151": 0.01, + "R152": 0.02, + "R153": 0.03, + "R154": 0.04, + "R155": 0.05, + "R156": 0.06, + "R157": 0.07, + "R158": 0.08, + "R159": 0.09, + "R160": 0.10, + "R161": 0.11, + "R162": 0.12, + "R163": 0.13, + "R164": 0.14, + "R165": 0.15, + "R166": 0.16, + "R167": 0.17, + "R168": 0.18, + "R169": 0.19, + "R170": 0.20, + "R171": 0.21, + "R172": 0.22, + "R173": 0.23, + "R174": 0.24, + "R175": 0.00, + "R176": 0.01, + "R177": 0.02, + "R178": 0.03, + "R179": 0.04, + "R180": 0.05, + "R181": 0.06, + "R182": 0.07, + "R183": 0.08, + "R184": 0.09, + "R185": 0.10, + "R186": 0.11, + "R187": 0.12, + "R188": 0.13, + "R189": 0.14, + "R190": 0.15, + "R191": 0.16, + "R192": 0.17, + "R193": 0.18, + "R194": 0.19, + "R195": 0.20, + "R196": 0.21, + "R197": 0.22, + "R198": 0.23, + "R199": 0.24, + "R200": 0.00, + "R201": 0.01, + "R202": 0.02, + "R203": 0.03, + "R204": 0.04, + "R205": 0.05, + "R206": 0.06, + "R207": 0.07, + "R208": 0.08, + "R209": 0.09, + "R210": 0.10, + "R211": 0.11, + "R212": 0.12, + "R213": 0.13, + "R214": 0.14, + "R215": 0.15, + "R216": 0.16, + "R217": 0.17, + "R218": 0.18, + "R219": 0.19, + "R220": 0.20, + "R221": 0.21, + "R222": 0.22, + "R223": 0.23, + "R224": 0.24, + "R225": 0.00, + "R226": 0.01, + "R227": 0.02, + "R228": 0.03, + "R229": 0.04, + "R230": 0.05, + "R231": 0.06, + "R232": 0.07, + "R233": 0.08, + "R234": 0.09, + "R235": 0.10, + "R236": 0.11, + "R237": 0.12, + "R238": 0.13, + "R239": 0.14, + "R240": 0.15, + "R241": 0.16, + "R242": 0.17, + "R243": 0.18, + "R244": 0.19, + "R245": 0.20, + "R246": 0.21, + "R247": 0.22, + "R248": 0.23, + "R249": 0.24, + "R250": 0.00, + "R251": 0.01, + "R252": 0.02, + "R253": 0.03, + "R254": 0.04, + "R255": 0.05, + "R256": 0.06, + "R257": 0.07, + "R258": 0.08, + "R259": 0.09, + "R260": 0.10, + "R261": 0.11, + "R262": 0.12, + "R263": 0.13, + "R264": 0.14, + "R265": 0.15, + "R266": 0.16, + "R267": 0.17, + "R268": 0.18, + "R269": 0.19, + "R270": 0.20, + "R271": 0.21, + "R272": 0.22, + "R273": 0.23, + "R274": 0.24, + "R275": 0.00, + "R276": 0.01, + "R277": 0.02, + "R278": 0.03, + "R279": 0.04, + "R280": 0.05, + "R281": 0.06, + "R282": 0.07, + "R283": 0.08, + "R284": 0.09, + "R285": 0.10, + "R286": 0.11, + "R287": 0.12, + "R288": 0.13, + "R289": 0.14, + "R290": 0.15, + "R291": 0.16, + "R292": 0.17, + "R293": 0.18, + "R294": 0.19, + "R295": 0.20, + "R296": 0.21, + "R297": 0.22, + "R298": 0.23, + "R299": 0.24, +} diff --git a/nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml new file mode 100644 index 0000000..3ac8ae3 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml @@ -0,0 +1,12 @@ +title: Find one failure in long test output +check: python -m pytest -q .bench_check +plan: | + Milestone 1: fix the one wrong entry in the rate table. + + 1. Run `python -m pytest -v`. Exactly one test fails, and its output names a + region code. + 2. The tests are correct. In `rates.py`, the rate for region code RNNN must be + (NNN mod 25) / 100. Correct only the failing region's entry; leave every + other line as it is. + 3. Run `python -m pytest -q` until every test passes. + 4. Call finish. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/hidden/test_hidden.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/hidden/test_hidden.py new file mode 100644 index 0000000..61187c4 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/hidden/test_hidden.py @@ -0,0 +1,33 @@ +import subprocess +import sys + +from greeter.cli import main +from greeter.core import greet + + +def test_default_is_unchanged() -> None: + assert greet("Bob") == "Hello, Bob!" + + +def test_shout() -> None: + assert greet("Bob", shout=True) == "HELLO, BOB!" + + +def test_cli_default(capsys) -> None: + assert main(["Ada"]) == 0 + assert capsys.readouterr().out == "Hello, Ada!\n" + + +def test_cli_shout(capsys) -> None: + assert main(["--shout", "Ada"]) == 0 + assert capsys.readouterr().out == "HELLO, ADA!\n" + + +def test_module_entry_point() -> None: + result = subprocess.run( + [sys.executable, "-m", "greeter", "--shout", "Bob"], + capture_output=True, + text=True, + check=True, + ) + assert result.stdout == "HELLO, BOB!\n" diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__init__.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__main__.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__main__.py new file mode 100644 index 0000000..5664529 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__main__.py @@ -0,0 +1,3 @@ +from greeter.cli import main + +raise SystemExit(main()) diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/cli.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/cli.py new file mode 100644 index 0000000..a38c398 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/cli.py @@ -0,0 +1,11 @@ +import argparse + +from greeter.core import greet + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="greeter") + parser.add_argument("name") + args = parser.parse_args(argv) + print(greet(args.name)) + return 0 diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/core.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/core.py new file mode 100644 index 0000000..1e21eeb --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/core.py @@ -0,0 +1,2 @@ +def greet(name: str) -> str: + return f"Hello, {name}!" diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/tests/test_greeter.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/tests/test_greeter.py new file mode 100644 index 0000000..54c92e2 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/tests/test_greeter.py @@ -0,0 +1,11 @@ +from greeter.cli import main +from greeter.core import greet + + +def test_greet() -> None: + assert greet("Bob") == "Hello, Bob!" + + +def test_cli_prints_the_greeting(capsys) -> None: + assert main(["Bob"]) == 0 + assert capsys.readouterr().out == "Hello, Bob!\n" diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/cli.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/cli.py new file mode 100644 index 0000000..82f856b --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/cli.py @@ -0,0 +1,12 @@ +import argparse + +from greeter.core import greet + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="greeter") + parser.add_argument("name") + parser.add_argument("--shout", action="store_true") + args = parser.parse_args(argv) + print(greet(args.name, shout=args.shout)) + return 0 diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/core.py b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/core.py new file mode 100644 index 0000000..819f836 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/core.py @@ -0,0 +1,3 @@ +def greet(name: str, shout: bool = False) -> str: + greeting = f"Hello, {name}!" + return greeting.upper() if shout else greeting diff --git a/nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml new file mode 100644 index 0000000..44f6902 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml @@ -0,0 +1,16 @@ +title: Thread a new option through several files +check: python -m pytest -q .bench_check +plan: | + Milestone 1: add a --shout option to the greeter. + + 1. In `greeter/core.py`, give `greet` a keyword argument `shout: bool = False`. + When it is true, return the whole greeting in upper case: + greet("Bob", shout=True) == "HELLO, BOB!". The default behaviour is unchanged. + 2. In `greeter/cli.py`, add a `--shout` flag (action="store_true") and pass it + to `greet`. + 3. Add two tests to `tests/test_greeter.py`: one for greet with shout=True, and + one that calls `main(["--shout", "Bob"])` and checks the printed output + using pytest's capsys fixture. + 4. Run `python -m pytest -q` until every test passes, and check that + `python -m greeter --shout Bob` prints `HELLO, BOB!`. + 5. Call finish. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/write-function/hidden/test_hidden.py b/nisse-dev/src/nisse_dev/bench/tasks/write-function/hidden/test_hidden.py new file mode 100644 index 0000000..cf925f6 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/write-function/hidden/test_hidden.py @@ -0,0 +1,19 @@ +import pytest +from slugify import slugify + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("Hello, World!", "hello-world"), + (" Nisse Agent ", "nisse-agent"), + ("a--b", "a-b"), + ("", ""), + ("Python 3.12 Release", "python-3-12-release"), + ("--leading and trailing--", "leading-and-trailing"), + ("MiXeD CaSe", "mixed-case"), + ("!!!", ""), + ], +) +def test_slugify(text: str, expected: str) -> None: + assert slugify(text) == expected diff --git a/nisse-dev/src/nisse_dev/bench/tasks/write-function/seed/README.md b/nisse-dev/src/nisse_dev/bench/tasks/write-function/seed/README.md new file mode 100644 index 0000000..e09c91f --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/write-function/seed/README.md @@ -0,0 +1 @@ +A small utilities project. diff --git a/nisse-dev/src/nisse_dev/bench/tasks/write-function/solution/slugify.py b/nisse-dev/src/nisse_dev/bench/tasks/write-function/solution/slugify.py new file mode 100644 index 0000000..c4d3700 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/write-function/solution/slugify.py @@ -0,0 +1,5 @@ +import re + + +def slugify(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") diff --git a/nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml b/nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml new file mode 100644 index 0000000..9955265 --- /dev/null +++ b/nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml @@ -0,0 +1,13 @@ +title: Write a small function and its tests +check: python -m pytest -q .bench_check +plan: | + Milestone 1: add a slugify function. + + 1. Create `slugify.py` in the workspace root, defining `slugify(text: str) -> str`. + 2. It lowercases the text, replaces every run of characters that are not a-z or + 0-9 with a single hyphen, and strips hyphens from both ends. Examples: + "Hello, World!" -> "hello-world"; " Nisse Agent " -> "nisse-agent"; + "a--b" -> "a-b"; "" -> "". + 3. Create `test_slugify.py` with pytest tests covering the four examples. + 4. Run `python -m pytest -q test_slugify.py`, and fix anything until it passes. + 5. Call finish. diff --git a/nisse-dev/src/nisse_dev/context.py b/nisse-dev/src/nisse_dev/context.py index d5c162b..d77eb61 100644 --- a/nisse-dev/src/nisse_dev/context.py +++ b/nisse-dev/src/nisse_dev/context.py @@ -230,6 +230,7 @@ class BudgetedClient: # Per model: how much the character estimate undercounted, as learned # from reported usage. Starts at 1 and only grows. self._factors: dict[str, float] = {} + self.trimmed_requests = 0 @property def last_usage(self) -> dict[str, int]: @@ -239,6 +240,10 @@ class BudgetedClient: def last_model(self) -> str | None: return getattr(self._inner, "last_model", None) + @property + def last_cost(self) -> float: + return float(getattr(self._inner, "last_cost", 0.0) or 0.0) + def budget_for(self, model: str) -> Budget: window, model_max_output = self._limits_for(model) ceiling = self._max_output_tokens @@ -261,6 +266,10 @@ class BudgetedClient: budget = self.budget_for(model) factor = self._factors.get(model, 1.0) fitted = fit(events, system, tools, budget, factor) + # Counted so the benchmark can report how often a model's window was + # the constraint: any request that had to be cut down or shortened. + if fitted != list(events): + self.trimmed_requests += 1 produced = self._inner.complete( fitted, tools, diff --git a/nisse-dev/src/nisse_dev/llm.py b/nisse-dev/src/nisse_dev/llm.py index 10c8877..1579a60 100644 --- a/nisse-dev/src/nisse_dev/llm.py +++ b/nisse-dev/src/nisse_dev/llm.py @@ -143,6 +143,7 @@ class LiteLLMClient: # protocol stays the same for every implementation. self.last_usage: dict[str, int] = {} self.last_model: str | None = None + self.last_cost: float = 0.0 def complete( self, @@ -171,6 +172,8 @@ class LiteLLMClient: raise LLMError( f"LLM request returned {response.status_code}: {response.text[:500]}" ) + # LiteLLM prices every call; kept so a caller can meter spending. + self.last_cost = _response_cost(response) return self._parse(response.json(), requested_model=model) def _parse( @@ -263,6 +266,7 @@ class AnthropicPassthroughClient: self._sleep = sleep self.last_usage: dict[str, int] = {} self.last_model: str | None = None + self.last_cost: float = 0.0 def complete( self, @@ -291,6 +295,8 @@ class AnthropicPassthroughClient: raise LLMError( f"passthrough returned {response.status_code}: {response.text[:500]}" ) + # LiteLLM prices every call; kept so a caller can meter spending. + self.last_cost = _response_cost(response) return self._parse(response.json(), requested_model=model) def _parse( @@ -358,6 +364,13 @@ def _post_with_rate_limit_retry( # Model families we have a native renderer adapter for. A13: delegations take # the vendor's passthrough route because a normalising middlebox is lossy # exactly where a long-running delegation lives. +def _response_cost(response: httpx.Response) -> float: + try: + return float(response.headers.get("x-litellm-response-cost") or 0.0) + except ValueError: + return 0.0 + + def _raise_for_upstream(response: httpx.Response) -> None: """Classify a non-200 as upstream trouble where it plainly is. diff --git a/nisse-dev/tests/test_bench.py b/nisse-dev/tests/test_bench.py new file mode 100644 index 0000000..a48eff1 --- /dev/null +++ b/nisse-dev/tests/test_bench.py @@ -0,0 +1,265 @@ +"""The executor benchmark.""" + +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from nisse_dev.bench.results import ( + append, + load, + render, + results_file, + summarise, + wilson, +) +from nisse_dev.bench.runner import MeteringClient, run_task +from nisse_dev.bench.tasks import CHECK_DIR, load_tasks +from nisse_dev.canonical import AssistantText, ToolCall +from nisse_dev.catalog import ModelFacts +from nisse_dev.config import ExecutorTuning, Tuning + +TASKS = load_tasks() +FACTS = ModelFacts( + alias="local-fast", + backing="openai/qwen3-8b-awq", + provider="openai", + supports_tools=True, + max_input_tokens=32768, + max_output_tokens=8192, + supports_tool_choice=True, +) + + +# --- the tasks themselves --------------------------------------------------- + + +def test_the_bundled_tasks_are_the_four_planned() -> None: + assert [t.name for t in TASKS] == [ + "fix-bug", + "long-output", + "multi-file", + "write-function", + ] + + +@pytest.mark.parametrize("task", TASKS, ids=lambda t: t.name) +def test_the_seed_fails_its_check_and_the_solution_passes(task, tmp_path: Path) -> None: + # A check that passes on the untouched seed measures nothing; a task its + # own reference solution can't pass is unfair to every model. + seed = tmp_path / "seed" + task.seed_into(seed) + assert not task.run_check(seed)[0], f"{task.name}: the seed already passes" + + solved = tmp_path / "solved" + task.seed_into(solved) + task.apply_solution(solved) + passed, output = task.run_check(solved) + assert passed, f"{task.name}: the reference solution fails:\n{output}" + + +@pytest.mark.parametrize("task", TASKS, ids=lambda t: t.name) +def test_the_hidden_check_is_not_in_the_seed(task, tmp_path: Path) -> None: + task.seed_into(tmp_path) + assert not (tmp_path / CHECK_DIR).exists() + assert "Call finish" in task.plan # plans end the way the executor must + + +def test_long_output_really_is_long(tmp_path: Path) -> None: + """The point of the task is output far past tools.max_output_chars.""" + import subprocess + import sys + + [task] = load_tasks(["long-output"]) + task.seed_into(tmp_path) + run = subprocess.run( + [sys.executable, "-m", "pytest", "-v"], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + assert len(run.stdout) > 4 * Tuning().tools.max_output_chars + assert "R237" in run.stdout[-3000:] # the failure is named at the end + + +def test_a_changed_task_gets_a_new_hash(tmp_path: Path) -> None: + [task] = load_tasks(["write-function"]) + shutil.copytree(task.path, tmp_path / "write-function") + [copy] = load_tasks(root=tmp_path) + assert copy.content_hash == task.content_hash + (copy.path / "seed" / "README.md").write_text("changed\n") + assert copy.content_hash != task.content_hash + + +def test_an_unknown_task_is_refused() -> None: + with pytest.raises(ValueError, match="unknown task"): + load_tasks(["nope"]) + + +# --- running a task ---------------------------------------------------------- + + +class Scripted: + """A model that plays back fixed turns, reporting itself as `answered`.""" + + def __init__(self, turns: list[list[Any]], answered: str = "local-fast") -> None: + self.turns = turns + self.last_usage = {"input": 1000, "output": 50} + self.last_cost = 0.0 + self.last_model = answered + self.trimmed_requests = 0 + + def complete(self, events, tools, model, system=None): + if not self.turns: + return [AssistantText(text="nothing left to do", model=model)] + return self.turns.pop(0) + + +def call(tool: str, cid: str, **arguments: Any) -> ToolCall: + return ToolCall(call_id=cid, tool=tool, arguments=arguments) + + +SOLVE_SLUGIFY = [ + [ + call( + "write_file", + "w", + path="slugify.py", + content=( + "import re\n\n\ndef slugify(text):\n" + " return re.sub(r'[^a-z0-9]+', '-', text.lower()).strip('-')\n" + ), + ) + ], + [call("run_command", "r", command="python -c 'import slugify'")], + [call("finish", "f", summary="added slugify")], +] + + +def test_a_solved_task_is_a_success(tmp_path: Path) -> None: + [task] = load_tasks(["write-function"]) + record = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning(), note="n") + + assert record.success + assert record.outcome == "completed" + assert record.turns == 3 + assert record.tokens_in == 3000 + assert record.task_hash == task.content_hash + assert record.model_info["supports_tool_choice"] is True + assert record.tuning["executor"]["max_turns"] == 24 + + +def test_saying_finished_is_not_success(tmp_path: Path) -> None: + # Success is the hidden check passing, never the model's own claim. + [task] = load_tasks(["write-function"]) + wrong = [ + [ + call( + "write_file", + "w", + path="slugify.py", + content="def slugify(t):\n return t\n", + ) + ], + [call("finish", "f", summary="done")], + ] + record = run_task(task, FACTS, Scripted(wrong), Tuning()) + assert record.outcome == "completed" + assert not record.success + + +def test_misbehaviour_is_counted(tmp_path: Path) -> None: + [task] = load_tasks(["write-function"]) + turns = [ + [call("chat", "x1")], + [call("run_command", "r1", command="exit 1")], + [AssistantText(text="let me explain", model="m")], + ] + # A higher failure limit, so the invented tool and the failed command + # don't escalate before the prose turn is reached. + tuning = Tuning(executor=ExecutorTuning(failure_limit=5)) + record = run_task(task, FACTS, Scripted(turns), tuning) + assert record.invented_tools == 1 + assert record.failed_commands == 1 + assert record.prose_turns >= 1 + assert not record.success + + +def test_a_fallback_answer_can_never_be_a_success(tmp_path: Path) -> None: + [task] = load_tasks(["write-function"]) + client = Scripted(list(SOLVE_SLUGIFY), answered="claude-sonnet-5") + record = run_task(task, FACTS, client, Tuning()) + assert record.fallbacks == ["claude-sonnet-5"] * 3 + assert not record.success # the check passed, but not for this model + + +# --- storing and reporting --------------------------------------------------- + + +def test_wilson_intervals_behave() -> None: + low, high = wilson(5, 5) + assert high == 1.0 and 0.5 < low < 0.6 + # 3 of 5 and 5 of 5 overlap: five runs do not separate them. + assert wilson(3, 5)[1] > wilson(5, 5)[0] + assert wilson(0, 0) == (0.0, 1.0) + + +def record_json(**overrides: Any) -> dict[str, Any]: + [task] = load_tasks(["write-function"]) + base = run_task(task, FACTS, Scripted(list(SOLVE_SLUGIFY)), Tuning()).to_json() + base.update(overrides) + return base + + +def test_results_round_trip_and_report(tmp_path: Path) -> None: + from nisse_dev.bench.runner import RunRecord + + rec = RunRecord(**record_json()) + path = results_file(tmp_path, "local-fast") + append(path, rec) + append(path, RunRecord(**record_json(success=False))) + stored = load(tmp_path) + assert len(stored) == 2 + assert json.loads(path.read_text().splitlines()[0])["model"] == "local-fast" + + current = {t.name: t.content_hash for t in TASKS} + [row] = summarise(stored, current) + assert (row.successes, row.runs) == (1, 2) + assert not row.stale + table = render([row]) + assert "openai/qwen3-8b-awq" in table + assert "1/2" in table + + +def test_runs_of_an_older_task_version_are_flagged() -> None: + current = {t.name: t.content_hash for t in TASKS} + rows = summarise([record_json(task_hash="0123456789abcdef")], current) + assert rows[0].stale + assert "(old)" in render(rows) + + +def test_models_served_differently_are_kept_apart() -> None: + current = {t.name: t.content_hash for t in TASKS} + rows = summarise( + [record_json(note="vLLM, AWQ"), record_json(note="llama.cpp, Q4_K_M")], current + ) + assert len(rows) == 2 + + +def test_metering_counts_prose_and_cost() -> None: + class Inner: + def __init__(self) -> None: + self.last_usage = {"input": 10, "output": 2} + self.last_cost = 0.5 + self.last_model = "local-fast" + + def complete(self, events, tools, model, system=None): + return [AssistantText(text="hi", model=model)] + + meter = MeteringClient(Inner(), FACTS) + meter.complete([], None, "local-fast") + meter.complete([], None, "local-fast") + assert (meter.calls, meter.prose_turns, meter.cost) == (2, 2, 1.0) diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100644 index 0000000..c694a80 --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,124 @@ +"""Benchmark executor models on canned milestones, and compare the results. + + uv run python scripts/benchmark.py run --models A,B [--runs 5] [--tasks T,...] + [--note "vLLM 0.29, AWQ, fp8 KV"] [--config PATH] + uv run python scripts/benchmark.py report [--models A,B] [--tasks T,...] + +`run` appends one JSON line per run to bench/results/, which is committed: +the report compares every run ever stored, so benchmarking a new model means +running only that model. See docs/Tuning-Guide.md, "Benchmarking models". +""" + +import argparse +import os +import sys +from pathlib import Path + +from nisse_dev.agent import make_llm_clients +from nisse_dev.bench.results import append, load, render, results_file, summarise +from nisse_dev.bench.runner import agent_commit, run_task +from nisse_dev.bench.tasks import load_tasks +from nisse_dev.catalog import ModelCatalog +from nisse_dev.config import AgentConfig, ConfigError +from nisse_dev.envfile import load_env_file + +REPO = Path(__file__).resolve().parents[1] +RESULTS = REPO / "bench" / "results" + + +def _names(raw: str | None) -> list[str] | None: + return [n.strip() for n in raw.split(",") if n.strip()] if raw else None + + +def run(args: argparse.Namespace) -> int: + load_env_file(Path(os.environ.get("NISSE_ENV", ".env")).expanduser()) + try: + config = ( + AgentConfig.from_file(Path(args.config)) + if args.config + else AgentConfig.load() + ) + except ConfigError as exc: + print(exc, file=sys.stderr) + return 2 + models = _names(args.models) or [] + tasks = load_tasks(_names(args.tasks)) + catalog = ModelCatalog.discover(config.llm_base_url, config.llm_token) + catalog.require_limits({m: m for m in models}) + normalized, _passthrough = make_llm_clients(config, catalog) + commit = agent_commit(REPO) + + for model in models: + facts = catalog.facts(model) + out = results_file(args.results, model) + for task in tasks: + for n in range(args.runs): + # The executor always takes the normalized route in the agent, + # so it does here too. + record = run_task( + task, + facts, + normalized, + config.tuning, + note=args.note, + commit=commit, + ) + append(out, record) + verdict = "pass" if record.success else f"FAIL ({record.outcome})" + extra = ( + f" fallback to {record.fallbacks[0]}!" if record.fallbacks else "" + ) + print( + f"{model} {task.name} run {n + 1}/{args.runs}: {verdict}, " + f"{record.turns} turns, {record.wall_s:.0f}s, " + f"${record.cost:.4f}{extra}", + flush=True, + ) + print() + return report(args) + + +def report(args: argparse.Namespace) -> int: + records = load(args.results) + models = _names(getattr(args, "models", None)) + tasks = _names(getattr(args, "tasks", None)) + if models: + records = [r for r in records if r["model"] in models or r["backing"] in models] + if tasks: + records = [r for r in records if r["task"] in tasks] + if not records: + print("no stored results match", file=sys.stderr) + return 1 + current = {task.name: task.content_hash for task in load_tasks()} + print(render(summarise(records, current))) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--results", type=Path, default=RESULTS, help=argparse.SUPPRESS) + sub = parser.add_subparsers(dest="command", required=True) + + run_p = sub.add_parser("run", help="benchmark models, then report") + run_p.add_argument( + "--models", required=True, help="LiteLLM aliases, comma separated" + ) + run_p.add_argument("--runs", type=int, default=5, help="runs per task (default 5)") + run_p.add_argument("--tasks", help="task names, comma separated (default: all)") + run_p.add_argument( + "--note", default="", help='how the model was served, e.g. "vLLM 0.29, AWQ"' + ) + run_p.add_argument("--config", help="agent config (default: NISSE_CONFIG)") + run_p.set_defaults(handler=run) + + report_p = sub.add_parser("report", help="compare stored results") + report_p.add_argument("--models", help="aliases or backing models to include") + report_p.add_argument("--tasks", help="tasks to include") + report_p.set_defaults(handler=report) + + args = parser.parse_args() + return args.handler(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/evaluate_model.sh b/scripts/evaluate_model.sh new file mode 100755 index 0000000..81dcf22 --- /dev/null +++ b/scripts/evaluate_model.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Evaluate a model for the executor role, from a declared LiteLLM alias to a +# benchmark report next to every model measured before it. +# +# scripts/evaluate_model.sh ALIAS "how it is served" [RUNS] [--config PATH] +# +# 1. The context probe: does the server's window match what LiteLLM declares? +# 2. The capability matrix: does it hold the tool contract, and is its +# supports_tool_choice declaration true? +# 3. The benchmark: canned milestones, RUNS times each (default 5). +# Stops at the first step that fails; there is no point benchmarking a model +# that can't hold the tool contract. +set -euo pipefail +cd "$(dirname "$0")/.." + +alias="${1:?usage: evaluate_model.sh ALIAS NOTE [RUNS] [--config PATH]}" +note="${2:?give a note on how the model is served, e.g. \"vLLM 0.29, AWQ\"}" +runs="${3:-5}" +shift $(( $# < 3 ? $# : 3 )) +config=("$@") + +echo "== 1/3 context probe" +uv run python scripts/probe_context.py "$alias" "${config[@]}" + +echo "== 2/3 capability matrix" +uv run pytest -m integration -q -rs tests/integration/test_model_capability.py \ + --capability-models="$alias" + +echo "== 3/3 benchmark ($runs runs per task)" +uv run python scripts/benchmark.py run --models "$alias" --runs "$runs" \ + --note "$note" "${config[@]}" diff --git a/tests/integration/test_benchmark_live.py b/tests/integration/test_benchmark_live.py new file mode 100644 index 0000000..38eeda6 --- /dev/null +++ b/tests/integration/test_benchmark_live.py @@ -0,0 +1,31 @@ +"""One live benchmark run: the simplest task, through the agent's own +client wiring, with the test config's executor.""" + +from pathlib import Path + +import pytest + +from nisse_dev.agent import make_llm_clients +from nisse_dev.bench.runner import run_task +from nisse_dev.bench.tasks import load_tasks +from nisse_dev.catalog import ModelCatalog +from nisse_dev.config import AgentConfig + +pytestmark = pytest.mark.integration + + +def test_the_simplest_task_passes_live( + live_config: AgentConfig, tmp_path: Path +) -> None: + catalog = ModelCatalog.discover(live_config.llm_base_url, live_config.llm_token) + normalized, _ = make_llm_clients(live_config, catalog) + facts = catalog.facts(live_config.roles.executor) + [task] = load_tasks(["write-function"]) + + record = run_task(task, facts, normalized, live_config.tuning, workdir=tmp_path) + + assert record.fallbacks == [] + assert record.success, ( + f"{record.outcome}: {record.outcome_note}\n{record.check_output}" + ) + assert record.cost >= 0