dongarandClaude Opus 5.5
4b6c48cc96
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m12s
Five runs per task at 320 W with max_turns 40, after the fixes in #50-#52, for qwen3.8-27b, qwen3.6-35b-a3b, gemma-4-26b-a4b, devstral-small-2-24b and mimo-v2.6-9b; the harder four also for claude-haiku-4-5 as the ceiling, and two calibration runs each for local-fast as the floor. Runs failed by the proxy's rate limiter were set aside, not stored. Earlier results move to bench/results/archive/, which the report does not read: they were made at other power caps, at max_turns 24, or on the RTX 4070, and are kept as the record. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
21 lines
53 KiB
JSON
21 lines
53 KiB
JSON
{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1578, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:41+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 63.2}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1576, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:43+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1576, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:44+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1574, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:46+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "29.0 \u00b1 2.9e-05\nE \nE comparison failed\nE Obtained: 0.35\nE Expected: 29.0 \u00b1 2.9e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n________________________ test_total_value[items2-37.5] _________________________\n\nitems = [Item(name='gear', unit_price=12.5, quantity=3)], expected = 37.5\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 12.5 == 37.5 \u00b1 3.7e-05\nE \nE comparison failed\nE Obtained: 12.5\nE Expected: 37.5 \u00b1 3.7e-05\n\n.bench_check/test_hidden.py:22: AssertionError\n_________________________ test_total_value[items3-0.0] _________________________\n\nitems = [Item(name='empty', unit_price=9.99, quantity=0)], expected = 0.0\n\n @pytest.mark.parametrize(\n (\"items\", \"expected\"),\n [\n ([], 0.0),\n ([Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)], 29.0),\n ([Item(\"gear\", 12.5, 3)], 37.5),\n ([Item(\"empty\", 9.99, 0)], 0.0),\n ],\n )\n def test_total_value(items: list[Item], expected: float) -> None:\n> assert total_value(items) == pytest.approx(expected)\nE assert 9.99 == 0.0 \u00b1 1.0e-12\nE \nE comparison failed\nE Obtained: 9.99\nE Expected: 0.0 \u00b1 1.0e-12\n\n.bench_check/test_hidden.py:22: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_total_value[items1-29.0] - assert 0....\nFAILED .bench_check/test_hidden.py::test_total_value[items2-37.5] - assert 12...\nFAILED .bench_check/test_hidden.py::test_total_value[items3-0.0] - assert 9.9...\n3 failed, 3 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF. [100%]\n=================================== FAILURES ===================================\n______________________ test_total_value_counts_every_unit ______________________\n\n def test_total_value_counts_every_unit() -> None:\n items = [Item(\"bolt\", 0.25, 100), Item(\"nut\", 0.10, 40)]\n> assert total_value(items) == 29.0\nE AssertionError: assert 0.35 =", "prose_turns": 0, "provider": "openai", "success": false, "task": "fix-bug", "task_hash": "b78248aff00ddc39", "tokens_in": 1578, "tokens_out": 50, "trimmed_requests": 0, "ts": "2026-09-25T01:00:48+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 2, "wall_s": 1.4}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmpl4mdto5g/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11000, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:00:52+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmpsqvx_94y/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11000, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:00:57+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmpe3eakwxj/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11013, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:01:01+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmprlm2camm/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 10987, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:01:06+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:2: in <module>\n from rates import RATES\nE File \"/tmp/tmpvzw7f5tn/work/rates.py\", line 1\nE \"R237\": 0.12,\nE IndentationError: unexpected indent\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 2\nstdout:\n\n==================================== ERRORS ====================================\n________________________ ERROR collecting test_rates.py ________________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__", "prose_turns": 0, "provider": "openai", "success": false, "task": "long-output", "task_hash": "8cfbf73fb6620613", "tokens_in": 11000, "tokens_out": 183, "trimmed_requests": 0, "ts": "2026-09-25T01:01:11+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 4.2}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n...F [100%]\n=================================== FAILURES ===================================\n_________________________ test_cli_prints_the_greeting _________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x713c88ce3c20>\n\n def test_cli_prints_the_greeting(capsys) -> None:\n> assert main([\"Bob\"])\nE AssertionError: assert 0\nE + wh", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 13039, "tokens_out": 566, "trimmed_requests": 0, "ts": "2026-09-25T01:01:19+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 8.4}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..FF [100%]\n=================================== FAILURES ===================================\n________________________________ test_cli_shout ________________________________\n\n def test_cli_shout() -> None:\n> from pytest import capsys\nE ImportError: cannot import name 'capsys' from 'pytest' (/root/nisse/.venv/lib/python3.12/site-packages/pytest/__init__.py)", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 11243, "tokens_out": 715, "trimmed_requests": 0, "ts": "2026-09-25T01:01:29+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 9.9}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "\n==================================== ERRORS ====================================\n_________________ ERROR collecting .bench_check/test_hidden.py _________________\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/usr/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n<frozen importlib._bootstrap>:1387: in _gcd_import\n ???\n<frozen importlib._bootstrap>:1360: in _find_and_load\n ???\n<frozen importlib._bootstrap>:1331: in _find_and_load_unlocked\n ???\n<frozen importlib._bootstrap>:935: in _load_unlocked\n ???\n/root/nisse/.venv/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\n.bench_check/test_hidden.py:4: in <module>\n from greeter.cli import main\nE File \"/tmp/tmp3f09u7jk/work/greeter/cli.py\", line 11\nE print(greet(args.name, shout=args.--shout))\nE ^\nE SyntaxError: invalid syntax\n=========================== short test summary info ============================\nERROR .bench_check/test_hidden.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.14s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n\nstderr:\nTraceback (most recent call last):\n File \"<frozen runpy>\", line 198, in _run_module_as_main\n File \"<frozen runpy>\", line 88, in _run_code\n File \"/tmp/tmp3f09u7jk/work/greeter/__main__.py\", line 1, in <module>\n from greeter.cli import main\n File \"/tmp/tmp3f09u7jk/work/greeter/cli.py\", line 11\n print(greet(args.name, shout=args.--shout))\n ^\nSyntaxError: invalid syntax\n", "prose_turns": 0, "provider": "openai", "success": false, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 9358, "tokens_out": 450, "trimmed_requests": 0, "ts": "2026-09-25T01:01:56+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 26.5}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "completed", "outcome_note": "Added --shout option to greeter. Tests pass.", "prose_turns": 0, "provider": "openai", "success": true, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 10784, "tokens_out": 549, "trimmed_requests": 0, "ts": "2026-09-25T01:02:04+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 7.5}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": " stderr arguments may not be used '\n 'with capture_output.')\n kwargs['stdout'] = PIPE\n kwargs['stderr'] = PIPE\n \n with Popen(*popenargs, **kwargs) as process:\n try:\n stdout, stderr = process.communicate(input, timeout=timeout)\n except TimeoutExpired as exc:\n process.kill()\n if _mswindows:\n # Windows accumulates the output in a single blocking\n # read() call run on child threads, with the timeout\n # being done in a join() on those threads. communicate()\n # _after_ kill() is required to collect that and add it\n # to the exception.\n exc.stdout, exc.stderr = process.communicate()\n else:\n # POSIX _communicate already populated the output so\n # far into the TimeoutExpired exception.\n process.wait()\n raise\n except: # Including KeyboardInterrupt, communicate handled that.\n process.kill()\n # We don't call process.wait() as .__exit__ does that for us.\n raise\n retcode = process.poll()\n if check and retcode:\n> raise CalledProcessError(retcode, process.args,\n output=stdout, stderr=stderr)\nE subprocess.CalledProcessError: Command '['/root/nisse/.venv/bin/python', '-m', 'greeter', '--shout', 'Bob']' returned non-zero exit status 2.\n\n/usr/lib/python3.12/subprocess.py:571: CalledProcessError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_cli_default - SystemExit: 2\nFAILED .bench_check/test_hidden.py::test_cli_shout - SystemExit: 2\nFAILED .bench_check/test_hidden.py::test_module_entry_point - subprocess.Call...\n3 failed, 2 passed in 0.19s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\n..F [100%]\n=================================== FAILURES ===================================\n________________________________ test_cli_shout ________________________________\n\ncapsys = <_pytest.capture.CaptureFixture object at 0x763f352e3980>\n\n def test_cli_shout(capsys) -> None:\n> assert main([\"--shout\", \"Bob\"]) == 0\n ^^^^^^^^^^^^^^^^^^^^^^^^\n\ntest", "prose_turns": 0, "provider": "openai", "success": false, "task": "multi-file", "task_hash": "76bc0dcf36a72599", "tokens_in": 13124, "tokens_out": 589, "trimmed_requests": 0, "ts": "2026-09-25T01:02:13+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 8.4}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": " (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'leading-and-trailing'\nE \nE - leading-and-trailing\n\n.bench_check/test_hidden.py:19: AssertionError\n_____________________ test_slugify[MiXeD CaSe-mixed-case] ______________________\n\ntext = 'MiXeD CaSe', expected = 'mixed-case'\n\n @pytest.mark.parametrize(\n (\"text\", \"expected\"),\n [\n (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'mixed-case'\nE \nE - mixed-case\n\n.bench_check/test_hidden.py:19: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_slugify[Hello, World!-hello-world]\nFAILED .bench_check/test_hidden.py::test_slugify[ Nisse Agent -nisse-agent]\nFAILED .bench_check/test_hidden.py::test_slugify[a--b-a-b] - AssertionError: ...\nFAILED .bench_check/test_hidden.py::test_slugify[Python 3.12 Release-python-3-12-release]\nFAILED .bench_check/test_hidden.py::test_slugify[--leading and trailing---leading-and-trailing]\nFAILED .bench_check/test_hidden.py::test_slugify[MiXeD CaSe-mixed-case] - Ass...\n6 failed, 2 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\nE AssertionError: assert '' == 'hello-world'\nE \nE - hello-world\n\ntest_slugify.py:6: AssertionEr", "prose_turns": 0, "provider": "openai", "success": false, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 4942, "tokens_out": 377, "trimmed_requests": 0, "ts": "2026-09-25T01:02:19+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 5.5}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\n ^^^^^^^\nE NameError: name 'slugify' is not defined\n\ntest_slugify.py:4: NameError\n===============", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3688, "tokens_out": 255, "trimmed_requests": 0, "ts": "2026-09-25T01:02:23+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 3.9}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\n ^^^^^^^\nE NameError: name 'slugify' is not defined\n\ntest_slugify.py:4: NameError\n===============", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3696, "tokens_out": 255, "trimmed_requests": 0, "ts": "2026-09-25T01:02:27+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 3.9}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": " (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'leading-and-trailing'\nE \nE - leading-and-trailing\n\n.bench_check/test_hidden.py:19: AssertionError\n_____________________ test_slugify[MiXeD CaSe-mixed-case] ______________________\n\ntext = 'MiXeD CaSe', expected = 'mixed-case'\n\n @pytest.mark.parametrize(\n (\"text\", \"expected\"),\n [\n (\"Hello, World!\", \"hello-world\"),\n (\" Nisse Agent \", \"nisse-agent\"),\n (\"a--b\", \"a-b\"),\n (\"\", \"\"),\n (\"Python 3.12 Release\", \"python-3-12-release\"),\n (\"--leading and trailing--\", \"leading-and-trailing\"),\n (\"MiXeD CaSe\", \"mixed-case\"),\n (\"!!!\", \"\"),\n ],\n )\n def test_slugify(text: str, expected: str) -> None:\n> assert slugify(text) == expected\nE AssertionError: assert '' == 'mixed-case'\nE \nE - mixed-case\n\n.bench_check/test_hidden.py:19: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_slugify[Hello, World!-hello-world]\nFAILED .bench_check/test_hidden.py::test_slugify[ Nisse Agent -nisse-agent]\nFAILED .bench_check/test_hidden.py::test_slugify[a--b-a-b] - AssertionError: ...\nFAILED .bench_check/test_hidden.py::test_slugify[Python 3.12 Release-python-3-12-release]\nFAILED .bench_check/test_hidden.py::test_slugify[--leading and trailing---leading-and-trailing]\nFAILED .bench_check/test_hidden.py::test_slugify[MiXeD CaSe-mixed-case] - Ass...\n6 failed, 2 passed in 0.03s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\nE AssertionError: assert '' == 'hello-world'\nE \nE - hello-world\n\ntest_slugify.py:6: AssertionEr", "prose_turns": 0, "provider": "openai", "success": false, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 4936, "tokens_out": 377, "trimmed_requests": 0, "ts": "2026-09-25T01:02:33+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 5.5}
|
|
{"backing": "openai/qwen3-8b-awq", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "50b4682", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "local-fast", "model_info": {"max_input_tokens": 32768, "max_output_tokens": 8192, "supports_function_calling": true, "supports_tool_choice": true}, "note": "vLLM 0.29, Qwen3-8B-AWQ, fp8 KV, 32k", "outcome": "escalate", "outcome_note": "2 consecutive command failures; last: exit code: 1\nstdout:\nF [100%]\n=================================== FAILURES ===================================\n_________________________________ test_slugify _________________________________\n\n def test_slugify():\n> assert slugify('Hello, World!') == 'hello-world'\n ^^^^^^^\nE NameError: name 'slugify' is not defined\n\ntest_slugify.py:4: NameError\n===============", "prose_turns": 0, "provider": "openai", "success": true, "task": "write-function", "task_hash": "ec722fb0f389a6f4", "tokens_in": 3686, "tokens_out": 251, "trimmed_requests": 0, "ts": "2026-09-25T01:02:37+00:00", "tuning": {"context": {"keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "executor": {"failure_limit": 2, "max_turns": 24}, "expert": {"consult_budget": 3}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 3.9}
|