Stream the chat route, so the timeout is on silence (#174)

A model call that passed request_timeout_s failed the whole run, though the
model was still writing. The chat route, which every local model takes, now
streams its replies and puts them back together into the response the
server would have sent, so the call record and parsing are unchanged.
request_timeout_s becomes the longest the model may go silent.

Measured on the 4090 through LiteLLM: an abandoned plain request kept the
server busy (a tiny request after it waited 51 s); an abandoned stream freed
it (0.3 s). A plain retry would have waited behind the abandoned call, so
there is none.

The passthrough route, hosted Anthropic models only, stays unstreamed: its
cost comes only in a header that streaming loses.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk

nisse-core/tests/test_llm.py

@@ -602,3 +602,223 @@ def test_a_budget_refusal_is_not_waited_on() -> None:
     with pytest.raises(BudgetExhausted):
         make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
     assert len(calls) == 1
+
+
+# --- streamed replies on the chat route (#174) -----------------------------------
+
+
+def sse(*chunks: dict, done: bool = True) -> httpx.Response:
+    lines = "".join(f"data: {json.dumps(c)}\n\n" for c in chunks)
+    return httpx.Response(
+        200,
+        headers={"content-type": "text/event-stream"},
+        content=lines + ("data: [DONE]\n\n" if done else ""),
+    )
+
+
+def piece(delta: dict | None = None, finish: str | None = None, **extra) -> dict:
+    choice: dict = {"index": 0, "delta": delta or {}}
+    if finish:
+        choice["finish_reason"] = finish
+    return {"model": "qwen-real", "choices": [choice], **extra}
+
+
+class Failing(httpx.SyncByteStream):
+    """A reply that sends its first piece, then fails."""
+
+    def __init__(self, first: bytes, failure: Exception) -> None:
+        self._first, self._failure = first, failure
+
+    def __iter__(self):
+        yield self._first
+        raise self._failure
+
+
+def test_the_chat_route_asks_for_a_stream_with_usage() -> None:
+    seen: dict = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.update(json.loads(request.content))
+        return sse(piece({"content": "hi"}, "stop"))
+
+    make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
+    assert seen["stream"] is True
+    assert seen["stream_options"] == {"include_usage": True}
+
+
+def test_a_streamed_reply_is_put_back_together() -> None:
+    usage = {"prompt_tokens": 7, "completion_tokens": 3, "cost": 0.25}
+    usage["prompt_tokens_details"] = {"cached_tokens": 4}
+    client = make_client(
+        lambda request: sse(
+            piece({"role": "assistant", "reasoning_content": "think"}),
+            piece({"reasoning_content": "ing"}),
+            piece({"content": "hel"}),
+            piece({"content": "lo"}),
+            piece(finish="stop"),
+            {"model": "qwen-real", "choices": [], "usage": usage},
+        )
+    )
+    [event] = client.complete(EVENTS, tools=None, model="local/qwen")
+    assert event == AssistantText(text="hello", model="qwen-real")
+    assert client.last_usage == {"input": 7, "output": 3, "cached": 4}
+    assert client.last_cost == 0.25
+    assert client.last_model == "qwen-real"
+    # The call record keeps the reply as one response, reasoning included.
+    message = client.last_exchange["response"]["choices"][0]["message"]
+    assert message["reasoning_content"] == "thinking"
+    assert message["content"] == "hello"
+
+
+def test_streamed_tool_calls_are_joined_from_their_pieces() -> None:
+    def call(index: int, **function) -> dict:
+        return {"tool_calls": [{"index": index, "function": function}]}
+
+    client = make_client(
+        lambda request: sse(
+            piece(call(0, name="write_file", arguments='{"path":')),
+            piece(call(0, arguments=' "a.txt"}')),
+            piece(call(1, name="read_file", arguments="{")),
+            piece(call(1, arguments='"path": "b"}')),
+            piece(finish="tool_calls"),
+        )
+    )
+    first, second = client.complete(EVENTS, tools=None, model="local/qwen")
+    assert isinstance(first, ToolCall) and isinstance(second, ToolCall)
+    assert (first.tool, first.arguments) == ("write_file", {"path": "a.txt"})
+    assert (second.tool, second.arguments) == ("read_file", {"path": "b"})
+
+
+def test_a_streamed_reply_cut_off_at_the_limit_is_truncated() -> None:
+    client = make_client(lambda request: sse(piece({"content": "half"}, "length")))
+    with pytest.raises(TruncatedResponse):
+        client.complete(EVENTS, tools=None, model="local/qwen")
+
+
+def test_a_stream_that_stops_before_it_finishes_is_upstream_trouble() -> None:
+    client = make_client(lambda request: sse(piece({"content": "half"}), done=False))
+    with pytest.raises(UpstreamUnavailable, match="stopped before it finished"):
+        client.complete(EVENTS, tools=None, model="local/qwen")
+
+
+def test_an_error_mid_stream_is_upstream_trouble() -> None:
+    broke = make_client(lambda request: sse({"error": {"message": "backend died"}}))
+    with pytest.raises(UpstreamUnavailable, match="backend died"):
+        broke.complete(EVENTS, tools=None, model="local/qwen")
+    spent = make_client(
+        lambda request: sse({"error": {"message": "Budget has been exceeded"}})
+    )
+    with pytest.raises(BudgetExhausted):
+        spent.complete(EVENTS, tools=None, model="local/qwen")
+
+
+def test_a_model_gone_quiet_mid_reply_times_out_once() -> None:
+    calls: list[int] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls.append(1)
+        first = f"data: {json.dumps(piece({'content': 'hel'}))}\n\n".encode()
+        return httpx.Response(
+            200,
+            headers={"content-type": "text/event-stream"},
+            stream=Failing(first, httpx.ReadTimeout("quiet")),
+        )
+
+    with pytest.raises(UpstreamUnavailable, match="said nothing for 600 s"):
+        make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
+    assert len(calls) == 1  # a hung model is not asked again
+
+
+def test_a_connection_lost_mid_reply_is_retried() -> None:
+    calls: list[int] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls.append(1)
+        if len(calls) == 1:
+            first = f"data: {json.dumps(piece({'content': 'hel'}))}\n\n".encode()
+            return httpx.Response(
+                200,
+                headers={"content-type": "text/event-stream"},
+                stream=Failing(first, httpx.RemoteProtocolError("peer closed")),
+            )
+        return sse(piece({"content": "hello"}, "stop"))
+
+    [event] = make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
+    assert event == AssistantText(text="hello", model="qwen-real") and len(calls) == 2
+
+
+def test_the_passthrough_still_asks_for_the_whole_reply() -> None:
+    seen: dict = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.update(json.loads(request.content))
+        return httpx.Response(
+            200,
+            json={"model": "m", "content": [{"type": "text", "text": "hi"}]},
+        )
+
+    AnthropicPassthroughClient(
+        "https://litellm.test", "tok", transport=httpx.MockTransport(handler)
+    ).complete(EVENTS, tools=None, model="claude")
+    assert "stream" not in seen
+
+
+class _Server:
+    """A real HTTP server that streams a reply in pieces, ``gap`` seconds
+    apart: the timeout is httpx's own, so only a real socket shows what it
+    measures."""
+
+    def __init__(self, pieces: int, gap: float) -> None:
+        import threading
+        from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+        class Handler(BaseHTTPRequestHandler):
+            def do_POST(self) -> None:
+                self.rfile.read(int(self.headers["content-length"]))
+                self.send_response(200)
+                self.send_header("content-type", "text/event-stream")
+                self.end_headers()
+                import time
+
+                try:
+                    for n in range(pieces):
+                        time.sleep(gap)
+                        chunk = piece({"content": str(n)})
+                        self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode())
+                        self.wfile.flush()
+                    self.wfile.write(b"data: [DONE]\n\n")
+                except (BrokenPipeError, ConnectionResetError):
+                    pass  # the client gave up, as it should
+
+            def log_message(self, *args: Any) -> None:
+                pass
+
+        self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+        self.url = f"http://127.0.0.1:{self.server.server_address[1]}"
+        threading.Thread(target=self.server.serve_forever, daemon=True).start()
+
+    def close(self) -> None:
+        self.server.shutdown()
+        self.server.server_close()
+
+
+def test_a_slow_reply_that_keeps_coming_outlasts_the_timeout() -> None:
+    # Six pieces 0.1 s apart take longer than the 0.35 s timeout, but no gap
+    # does: a model still writing is never cut off (#174).
+    server = _Server(pieces=6, gap=0.1)
+    try:
+        client = LiteLLMClient(server.url, "tok", timeout_s=0.35)
+        [event] = client.complete(EVENTS, tools=None, model="local/qwen")
+    finally:
+        server.close()
+    assert isinstance(event, AssistantText) and event.text == "012345"
+
+
+def test_a_reply_that_goes_quiet_times_out() -> None:
+    server = _Server(pieces=2, gap=1.0)
+    try:
+        client = LiteLLMClient(server.url, "tok", timeout_s=0.2)
+        with pytest.raises(UpstreamUnavailable, match="said nothing"):
+            client.complete(EVENTS, tools=None, model="local/qwen")
+    finally:
+        server.close()