dev/nisse
2
1 Fork 0
Code Issues 23 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

M7.1: extract nisse-core
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m42s
Details
CI / e2e (pull_request) Successful in 4m0s
Details

Browse Source 
What isn't the dev agent's moves to nisse-core, unchanged in behaviour,
with its tests: canonical, render, render_anthropic, llm, catalog, probe,
context, views, state, reconcile, worker, cancel, dump, transport, matrix,
execution, testing, vault, memory, oplog, envfile and gitea (whole).

- nisse_core.state.StateStore holds the tables every agent has; an
  agent's subclass adds its tables (version, migrations), roles and
  views. nisse_dev.state.StateStore adds the build VM blocks (v5).
- views.project takes what each role is shown, rather than a fixed table.
- Vault takes the agent's name (agent=), for provenance.
- A test that nisse_core never imports an agent.
- The loop, config, git, delegation, expert, tools and evals stay in
  nisse-dev until nisse-ops has a loop to share.

Docs: the Development Plan marks M7.1 done and notes that Phase 6 waits on
the agent host and Terraform; Architecture Part L names the package.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
dongarandClaude Opus 5.5 committed 2026-10-13 18:37:01 +00:00
1 parent 4bbcb09c8e
commit 154a0c3fa8
119 files changed
+1086 -912
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

nisse-dev/tests/model_contract.py
+2 -2
View File
@@ -10,8 +10,8 @@ unit tests and be rejected live; this makes it fail here.
from typing import Any
from nisse_dev.render import OpenAIChatRenderer
from nisse_dev.render_anthropic import AnthropicMessagesRenderer
from nisse_core.render import OpenAIChatRenderer
from nisse_core.render_anthropic import AnthropicMessagesRenderer
def assert_renderable(events: list[Any]) -> None:
nisse-dev/tests/test_agent_build.py
+5 -5
View File
@@ -11,13 +11,13 @@ import httpx
import pytest
from test_config import SECRETS, P, tree
from nisse_dev.catalog import ModelCatalog, UnknownModel
from nisse_core.catalog import ModelCatalog, UnknownModel
from nisse_core.execution import SSHHost
from nisse_core.llm import AnthropicPassthroughClient, LiteLLMClient
from nisse_core.oplog import OpLog
from nisse_dev.config import AgentConfig
from nisse_dev.execution import SSHHost
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient
from nisse_dev.loop.build import RunningLoop, build_loop_agent, discover_models
from nisse_dev.loop.build import prepare_workspace as prepare
from nisse_dev.oplog import OpLog
# What the test roles resolve to, as LiteLLM would report them.
TEST_CATALOG = ModelCatalog.from_payload(
@@ -143,7 +143,7 @@ def offline(monkeypatch: Any, origins: dict[str, str] | str, catalog=None) -> No
"""Only the network boundaries faked: Gitea, LiteLLM's model list, Matrix."""
# The proxy here doesn't exist: a call to it fails at once rather than
# waiting out an outage's backoff (L5).
monkeypatch.setattr("nisse_dev.llm.OUTAGE_BACKOFF_S", ())
monkeypatch.setattr("nisse_core.llm.OUTAGE_BACKOFF_S", ())
monkeypatch.setattr(
"nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing"
)
nisse-dev/tests/test_bench.py
+3 -3
View File
@@ -7,6 +7,9 @@ from typing import Any
import pytest
from nisse_core.canonical import AssistantText, ToolCall
from nisse_core.catalog import ModelFacts
from nisse_core.execution import SSHHost
from nisse_dev.bench.results import (
append,
load,
@@ -17,10 +20,7 @@ from nisse_dev.bench.results import (
)
from nisse_dev.bench.runner import MeteringClient, run_task
from nisse_dev.bench.tasks import CHECK_DIR, load_tasks
from nisse_dev.canonical import AssistantText, ToolCall
from nisse_dev.catalog import ModelFacts
from nisse_dev.config import DriverTuning, Tuning
from nisse_dev.execution import SSHHost
# The build VM for every test here: the session's throwaway sshd.
HOST: SSHHost
nisse-dev/tests/test_canonical.py
-120
View File
@@ -1,120 +0,0 @@
import sqlite3
from pathlib import Path
import pytest
from nisse_dev.canonical import (
AssistantText,
Compaction,
DelegationCompleted,
DelegationStarted,
GateOpened,
PlanSubmitted,
ToolCall,
ToolResult,
UserMessage,
from_payload,
mint_call_id,
)
from nisse_dev.state import StateSchemaError, StateStore
ALL_EVENTS = [
UserMessage(text="use demo", source="matrix:$abc"),
AssistantText(text="starting", model="local/qwen"),
ToolCall(call_id="call-1", tool="run_tests", arguments={"path": "tests/"}),
ToolResult(call_id="call-1", result="3 passed", success=True),
PlanSubmitted(plan="the plan", model="claude-opus-5"),
GateOpened(gate="arch", reason="plan submitted (rev-1)"),
DelegationStarted(
brief="fix the flaky test",
role="helper",
model="claude-opus-5",
caps={"turns": 20, "tokens": 100000},
),
DelegationCompleted(
handback="test fixed; root cause was a shared tmpdir",
artifacts=["nisse-dev/tests/test_x.py"],
tokens=51234,
duration_s=61.5,
exit_status="success",
),
Compaction(summary="earlier work summarized", model="local/qwen", replaces=[1, 4]),
]
def test_all_event_types_round_trip(tmp_path: Path) -> None:
with StateStore(tmp_path / "state.db") as store:
for event in ALL_EVENTS:
store.transcript("t", "executor").append(event)
with StateStore(tmp_path / "state.db") as store:
stored = store.transcript("t", "executor").read()
assert [s.event for s in stored] == ALL_EVENTS
assert [s.id for s in stored] == sorted(s.id for s in stored)
assert {(s.work_item, s.actor) for s in stored} == {("t", "executor")}
def test_read_events_after_id(tmp_path: Path) -> None:
with StateStore(tmp_path / "state.db") as store:
log = store.transcript("t", "operator")
first = log.append(UserMessage(text="one", source="m:1"))
log.append(UserMessage(text="two", source="m:2"))
tail = log.read(after=first)
assert [s.event.text for s in tail] == ["two"]
def test_mint_call_id_is_unique_and_prefixed() -> None:
ids = {mint_call_id() for _ in range(100)}
assert len(ids) == 100
assert all(i.startswith("call-") for i in ids)
def test_unknown_event_type_rejected() -> None:
with pytest.raises(ValueError, match="unknown canonical event type"):
from_payload("mystery", {})
def test_unknown_fields_rejected() -> None:
with pytest.raises(ValueError, match="unknown fields"):
from_payload("user_message", {"text": "hi", "source": "m", "extra": 1})
# The v1 schema as it shipped (PR #2), frozen here to prove a newer image
# opens and upgrades a database written by the older one (L8).
V1_SCHEMA = [
"CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)",
"""
CREATE TABLE agent_state (
id INTEGER PRIMARY KEY CHECK (id = 1),
state TEXT NOT NULL,
project TEXT,
data TEXT NOT NULL DEFAULT '{}',
updated_at TEXT NOT NULL
)
""",
"""
CREATE TABLE intent_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL,
payload TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
created_at TEXT NOT NULL,
resolved_at TEXT
)
""",
]
def test_a_store_from_before_work_items_is_refused_clearly(tmp_path: Path) -> None:
# No production data predates work items, so there is no migration from
# these versions (Development Plan M1.1): the store must say so rather
# than open half-understood.
db = tmp_path / "state.db"
conn = sqlite3.connect(db)
for statement in V1_SCHEMA:
conn.execute(statement)
conn.execute("INSERT INTO meta (key, value) VALUES ('schema_version', '3')")
conn.commit()
conn.close()
with pytest.raises(StateSchemaError, match="predates work items"):
StateStore(db)
nisse-dev/tests/test_catalog.py
-202
View File
@@ -1,202 +0,0 @@
"""Runtime model discovery (architecture L9).
The point of these tests is that nothing here reads the model's *name*. A
semantic alias like "remote-architecture" is free to point anywhere, so the
route has to come from what LiteLLM says the model is.
"""
import httpx
import pytest
from nisse_dev.catalog import (
CatalogUnavailable,
ModelCatalog,
UnknownModel,
select_client,
)
def entry(alias: str, backing: str, provider: str | None, tools: bool | None = None):
"""One /model/info row, shaped as the live proxy returns it."""
info: dict = {"supports_function_calling": tools}
if provider is not None:
info["litellm_provider"] = provider
return {
"model_name": alias,
"litellm_params": {"model": backing, "api_base": "http://proxy.test/v1"},
"model_info": info,
}
def catalog(*entries) -> ModelCatalog:
return ModelCatalog.from_payload({"data": list(entries)})
# --- parsing -------------------------------------------------------------
def test_facts_are_read_from_the_payload() -> None:
cat = catalog(
entry("claude-haiku-4-5", "anthropic/claude-haiku-4-5", "anthropic", True)
)
facts = cat.facts("claude-haiku-4-5")
assert facts.backing == "anthropic/claude-haiku-4-5"
assert facts.provider == "anthropic"
assert facts.supports_tools is True
assert facts.native_anthropic
def test_provider_falls_back_to_the_backing_prefix() -> None:
# litellm_provider is absent on some deployments; the backing model's
# own "vendor/model" prefix says the same thing.
cat = catalog(entry("mystery", "anthropic/claude-opus-5", None))
assert cat.facts("mystery").provider == "anthropic"
def test_unnamed_entries_are_skipped() -> None:
cat = ModelCatalog.from_payload({"data": [{"litellm_params": {"model": "x"}}]})
assert cat.aliases() == []
def test_missing_tool_support_is_unknown_not_false() -> None:
# LiteLLM reports null for models it has no capability data on. That is
# not the same as declaring the model tool-less, and the startup warning
# depends on the difference.
cat = catalog(entry("local-fast", "openai/qwen3-8b-awq", "openai", None))
assert cat.facts("local-fast").supports_tools is None
# --- route selection by provider, never by name --------------------------
def test_a_semantic_alias_routes_by_its_provider() -> None:
# The whole feature: nothing in "remote-architecture" says Anthropic.
cat = catalog(entry("remote-architecture", "anthropic/claude-opus-5", "anthropic"))
normalized, passthrough = object(), object()
chosen = select_client("remote-architecture", normalized, passthrough, cat)
assert chosen is passthrough
def test_the_same_alias_reroutes_when_it_is_repointed() -> None:
# L9: provisionally accept LiteLLM's normalization for a vendor with no
# native adapter, rather than sending it Anthropic-shaped bodies.
cat = catalog(entry("remote-architecture", "openai/gpt-6-astra", "openai"))
normalized, passthrough = object(), object()
assert (
select_client("remote-architecture", normalized, passthrough, cat) is normalized
)
def test_a_name_that_merely_mentions_a_vendor_is_not_that_vendor() -> None:
cat = catalog(entry("claude-clone-9", "openai/some-clone", "openai"))
normalized, passthrough = object(), object()
assert select_client("claude-clone-9", normalized, passthrough, cat) is normalized
def test_local_models_take_the_normalized_route() -> None:
cat = catalog(
entry("local-coder", "openai/qwen3-14b", "openai"),
entry("local-fast", "openai/qwen3-8b-awq", "openai"),
)
normalized, passthrough = object(), object()
for alias in ("local-coder", "local-fast"):
assert select_client(alias, normalized, passthrough, cat) is normalized
# --- misconfiguration is loud --------------------------------------------
def test_an_unknown_model_raises_rather_than_guessing_a_route() -> None:
cat = catalog(entry("local-fast", "openai/qwen3-8b-awq", "openai"))
with pytest.raises(UnknownModel, match="local-fsat"):
select_client("local-fsat", object(), object(), cat)
def test_the_error_lists_what_is_available() -> None:
cat = catalog(
entry("local-fast", "openai/qwen3-8b-awq", "openai"),
entry("claude-opus-5", "anthropic/claude-opus-5", "anthropic"),
)
with pytest.raises(UnknownModel, match="claude-opus-5, local-fast"):
cat.facts("typo")
def test_validate_names_every_bad_role_at_once() -> None:
# One boot should surface the whole list, not make the operator
# rediscover the next typo on the next start.
cat = catalog(entry("local-fast", "openai/qwen3-8b-awq", "openai"))
with pytest.raises(UnknownModel) as caught:
cat.validate(
{
"executor": "local-fast",
"planner": "clause-opus-5",
"helper": "remote-architecture",
"expert": "local-fast",
}
)
message = str(caught.value)
assert "planner='clause-opus-5'" in message
assert "helper='remote-architecture'" in message
assert "executor" not in message # the good ones are not noise
def test_validate_passes_a_whole_good_mapping() -> None:
cat = catalog(
entry("local-fast", "openai/qwen3-8b-awq", "openai"),
entry("claude-opus-5", "anthropic/claude-opus-5", "anthropic"),
)
cat.validate({"executor": "local-fast", "planner": "claude-opus-5"})
# --- an outage is not a misconfiguration ---------------------------------
def test_an_unreachable_proxy_is_reported_as_an_outage() -> None:
def handler(request: httpx.Request) -> httpx.Response:
raise httpx.ConnectError("no route to host")
with pytest.raises(CatalogUnavailable, match="could not reach"):
ModelCatalog.discover(
"http://litellm.test", "sk-test", transport=httpx.MockTransport(handler)
)
def test_a_refused_listing_is_reported_as_an_outage() -> None:
cat = httpx.MockTransport(lambda request: httpx.Response(401, text="bad key"))
with pytest.raises(CatalogUnavailable, match="401"):
ModelCatalog.discover("http://litellm.test", "sk-test", transport=cat)
def test_discovery_asks_the_proxy_with_the_token() -> None:
seen: dict = {}
def handler(request: httpx.Request) -> httpx.Response:
seen["url"] = str(request.url)
seen["auth"] = request.headers.get("Authorization")
return httpx.Response(
200, json={"data": [entry("local-fast", "openai/qwen3-8b-awq", "openai")]}
)
cat = ModelCatalog.discover(
"http://litellm.test/", "sk-test", transport=httpx.MockTransport(handler)
)
assert seen["url"] == "http://litellm.test/model/info"
assert seen["auth"] == "Bearer sk-test"
assert cat.aliases() == ["local-fast"]
# --- what gets pinned ----------------------------------------------------
def test_pinning_records_what_the_alias_resolved_to() -> None:
# An alias may point somewhere else tomorrow; eval history (L9) keyed on
# the alias alone would average across whatever it pointed at.
cat = catalog(entry("remote-architecture", "anthropic/claude-opus-5", "anthropic"))
pinned = cat.pinned({"planner": "remote-architecture"})
assert pinned == {
"planner": {
"alias": "remote-architecture",
"backing": "anthropic/claude-opus-5",
"provider": "anthropic",
}
}
nisse-dev/tests/test_config.py
+2 -2
View File
@@ -5,8 +5,8 @@ from typing import Any
import pytest
from nisse_core.llm import DEFAULT_REQUEST_TIMEOUT_S
from nisse_dev.config import AgentConfig, ConfigError, Project
from nisse_dev.llm import DEFAULT_REQUEST_TIMEOUT_S
# A complete config as the YAML file would give it, with secrets by reference.
COMPLETE: dict[str, Any] = {
@@ -142,7 +142,7 @@ def test_request_timeout_is_configurable_and_checked() -> None:
def test_the_request_timeout_reaches_both_clients() -> None:
from nisse_dev.catalog import ModelCatalog
from nisse_core.catalog import ModelCatalog
from nisse_dev.loop.build import make_llm_clients
normalized, passthrough = make_llm_clients(
nisse-dev/tests/test_context.py
-300
View File
@@ -1,300 +0,0 @@
"""The context budget (architecture L6)."""
import io
from pathlib import Path
from typing import Any
import pytest
from nisse_dev.canonical import (
AssistantText,
CanonicalEvent,
StoredEvent,
ToolCall,
ToolResult,
UserMessage,
)
from nisse_dev.catalog import ModelCatalog, UnknownLimits
from nisse_dev.context import (
Budget,
BudgetedClient,
ContextOverflow,
estimate_tokens,
fit,
head_and_tail,
)
from nisse_dev.loop.driver import Driver
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore
def stored(events: list[CanonicalEvent]) -> list[StoredEvent]:
return [StoredEvent(id=i + 1, ts="t", event=e) for i, e in enumerate(events)]
def turn(n: int, output: str) -> list[CanonicalEvent]:
"""One executor turn: a command and its (possibly long) output."""
return [
ToolCall(call_id=f"c{n}", tool="run_command", arguments={"command": "make"}),
ToolResult(call_id=f"c{n}", result=output, success=True),
]
def transcript(turns: int, output_chars: int) -> list[StoredEvent]:
events: list[CanonicalEvent] = [UserMessage(text="start milestone 1", source="t")]
for n in range(turns):
events += turn(n, f"output {n} " + "x" * output_chars)
return stored(events)
def budget(window: int, keep: int = 2) -> Budget:
return Budget(
window=window, max_output=500, keep_recent_turns=keep, safety_margin=0.1
)
def results(events: list[StoredEvent]) -> dict[str, str]:
return {
e.event.call_id: e.event.result
for e in events
if isinstance(e.event, ToolResult)
}
# --- keeping the end of long output ---------------------------------------
def test_head_and_tail_keeps_the_failure_summary_at_the_end() -> None:
output = (
"exit code: 1\nstdout:\n"
+ "collected 400 items\n" * 500
+ "FAILED tests/test_app.py::test_greet - AssertionError\n"
+ "==== 1 failed, 399 passed ====\n"
)
kept = head_and_tail(output, 1000)
assert len(kept) < 1100
assert kept.startswith("exit code: 1") # the head keeps the exit code
assert "FAILED tests/test_app.py::test_greet" in kept
assert "1 failed, 399 passed" in kept
assert "characters omitted" in kept
def test_short_text_is_untouched() -> None:
assert head_and_tail("fine", 1000) == "fine"
# --- fitting, step by step -------------------------------------------------
def test_a_transcript_that_fits_is_sent_unchanged() -> None:
events = transcript(turns=3, output_chars=100)
assert fit(events, "system", None, budget(100_000)) == events
def test_step_one_cuts_older_tool_results_and_keeps_recent_ones_whole() -> None:
events = transcript(turns=6, output_chars=3000)
fitted = fit(events, "system", None, budget(4000, keep=2))
kept = results(fitted)
assert len(kept) == 6 # nothing dropped yet
for older in ("c0", "c1", "c2", "c3"):
assert "characters omitted" in kept[older]
for recent in ("c4", "c5"):
assert "characters omitted" not in kept[recent]
def test_step_two_drops_the_oldest_turns_and_says_so() -> None:
events = transcript(turns=20, output_chars=3000)
fitted = fit(events, "system", None, budget(4000, keep=2))
first = fitted[0].event
assert isinstance(first, UserMessage)
assert "omitted to fit the context window" in first.text
kept = results(fitted)
assert "c0" not in kept # the oldest went
assert {"c18", "c19"} <= kept.keys() # the recent turns stayed
assert "characters omitted" not in kept["c19"]
# Every surviving result still has its call: a provider rejects an
# answer to a call it never saw.
calls = {e.event.call_id for e in fitted if isinstance(e.event, ToolCall)}
assert kept.keys() <= calls
assert estimate_tokens(fitted, "system", None) <= budget(4000).for_prompt
def test_step_three_refuses_what_cannot_fit() -> None:
# Even the two most recent turns are bigger than the whole window.
events = transcript(turns=3, output_chars=20_000)
with pytest.raises(ContextOverflow, match="last 2 turns"):
fit(events, "system", None, budget(2000, keep=2))
def test_a_huge_system_prompt_alone_overflows() -> None:
events = transcript(turns=1, output_chars=10)
with pytest.raises(ContextOverflow):
fit(events, "s" * 50_000, None, budget(4096))
def test_the_output_ceiling_and_margin_come_out_of_the_window() -> None:
b = Budget(window=4096, max_output=1024, keep_recent_turns=6, safety_margin=0.1)
assert b.for_prompt == int(4096 * 0.9) - 1024
# --- the client wrapper ----------------------------------------------------
class RecordingClient:
def __init__(self, reported_input: int = 0) -> None:
self.calls: list[dict[str, Any]] = []
self.last_usage = {"input": reported_input, "output": 5}
def complete(self, events, tools, model, system=None, *, max_tokens=None, **kwargs):
self.calls.append({"events": events, "max_tokens": max_tokens, **kwargs})
return [AssistantText(text="ok", model=model)]
def test_the_ceiling_is_the_smaller_of_config_and_model() -> None:
inner = RecordingClient()
client = BudgetedClient(inner, lambda m: (4096, 1024), max_output_tokens=16384)
client.complete(transcript(1, 10), None, "llama3.2")
assert inner.calls[0]["max_tokens"] == 1024
client = BudgetedClient(inner, lambda m: (200_000, 64_000), max_output_tokens=16384)
client.complete(transcript(1, 10), None, "claude-haiku-4-5")
assert inner.calls[1]["max_tokens"] == 16384
def test_the_wrapper_sends_the_fitted_events() -> None:
inner = RecordingClient()
client = BudgetedClient(
inner, lambda m: (4000, 500), keep_recent_turns=2, safety_margin=0.1
)
client.complete(transcript(20, 3000), None, "m", system="system")
sent = inner.calls[0]["events"]
assert "omitted to fit" in sent[0].event.text
def test_reported_usage_only_ever_raises_the_estimate() -> None:
events = transcript(1, 10)
estimate = estimate_tokens(events, None, None)
under = RecordingClient(reported_input=estimate * 2) # we undercounted
client = BudgetedClient(under, lambda m: (100_000, None))
client.complete(events, None, "m")
assert client._factors["m"] == pytest.approx(2.0, rel=0.05)
over = RecordingClient(reported_input=estimate // 2) # we overcounted
under_then_over = BudgetedClient(over, lambda m: (100_000, None))
under_then_over._factors["m"] = 2.0
under_then_over.complete(events, None, "m")
assert under_then_over._factors["m"] == 2.0 # never loosened
def test_last_usage_passes_through() -> None:
client = BudgetedClient(RecordingClient(reported_input=7), lambda m: (100, None))
assert client.last_usage["input"] == 7
# --- the limits come from LiteLLM's catalog --------------------------------
CATALOG = ModelCatalog.from_payload(
{
"data": [
{
"model_name": "claude-haiku-4-5",
"litellm_params": {"model": "anthropic/claude-haiku-4-5"},
"model_info": {"max_input_tokens": 200000, "max_output_tokens": 64000},
},
{
"model_name": "llama3.2",
"litellm_params": {"model": "ollama/llama3.2"},
"model_info": {"max_input_tokens": 4096, "max_output_tokens": 1024},
},
{
"model_name": "undeclared",
"litellm_params": {"model": "ollama/qwen3"},
"model_info": {"max_input_tokens": None},
},
]
}
)
def test_the_catalog_reads_each_models_limits() -> None:
facts = CATALOG.facts("llama3.2")
assert (facts.max_input_tokens, facts.max_output_tokens) == (4096, 1024)
assert CATALOG.facts("undeclared").max_input_tokens is None
def test_a_role_on_a_model_without_a_window_is_refused() -> None:
CATALOG.require_limits({"driver": "llama3.2", "helper": "claude-haiku-4-5"})
with pytest.raises(UnknownLimits, match="driver='undeclared'"):
CATALOG.require_limits({"driver": "undeclared", "helper": "llama3.2"})
def test_a_request_that_cannot_fit_ends_the_run(tmp_path: Path) -> None:
# Never sent: the driver's run fails and the item waits on the operator,
# like any failed call, rather than a server getting a prompt it would
# silently cut.
store = StateStore(tmp_path / "state.db", oplog=OpLog("t", io.StringIO()))
store.transcript("t", "operator").append(UserMessage(text="go", source="t"))
inner = RecordingClient()
client = BudgetedClient(inner, lambda m: (4096, 1024))
outcome = Driver(client, "m").run(
store.transcript("t", "driver"),
system="s" * 20_000, # a system prompt too big to fit
tools=[],
workspace=None,
)
assert outcome.kind == "failed"
assert "fit" in outcome.note
assert inner.calls == []
# --- tool_choice is only asked of servers that enforce it -------------------
FINISH = [{"type": "function", "function": {"name": "finish", "parameters": {}}}]
def test_required_is_asked_only_of_models_declared_to_enforce_it() -> None:
# Measured: qwen3-14b under "required" cannot stop and rambles to the
# output ceiling, so a model that is not declared is never asked.
inner = RecordingClient()
client = BudgetedClient(
inner, lambda m: (32768, 8192), enforces_tool_choice=lambda m: m == "local-fast"
)
client.complete(transcript(1, 10), FINISH, "local-fast")
client.complete(transcript(1, 10), FINISH, "local-coder")
client.complete(transcript(1, 10), None, "local-fast")
assert [c["require_tool_call"] for c in inner.calls] == [True, False, False]
def test_the_catalog_reads_supports_tool_choice() -> None:
catalog = ModelCatalog.from_payload(
{
"data": [
{"model_name": "a", "model_info": {"supports_tool_choice": True}},
{"model_name": "b", "model_info": {"supports_tool_choice": False}},
{"model_name": "c", "model_info": {}},
]
}
)
assert [catalog.facts(m).supports_tool_choice for m in "abc"] == [True, False, None]
def test_a_fallback_answer_is_recognised() -> None:
facts = ModelCatalog.from_payload(
{
"data": [
{
"model_name": "claude-haiku-4-5",
"litellm_params": {"model": "anthropic/claude-haiku-4-5"},
}
]
}
).facts("claude-haiku-4-5")
assert facts.answered("claude-haiku-4-5") # LiteLLM's alias
assert facts.answered("claude-haiku-4-5-20251001") # the provider's own name
assert facts.answered(None) # nothing reported
assert not facts.answered("claude-sonnet-5") # somebody else answered
nisse-dev/tests/test_delegation.py
+7 -6
View File
@@ -4,8 +4,8 @@ from typing import Any
import pytest
from nisse_dev.cancel import STOPPED_RESULT, Cancel
from nisse_dev.canonical import (
from nisse_core.cancel import STOPPED_RESULT, Cancel
from nisse_core.canonical import (
AssistantText,
CanonicalEvent,
DelegationCompleted,
@@ -14,6 +14,10 @@ from nisse_dev.canonical import (
ToolCall,
ToolResult,
)
from nisse_core.execution import SSHHost
from nisse_core.llm import BudgetExhausted, LLMError, UpstreamUnavailable
from nisse_core.oplog import OpLog
from nisse_core.state import Transcript
from nisse_dev.delegation import (
DELEGATE_TOOLS,
DELEGATION_INTENT,
@@ -22,11 +26,8 @@ from nisse_dev.delegation import (
DelegationResult,
delegation_reconciler,
)
from nisse_dev.execution import SSHHost
from nisse_dev.git import GitRepo
from nisse_dev.llm import BudgetExhausted, LLMError, UpstreamUnavailable
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore, Transcript
from nisse_dev.state import StateStore
from nisse_dev.tools import Workspace
nisse-dev/tests/test_dev_state.py
+61
View File
@@ -0,0 +1,61 @@
"""nisse-dev's state store: the shared one, with its own roles, views and
build VM blocks (D3)."""
from pathlib import Path
import pytest
from nisse_core.canonical import ToolCall
from nisse_core.state import StateStore as CoreStore
from nisse_dev.state import StateStore
@pytest.fixture
def db(tmp_path: Path) -> Path:
return tmp_path / "state.db"
def test_a_vm_block_persists_across_reopen(db: Path) -> None:
with StateStore(db) as store:
store.block_vm("root@vm1", cause="broken", reason="pip crashes", work_item="x")
store.block_vm("root@vm1", cause="broken", reason="later", work_item="y")
with StateStore(db) as store:
[block] = store.vm_blocks().values()
assert (block.host, block.reason, block.work_item) == (
"root@vm1",
"pip crashes", # the first block is kept
"x",
)
assert store.unblock_vm("root@vm1", cause="test") == block
assert store.unblock_vm("root@vm1", cause="test") is None
assert store.vm_blocks() == {}
def test_a_version_4_store_gains_vm_blocks(db: Path) -> None:
with StateStore(db, schema_version=4) as store:
store._conn.execute("DROP TABLE vm_blocks") # as a real version 4 store
with StateStore(db) as store:
assert store.schema_version == 5
store.block_vm("root@vm1", cause="broken", reason="r", work_item=None)
assert list(store.vm_blocks()) == ["root@vm1"]
def test_the_dev_roles_are_the_dev_stores_own(db: Path) -> None:
with StateStore(db) as store:
store.transcript("x", "toolchain")
assert store.oplog.agent == "nisse-dev"
core = CoreStore(db.with_name("core.db"))
with core, pytest.raises(ValueError, match="unknown actor"):
core.transcript("x", "toolchain")
def test_each_role_is_shown_the_others_as_the_dev_store_says(db: Path) -> None:
with StateStore(db) as store:
for actor in ("driver", "toolchain"):
store.transcript("x", actor).append(
ToolCall(call_id=actor, tool="read_file", arguments={"path": "a"})
)
[seen] = store.transcript("x", "memory").view() # the driver's, told
assert seen.actor == "driver" and "read_file(a)" in seen.event.text
[own] = store.transcript("x", "driver").view() # not the toolchain's
assert own.event.call_id == "driver"
nisse-dev/tests/test_dump.py
-80
View File
@@ -1,80 +0,0 @@
import io
from pathlib import Path
from nisse_dev.canonical import Fact, PlanSubmitted, ToolCall, ToolResult, UserMessage
from nisse_dev.dump import main, render
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore
def store_with_a_run(path: Path) -> None:
store = StateStore(path, oplog=OpLog("nisse-dev", io.StringIO()))
store.transcript(None, "operator").append(
UserMessage(text="hello before use", source="matrix")
)
store.open_work_item(
"demo-1",
title="Add a truncate helper",
scope="demo",
position="WAITING",
data={"loop": {"wait": "go-ahead", "rev": 1}},
cause="test",
)
store.set_active("demo-1", cause="test")
log = store.transcript("demo-1", "operator")
log.append(UserMessage(text="Add a truncate helper", source="matrix"))
driver = log.as_actor("driver")
driver.append(PlanSubmitted(plan="x" * 500, model="opus"))
driver.append(
ToolCall(call_id="c1", tool="run_command", arguments={"command": "ls"})
)
driver.append(ToolResult(call_id="c1", result="README.md", success=False))
log.begin_intent("open_pr", {"branch": "b"}, cause="test")
store.close()
def test_the_dump_shows_items_intents_and_each_event_with_its_actor(
tmp_path: Path,
) -> None:
db = tmp_path / "state.db"
store_with_a_run(db)
text = render(db)
assert "active work item: demo-1" in text
assert (
"demo-1 open position: WAITING scope: demo rev-1 waits for: go-ahead"
in text
)
assert "pending intent #1 open_pr [demo-1]" in text
assert "[agent/operator] USER [matrix]: hello before use" in text
assert "[demo-1/operator] USER [matrix]: Add a truncate helper" in text
assert "[demo-1/driver] PLAN (opus, 500 chars): " in text and "[340 more]" in text
assert '[demo-1/driver] CALL run_command {"command": "ls"}' in text
assert "result (FAILED): README.md" in text
def test_full_shows_long_texts_whole(tmp_path: Path) -> None:
db = tmp_path / "state.db"
store_with_a_run(db)
assert "x" * 500 in render(db, full=True)
def test_facts_are_shown_as_the_orchestrators(tmp_path: Path) -> None:
db = tmp_path / "state.db"
store_with_a_run(db)
store = StateStore(db, oplog=OpLog("nisse-dev", io.StringIO()))
store.transcript("demo-1", "orchestrator").append(Fact(text="PR #3 opened"))
store.close()
assert "[demo-1/orchestrator] FACT: PR #3 opened" in render(db)
def test_the_dump_never_writes_to_the_store(tmp_path: Path) -> None:
db = tmp_path / "state.db"
store_with_a_run(db)
before = db.read_bytes()
render(db)
assert db.read_bytes() == before
def test_a_missing_store_is_reported(tmp_path: Path, capsys) -> None:
assert main([str(tmp_path / "nope.db")]) == 1
assert "no state store" in capsys.readouterr().err
nisse-dev/tests/test_evals.py
+2 -2
View File
@@ -4,13 +4,13 @@ from pathlib import Path
import pytest
from nisse_dev.canonical import (
from nisse_core.canonical import (
AssistantText,
DelegationCompleted,
DelegationStarted,
)
from nisse_core.oplog import OpLog
from nisse_dev.evals import UNMARKED, collect, summarise, to_jsonl
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore
nisse-dev/tests/test_execution.py
-108
View File
@@ -1,108 +0,0 @@
"""Running commands on a tool VM over SSH (L13), against a throwaway sshd."""
import os
from pathlib import Path
import pytest
from nisse_dev.execution import REDACTED, HostUnavailable, SSHHost, redact
def test_exit_codes_and_streams_come_back(ssh_host: SSHHost) -> None:
result = ssh_host.execute("echo out; echo err >&2; exit 3")
assert (result.exit_code, result.stdout, result.stderr) == (3, "out\n", "err\n")
assert not result.ok
def test_cwd_and_stdin(ssh_host: SSHHost, tmp_path: Path) -> None:
result = ssh_host.execute("pwd; cat", cwd=str(tmp_path), stdin="line one\nline two")
assert result.stdout == f"{tmp_path}\nline one\nline two\n"
def test_stdin_that_looks_like_a_delimiter_survives(ssh_host: SSHHost) -> None:
payload = "NISSE_EOF\nEOF\n'quoted' $notexpanded `nor this`"
assert ssh_host.execute("cat", stdin=payload).stdout == payload + "\n"
def test_without_stdin_a_command_cannot_read_the_script(ssh_host: SSHHost) -> None:
# The script arrives on the same channel; the command must see /dev/null.
result = ssh_host.execute("cat; echo done")
assert result.stdout == "done\n"
def test_a_missing_cwd_fails_before_running(ssh_host: SSHHost, tmp_path: Path) -> None:
marker = tmp_path / "ran"
result = ssh_host.execute(f"touch {marker}", cwd=str(tmp_path / "absent"))
assert result.exit_code == 125
assert not marker.exists()
def test_a_command_exiting_255_is_not_mistaken_for_ssh(ssh_host: SSHHost) -> None:
assert ssh_host.execute("exit 255").exit_code == 254
def test_an_unreachable_host_raises(tmp_path: Path) -> None:
host = SSHHost("127.0.0.1", key=tmp_path / "k", state_dir=tmp_path / "s", port=1)
with pytest.raises(HostUnavailable):
host.execute("true")
def test_a_timeout_is_reported(ssh_host: SSHHost) -> None:
result = ssh_host.execute("sleep 30", timeout_s=1)
assert result.timed_out
def test_secrets_reach_the_command_and_never_its_output(ssh_host: SSHHost) -> None:
value = "s-" + os.urandom(12).hex()
result = ssh_host.execute(
'echo "$API_KEY"; echo "$API_KEY" >&2', secrets={"API_KEY": value}
)
assert result.ok
assert value not in result.stdout + result.stderr
assert result.stdout == f"{REDACTED}\n"
def test_secrets_are_never_on_a_command_line(ssh_host: SSHHost, tmp_path: Path) -> None:
# Made up here, so no command line in this test run can contain it except
# by leaking it.
value = "s-" + os.urandom(12).hex()
seen = tmp_path / "ps.txt"
ssh_host.execute(f"ps -eo args > {seen}", secrets={"API_KEY": value})
assert "bash -s" in seen.read_text() # the listing did include our shell
assert value not in seen.read_text()
def test_redact_masks_longest_values_first() -> None:
assert redact("abc-abcdef", ["abcdef", "abc-abcdef"]) == REDACTED
assert redact("short ab stays", ["ab"]) == "short ab stays" # too short to mask
def test_a_changed_host_key_is_its_own_failure_and_only_forgetting_clears_it(
ssh_host: SSHHost, tmp_path: Path
) -> None:
import subprocess
from nisse_dev.execution import HostKeyChanged
host = SSHHost(
ssh_host.address,
key=ssh_host._key,
user=ssh_host.user,
port=ssh_host.port,
state_dir=tmp_path / "state",
)
assert host.execute("true").ok # first contact pins the key
host.close()
# Someone else's key where the pinned one was: a rebuilt VM, or not it.
subprocess.run(
["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(tmp_path / "other")],
check=True,
)
other = (tmp_path / "other.pub").read_text().split()
known = tmp_path / "state" / "known_hosts"
name = host.address if host.port == 22 else f"[{host.address}]:{host.port}"
known.write_text(f"{name} {other[0]} {other[1]}\n")
with pytest.raises(HostKeyChanged):
host.execute("true")
host.forget_host_key()
assert host.execute("true").ok # the real key pinned again
nisse-dev/tests/test_expert.py
+5 -4
View File
@@ -4,13 +4,16 @@ from typing import Any
import pytest
from nisse_dev.canonical import (
from nisse_core.canonical import (
AssistantText,
CanonicalEvent,
DelegationCompleted,
DelegationStarted,
StoredEvent,
)
from nisse_core.llm import LLMError, RoleMap
from nisse_core.oplog import OpLog
from nisse_core.state import Transcript
from nisse_dev.expert import (
DEFAULT_BUDGET,
BudgetExhausted,
@@ -18,9 +21,7 @@ from nisse_dev.expert import (
ExpertDesk,
parse_answer,
)
from nisse_dev.llm import LLMError, RoleMap
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore, Transcript
from nisse_dev.state import StateStore
ROLES = RoleMap(
driver="haiku",
nisse-dev/tests/test_git.py
+2 -2
View File
@@ -6,7 +6,7 @@ from pathlib import Path
import pytest
from nisse_dev.execution import SSHHost
from nisse_core.execution import SSHHost
from nisse_dev.git import GitError, GitRepo, HTTPSToken, SSHKey
@@ -170,7 +170,7 @@ class RecordingHost:
self.calls: list[dict] = []
def execute(self, command: str, **kwargs):
from nisse_dev.execution import Result
from nisse_core.execution import Result
self.calls.append({"command": command, **kwargs})
return Result(0, "", "")
nisse-dev/tests/test_gitea.py
-321
View File
@@ -1,321 +0,0 @@
import json
import httpx
import pytest
from nisse_dev.gitea import (
PR_PAGE_SIZE,
Comment,
GiteaClient,
GiteaError,
pull_request_reconciler,
)
from nisse_dev.state import Intent
INTENT = Intent(
id=1,
kind="open_pr",
payload={"branch": "milestone-1"},
status="pending",
created_at="t",
resolved_at=None,
)
def client(handler) -> GiteaClient:
return GiteaClient(
"https://gitea.test/", "tok", transport=httpx.MockTransport(handler)
)
def pr_json(number: int, head: str, state: str = "open") -> dict:
return {
"number": number,
"state": state,
"head": {"ref": head},
"html_url": f"https://gitea.test/o/r/pulls/{number}",
}
def test_open_pull_request_posts_expected_body() -> None:
seen: dict = {}
def handler(request: httpx.Request) -> httpx.Response:
seen["url"] = str(request.url)
seen["auth"] = request.headers.get("authorization")
seen["body"] = json.loads(request.content)
return httpx.Response(201, json=pr_json(7, "milestone-1"))
pr = client(handler).open_pull_request(
"o/r", head="milestone-1", base="main", title="T", body="B"
)
assert seen["url"] == "https://gitea.test/api/v1/repos/o/r/pulls"
assert seen["auth"] == "token tok"
assert seen["body"] == {
"head": "milestone-1",
"base": "main",
"title": "T",
"body": "B",
}
assert pr.number == 7
assert pr.head == "milestone-1"
def test_find_pull_request_matches_head_branch() -> None:
handler = lambda request: httpx.Response(200, json=[pr_json(3, "other")])
assert client(handler).find_pull_request("o/r", "milestone-1") is None
def test_find_pull_request_also_finds_closed() -> None:
# A crash after opening must not be answered with "nothing open, open
# another" — the effect already happened.
handler = lambda request: httpx.Response(
200, json=[pr_json(4, "milestone-1", "closed")]
)
found = client(handler).find_pull_request("o/r", "milestone-1")
assert found is not None
assert found.number == 4
def test_find_pull_request_pages_past_the_first_page() -> None:
# The match sits on page 2; sampling only the newest page would report
# "absent" and licence a duplicate PR.
def handler(request: httpx.Request) -> httpx.Response:
page = int(request.url.params.get("page", 1))
if page == 1:
return httpx.Response(
200, json=[pr_json(i, f"other-{i}") for i in range(PR_PAGE_SIZE)]
)
return httpx.Response(200, json=[pr_json(999, "milestone-1", "closed")])
found = client(handler).find_pull_request("o/r", "milestone-1")
assert found is not None
assert found.number == 999
def test_short_page_ends_the_search_definitively() -> None:
calls: list[int] = []
def handler(request: httpx.Request) -> httpx.Response:
calls.append(int(request.url.params.get("page", 1)))
return httpx.Response(200, json=[pr_json(1, "other")])
assert client(handler).find_pull_request("o/r", "milestone-1") is None
assert calls == [1] # one short page proves absence; no further paging
def test_unbounded_history_raises_rather_than_guessing() -> None:
# Never answer "absent" from an incomplete search: the reconciler turns
# this error into "unverifiable" and gates to the human.
handler = lambda request: httpx.Response(
200, json=[pr_json(i, f"other-{i}") for i in range(PR_PAGE_SIZE)]
)
with pytest.raises(GiteaError, match="without reaching the end"):
client(handler).find_pull_request("o/r", "milestone-1")
def test_error_status_raises_gitea_error() -> None:
handler = lambda request: httpx.Response(403, text="forbidden")
with pytest.raises(GiteaError, match="403"):
client(handler).whoami()
def test_list_and_post_comments() -> None:
def handler(request: httpx.Request) -> httpx.Response:
if request.method == "POST":
return httpx.Response(
201,
json={
"id": 9,
"user": {"login": "don"},
"body": "hi",
"created_at": "t",
},
)
return httpx.Response(
200,
json=[
{
"id": 8,
"user": {"login": "don"},
"body": "approve rev-1",
"created_at": "t",
}
],
)
c = client(handler)
assert c.list_comments("o/r", 1) == [
Comment(id=8, author="don", body="approve rev-1", created_at="t")
]
assert c.post_comment("o/r", 1, "hi").id == 9
def test_list_comments_passes_since() -> None:
seen: dict = {}
def handler(request: httpx.Request) -> httpx.Response:
seen["since"] = request.url.params.get("since")
return httpx.Response(200, json=[])
client(handler).list_comments("o/r", 1, since="2026-09-07T00:00:00Z")
assert seen["since"] == "2026-09-07T00:00:00Z"
# --- reconciler (L8 external truth) --------------------------------------
def test_reconciler_reports_existing_pr_as_done() -> None:
handler = lambda request: httpx.Response(200, json=[pr_json(5, "milestone-1")])
assert pull_request_reconciler(client(handler), "o/r")(INTENT) is True
def test_reconciler_reports_missing_pr_as_not_done() -> None:
handler = lambda request: httpx.Response(200, json=[])
assert pull_request_reconciler(client(handler), "o/r")(INTENT) is False
def test_unreachable_gitea_is_unverifiable_not_a_guess() -> None:
def handler(request: httpx.Request) -> httpx.Response:
raise httpx.ConnectError("refused")
# None gates to the human: a delayed decision is cheap, a duplicate PR
# is not.
assert pull_request_reconciler(client(handler), "o/r")(INTENT) is None
def test_intent_without_a_branch_is_unverifiable() -> None:
handler = lambda request: httpx.Response(200, json=[])
blank = Intent(
id=2,
kind="open_pr",
payload={},
status="pending",
created_at="t",
resolved_at=None,
)
assert pull_request_reconciler(client(handler), "o/r")(blank) is None
def test_closing_an_issue_and_a_pr() -> None:
seen: list[tuple[str, str, dict]] = []
def handler(request: httpx.Request) -> httpx.Response:
body = json.loads(request.content) if request.content else {}
seen.append((request.method, request.url.path, body))
if request.method == "POST":
return httpx.Response(
201, json={"id": 5, "user": {"login": "bot"}, "body": body["body"]}
)
return httpx.Response(201, json={})
gitea = client(handler)
gitea.post_comment("o/r", 10, "abandoned by don")
gitea.close_issue("o/r", 10)
gitea.post_comment("o/r", 20, "abandoned by don")
gitea.close_pull_request("o/r", 20)
assert seen == [
("POST", "/api/v1/repos/o/r/issues/10/comments", {"body": "abandoned by don"}),
("PATCH", "/api/v1/repos/o/r/issues/10", {"state": "closed"}),
("POST", "/api/v1/repos/o/r/issues/20/comments", {"body": "abandoned by don"}),
("PATCH", "/api/v1/repos/o/r/pulls/20", {"state": "closed"}),
]
def test_assigned_at_is_the_last_assignment_across_pages() -> None:
def assigned(login: str, at: str, removed: bool = False) -> dict:
return {
"type": "assignees",
"assignee": {"login": login},
"removed_assignee": removed,
"created_at": at,
}
filler = {"type": "comment", "created_at": "t0"}
pages = {
1: [assigned("dev-agent", "t1")] + [filler] * (PR_PAGE_SIZE - 1),
2: [
assigned("dev-agent", "t2", removed=True),
assigned("someone", "t3"),
assigned("dev-agent", "t4"),
],
}
def handler(request: httpx.Request) -> httpx.Response:
assert request.url.path == "/api/v1/repos/o/r/issues/6/timeline"
return httpx.Response(200, json=pages[int(request.url.params["page"])])
assert client(handler).assigned_at("o/r", 6, "dev-agent") == "t4"
assert client(handler).assigned_at("o/r", 6, "nobody") is None
def test_approvals_are_the_standing_approving_reviews() -> None:
reviews = [
{"state": "APPROVED", "user": {"login": "don"}, "commit_id": "abc"},
{
"state": "APPROVED",
"user": {"login": "don"},
"commit_id": "old",
"stale": True,
},
{
"state": "APPROVED",
"user": {"login": "x"},
"commit_id": "abc",
"dismissed": True,
},
{"state": "REQUEST_CHANGES", "user": {"login": "don"}, "commit_id": "abc"},
{"state": "COMMENT", "user": {"login": "don"}, "commit_id": "abc"},
]
handler = lambda request: httpx.Response(200, json=reviews)
approvals = client(handler).approvals("o/r", 3)
assert [(a.author, a.commit, a.stale) for a in approvals] == [
("don", "abc", False),
("don", "old", True),
]
def test_checks_are_the_combined_status_or_none() -> None:
def handler(request: httpx.Request) -> httpx.Response:
assert request.url.path == "/api/v1/repos/o/r/commits/abc/status"
return httpx.Response(200, json=answer)
answer: dict = {"state": "", "total_count": 0, "statuses": []}
assert client(handler).checks("o/r", "abc") == "none"
answer = {"state": "pending", "total_count": 1, "statuses": [{}]}
assert client(handler).checks("o/r", "abc") == "pending"
answer = {"state": "success", "total_count": 2, "statuses": [{}, {}]}
assert client(handler).checks("o/r", "abc") == "success"
def test_a_pull_request_carries_its_head_commit_and_mergeability() -> None:
data = {
**pr_json(5, "m-1"),
"head": {"ref": "m-1", "sha": "f00"},
"mergeable": True,
}
pr = client(lambda request: httpx.Response(200, json=data)).pull_request("o/r", 5)
assert (pr.head_sha, pr.mergeable) == ("f00", True)
def test_a_check_branch_is_made_through_the_api() -> None:
seen: list = []
def handler(request: httpx.Request) -> httpx.Response:
seen.append((request.method, request.url.path, request.content))
return httpx.Response(201, json={})
gitea = client(handler)
gitea.branch_with_file(
"o/r", base="main", branch="chk", path="f", text="x", message="m"
)
gitea.delete_branch("o/r", "chk")
body = json.loads(seen[0][2])
assert (seen[0][0], seen[0][1]) == ("POST", "/api/v1/repos/o/r/contents/f")
assert (body["branch"], body["new_branch"], body["content"]) == (
"main",
"chk",
"eA==",
)
assert seen[1][:2] == ("DELETE", "/api/v1/repos/o/r/branches/chk")
nisse-dev/tests/test_llm.py
-516
View File
@@ -1,516 +0,0 @@
import json
from pathlib import Path
from typing import Any
import httpx
import pytest
from nisse_dev.canonical import (
AssistantText,
StoredEvent,
ToolCall,
UserMessage,
)
from nisse_dev.envfile import load_env_file
from nisse_dev.llm import (
DEFAULT_MAX_OUTPUT_TOKENS,
OUTAGE_BACKOFF_S,
RATE_LIMIT_RETRY_S,
AnthropicPassthroughClient,
BudgetExhausted,
LiteLLMClient,
LLMError,
RoleMap,
TruncatedResponse,
UpstreamUnavailable,
)
EVENTS = [StoredEvent(id=1, ts="t", event=UserMessage(text="hi", source="m:1"))]
def make_client(handler, sleep=lambda _s: None) -> LiteLLMClient:
return LiteLLMClient(
"http://litellm.test/v1",
"sk-test",
transport=httpx.MockTransport(handler),
sleep=sleep, # never wait in unit tests
)
def completion(message: dict) -> httpx.Response:
return httpx.Response(
200, json={"model": "gpt-test-1", "choices": [{"message": message}]}
)
def test_request_shape_and_auth() -> None:
seen: dict = {}
def handler(request: httpx.Request) -> httpx.Response:
seen["url"] = str(request.url)
seen["auth"] = request.headers.get("authorization")
seen["body"] = json.loads(request.content)
return completion({"content": "hello"})
make_client(handler).complete(EVENTS, tools=None, model="local/qwen", system="s")
assert seen["url"] == "http://litellm.test/v1/chat/completions"
assert seen["auth"] == "Bearer sk-test"
assert seen["body"]["model"] == "local/qwen"
assert seen["body"]["messages"][0] == {"role": "system", "content": "s"}
assert seen["body"]["messages"][1] == {"role": "user", "content": "hi"}
def test_assistant_text_parsed_with_producing_model() -> None:
client = make_client(lambda request: completion({"content": "hello"}))
[event] = client.complete(EVENTS, tools=None, model="local/qwen")
assert event == AssistantText(text="hello", model="gpt-test-1")
def test_tool_calls_get_locally_minted_canonical_ids() -> None:
message = {
"content": None,
"tool_calls": [
{
"id": "call_provider_abc",
"type": "function",
"function": {
"name": "run_tests",
"arguments": '{"path": "tests/"}',
},
}
],
}
client = make_client(lambda request: completion(message))
[event] = client.complete(EVENTS, tools=None, model="local/qwen")
assert isinstance(event, ToolCall)
assert event.tool == "run_tests"
assert event.arguments == {"path": "tests/"}
assert event.call_id != "call_provider_abc" # provider ID never a join key
assert event.call_id.startswith("call-")
def test_truncated_tool_arguments_raise_llm_error() -> None:
message = {
"content": None,
"tool_calls": [
{
"id": "c",
"type": "function",
"function": {"name": "submit_plan", "arguments": '{"content": "'},
}
],
}
client = make_client(lambda request: completion(message))
with pytest.raises(LLMError, match="non-JSON arguments"):
client.complete(EVENTS, tools=None, model="local/qwen")
def test_non_object_tool_arguments_raise_llm_error() -> None:
message = {
"content": None,
"tool_calls": [
{
"id": "c",
"type": "function",
"function": {"name": "submit_plan", "arguments": '"just a string"'},
}
],
}
client = make_client(lambda request: completion(message))
with pytest.raises(LLMError, match="expected object"):
client.complete(EVENTS, tools=None, model="local/qwen")
def test_http_error_status_raises_llm_error() -> None:
client = make_client(lambda request: httpx.Response(500, text="boom"))
with pytest.raises(LLMError, match="500"):
client.complete(EVENTS, tools=None, model="local/qwen")
def test_rate_limit_is_retried_once_then_surfaced() -> None:
# A 429 is transient. Losing a long delegation to one costs far more
# than waiting out the window, but retrying forever would hide a real
# ceiling, so it is retried exactly once.
attempts: list[int] = []
waited: list[float] = []
def handler(request: httpx.Request) -> httpx.Response:
attempts.append(1)
return httpx.Response(429, text="rate limited")
client = make_client(handler, sleep=waited.append)
with pytest.raises(LLMError, match="429"):
client.complete(EVENTS, tools=None, model="local/qwen")
assert len(attempts) == 2
assert waited == [RATE_LIMIT_RETRY_S]
# Reported even though the call failed, so a caller can leave the wait
# out of how long the model took.
assert client.last_rate_limit_wait_s == RATE_LIMIT_RETRY_S
def test_rate_limit_that_clears_succeeds() -> None:
responses = [httpx.Response(429, text="slow down"), completion({"content": "hi"})]
client = make_client(lambda request: responses.pop(0), sleep=lambda _s: None)
[event] = client.complete(EVENTS, tools=None, model="local/qwen")
assert isinstance(event, AssistantText)
assert client.last_rate_limit_wait_s == RATE_LIMIT_RETRY_S
def test_the_rate_limit_wait_is_per_call() -> None:
responses = [
httpx.Response(429, text="slow down"),
completion({"content": "one"}),
completion({"content": "two"}),
]
client = make_client(lambda request: responses.pop(0), sleep=lambda _s: None)
client.complete(EVENTS, tools=None, model="local/qwen")
client.complete(EVENTS, tools=None, model="local/qwen")
assert client.last_rate_limit_wait_s == 0.0
def test_connection_failure_raises_llm_error() -> None:
def handler(request: httpx.Request) -> httpx.Response:
raise httpx.ConnectError("refused")
with pytest.raises(LLMError, match="request failed"):
make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
def test_malformed_response_raises_llm_error() -> None:
client = make_client(lambda request: httpx.Response(200, json={"oops": True}))
with pytest.raises(LLMError, match="malformed"):
client.complete(EVENTS, tools=None, model="local/qwen")
ROLES = RoleMap(
driver="local/qwen",
helper="anthropic/claude-sonnet-5",
expert="anthropic/claude-opus-5",
)
def test_role_map_resolves_and_pins() -> None:
roles = ROLES
assert roles.resolve("driver") == "local/qwen"
assert roles.pinned() == {
"driver": "local/qwen",
"helper": "anthropic/claude-sonnet-5",
"expert": "anthropic/claude-opus-5",
# Unset, the classifier falls back to the expert's model (L9).
"classifier": "anthropic/claude-opus-5",
}
def test_a_pinned_map_from_before_the_driver_reads_its_own_roles() -> None:
# Pinned before M2.8: the old loop's roles are there, and are ignored.
old = {"planner": "p", "executor": "e", "driver": "d", "helper": "h"}
roles = RoleMap.from_pinned(old, ROLES)
assert (roles.driver, roles.helper, roles.expert) == ("d", "h", ROLES.expert)
def test_unknown_role_raises() -> None:
with pytest.raises(LLMError, match="unknown role"):
ROLES.resolve("boss")
def test_load_env_file(tmp_path: Path) -> None:
envfile = tmp_path / ".env"
envfile.write_text(
"# comment\n\nNISSE_LLM_TOKEN=sk-abc\nNISSE_LLM_BASE_URL = http://x \n"
)
env: dict[str, str] = {"NISSE_LLM_TOKEN": "sk-already-set"}
load_env_file(envfile, env)
assert env["NISSE_LLM_TOKEN"] == "sk-already-set" # real env wins
assert env["NISSE_LLM_BASE_URL"] == "http://x"
def test_load_env_file_missing_is_noop(tmp_path: Path) -> None:
env: dict[str, str] = {}
load_env_file(tmp_path / "nope.env", env)
assert env == {}
# --- truncation (silent until measured against Opus) ---------------------
def test_openai_route_raises_on_truncation() -> None:
# finish_reason "length" means the answer was cut off; parsing it as
# complete yields a tool call with required arguments simply absent.
truncated = httpx.Response(
200,
json={
"model": "gpt-test-1",
"choices": [
{
"finish_reason": "length",
"message": {
"content": None,
"tool_calls": [
{
"id": "c",
"type": "function",
"function": {
"name": "submit_plan",
"arguments": '{"type": "arch"}',
},
}
],
},
}
],
},
)
client = make_client(lambda request: truncated)
with pytest.raises(TruncatedResponse, match="output limit"):
client.complete(EVENTS, tools=None, model="gpt-6-astra")
def test_anthropic_route_raises_on_truncation() -> None:
# Observed for real: Opus spent 4,064 output tokens against a 4,096
# ceiling, and a longer plan came back as submit_plan with no content.
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json={
"model": "claude-opus-5",
"stop_reason": "max_tokens",
"content": [
{
"type": "tool_use",
"id": "t",
"name": "submit_plan",
"input": {"type": "arch"},
}
],
},
)
client = AnthropicPassthroughClient(
"https://litellm.test",
"tok",
transport=httpx.MockTransport(handler),
sleep=lambda _s: None,
)
with pytest.raises(TruncatedResponse, match="output limit"):
client.complete(EVENTS, tools=None, model="claude-opus-5")
def test_a_complete_answer_is_not_mistaken_for_truncation() -> None:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json={
"model": "claude-opus-5",
"stop_reason": "tool_use",
"content": [
{
"type": "tool_use",
"id": "t",
"name": "submit_plan",
"input": {"type": "arch", "content": "a real plan"},
}
],
},
)
client = AnthropicPassthroughClient(
"https://litellm.test", "tok", transport=httpx.MockTransport(handler)
)
[call] = client.complete(EVENTS, tools=None, model="claude-opus-5")
assert call.arguments["content"] == "a real plan"
def test_output_ceiling_is_generous_by_default() -> None:
# Output is billed by what is produced, so a low ceiling buys nothing and
# costs silent truncation. The configured value is tested in test_config.
assert DEFAULT_MAX_OUTPUT_TOKENS >= 8192
assert LiteLLMClient("http://x", "t")._max_tokens == DEFAULT_MAX_OUTPUT_TOKENS
def test_truncation_is_an_llm_error_so_callers_already_handle_it() -> None:
# The planner retries, the executor and delegation gate. None of them
# need to learn a new exception type.
assert issubclass(TruncatedResponse, LLMError)
def test_a_5xx_is_upstream_trouble() -> None:
client = make_client(lambda request: httpx.Response(503, text="bad gateway"))
with pytest.raises(UpstreamUnavailable, match="503"):
client.complete(EVENTS, tools=None, model="local-fast")
def test_a_connection_failure_is_upstream_trouble() -> None:
def handler(request: httpx.Request) -> httpx.Response:
raise httpx.ConnectError("refused")
with pytest.raises(UpstreamUnavailable, match="request failed"):
make_client(handler).complete(EVENTS, tools=None, model="local-fast")
def test_a_persistent_rate_limit_is_upstream_trouble() -> None:
# It cleared for neither attempt, so the caller should keep its work
# rather than discard it.
client = make_client(lambda request: httpx.Response(429, text="slow down"))
with pytest.raises(UpstreamUnavailable):
client.complete(EVENTS, tools=None, model="local-fast")
def test_a_bad_request_is_not_upstream_trouble() -> None:
client = make_client(lambda request: httpx.Response(400, text="bad schema"))
with pytest.raises(LLMError) as excinfo:
client.complete(EVENTS, tools=None, model="local-fast")
assert not isinstance(excinfo.value, UpstreamUnavailable)
def test_budget_refusal_is_a_kind_of_upstream_trouble() -> None:
# So one catch covers the family, but it can still be told apart where
# the difference matters — it will not clear within the hour.
assert issubclass(BudgetExhausted, UpstreamUnavailable)
# --- tool_choice (L9: constrained decoding) --------------------------------
TOOLS = [
{
"type": "function",
"function": {
"name": "finish",
"description": "done",
"parameters": {"type": "object", "properties": {}},
},
}
]
def capture_bodies() -> tuple[list[dict], Any]:
bodies: list[dict] = []
def handler(request: httpx.Request) -> httpx.Response:
bodies.append(json.loads(request.content))
return completion({"role": "assistant", "content": "ok"})
return bodies, handler
def test_required_is_sent_only_when_asked_and_tools_are_offered() -> None:
bodies, handler = capture_bodies()
client = make_client(handler)
client.complete(EVENTS, tools=TOOLS, model="m", require_tool_call=True)
client.complete(EVENTS, tools=TOOLS, model="m")
client.complete(EVENTS, tools=None, model="m", require_tool_call=True)
assert bodies[0]["tool_choice"] == "required"
assert "tool_choice" not in bodies[1] # the server's default, auto
assert "tool_choice" not in bodies[2] # nothing to choose from
def test_the_passthrough_asks_in_anthropics_terms() -> None:
bodies: list[dict] = []
def handler(request: httpx.Request) -> httpx.Response:
bodies.append(json.loads(request.content))
return httpx.Response(
200,
json={
"model": "claude-haiku-4-5",
"stop_reason": "tool_use",
"content": [
{"type": "tool_use", "id": "t", "name": "finish", "input": {}}
],
},
)
client = AnthropicPassthroughClient(
"https://litellm.test", "tok", transport=httpx.MockTransport(handler)
)
client.complete(
EVENTS, tools=TOOLS, model="claude-haiku-4-5", require_tool_call=True
)
assert bodies[0]["tool_choice"] == {"type": "any"}
def test_the_answering_model_is_recorded() -> None:
# A proxy fallback answers under another model's name; this is how a
# caller can tell.
client = make_client(
lambda request: completion({"role": "assistant", "content": "hi"})
)
client.complete(EVENTS, tools=None, model="local-fast")
assert client.last_model == "gpt-test-1"
# --- outages delay, never fail (L5) --------------------------------------------
def test_a_gateway_error_is_waited_out_with_backoff() -> None:
responses = [
httpx.Response(502, text="bad gateway"),
httpx.Response(503, text="unavailable"),
completion({"content": "back"}),
]
waited: list[float] = []
client = make_client(lambda request: responses.pop(0), sleep=waited.append)
[event] = client.complete(EVENTS, tools=None, model="local/qwen")
assert isinstance(event, AssistantText) and event.text == "back"
assert waited == list(OUTAGE_BACKOFF_S[:2])
# Waiting isn't the model's time.
assert client.last_rate_limit_wait_s == sum(OUTAGE_BACKOFF_S[:2])
def test_an_outage_that_outlasts_the_backoff_is_surfaced() -> None:
attempts: list[int] = []
waited: list[float] = []
def handler(request: httpx.Request) -> httpx.Response:
attempts.append(1)
return httpx.Response(502, text="bad gateway")
client = make_client(handler, sleep=waited.append)
with pytest.raises(UpstreamUnavailable, match="502"):
client.complete(EVENTS, tools=None, model="local/qwen")
assert len(attempts) == len(OUTAGE_BACKOFF_S) + 1
assert waited == list(OUTAGE_BACKOFF_S)
def test_a_dropped_connection_is_retried() -> None:
calls: list[int] = []
def handler(request: httpx.Request) -> httpx.Response:
calls.append(1)
if len(calls) == 1:
raise httpx.ConnectError("refused")
return completion({"content": "ok"})
[event] = make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
assert isinstance(event, AssistantText) and len(calls) == 2
def test_a_500_or_a_timeout_is_not_retried() -> None:
# A 500 may be the request's fault; a timeout already had its full
# request_timeout_s.
for failure in (
lambda request: httpx.Response(500, text="boom"),
lambda request: (_ for _ in ()).throw(httpx.ReadTimeout("slow")),
):
calls: list[int] = []
def handler(
request: httpx.Request, failure=failure, calls=calls
) -> httpx.Response:
calls.append(1)
return failure(request)
with pytest.raises(LLMError):
make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
assert len(calls) == 1
def test_a_budget_refusal_is_not_waited_on() -> None:
calls: list[int] = []
def handler(request: httpx.Request) -> httpx.Response:
calls.append(1)
return httpx.Response(503, text="Budget has been exceeded! Current cost: 5.1")
with pytest.raises(BudgetExhausted):
make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
assert len(calls) == 1
nisse-dev/tests/test_loop.py
+8 -8
View File
@@ -8,25 +8,25 @@ from typing import Any
import pytest
from model_contract import assert_renderable
from nisse_dev.cancel import Cancel
from nisse_dev.canonical import (
from nisse_core.cancel import Cancel
from nisse_core.canonical import (
AssistantText,
CanonicalEvent,
Fact,
ToolCall,
ToolResult,
)
from nisse_dev.gitea import Issue
from nisse_dev.llm import LLMError
from nisse_core.gitea import Issue
from nisse_core.llm import LLMError
from nisse_core.oplog import OpLog
from nisse_core.transport import InMemoryTransport
from nisse_core.worker import InlineWorker
from nisse_dev.loop.agent import LoopAgent
from nisse_dev.loop.driver import Driver
from nisse_dev.loop.items import CLOSED, GO_AHEAD, READY, RUNNING, WAITING, loop_of
from nisse_dev.loop.reader import command_form
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore
from nisse_dev.tools import ToolOutcome
from nisse_dev.transport import InMemoryTransport
from nisse_dev.worker import InlineWorker
DON = "@don:test"
@@ -250,7 +250,7 @@ def test_a_go_for_an_earlier_revision_is_not_applied(h: Harness) -> None:
h.say("put it in its own module") # changes, so the driver restates rev-2
assert loop_of(h.item()).rev == 2
# A go sent while rev-1 was on screen, arriving now.
from nisse_dev.transport import InboundMessage
from nisse_core.transport import InboundMessage
h.agent.handle(InboundMessage(DON, "go", thread=h.thread()), about=1)
assert not loop_of(h.item()).started
nisse-dev/tests/test_loop_actions.py
+1 -1
View File
@@ -12,7 +12,7 @@ from test_loop_milestones import item as the_item
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from nisse_core.actions import Action
from nisse_dev.canonical import ActionVerdict, ToolCall, ToolResult
from nisse_core.canonical import ActionVerdict, ToolCall, ToolResult
from nisse_dev.loop.actions import classify, commits
from nisse_dev.loop.items import ANSWER, loop_of
nisse-dev/tests/test_loop_classifier.py
+3 -3
View File
@@ -14,8 +14,8 @@ from test_loop_milestones import item as the_item
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from nisse_core.actions import Action
from nisse_dev.canonical import ActionVerdict, ToolCall
from nisse_dev.llm import LLMError
from nisse_core.canonical import ActionVerdict, ToolCall
from nisse_core.llm import LLMError
from nisse_dev.loop.classifier import ActionClassifier, Scope
from nisse_dev.loop.items import ANSWER, loop_of
@@ -312,7 +312,7 @@ def test_the_export_script_reads_a_store(tmp_path: Path) -> None:
import subprocess
import sys
from nisse_dev.oplog import OpLog
from nisse_core.oplog import OpLog
from nisse_dev.state import StateStore
store = StateStore(tmp_path / "state.db", oplog=OpLog("t", io.StringIO()))
nisse-dev/tests/test_loop_conversations.py
+2 -2
View File
@@ -9,8 +9,8 @@ from typing import Any
from test_loop_milestones import DON, Forge, ScriptedModel, call, loop_agent
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from nisse_dev.canonical import Fact, ToolResult
from nisse_dev.gitea import Issue
from nisse_core.canonical import Fact, ToolResult
from nisse_core.gitea import Issue
from nisse_dev.loop.agent import FILE_ISSUE
from nisse_dev.loop.items import CHAT, GO_AHEAD, MERGE, WAITING, loop_of
nisse-dev/tests/test_loop_help.py
+2 -2
View File
@@ -18,16 +18,16 @@ from test_loop_milestones import (
)
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from nisse_dev.canonical import (
from nisse_core.canonical import (
AssistantText,
DelegationStarted,
Fact,
ToolCall,
ToolResult,
)
from nisse_core.llm import RoleMap
from nisse_dev.config import Profile
from nisse_dev.delegation import DELEGATION_INTENT, Caps
from nisse_dev.llm import RoleMap
from nisse_dev.loop.help import Helpers
from nisse_dev.loop.items import ANSWER, MERGE, loop_of
nisse-dev/tests/test_loop_labels.py
+3 -3
View File
@@ -9,8 +9,8 @@ import pytest
from test_loop import DON, RESTATED, FakeGitea, Harness, ScriptedModel, call, issue
from test_loop import h as h # noqa: PLC0414 -- the fixture
from nisse_dev.canonical import Fact, ToolCall
from nisse_dev.llm import LLMError
from nisse_core.canonical import Fact, ToolCall
from nisse_core.llm import LLMError
from nisse_dev.loop.items import ANSWER, GO_AHEAD, READY, WAITING, loop_of
from nisse_dev.loop.reader import GoAheadReader
@@ -284,7 +284,7 @@ def test_an_unlabelled_issue_waits_for_go_with_no_countdown(h: Harness) -> None:
def _msg(text: str, thread: str) -> Any:
from nisse_dev.transport import InboundMessage
from nisse_core.transport import InboundMessage
return InboundMessage(DON, text, thread=thread)
nisse-dev/tests/test_loop_live_fixes.py
+3 -3
View File
@@ -16,9 +16,9 @@ from test_loop_milestones import RESTATE, Forge, loop_agent, say
from test_loop_milestones import item as the_item
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from nisse_dev.canonical import AssistantText, Fact
from nisse_core.canonical import AssistantText, Fact
from nisse_core.gitea import Comment
from nisse_dev.delegation import Caps
from nisse_dev.gitea import Comment
from nisse_dev.loop.help import Helpers
from nisse_dev.loop.items import GO_AHEAD, MERGE, WAITING, loop_of
@@ -231,7 +231,7 @@ def test_an_issue_assigned_again_after_its_item_closed_is_taken_again(
def test_abandoning_a_blocked_issue_says_why_it_stays_open(tmp_path: Path) -> None:
from nisse_dev.gitea import GiteaError
from nisse_core.gitea import GiteaError
class Strict(FakeGitea):
def close_issue(self, repo: str, index: int) -> None:
nisse-dev/tests/test_loop_memory.py
+5 -5
View File
@@ -12,17 +12,17 @@ from test_loop_milestones import item as the_item
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from test_loop_notes import Spy
from nisse_dev.canonical import AssistantText, Fact
from nisse_dev.llm import RoleMap
from nisse_dev.memory import Memory
from nisse_dev.vault import Vault
from nisse_core.canonical import AssistantText, Fact
from nisse_core.llm import RoleMap
from nisse_core.memory import Memory
from nisse_core.vault import Vault
FOLDER = "memory/nisse-dev"
@pytest.fixture
def memory(tmp_path: Path) -> Memory:
return Memory(Vault(tmp_path / "v", {FOLDER: "rw"}), FOLDER)
return Memory(Vault(tmp_path / "v", {FOLDER: "rw"}, agent="nisse-dev"), FOLDER)
def facts(world: dict[str, Any]) -> list[str]:
nisse-dev/tests/test_loop_milestones.py
+8 -8
View File
@@ -10,11 +10,10 @@ from typing import Any
import pytest
from model_contract import assert_renderable
from nisse_dev.cancel import Cancel
from nisse_dev.canonical import ToolCall
from nisse_dev.execution import SSHHost
from nisse_dev.git import GitRepo
from nisse_dev.gitea import (
from nisse_core.cancel import Cancel
from nisse_core.canonical import ToolCall
from nisse_core.execution import SSHHost
from nisse_core.gitea import (
Approval,
Comment,
GiteaError,
@@ -22,15 +21,16 @@ from nisse_dev.gitea import (
PullRequest,
ReviewNote,
)
from nisse_core.oplog import OpLog
from nisse_core.transport import InMemoryTransport
from nisse_core.worker import InlineWorker
from nisse_dev.git import GitRepo
from nisse_dev.loop.agent import LoopAgent
from nisse_dev.loop.driver import Driver
from nisse_dev.loop.items import ANSWER, MERGE, WAITING, loop_of
from nisse_dev.loop.milestones import OPEN_PR, Milestones
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore
from nisse_dev.tools import Workspace
from nisse_dev.transport import InMemoryTransport
from nisse_dev.worker import InlineWorker
DON = "@don:test"
nisse-dev/tests/test_loop_multi.py
+2 -2
View File
@@ -18,8 +18,8 @@ from test_loop_milestones import (
)
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from nisse_dev.canonical import ToolCall, ToolResult
from nisse_dev.gitea import Issue
from nisse_core.canonical import ToolCall, ToolResult
from nisse_core.gitea import Issue
from nisse_dev.loop.items import MERGE, WAITING, loop_of
nisse-dev/tests/test_loop_new_projects.py
+5 -5
View File
@@ -14,16 +14,16 @@ import yaml
from test_loop_milestones import DON, Forge, ScriptedModel, call, make_world
from test_loop_projects import Router
from nisse_dev.cancel import Cancel
from nisse_core.cancel import Cancel
from nisse_core.execution import SSHHost
from nisse_core.oplog import OpLog
from nisse_core.transport import InMemoryTransport
from nisse_core.worker import InlineWorker
from nisse_dev.config import _parse_projects
from nisse_dev.execution import SSHHost
from nisse_dev.loop.agent import LoopAgent, Project
from nisse_dev.loop.driver import Driver
from nisse_dev.loop.items import CHAT, GO_AHEAD, MERGE, PROVISION, loop_of
from nisse_dev.oplog import OpLog
from nisse_dev.tools import Workspace
from nisse_dev.transport import InMemoryTransport
from nisse_dev.worker import InlineWorker
PROPOSAL = call(
"propose_project",
nisse-dev/tests/test_loop_notes.py
+3 -3
View File
@@ -11,9 +11,9 @@ from test_loop_milestones import DON, RESTATE, ScriptedModel, call, loop_agent
from test_loop_milestones import item as the_item
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from nisse_dev.canonical import ToolResult
from nisse_core.canonical import ToolResult
from nisse_core.vault import Vault
from nisse_dev.loop.items import GO_AHEAD, loop_of
from nisse_dev.vault import Vault
@pytest.fixture
@@ -21,7 +21,7 @@ def vault(tmp_path: Path) -> Vault:
root = tmp_path / "vault"
(root / "notes").mkdir(parents=True)
(root / "notes" / "Style.md").write_text("# Style\n\nCount words with split.\n")
return Vault(root, {"memory/nisse-dev": "rw", "notes": "ro"})
return Vault(root, {"memory/nisse-dev": "rw", "notes": "ro"}, agent="nisse-dev")
class Spy(ScriptedModel):
nisse-dev/tests/test_loop_projects.py
+5 -5
View File
@@ -10,15 +10,15 @@ from typing import Any
import pytest
from test_loop_milestones import DON, Forge, ScriptedModel, call, git, make_world
from nisse_dev.cancel import Cancel
from nisse_dev.execution import SSHHost
from nisse_core.cancel import Cancel
from nisse_core.execution import SSHHost
from nisse_core.oplog import OpLog
from nisse_core.transport import InMemoryTransport
from nisse_core.worker import InlineWorker
from nisse_dev.loop.agent import LoopAgent, Project
from nisse_dev.loop.driver import Driver
from nisse_dev.loop.items import ANSWER, GO_AHEAD, MERGE, item_for_issue, loop_of
from nisse_dev.oplog import OpLog
from nisse_dev.tools import Workspace
from nisse_dev.transport import InMemoryTransport
from nisse_dev.worker import InlineWorker
REPOS = {"alpha": "don/alpha", "beta": "don/beta"}
nisse-dev/tests/test_loop_properties.py
+15 -11
View File
@@ -24,8 +24,8 @@ from test_loop_milestones import DON, Forge, call, make_world
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from test_loop_projects import Router
from nisse_dev.cancel import Cancel
from nisse_dev.canonical import (
from nisse_core.cancel import Cancel
from nisse_core.canonical import (
ActionVerdict,
AssistantText,
Fact,
@@ -34,10 +34,15 @@ from nisse_dev.canonical import (
ToolResult,
UserMessage,
)
from nisse_core.execution import HostUnavailable, SSHHost
from nisse_core.gitea import Comment, Issue
from nisse_core.llm import LLMError, RoleMap
from nisse_core.memory import Memory
from nisse_core.oplog import OpLog
from nisse_core.transport import InboundMessage, InMemoryTransport
from nisse_core.vault import Vault, split_frontmatter
from nisse_core.worker import InlineWorker
from nisse_dev.delegation import DELEGATION_INTENT, Caps
from nisse_dev.execution import HostUnavailable, SSHHost
from nisse_dev.gitea import Comment, Issue
from nisse_dev.llm import LLMError, RoleMap
from nisse_dev.loop.agent import LoopAgent, Project
from nisse_dev.loop.classifier import ActionClassifier
from nisse_dev.loop.driver import Driver
@@ -53,12 +58,7 @@ from nisse_dev.loop.items import (
WAITING,
loop_of,
)
from nisse_dev.memory import Memory
from nisse_dev.oplog import OpLog
from nisse_dev.tools import Workspace
from nisse_dev.transport import InboundMessage, InMemoryTransport
from nisse_dev.vault import Vault, split_frontmatter
from nisse_dev.worker import InlineWorker
# How many walks, and from which seed: all 48 by default; CI runs fewer on a
# PR, from a seed that moves with the run number, so PRs still cover new
@@ -360,7 +360,11 @@ class Walk:
(vault_root / "notes").mkdir(parents=True, exist_ok=True)
(vault_root / "notes" / "Ops.md").write_text("the operator's\n")
self.memory = Memory(
Vault(vault_root, {"memory/nisse-dev": "rw", "notes": "ro"}),
Vault(
vault_root,
{"memory/nisse-dev": "rw", "notes": "ro"},
agent="nisse-dev",
),
"memory/nisse-dev",
)
real = SSHHost.execute
nisse-dev/tests/test_loop_queue.py
+2 -2
View File
@@ -7,7 +7,8 @@ from typing import Any
from test_loop_milestones import DON, Forge, ScriptedModel, call, git, loop_agent
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from nisse_dev.gitea import Comment, Issue
from nisse_core.gitea import Comment, Issue
from nisse_core.transport import InboundMessage, InMemoryTransport
from nisse_dev.loop.items import (
ANSWER,
BLOCKED,
@@ -18,7 +19,6 @@ from nisse_dev.loop.items import (
item_for_issue,
loop_of,
)
from nisse_dev.transport import InboundMessage, InMemoryTransport
def two_issues(forge: Forge) -> None:
nisse-dev/tests/test_loop_talk.py
+1 -1
View File
@@ -8,7 +8,7 @@ from typing import Any
from test_loop import DON, RESTATED, FakeGitea, Harness, ScriptedModel, call, issue
from nisse_dev.canonical import AssistantText
from nisse_core.canonical import AssistantText
from nisse_dev.loop.items import ANSWER, GO_AHEAD, WAITING, loop_of
nisse-dev/tests/test_loop_toolchain.py
+8 -8
View File
@@ -26,18 +26,18 @@ from test_loop_milestones import (
from test_loop_milestones import item as the_item
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from nisse_dev.cancel import Cancel
from nisse_dev.canonical import Fact, ToolResult
from nisse_dev.execution import HostKeyChanged
from nisse_dev.gitea import Issue
from nisse_core.cancel import Cancel
from nisse_core.canonical import Fact, ToolResult
from nisse_core.execution import HostKeyChanged
from nisse_core.gitea import Issue
from nisse_core.oplog import OpLog
from nisse_core.transport import InMemoryTransport
from nisse_core.worker import InlineWorker
from nisse_dev.loop import preflight
from nisse_dev.loop.agent import LoopAgent, Project
from nisse_dev.loop.driver import Driver
from nisse_dev.loop.items import ANSWER, GO_AHEAD, WAITING, loop_of
from nisse_dev.oplog import OpLog
from nisse_dev.tools import Workspace
from nisse_dev.transport import InMemoryTransport
from nisse_dev.worker import InlineWorker
PYTHON = {"name": "python", "version_command": "python3 --version"}
@@ -194,7 +194,7 @@ def test_what_was_installed_or_is_missing_is_said(
def test_an_unfinished_step_is_said_and_work_goes_on(
world: dict[str, Any], project: str
) -> None:
from nisse_dev.canonical import AssistantText
from nisse_core.canonical import AssistantText
prose = [AssistantText(text="hmm", model="m")]
model = ScriptedModel(prose, prose, RESTATE)
nisse-dev/tests/test_loop_trust.py
+5 -5
View File
@@ -19,14 +19,14 @@ from test_loop_milestones import (
from test_loop_milestones import item as the_item
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from nisse_dev.cancel import Cancel
from nisse_core.cancel import Cancel
from nisse_core.oplog import OpLog
from nisse_core.transport import InMemoryTransport
from nisse_core.worker import InlineWorker
from nisse_dev.loop.agent import LoopAgent, Project
from nisse_dev.loop.driver import Driver
from nisse_dev.loop.items import MERGE, loop_of
from nisse_dev.oplog import OpLog
from nisse_dev.tools import Workspace
from nisse_dev.transport import InMemoryTransport
from nisse_dev.worker import InlineWorker
CLOSE = [call("close_item", summary="added and merged")]
@@ -169,7 +169,7 @@ def test_unprotected_means_the_operator_merges(world: dict[str, Any]) -> None:
def test_a_refused_merge_is_said_once_and_left_to_the_operator(
world: dict[str, Any],
) -> None:
from nisse_dev.gitea import GiteaError
from nisse_core.gitea import GiteaError
forge: Forge = world["forge"]
agent, transport = to_review(world)
nisse-dev/tests/test_loop_vm.py
+3 -3
View File
@@ -19,9 +19,9 @@ from test_loop_toolchain import (
)
from test_loop_toolchain import project as project # noqa: PLC0414 -- the fixture
from nisse_dev.canonical import ToolResult
from nisse_dev.execution import HostKeyChanged, HostUnavailable
from nisse_dev.gitea import Issue
from nisse_core.canonical import ToolResult
from nisse_core.execution import HostKeyChanged, HostUnavailable
from nisse_core.gitea import Issue
from nisse_dev.loop.items import GO_AHEAD, READY, loop_of
RETRY, SETTLE = 300.0, 180.0
nisse-dev/tests/test_matrix.py
-361
View File
@@ -1,361 +0,0 @@
import json
import threading
import httpx
import pytest
from nisse_dev.matrix import MatrixConfig, MatrixTransport, TransportError
from nisse_dev.transport import InboundMessage
CONFIG = MatrixConfig(
homeserver="https://matrix.test",
user_id="@nisse-dev:test",
token="syt_secret",
room_id="!room:test",
)
def sync_body(events: list[dict], next_batch: str) -> dict:
return {
"next_batch": next_batch,
"rooms": {"join": {"!room:test": {"timeline": {"events": events}}}},
}
def message_event(sender: str, body: str) -> dict:
return {
"type": "m.room.message",
"sender": sender,
"content": {"msgtype": "m.text", "body": body},
}
def test_send_message_puts_event_with_auth_and_unique_txn() -> None:
seen: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
seen.append(request)
return httpx.Response(200, json={"event_id": "$e"})
transport = MatrixTransport(CONFIG, http_transport=httpx.MockTransport(handler))
transport.send_message("hello")
transport.send_approval_request("plan rev-1")
first, second = seen
assert first.headers["authorization"] == "Bearer syt_secret"
assert "/rooms/!room:test/send/m.room.message/" in str(first.url)
assert json.loads(first.content) == {"msgtype": "m.text", "body": "hello"}
assert str(first.url) != str(second.url) # txn ids must be unique
def test_send_failure_raises_transport_error() -> None:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(403, text="forbidden")
transport = MatrixTransport(CONFIG, http_transport=httpx.MockTransport(handler))
with pytest.raises(TransportError, match="403"):
transport.send_message("hello")
class SyncScript:
"""Serves scripted /sync responses, then endless empty ones."""
def __init__(self, responses: list[dict]) -> None:
self._responses = responses
self._i = 0
self.batches_seen: list[str | None] = []
def __call__(self, request: httpx.Request) -> httpx.Response:
assert request.url.path.endswith("/sync")
self.batches_seen.append(request.url.params.get("since"))
if self._i < len(self._responses):
body = self._responses[self._i]
self._i += 1
return httpx.Response(200, json=body)
return httpx.Response(200, json=sync_body([], f"b{self._i}"))
def collect_messages(
responses: list[dict], expected: int, timeout_s: float = 5.0
) -> list[InboundMessage]:
received: list[InboundMessage] = []
done = threading.Event()
def on_message(msg: InboundMessage) -> None:
received.append(msg)
if len(received) >= expected:
done.set()
transport = MatrixTransport(
CONFIG,
poll_timeout_ms=10,
http_transport=httpx.MockTransport(SyncScript(responses)),
)
transport.on_message(on_message)
transport.start()
done.wait(timeout_s)
transport.close()
return received
def test_listener_delivers_operator_messages() -> None:
responses = [
sync_body([message_event("@don:test", "backlog: never delivered")], "b1"),
sync_body([message_event("@don:test", "approve")], "b2"),
]
received = collect_messages(responses, expected=1)
assert received == [InboundMessage(sender="@don:test", text="approve")]
def test_own_and_non_message_events_are_filtered() -> None:
responses = [
sync_body([], "b1"),
sync_body(
[
message_event("@nisse-dev:test", "my own echo"),
{"type": "m.room.member", "sender": "@don:test", "content": {}},
message_event("@don:test", "status"),
],
"b2",
),
]
received = collect_messages(responses, expected=1)
assert received == [InboundMessage(sender="@don:test", text="status")]
def test_first_sync_learns_position_without_delivering() -> None:
script = SyncScript([sync_body([message_event("@don:test", "old")], "b1")])
transport = MatrixTransport(
CONFIG, poll_timeout_ms=10, http_transport=httpx.MockTransport(script)
)
got = threading.Event()
transport.on_message(lambda msg: got.set())
transport.start()
assert not got.wait(0.3)
transport.close()
assert script.batches_seen[0] is None # initial sync carries no since
assert "b1" in script.batches_seen # later polls resume from its batch
def test_start_without_handler_raises() -> None:
transport = MatrixTransport(
CONFIG, http_transport=httpx.MockTransport(lambda r: httpx.Response(200))
)
with pytest.raises(TransportError, match="no message handler"):
transport.start()
# --- leading mentions --------------------------------------------------------
def own_member_state(name: str) -> dict:
"""The first sync's room state, which carries the agent's display name."""
body = sync_body([], "b1")
body["rooms"]["join"]["!room:test"]["state"] = {
"events": [
{
"type": "m.room.member",
"state_key": "@nisse-dev:test",
"sender": "@nisse-dev:test",
"content": {"membership": "join", "displayname": name},
}
]
}
return body
def pill_event(sender: str, user_id: str, label: str, rest: str) -> dict:
"""A message as Element sends one that starts by mentioning user_id."""
return {
"type": "m.room.message",
"sender": sender,
"content": {
"msgtype": "m.text",
"body": f"{label}: {rest}",
"format": "org.matrix.custom.html",
"formatted_body": (
f'<a href="https://matrix.to/#/{user_id}">{label}</a>: {rest}'
),
},
}
def test_a_leading_pill_for_the_agent_is_removed_before_matching() -> None:
responses = [
sync_body([], "b1"),
sync_body(
[pill_event("@don:test", "@nisse-dev:test", "Nisse Dev", "approve")],
"b2",
),
]
[msg] = collect_messages(responses, expected=1)
assert msg.sender == "@don:test" # from the event, never from the text
assert msg.text == "approve"
assert msg.original == "Nisse Dev: approve"
def test_the_agents_display_name_typed_in_front_is_removed() -> None:
responses = [
own_member_state("Nisse Dev"),
sync_body([message_event("@don:test", "Nisse Dev: re-plan")], "b2"),
]
[msg] = collect_messages(responses, expected=1)
assert msg.text == "re-plan"
def test_the_agents_user_id_typed_in_front_is_removed() -> None:
responses = [
sync_body([], "b1"),
sync_body([message_event("@don:test", "@nisse-dev:test: status")], "b2"),
]
[msg] = collect_messages(responses, expected=1)
assert msg.text == "status"
def test_messages_addressed_to_someone_else_are_skipped() -> None:
responses = [
sync_body([], "b1"),
sync_body(
[
pill_event("@don:test", "@nisse-ops:test", "Nisse Ops", "approve"),
message_event("@don:test", "@alice:test: looks good"),
message_event("@don:test", "status"),
],
"b2",
),
]
received = collect_messages(responses, expected=1)
assert received == [InboundMessage(sender="@don:test", text="status")]
def test_a_mention_later_in_the_message_is_left_alone() -> None:
responses = [
sync_body([], "b1"),
sync_body([message_event("@don:test", "approve, cc @alice:test")], "b2"),
]
received = collect_messages(responses, expected=1)
assert received == [
InboundMessage(sender="@don:test", text="approve, cc @alice:test")
]
# --- threads (L5): a work item's conversation is a thread --------------------
def threaded_transport() -> tuple[MatrixTransport, list[dict], list[InboundMessage]]:
sent: list[dict] = []
ids = iter(f"$sent{n}" for n in range(1, 100))
def handler(request: httpx.Request) -> httpx.Response:
sent.append(json.loads(request.content))
return httpx.Response(200, json={"event_id": next(ids)})
transport = MatrixTransport(CONFIG, http_transport=httpx.MockTransport(handler))
received: list[InboundMessage] = []
transport.on_message(received.append)
return transport, sent, received
def event(event_id: str, body: str, relates: dict | None = None) -> dict:
content: dict = {"msgtype": "m.text", "body": body}
if relates is not None:
content["m.relates_to"] = relates
return {
"type": "m.room.message",
"sender": "@don:test",
"event_id": event_id,
"content": content,
}
def deliver(transport: MatrixTransport, *events: dict) -> None:
transport._deliver(sync_body(list(events), "b"))
def test_post_returns_the_event_id_and_threads_a_reply() -> None:
transport, sent, _ = threaded_transport()
root = transport.post("queued sandbox#14")
transport.post("As I understand it: ...", thread=root)
assert root == "$sent1"
assert "m.relates_to" not in sent[0]
assert sent[1]["m.relates_to"] == {
"rel_type": "m.thread",
"event_id": "$sent1",
# Clients without threads show it as a reply to the root.
"is_falling_back": True,
"m.in_reply_to": {"event_id": "$sent1"},
}
def test_a_message_in_a_thread_carries_its_root() -> None:
transport, _, received = threaded_transport()
deliver(
transport,
event("$top", "status"),
event("$in", "go", {"rel_type": "m.thread", "event_id": "$root"}),
)
assert [(m.text, m.event_id, m.thread) for m in received] == [
("status", "$top", None),
("go", "$in", "$root"),
]
def test_a_plain_reply_to_a_threaded_message_joins_its_thread() -> None:
# Clients without thread support send an ordinary reply, quoting.
transport, _, received = threaded_transport()
root = transport.post("queued sandbox#14")
in_thread = transport.post("As I understand it: ...", thread=root)
deliver(
transport,
event(
"$r1",
"> <@nisse-dev:test> As I understand it: ...\n\nyes, go",
{"m.in_reply_to": {"event_id": in_thread}},
),
event(
"$r2",
"> <@nisse-dev:test> queued sandbox#14\n\nthanks",
{"m.in_reply_to": {"event_id": root}},
),
)
assert [(m.text, m.thread) for m in received] == [
("yes, go", root), # the quote is not what was said
("thanks", root), # a reply to the root joins its thread
]
def test_a_reply_to_an_unknown_message_stays_at_top_level() -> None:
transport, _, received = threaded_transport()
deliver(
transport,
event("$r", "> <@x:test> old\n\nhm", {"m.in_reply_to": {"event_id": "$old"}}),
)
assert [(m.text, m.thread) for m in received] == [("hm", None)]
def test_a_reply_to_the_operators_own_threaded_message_joins_the_thread() -> None:
transport, _, received = threaded_transport()
deliver(
transport,
event("$t", "first", {"rel_type": "m.thread", "event_id": "$root"}),
event(
"$r",
"> <@don:test> first\n\nand also",
{"m.in_reply_to": {"event_id": "$t"}},
),
)
assert [m.thread for m in received] == ["$root", "$root"]
def test_in_memory_transport_plays_threads() -> None:
from nisse_dev.transport import InMemoryTransport
transport = InMemoryTransport()
seen: list[InboundMessage] = []
transport.on_message(seen.append)
root = transport.post("queued #14")
transport.post("restated", thread=root)
reply = transport.receive("@don:test", "go", thread=root)
assert transport.outbound[-1].thread == root
assert (seen[0].thread, seen[0].event_id) == (root, reply)
nisse-dev/tests/test_memory.py
-95
View File
@@ -1,95 +0,0 @@
"""Agent memory (L6.4, M5.2): one note per fact in the agent's folder, an
index the harness keeps, and a snapshot of it for the transcript."""
from pathlib import Path
import pytest
from nisse_dev.memory import Memory
from nisse_dev.vault import Vault, VaultError, split_frontmatter
FOLDER = "memory/nisse-dev"
@pytest.fixture
def memory(tmp_path: Path) -> Memory:
return Memory(Vault(tmp_path / "v", {FOLDER: "rw", "notes": "ro"}), FOLDER)
def note(memory: Memory, name: str) -> tuple[dict, str]:
path = memory._vault.root / FOLDER / f"{name}.md"
return split_frontmatter(path.read_text())
def test_a_fact_is_a_note_with_its_description_and_provenance(memory: Memory) -> None:
replaced = memory.remember(
"prefers-small-prs",
"The operator prefers small PRs",
"Don said PRs over 400 lines are hard to review.\n",
provenance={"source": "work item x"},
)
assert not replaced
front, body = note(memory, "prefers-small-prs")
assert front["description"] == "The operator prefers small PRs"
assert front["provenance"]["source"] == "work item x"
assert body == "Don said PRs over 400 lines are hard to review.\n"
def test_the_same_name_replaces_and_the_index_follows(memory: Memory) -> None:
memory.remember("b-fact", "Bee", "b", provenance={})
memory.remember("a-fact", "Ay", "a", provenance={})
assert memory.remember("a-fact", "Ay, again", "a2", provenance={})
index = memory.snapshot().text
assert index.endswith(
"- [[memory/nisse-dev/a-fact]] — Ay, again\n- [[memory/nisse-dev/b-fact]] — Bee"
)
memory.forget("a-fact")
assert "a-fact" not in memory.snapshot().text
with pytest.raises(VaultError, match="no such note"):
memory.forget("a-fact")
@pytest.mark.parametrize("name", ["index", "Caps", "../x", "a/b", "", "-x"])
def test_names_are_plain(memory: Memory, name: str) -> None:
with pytest.raises(VaultError, match="a note's name"):
memory.remember(name, "d", "f", provenance={})
def test_a_note_needs_a_description_and_a_fact(memory: Memory) -> None:
with pytest.raises(VaultError, match="description and the fact"):
memory.remember("x", " ", "f", provenance={})
def test_the_snapshot_is_hashed_and_made_if_missing(memory: Memory) -> None:
first = memory.snapshot()
assert first.paths == (f"{FOLDER}/index.md",)
assert "Nothing yet." in first.text
assert memory.snapshot().sha == first.sha # same index, same hash
memory.remember("x", "Ex", "f", provenance={})
assert memory.snapshot().sha != first.sha
def test_a_projects_notes_have_their_own_folder_and_index(memory: Memory) -> None:
memory.remember("general-one", "General", "g", provenance={})
memory.remember("deploy", "Deploys by tag", "d", provenance={}, project="sandbox")
memory.remember("other", "Other's", "o", provenance={}, project="other")
root = memory._vault.root / FOLDER
assert (root / "projects" / "sandbox" / "deploy.md").exists()
top = memory.snapshot().text
assert "- [[memory/nisse-dev/general-one]] — General" in top
assert "deploy" not in top.split("Projects:")[0]
assert "- [[memory/nisse-dev/projects/sandbox/index]] — notes about sandbox" in top
seen = memory.snapshot("sandbox")
assert seen.paths == (
f"{FOLDER}/index.md",
f"{FOLDER}/projects/sandbox/index.md",
)
assert (
"- [[memory/nisse-dev/projects/sandbox/deploy]] — Deploys by tag" in seen.text
)
assert "projects/other/other]]" not in seen.text
memory.forget("deploy", "sandbox")
assert memory.snapshot("sandbox").paths == (f"{FOLDER}/index.md",)
assert "sandbox/index" not in memory.snapshot().text
with pytest.raises(VaultError, match="not a project's name"):
memory.remember("x", "d", "f", provenance={}, project="../x")
nisse-dev/tests/test_oplog.py
-183
View File
@@ -1,183 +0,0 @@
"""State-change log conventions (architecture L12): every durable mutation emits one
JSON-lines event with the fixed envelope, from the store's own write path."""
import io
import json
import sqlite3
from pathlib import Path
import pytest
from nisse_dev.oplog import OpLog
from nisse_dev.reconcile import reconcile_on_boot
from nisse_dev.state import StateStore
ENVELOPE_FIELDS = {"ts", "agent", "event", "task_id", "cause"}
@pytest.fixture
def stream() -> io.StringIO:
return io.StringIO()
@pytest.fixture
def store(tmp_path: Path, stream: io.StringIO) -> StateStore:
return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", stream))
def events(stream: io.StringIO) -> list[dict]:
return [json.loads(line) for line in stream.getvalue().splitlines()]
def start_item(
store: StateStore, item_id: str, *, scope: str | None = None, cause: str
) -> None:
"""Open a work item, make it the active one and set it running."""
store.open_work_item(
item_id, title=item_id, scope=scope, position="IDLE", cause=cause
)
store.set_active(item_id, cause=cause)
store.update_work_item(item_id, position="RUNNING", data={}, cause=cause)
def test_every_event_carries_the_envelope(
store: StateStore, stream: io.StringIO
) -> None:
start_item(store, "demo-1", cause="issue assigned")
intent_id = store.begin_intent("push_branch", {}, cause="milestone execution")
store.commit_intent(intent_id, cause="push verified")
for event in events(stream):
assert ENVELOPE_FIELDS <= event.keys()
assert event["agent"] == "nisse-dev"
def test_a_position_change_logs_from_to_and_cause(
store: StateStore, stream: io.StringIO
) -> None:
start_item(store, "d-1", scope="d", cause="issue assigned")
store.update_work_item("d-1", position="WAITING", data={}, cause="restated")
# Opening the work item logs its own events first (work_item.open, .switch).
first, second = [e for e in events(stream) if e["event"] == "state.transition"]
assert (first["from"], first["to"]) == ("IDLE", "RUNNING")
assert (second["from"], second["to"], second["project"]) == (
"RUNNING",
"WAITING",
"d",
)
assert second["cause"] == "restated"
def test_intent_lifecycle_events(store: StateStore, stream: io.StringIO) -> None:
intent_id = store.begin_intent("push_branch", {"b": "m2"}, cause="milestone")
store.abort_intent(intent_id, cause="discarded")
begin, abort = events(stream)
assert begin["event"] == "intent.begin"
assert abort["event"] == "intent.abort"
assert begin["intent_id"] == abort["intent_id"] == intent_id
assert begin["kind"] == abort["kind"] == "push_branch"
def test_task_id_threads_through_events(store: StateStore, stream: io.StringIO) -> None:
store.open_work_item("general", title="general", position="IDLE", cause="boot")
store.oplog.task_id = "task-42"
start_item(store, "task-42", cause="issue assigned")
before, *during = events(stream)
assert before["task_id"] is None
assert {e["task_id"] for e in during} == {"task-42"}
def test_migration_emits_event(tmp_path: Path, stream: io.StringIO) -> None:
db = tmp_path / "state.db"
with StateStore(db, schema_version=1):
pass
def add_notes(conn: sqlite3.Connection) -> None:
conn.execute("CREATE TABLE notes (id INTEGER PRIMARY KEY)")
with StateStore(
db,
schema_version=2,
migrations={1: add_notes},
oplog=OpLog("nisse-dev", stream),
):
pass
[event] = events(stream)
assert event["event"] == "schema.migrate"
assert (event["from_version"], event["to_version"]) == (1, 2)
def test_boot_reconcile_emits_action_and_verdicts(
store: StateStore, stream: io.StringIO
) -> None:
store.begin_intent("push_branch", {}, cause="milestone")
reconcile_on_boot(store, {"push_branch": lambda intent: True})
boot = [e for e in events(stream) if e["event"] == "boot.reconcile"]
assert len(boot) == 1
assert boot[0]["action"] == "resume"
assert "effect verified" in boot[0]["verdicts"][0]
commit = [e for e in events(stream) if e["event"] == "intent.commit"]
assert commit[0]["cause"] == "reconciliation verdict: effect verified"
def test_literal_log_lines_for_human_readability(
store: StateStore, stream: io.StringIO, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Asserts the exact bytes written per line, so the on-disk format is
visible here rather than only in a running system."""
class _FrozenDatetime:
@staticmethod
def now(tz: object) -> object:
from datetime import UTC, datetime
return datetime(2026, 9, 5, 12, 0, 0, tzinfo=UTC)
monkeypatch.setattr("nisse_dev.oplog.datetime", _FrozenDatetime)
start_item(store, "demo-m2", scope="demo", cause="issue assigned: demo#2")
store.oplog.task_id = "demo-m2"
intent_id = store.begin_intent(
"push_branch", {"branch": "milestone-2"}, cause="milestone 2 execution"
)
store.commit_intent(intent_id, cause="push verified by Gitea query")
assert stream.getvalue().splitlines() == [
(
'{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",'
' "event": "work_item.open", "task_id": null,'
' "cause": "issue assigned: demo#2",'
' "work_item": "demo-m2", "scope": "demo"}'
),
(
'{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",'
' "event": "work_item.switch", "task_id": null,'
' "cause": "issue assigned: demo#2",'
' "from": null, "to": "demo-m2"}'
),
(
'{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",'
' "event": "state.transition", "task_id": null,'
' "cause": "issue assigned: demo#2",'
' "from": "IDLE", "to": "RUNNING", "project": "demo"}'
),
(
'{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",'
' "event": "intent.begin", "task_id": "demo-m2",'
' "cause": "milestone 2 execution", "intent_id": 1,'
' "kind": "push_branch"}'
),
(
'{"ts": "2026-09-05T12:00:00+00:00", "agent": "nisse-dev",'
' "event": "intent.commit", "task_id": "demo-m2",'
' "cause": "push verified by Gitea query",'
' "intent_id": 1, "kind": "push_branch"}'
),
]
def test_failed_resolution_emits_nothing(
store: StateStore, stream: io.StringIO
) -> None:
with pytest.raises(KeyError):
store.commit_intent(9999, cause="test")
assert events(stream) == []
nisse-dev/tests/test_preflight.py
+1 -1
View File
@@ -5,7 +5,7 @@ import uuid
import pytest
from nisse_dev.execution import SSHHost
from nisse_core.execution import SSHHost
from nisse_dev.loop import preflight
from nisse_dev.loop.preflight import Tool
nisse-dev/tests/test_probe.py
-89
View File
@@ -1,89 +0,0 @@
import json
import httpx
from nisse_dev.probe import interpret, probe
def test_a_cut_at_the_declared_window_matches() -> None:
result = interpret(declared=4096, sent=6144, reported=4096, error=None)
assert result.verdict == "matches"
assert result.server_window == 4096
def test_a_cut_below_the_declaration_says_what_to_set() -> None:
# The dangerous case: prompts between the two would be silently cut.
result = interpret(declared=8192, sent=12288, reported=4096, error=None)
assert result.verdict == "declared_too_high"
assert "set max_input_tokens to 4096" in result.message
def test_a_cut_above_the_declaration_is_conservative() -> None:
result = interpret(declared=4096, sent=6144, reported=5000, error=None)
assert result.verdict == "declared_conservative"
def test_an_uncut_prompt_means_the_server_allows_more() -> None:
result = interpret(declared=4096, sent=6144, reported=6100, error=None)
assert result.verdict == "declared_conservative"
assert result.server_window is None
def test_a_refusal_is_the_safe_behaviour() -> None:
result = interpret(declared=4096, sent=6144, reported=None, error="400: too long")
assert result.verdict == "refuses"
def test_probe_calibrates_then_sends_half_again_the_window() -> None:
"""Against a fake Ollama: 2 tokens a word, silently cut at 4096."""
sizes: list[int] = []
def handler(request: httpx.Request) -> httpx.Response:
words = len(json.loads(request.content)["messages"][0]["content"].split())
sizes.append(words * 2)
reported = min(words * 2, 4096)
return httpx.Response(
200, json={"model": "llama3.2", "usage": {"prompt_tokens": reported}}
)
with httpx.Client(
base_url="http://litellm", transport=httpx.MockTransport(handler)
) as http:
result = probe(http, "llama3.2", declared=4096)
assert result.verdict == "matches"
assert sizes[1] >= 4096 * 1.4 # the probe really exceeded the window
def test_a_fallback_answer_is_never_credited_to_the_model() -> None:
"""Measured: LiteLLM answered an oversized prompt to local-coder with
claude-sonnet-5, and the probe called the refusing server roomy."""
def handler(request: httpx.Request) -> httpx.Response:
words = len(json.loads(request.content)["messages"][0]["content"].split())
model = "local-coder" if words < 1000 else "claude-sonnet-5"
return httpx.Response(
200, json={"model": model, "usage": {"prompt_tokens": words * 2}}
)
with httpx.Client(
base_url="http://litellm", transport=httpx.MockTransport(handler)
) as http:
result = probe(http, "local-coder", declared=32768, backing="openai/qwen3-14b")
assert result.verdict == "fallback"
assert "claude-sonnet-5 answered instead of local-coder" in result.message
def test_a_provider_reporting_its_own_model_name_is_not_a_fallback() -> None:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200, json={"model": "qwen3-14b", "usage": {"prompt_tokens": 400}}
)
with httpx.Client(
base_url="http://litellm", transport=httpx.MockTransport(handler)
) as http:
result = probe(http, "local-coder", declared=32768, backing="openai/qwen3-14b")
assert result.verdict != "fallback"
nisse-dev/tests/test_reconcile.py
-68
View File
@@ -1,68 +0,0 @@
from pathlib import Path
import pytest
from nisse_dev.reconcile import reconcile_on_boot
from nisse_dev.state import Intent, StateStore
@pytest.fixture
def store(tmp_path: Path) -> StateStore:
return StateStore(tmp_path / "state.db")
def test_no_pending_intents_resumes(store: StateStore) -> None:
report = reconcile_on_boot(store, {})
assert report.action == "resume"
assert report.notes == []
def test_verified_effect_commits_intent_and_resumes(store: StateStore) -> None:
store.begin_intent("push_branch", {"branch": "m2"}, cause="test")
report = reconcile_on_boot(store, {"push_branch": lambda intent: True})
assert report.action == "resume"
assert store.pending_intents() == []
def test_verified_no_effect_aborts_intent_and_resumes(store: StateStore) -> None:
store.begin_intent("push_branch", {"branch": "m2"}, cause="test")
report = reconcile_on_boot(store, {"push_branch": lambda intent: False})
assert report.action == "resume"
assert store.pending_intents() == []
def test_unverifiable_intent_gates_to_await_input(store: StateStore) -> None:
store.begin_intent("push_branch", {"branch": "m2"}, cause="test")
report = reconcile_on_boot(store, {"push_branch": lambda intent: None})
assert report.action == "await_input"
assert [i.kind for i in store.pending_intents()] == ["push_branch"]
def test_unknown_intent_kind_gates_to_await_input(store: StateStore) -> None:
store.begin_intent("mystery", {}, cause="test")
report = reconcile_on_boot(store, {})
assert report.action == "await_input"
assert "no reconciler registered" in report.notes[0]
def test_reconciler_receives_the_intent(store: StateStore) -> None:
store.begin_intent("push_branch", {"branch": "m2"}, cause="test")
seen: list[Intent] = []
def check(intent: Intent) -> bool:
seen.append(intent)
return True
reconcile_on_boot(store, {"push_branch": check})
assert seen[0].payload == {"branch": "m2"}
def test_mixed_verdicts_still_gate(store: StateStore) -> None:
store.begin_intent("push_branch", {}, cause="test")
store.begin_intent("open_pr", {}, cause="test")
report = reconcile_on_boot(
store,
{"push_branch": lambda intent: True, "open_pr": lambda intent: None},
)
assert report.action == "await_input"
assert [i.kind for i in store.pending_intents()] == ["open_pr"]
nisse-dev/tests/test_render.py
-151
View File
@@ -1,151 +0,0 @@
import json
from itertools import pairwise
from nisse_dev.canonical import (
AssistantText,
Compaction,
GateOpened,
StoredEvent,
ToolCall,
ToolResult,
UserMessage,
)
from nisse_dev.render import OpenAIChatRenderer, effective_events
def stored(events: list) -> list[StoredEvent]:
return [StoredEvent(id=i + 1, ts="t", event=e) for i, e in enumerate(events)]
def test_renders_mixed_transcript_in_order() -> None:
events = stored(
[
UserMessage(text="run the tests", source="m:1"),
ToolCall(call_id="call-9", tool="run_tests", arguments={"path": "x"}),
ToolResult(call_id="call-9", result="1 passed", success=True),
AssistantText(text="all green", model="local/qwen"),
]
)
body = OpenAIChatRenderer().render(events, tools=None, model="local/qwen")
assert body["model"] == "local/qwen"
roles = [m["role"] for m in body["messages"]]
assert roles == ["user", "assistant", "tool", "assistant"]
call = body["messages"][1]["tool_calls"][0]
assert call["id"] == "call-9"
assert call["function"]["name"] == "run_tests"
assert json.loads(call["function"]["arguments"]) == {"path": "x"}
assert body["messages"][2]["tool_call_id"] == "call-9"
assert "tools" not in body
def test_system_prompt_and_tools_pass_through() -> None:
tools = [{"type": "function", "function": {"name": "run_tests"}}]
body = OpenAIChatRenderer().render(
stored([UserMessage(text="hi", source="m:1")]),
tools=tools,
model="local/qwen",
system="you are nisse-dev",
)
assert body["messages"][0] == {"role": "system", "content": "you are nisse-dev"}
assert body["tools"] == tools
def test_compaction_replaces_its_range_only() -> None:
events = stored(
[
UserMessage(text="old-1", source="m:1"),
AssistantText(text="old-2", model="q"),
UserMessage(text="kept", source="m:3"),
]
)
events.append(
StoredEvent(
id=4,
ts="t",
event=Compaction(summary="the old stuff", model="q", replaces=[1, 2]),
)
)
remaining = effective_events(events)
assert [e.id for e in remaining] == [3, 4]
body = OpenAIChatRenderer().render(events, tools=None, model="q")
contents = [m["content"] for m in body["messages"]]
# Both render as user messages, so they share one: roles must alternate.
assert contents == ["kept\n\n[summary of earlier conversation]\nthe old stuff"]
def test_tool_call_without_a_result_gets_a_placeholder() -> None:
# Providers reject a dangling tool_use; this happens after a crash (L8)
# or with turn-ending gate tools like submit_plan.
events = stored(
[
UserMessage(text="go", source="m:1"),
ToolCall(call_id="call-x", tool="submit_plan", arguments={}),
]
)
body = OpenAIChatRenderer().render(events, tools=None, model="q")
assert [m["role"] for m in body["messages"]] == ["user", "assistant", "tool"]
assert body["messages"][2]["tool_call_id"] == "call-x"
assert body["messages"][2]["content"] == "[no result recorded]"
def test_answered_tool_call_gets_no_placeholder() -> None:
events = stored(
[
ToolCall(call_id="call-x", tool="run_command", arguments={}),
ToolResult(call_id="call-x", result="ok", success=True),
]
)
body = OpenAIChatRenderer().render(events, tools=None, model="q")
assert [m["role"] for m in body["messages"]] == ["assistant", "tool"]
assert body["messages"][1]["content"] == "ok"
def test_gate_event_renders_as_context() -> None:
events = stored([GateOpened(gate="arch", reason="plan submitted (rev-1)")])
body = OpenAIChatRenderer().render(events, tools=None, model="q")
assert body["messages"][0]["role"] == "user"
assert "arch" in body["messages"][0]["content"]
def _roles(body: dict) -> list[str]:
return [m["role"] for m in body["messages"]]
def test_one_turn_is_one_assistant_message() -> None:
# Text and two parallel tool calls from one model turn: the OpenAI format
# carries them as a single assistant message with a tool_calls list.
events = stored(
[
UserMessage(text="go", source="m:1"),
AssistantText(text="Looking around first.", model="q"),
ToolCall(call_id="a", tool="list_dir", arguments={"path": "."}),
ToolCall(call_id="b", tool="read_file", arguments={"path": "README.md"}),
ToolResult(call_id="a", result="README.md", success=True),
ToolResult(call_id="b", result="hello", success=True),
]
)
body = OpenAIChatRenderer().render(events, tools=None, model="q")
assert _roles(body) == ["user", "assistant", "tool", "tool"]
turn = body["messages"][1]
assert turn["content"] == "Looking around first."
assert [c["id"] for c in turn["tool_calls"]] == ["a", "b"]
def test_roles_never_repeat_for_strict_templates() -> None:
"""Mistral's chat template raises on two user or two assistant messages
in a row; every mix of events must render without one."""
events = stored(
[
UserMessage(text="one", source="m:1"),
UserMessage(text="two", source="m:2"),
AssistantText(text="thinking", model="q"),
ToolCall(call_id="a", tool="list_dir", arguments={}),
ToolResult(call_id="a", result="x", success=True),
AssistantText(text="more", model="q"),
AssistantText(text="and more", model="q"),
ToolCall(call_id="b", tool="finish", arguments={}),
]
)
roles = _roles(OpenAIChatRenderer().render(events, tools=None, model="q"))
for first, second in pairwise(roles):
assert not (first == second and first in ("user", "assistant")), roles
nisse-dev/tests/test_render_anthropic.py
-196
View File
@@ -1,196 +0,0 @@
"""The Anthropic Messages shape differs structurally from the OpenAI one;
these tests pin the differences that a normalising middlebox would blur."""
import httpx
import pytest
from nisse_dev.canonical import (
AssistantText,
Compaction,
StoredEvent,
ToolCall,
ToolResult,
UserMessage,
)
from nisse_dev.llm import AnthropicPassthroughClient, LLMError
from nisse_dev.render_anthropic import (
AnthropicMessagesRenderer,
to_anthropic_tools,
)
TOOLS = [
{
"type": "function",
"function": {
"name": "run_command",
"description": "Run a shell command.",
"parameters": {
"type": "object",
"properties": {"command": {"type": "string"}},
"required": ["command"],
},
},
}
]
def stored(events: list) -> list[StoredEvent]:
return [StoredEvent(id=i + 1, ts="t", event=e) for i, e in enumerate(events)]
def render(events: list, **kw) -> dict:
return AnthropicMessagesRenderer().render(
stored(events), kw.pop("tools", None), "claude-haiku-4-5", **kw
)
def test_system_is_a_top_level_field_not_a_message() -> None:
body = render([UserMessage(text="hi", source="m")], system="you are nisse")
assert body["system"] == "you are nisse"
assert all(m["role"] != "system" for m in body["messages"])
def test_max_tokens_is_always_present() -> None:
# The Messages API rejects a request without it.
assert render([UserMessage(text="hi", source="m")])["max_tokens"] > 0
def test_tool_call_is_a_content_block_not_an_array() -> None:
body = render(
[ToolCall(call_id="call-1", tool="run_command", arguments={"command": "ls"})]
)
block = body["messages"][0]["content"][0]
assert block["type"] == "tool_use"
assert block["id"] == "call-1"
assert block["input"] == {"command": "ls"}
assert "tool_calls" not in body["messages"][0]
def test_tool_result_is_a_user_block_directly_after_the_call() -> None:
body = render(
[
ToolCall(call_id="call-1", tool="run_command", arguments={}),
ToolResult(call_id="call-1", result="exit code: 0", success=True),
]
)
assistant, user = body["messages"]
assert assistant["role"] == "assistant"
assert assistant["content"][0]["type"] == "tool_use"
assert user["role"] == "user" # not a "tool" role, as OpenAI would have
assert user["content"][0] == {
"type": "tool_result",
"tool_use_id": "call-1",
"content": "exit code: 0",
}
def test_adjacent_same_role_events_are_merged() -> None:
body = render(
[
UserMessage(text="one", source="m"),
UserMessage(text="two", source="m"),
AssistantText(text="reply", model="m"),
]
)
assert [m["role"] for m in body["messages"]] == ["user", "assistant"]
assert [b["text"] for b in body["messages"][0]["content"]] == ["one", "two"]
def test_dangling_tool_call_gets_a_placeholder_result() -> None:
body = render([ToolCall(call_id="call-x", tool="submit_plan", arguments={})])
assert [m["role"] for m in body["messages"]] == ["assistant", "user"]
assert body["messages"][1]["content"][0]["tool_use_id"] == "call-x"
def test_tools_are_converted_to_input_schema() -> None:
[tool] = to_anthropic_tools(TOOLS)
assert tool["name"] == "run_command"
assert tool["input_schema"]["properties"] == {"command": {"type": "string"}}
assert "parameters" not in tool
assert "function" not in tool
def test_compaction_still_replaces_its_range() -> None:
events = stored(
[
UserMessage(text="old", source="m"),
UserMessage(text="kept", source="m"),
]
)
events.append(
StoredEvent(
id=3,
ts="t",
event=Compaction(summary="summary", model="m", replaces=[1, 1]),
)
)
body = AnthropicMessagesRenderer().render(events, None, "claude-haiku-4-5")
texts = [b["text"] for m in body["messages"] for b in m["content"]]
assert "old" not in texts
assert any("summary" in t for t in texts)
# --- passthrough client --------------------------------------------------
def client(handler) -> AnthropicPassthroughClient:
return AnthropicPassthroughClient(
"https://litellm.test", "tok", transport=httpx.MockTransport(handler)
)
def test_client_posts_to_the_messages_route_with_version_header() -> None:
seen: dict = {}
def handler(request: httpx.Request) -> httpx.Response:
seen["url"] = str(request.url)
seen["version"] = request.headers.get("anthropic-version")
return httpx.Response(
200,
json={
"model": "claude-haiku-4-5",
"content": [{"type": "text", "text": "hi"}],
},
)
events = stored([UserMessage(text="hello", source="m")])
[event] = client(handler).complete(events, None, "claude-haiku-4-5")
assert seen["url"] == "https://litellm.test/v1/messages"
assert seen["version"] == "2023-06-01"
assert event == AssistantText(text="hi", model="claude-haiku-4-5")
def test_client_maps_tool_use_to_local_call_ids() -> None:
response = {
"model": "claude-haiku-4-5",
"content": [
{"type": "text", "text": "running it"},
{
"type": "tool_use",
"id": "toolu_vendor_xyz",
"name": "run_command",
"input": {"command": "pytest"},
},
],
}
events = stored([UserMessage(text="go", source="m")])
text, call = client(lambda r: httpx.Response(200, json=response)).complete(
events, TOOLS, "claude-haiku-4-5"
)
assert isinstance(text, AssistantText)
assert isinstance(call, ToolCall)
assert call.arguments == {"command": "pytest"}
assert call.call_id != "toolu_vendor_xyz" # provider id never a join key
assert call.call_id.startswith("call-")
def test_client_errors_are_llm_errors() -> None:
events = stored([UserMessage(text="go", source="m")])
with pytest.raises(LLMError, match="400"):
client(lambda r: httpx.Response(400, text="bad")).complete(
events, None, "claude-haiku-4-5"
)
with pytest.raises(LLMError, match="malformed"):
client(lambda r: httpx.Response(200, json={"oops": 1})).complete(
events, None, "claude-haiku-4-5"
)
nisse-dev/tests/test_state.py
-148
View File
@@ -1,148 +0,0 @@
import sqlite3
from pathlib import Path
import pytest
from nisse_dev.state import StateSchemaError, StateStore
@pytest.fixture
def db(tmp_path: Path) -> Path:
return tmp_path / "state.db"
def test_wal_mode_and_full_sync(db: Path) -> None:
with StateStore(db) as store:
journal = store._conn.execute("PRAGMA journal_mode").fetchone()[0]
sync = store._conn.execute("PRAGMA synchronous").fetchone()[0]
assert journal == "wal"
assert sync == 2 # FULL
def test_fresh_store_stamps_schema_version(db: Path) -> None:
with StateStore(db, schema_version=1) as store:
assert store.schema_version == 1
def test_a_work_item_persists_across_reopen(db: Path) -> None:
with StateStore(db) as store:
store.open_work_item(
"demo-1", title="t", scope="demo", position="WAITING", cause="test"
)
store.set_active("demo-1", cause="test")
store.update_work_item(
"demo-1", position="RUNNING", data={"milestone": 2}, cause="test"
)
with StateStore(db) as store:
item = store.active_item()
assert item is not None
assert (item.id, item.scope, item.position) == ("demo-1", "demo", "RUNNING")
assert item.data == {"milestone": 2}
def test_a_fresh_store_has_no_active_item(db: Path) -> None:
with StateStore(db) as store:
assert store.active_item() is None
def test_intent_lifecycle(db: Path) -> None:
with StateStore(db) as store:
intent_id = store.begin_intent("push_branch", {"branch": "m2"}, cause="test")
[pending] = store.pending_intents()
assert pending.id == intent_id
assert pending.kind == "push_branch"
assert pending.payload == {"branch": "m2"}
store.commit_intent(intent_id, cause="test")
assert store.pending_intents() == []
def test_pending_intent_survives_reopen(db: Path) -> None:
with StateStore(db) as store:
store.begin_intent("open_pr", {"title": "hello"}, cause="test")
with StateStore(db) as store:
[pending] = store.pending_intents()
assert pending.kind == "open_pr"
def test_aborted_intent_is_not_pending(db: Path) -> None:
with StateStore(db) as store:
intent_id = store.begin_intent("push_branch", {}, cause="test")
store.abort_intent(intent_id, cause="test")
assert store.pending_intents() == []
def test_resolving_nonpending_intent_raises(db: Path) -> None:
with StateStore(db) as store:
intent_id = store.begin_intent("push_branch", {}, cause="test")
store.commit_intent(intent_id, cause="test")
with pytest.raises(KeyError):
store.commit_intent(intent_id, cause="test")
with pytest.raises(KeyError):
store.abort_intent(9999, cause="test")
def test_migration_runs_and_bumps_version(db: Path) -> None:
with StateStore(db, schema_version=1):
pass
def add_notes_table(conn: sqlite3.Connection) -> None:
conn.execute("CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT)")
with StateStore(db, schema_version=2, migrations={1: add_notes_table}) as store:
assert store.schema_version == 2
store._conn.execute("SELECT * FROM notes")
def test_missing_migration_raises(db: Path) -> None:
with StateStore(db, schema_version=1):
pass
with pytest.raises(StateSchemaError, match="no migration"):
StateStore(db, schema_version=2, migrations={})
def test_failed_migration_leaves_version_unchanged(db: Path) -> None:
with StateStore(db, schema_version=1):
pass
def broken(conn: sqlite3.Connection) -> None:
conn.execute("CREATE TABLE notes (id INTEGER PRIMARY KEY)")
raise RuntimeError("boom")
with pytest.raises(RuntimeError, match="boom"):
StateStore(db, schema_version=2, migrations={1: broken})
with StateStore(db, schema_version=1) as store:
assert store.schema_version == 1
with pytest.raises(sqlite3.OperationalError):
store._conn.execute("SELECT * FROM notes")
def test_newer_schema_is_refused(db: Path) -> None:
with StateStore(db, schema_version=3):
pass
with pytest.raises(StateSchemaError, match="newer"):
StateStore(db, schema_version=1)
def test_a_vm_block_persists_across_reopen(db: Path) -> None:
with StateStore(db) as store:
store.block_vm("root@vm1", cause="broken", reason="pip crashes", work_item="x")
store.block_vm("root@vm1", cause="broken", reason="later", work_item="y")
with StateStore(db) as store:
[block] = store.vm_blocks().values()
assert (block.host, block.reason, block.work_item) == (
"root@vm1",
"pip crashes", # the first block is kept
"x",
)
assert store.unblock_vm("root@vm1", cause="test") == block
assert store.unblock_vm("root@vm1", cause="test") is None
assert store.vm_blocks() == {}
def test_a_version_4_store_gains_vm_blocks(db: Path) -> None:
with StateStore(db, schema_version=4) as store:
store._conn.execute("DROP TABLE vm_blocks") # as a real version 4 store
with StateStore(db) as store:
assert store.schema_version == 5
store.block_vm("root@vm1", cause="broken", reason="r", work_item=None)
assert list(store.vm_blocks()) == ["root@vm1"]
nisse-dev/tests/test_tools.py
+1 -1
View File
@@ -4,7 +4,7 @@ from pathlib import Path
import pytest
from nisse_dev.execution import REDACTED, SSHHost
from nisse_core.execution import REDACTED, SSHHost
from nisse_dev.tools import EXECUTION_TOOLS, MAX_OUTPUT_CHARS, Workspace
nisse-dev/tests/test_vault.py
-144
View File
@@ -1,144 +0,0 @@
"""The knowledge vault (L10, M5.1): only configured folders exist, paths
can't escape them, read-only folders refuse writes, writes are atomic and
record provenance, and the editor's own files are never notes."""
import os
from datetime import UTC, datetime
from pathlib import Path
import pytest
import yaml
from nisse_dev import vault as vault_module
from nisse_dev.vault import Vault, VaultError, split_frontmatter
@pytest.fixture
def vault(tmp_path: Path) -> Vault:
root = tmp_path / "vault"
(root / "notes" / "deep").mkdir(parents=True)
(root / "notes" / "Procedures.md").write_text("# Procedures\n\nRestart the hub.\n")
(root / "notes" / "deep" / "Hub.md").write_text(
"The hub is in the shed.\nhub hub\n"
)
(root / "notes" / ".silverbullet").mkdir()
(root / "notes" / ".silverbullet" / "state.md").write_text("hub")
(root / "notes" / "_plug").mkdir()
(root / "notes" / "_plug" / "x.md").write_text("hub")
(root / "private").mkdir()
(root / "private" / "Secret.md").write_text("the hub password")
(root / "memory" / "nisse-ops").mkdir(parents=True)
(root / "memory" / "nisse-ops" / "Ops.md").write_text("ops hub")
return Vault(
root,
{"memory/nisse-dev": "rw", "notes": "ro"},
clock=lambda: datetime(2026, 10, 8, 12, 0, tzinfo=UTC),
)
def test_only_configured_folders_exist(vault: Vault) -> None:
assert vault.read("notes/Procedures.md").startswith("# Procedures")
for path in ("private/Secret.md", "memory/nisse-ops/Ops.md", "Top.md"):
with pytest.raises(VaultError, match="not in a folder you can see"):
vault.read(path)
assert vault.notes() == ["notes/Procedures.md", "notes/deep/Hub.md"]
assert vault.notes("notes/deep") == ["notes/deep/Hub.md"]
@pytest.mark.parametrize(
"path",
["/etc/passwd", "notes/../private/Secret.md", "", "notes\\x.md", "../x.md"],
)
def test_paths_cant_escape(vault: Vault, path: str) -> None:
with pytest.raises(VaultError):
vault.read(path)
def test_a_link_out_of_its_folder_is_refused(vault: Vault) -> None:
os.symlink(vault.root / "private" / "Secret.md", vault.root / "notes" / "Ln.md")
with pytest.raises(VaultError, match="leads outside its folder"):
vault.read("notes/Ln.md")
assert "notes/Ln.md" not in vault.notes()
assert all(f.path != "notes/Ln.md" for f in vault.search("password"))
def test_the_editors_files_are_not_notes(vault: Vault) -> None:
with pytest.raises(VaultError, match="editor's own files"):
vault.read("notes/.silverbullet/state.md")
with pytest.raises(VaultError, match="editor's own files"):
vault.read("notes/_plug/x.md")
with pytest.raises(VaultError, match="Markdown"):
vault.read("notes/deep")
def test_search_ranks_matches_and_sees_only_what_it_may(vault: Vault) -> None:
found = vault.search("HUB")
assert [f.path for f in found] == ["notes/deep/Hub.md", "notes/Procedures.md"]
assert found[0].lines == ("The hub is in the shed.", "hub hub")
with pytest.raises(VaultError):
vault.search(" ")
def test_a_long_note_is_cut_and_says_so(vault: Vault) -> None:
(vault.root / "notes" / "Long.md").write_text("x" * 30_000)
text = vault.read("notes/Long.md")
assert text.endswith("[10000 more characters not shown]")
def test_read_only_folders_refuse_writes(vault: Vault) -> None:
with pytest.raises(VaultError, match="read-only"):
vault.write("notes/New.md", "hi", provenance={"source": "t"})
assert not (vault.root / "notes" / "New.md").exists()
def test_a_write_records_provenance_and_keeps_the_rest(vault: Vault) -> None:
text = "---\ntags: [hub]\naliases: [h]\n---\n# Hub\n\nIn the shed.\n"
vault.write(
"memory/nisse-dev/sub/Hub.md",
text,
provenance={"source": "matrix", "work_item": "sandbox-14"},
)
written = (vault.root / "memory" / "nisse-dev" / "sub" / "Hub.md").read_text()
front, body = split_frontmatter(written)
assert front["tags"] == ["hub"] and front["aliases"] == ["h"]
assert front["provenance"] == {
"by": "nisse-dev",
"source": "matrix",
"work_item": "sandbox-14",
"written": "2026-10-08T12:00:00+00:00",
}
assert body == "# Hub\n\nIn the shed.\n"
assert vault.notes("memory/nisse-dev") == ["memory/nisse-dev/sub/Hub.md"]
def test_an_interrupted_write_leaves_the_old_note_and_no_litter(
vault: Vault, monkeypatch: pytest.MonkeyPatch
) -> None:
vault.write("memory/nisse-dev/N.md", "old\n", provenance={"source": "t"})
folder = vault.root / "memory" / "nisse-dev"
before = (folder / "N.md").read_text()
def crash(src: str, dst: str) -> None:
raise OSError("disk gone")
monkeypatch.setattr(vault_module.os, "replace", crash)
with pytest.raises(VaultError, match="couldn't write it"):
vault.write("memory/nisse-dev/N.md", "new\n", provenance={"source": "t"})
assert (folder / "N.md").read_text() == before
assert sorted(p.name for p in folder.iterdir()) == ["N.md"]
def test_bad_frontmatter_is_refused(vault: Vault) -> None:
with pytest.raises(VaultError, match="valid YAML"):
vault.write(
"memory/nisse-dev/B.md", "---\n: [\n---\nx\n", provenance={"source": "t"}
)
with pytest.raises(VaultError, match="mapping"):
vault.write("memory/nisse-dev/B.md", "---\n- a\n---\nx\n", provenance={})
def test_read_write_folders_are_made(tmp_path: Path) -> None:
Vault(tmp_path / "v", {"memory/nisse-dev": "rw", "notes": "ro"})
assert (tmp_path / "v" / "memory" / "nisse-dev").is_dir()
assert not (tmp_path / "v" / "notes").exists() # not ours to make
assert yaml # frontmatter is YAML
nisse-dev/tests/test_views.py
-142
View File
@@ -1,142 +0,0 @@
"""Per-role transcript views (architecture L6; Development Plan M1.2)."""
import io
from pathlib import Path
from nisse_dev.canonical import (
AssistantText,
DelegationCompleted,
DelegationStarted,
Fact,
PlanSubmitted,
StoredEvent,
ToolCall,
ToolResult,
UserMessage,
)
from nisse_dev.oplog import OpLog
from nisse_dev.render import OpenAIChatRenderer
from nisse_dev.render_anthropic import AnthropicMessagesRenderer
from nisse_dev.state import StateStore
def new_store(tmp_path: Path) -> StateStore:
return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
def texts(events: list[StoredEvent]) -> list[str]:
return [getattr(e.event, "text", "") for e in events]
# --- the projection, per role ----------------------------------------------------
def delegated_milestone(store: StateStore) -> None:
"""A milestone in which the driver failed and handed off to a helper."""
store.transcript("a", "driver").append(PlanSubmitted(plan="1. fix it", model="y"))
driver = store.transcript("a", "driver")
driver.append(
ToolCall(call_id="e1", tool="run_command", arguments={"command": "make"})
)
driver.append(ToolResult(call_id="e1", result="error: missing ;", success=False))
helper = driver.as_actor("helper")
helper.append(
DelegationStarted(brief="make fails", role="helper", model="y", caps={})
)
helper.append(AssistantText(text="looking at main.c", model="y"))
helper.append(
ToolCall(call_id="h1", tool="read_file", arguments={"path": "main.c"})
)
helper.append(ToolResult(call_id="h1", result="int main()", success=True))
helper.append(
DelegationCompleted(
handback="fixed the missing semicolon",
artifacts=["main.c"],
tokens=0,
duration_s=1.0,
exit_status="completed",
)
)
def test_the_driver_sees_the_helpers_report_not_its_turns(tmp_path: Path) -> None:
store = new_store(tmp_path)
delegated_milestone(store)
view = store.transcript("a", "driver").view()
assert [type(e.event).__name__ for e in view] == [
"ToolCall",
"ToolResult",
"UserMessage",
]
assert view[-1].event.text == (
"[helper report, completed] fixed the missing semicolon"
)
def test_the_helper_sees_the_plan_and_the_drivers_work_as_narration(
tmp_path: Path,
) -> None:
store = new_store(tmp_path)
delegated_milestone(store)
view = store.transcript("a", "helper").view()
said = texts(view)
assert said[0] == "[approved plan] 1. fix it"
assert said[1] == "[driver] run_command(make) → failed: error: missing ;"
# Its own events are unchanged, tool calls and all.
assert any(isinstance(e.event, ToolCall) and e.event.call_id == "h1" for e in view)
assert not any(
isinstance(e.event, ToolCall) and e.event.call_id == "e1" for e in view
)
def test_a_view_keeps_log_ids_so_cursors_still_work(tmp_path: Path) -> None:
store = new_store(tmp_path)
delegated_milestone(store)
raw = store.transcript("a", "driver").read()
view = store.transcript("a", "driver").view(after=raw[2].id)
assert [e.id for e in view] == [raw[-1].id] # the helper's report
# --- rendering ---------------------------------------------------------------------
def test_both_renderers_show_a_fact_as_a_labelled_user_message() -> None:
events = [StoredEvent(id=1, ts="", event=Fact(text="PR #3 opened"))]
openai = OpenAIChatRenderer().render(events, tools=None, model="m")
assert openai["messages"] == [
{"role": "user", "content": "[orchestrator] PR #3 opened"}
]
anthropic = AnthropicMessagesRenderer().render(events, tools=None, model="m")
assert anthropic["messages"] == [
{
"role": "user",
"content": [{"type": "text", "text": "[orchestrator] PR #3 opened"}],
}
]
def test_operator_messages_carry_their_channel(tmp_path: Path) -> None:
store = new_store(tmp_path)
operator = store.transcript("a", "operator")
for source in ("transport:@don", "gitea:issue:9:comment:4", "gitea:comment:5"):
operator.append(UserMessage(text="hi", source=source))
assert texts(store.transcript("a", "driver").view()) == [
"[operator, Matrix] hi",
"[operator, comment on plan issue #9] hi",
"[operator, Gitea comment] hi",
]
def test_something_written_mid_call_moves_after_the_result(tmp_path: Path) -> None:
# Live, M2.1: start_milestone's fact landed between the call and its
# result, and the model API rejected every later request in the run.
store = new_store(tmp_path)
driver = store.transcript("a", "driver")
driver.append(ToolCall(call_id="c1", tool="start_milestone", arguments={}))
store.transcript("a", "orchestrator").append(Fact(text="milestone 1 started"))
driver.append(ToolResult(call_id="c1", result="started", success=True))
kinds = [type(e.event).__name__ for e in driver.view()]
assert kinds == ["ToolCall", "ToolResult", "UserMessage"]
body = AnthropicMessagesRenderer().render(driver.view(), tools=None, model="m")
result_first = body["messages"][1]["content"][0]
assert result_first["type"] == "tool_result"
nisse-dev/tests/test_worker.py
-83
View File
@@ -1,83 +0,0 @@
import io
import threading
from pathlib import Path
import pytest
from nisse_dev.oplog import OpLog
from nisse_dev.state import StateStore
from nisse_dev.worker import InlineWorker, SerialWorker
@pytest.fixture
def worker() -> SerialWorker:
w = SerialWorker()
w.start()
yield w
w.stop(timeout=5)
def test_jobs_run_in_order_on_the_worker_thread(worker: SerialWorker) -> None:
seen: list[tuple[int, str]] = []
for n in range(3):
worker.submit(lambda n=n: seen.append((n, threading.current_thread().name)))
worker.call(lambda: None) # drains everything queued before it
assert [n for n, _ in seen] == [0, 1, 2]
assert {name for _, name in seen} == {"nisse-worker"}
def test_call_returns_the_result_and_raises_the_error(worker: SerialWorker) -> None:
assert worker.call(lambda: 42) == 42
def boom() -> None:
raise ValueError("bad job")
with pytest.raises(ValueError, match="bad job"):
worker.call(boom)
def test_a_failed_job_is_reported_and_the_worker_carries_on() -> None:
errors: list[BaseException] = []
worker = SerialWorker(on_error=errors.append)
worker.start()
def boom() -> None:
raise RuntimeError("lost job")
worker.submit(boom)
assert worker.call(lambda: "still here") == "still here"
assert [str(e) for e in errors] == ["lost job"]
worker.stop(timeout=5)
def test_call_from_the_worker_itself_runs_inline(worker: SerialWorker) -> None:
# Queueing behind itself would deadlock.
assert worker.call(lambda: worker.call(lambda: "nested")) == "nested"
def test_the_store_is_usable_only_from_the_thread_that_owns_it(
worker: SerialWorker, tmp_path: Path
) -> None:
store = worker.call(
lambda: StateStore(tmp_path / "state.db", oplog=OpLog("t", io.StringIO()))
)
assert worker.call(store.active_item) is None
# The tripwire the worker design keeps on: any other thread fails loudly.
with pytest.raises(Exception, match="same thread"):
store.active_item()
worker.call(store.close)
def test_inline_worker_defers_a_job_submitted_from_inside_a_job() -> None:
worker = InlineWorker()
order: list[str] = []
def outer() -> None:
order.append("outer start")
worker.submit(lambda: order.append("inner"))
order.append("outer end")
worker.submit(outer)
assert order == ["outer start", "outer end", "inner"]
assert not worker.busy