Work items in the state store, with a transcript bound to each (M1.1) #87
No files matched your search
@@ -25,8 +25,8 @@ llm: # L9: LiteLLM is the only LLM destination
|
|||||||
roles: # L9: role -> LiteLLM model alias
|
roles: # L9: role -> LiteLLM model alias
|
||||||
executor: qwen3.8-27b # the local tier: best executor in the benchmark (PR #55)
|
executor: qwen3.8-27b # the local tier: best executor in the benchmark (PR #55)
|
||||||
planner: claude-opus-5-5 # high-end judgement: plans a weaker executor follows
|
planner: claude-opus-5-5 # high-end judgement: plans a weaker executor follows
|
||||||
delegate: claude-opus-5-5 # takes over what the executor got stuck on
|
helper: claude-opus-5-5 # takes over what the executor got stuck on
|
||||||
consultant: claude-haiku-4-5 # one question at a time, no tools: cheap is enough
|
expert: claude-haiku-4-5 # one question at a time, no tools: cheap is enough
|
||||||
|
|
||||||
matrix: # L5
|
matrix: # L5
|
||||||
homeserver: https://matrix.example
|
homeserver: https://matrix.example
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ from nisse_dev.worker import SerialWorker
|
|||||||
# manager allows for a stop, and a hard exit mid-work is already safe.
|
# manager allows for a stop, and a hard exit mid-work is already safe.
|
||||||
SHUTDOWN_WAIT_S = 10.0
|
SHUTDOWN_WAIT_S = 10.0
|
||||||
|
|
||||||
TOOL_DRIVING_ROLES = ("planner", "executor", "delegate")
|
TOOL_DRIVING_ROLES = ("planner", "executor", "helper")
|
||||||
|
|
||||||
|
|
||||||
def discover_models(config: AgentConfig, oplog: OpLog) -> ModelCatalog:
|
def discover_models(config: AgentConfig, oplog: OpLog) -> ModelCatalog:
|
||||||
@@ -220,7 +220,7 @@ def build_agent(
|
|||||||
expert = ExpertDesk(
|
expert = ExpertDesk(
|
||||||
normalized, store, config.roles, budget=tuning.expert.consult_budget
|
normalized, store, config.roles, budget=tuning.expert.consult_budget
|
||||||
)
|
)
|
||||||
delegate_model = config.roles.resolve("delegate")
|
delegate_model = config.roles.resolve("helper")
|
||||||
delegation = Delegation(
|
delegation = Delegation(
|
||||||
select_client(delegate_model, normalized, passthrough, catalog),
|
select_client(delegate_model, normalized, passthrough, catalog),
|
||||||
store,
|
store,
|
||||||
|
|||||||
@@ -174,7 +174,7 @@ def run_task(
|
|||||||
meter = MeteringClient(client, facts)
|
meter = MeteringClient(client, facts)
|
||||||
trims_before = getattr(client, "trimmed_requests", 0)
|
trims_before = getattr(client, "trimmed_requests", 0)
|
||||||
model = facts.alias
|
model = facts.alias
|
||||||
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
roles = RoleMap(executor=model, planner=model, helper=model, expert=model)
|
||||||
executor = LLMExecutor(
|
executor = LLMExecutor(
|
||||||
meter,
|
meter,
|
||||||
store,
|
store,
|
||||||
|
|||||||
@@ -36,8 +36,8 @@ REQUIRED_KEYS = (
|
|||||||
"llm.token",
|
"llm.token",
|
||||||
"roles.executor",
|
"roles.executor",
|
||||||
"roles.planner",
|
"roles.planner",
|
||||||
"roles.delegate",
|
"roles.helper",
|
||||||
"roles.consultant",
|
"roles.expert",
|
||||||
"matrix.homeserver",
|
"matrix.homeserver",
|
||||||
"matrix.user_id",
|
"matrix.user_id",
|
||||||
"matrix.token",
|
"matrix.token",
|
||||||
@@ -312,8 +312,8 @@ class AgentConfig:
|
|||||||
roles=RoleMap(
|
roles=RoleMap(
|
||||||
executor=str(values["roles.executor"]),
|
executor=str(values["roles.executor"]),
|
||||||
planner=str(values["roles.planner"]),
|
planner=str(values["roles.planner"]),
|
||||||
delegate=str(values["roles.delegate"]),
|
helper=str(values["roles.helper"]),
|
||||||
consultant=str(values["roles.consultant"]),
|
expert=str(values["roles.expert"]),
|
||||||
),
|
),
|
||||||
matrix=MatrixConfig(
|
matrix=MatrixConfig(
|
||||||
homeserver=str(values["matrix.homeserver"]).rstrip("/"),
|
homeserver=str(values["matrix.homeserver"]).rstrip("/"),
|
||||||
@@ -371,5 +371,5 @@ class AgentConfig:
|
|||||||
"repo": self.gitea_repo,
|
"repo": self.gitea_repo,
|
||||||
"planner": self.roles.planner,
|
"planner": self.roles.planner,
|
||||||
"executor": self.roles.executor,
|
"executor": self.roles.executor,
|
||||||
"delegate": self.roles.delegate,
|
"helper": self.roles.helper,
|
||||||
}
|
}
|
||||||
@@ -184,9 +184,7 @@ class Delegation:
|
|||||||
# a reason, and the next step is to talk about that reason.
|
# a reason, and the next step is to talk about that reason.
|
||||||
self._cancel = cancel or Cancel()
|
self._cancel = cancel or Cancel()
|
||||||
|
|
||||||
def run(
|
def run(self, *, brief: str, branch: str, role: str = "helper") -> DelegationResult:
|
||||||
self, *, brief: str, branch: str, role: str = "delegate"
|
|
||||||
) -> DelegationResult:
|
|
||||||
started_at = self._now()
|
started_at = self._now()
|
||||||
base_branch = self._git.current_branch()
|
base_branch = self._git.current_branch()
|
||||||
# Recorded before the branch exists, so a crash anywhere after this
|
# Recorded before the branch exists, so a crash anywhere after this
|
||||||
|
|||||||
@@ -165,7 +165,7 @@ class LLMExecutor:
|
|||||||
# that it is circling and should get on with the work or fail.
|
# that it is circling and should get on with the work or fail.
|
||||||
return ToolOutcome(str(exc), success=False)
|
return ToolOutcome(str(exc), success=False)
|
||||||
except LLMError as exc:
|
except LLMError as exc:
|
||||||
return ToolOutcome(f"the consultant did not answer: {exc}", success=False)
|
return ToolOutcome(f"the expert did not answer: {exc}", success=False)
|
||||||
|
|
||||||
def _scope(self) -> tuple[str | None, int]:
|
def _scope(self) -> tuple[str | None, int]:
|
||||||
"""The approved plan, and where this milestone's events begin.
|
"""The approved plan, and where this milestone's events begin.
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ memory: it survives a restart, and it is observable from outside the model,
|
|||||||
which is the same reasoning L9 applies to escalation triggers.
|
which is the same reasoning L9 applies to escalation triggers.
|
||||||
|
|
||||||
Consults are recorded as ``delegation_started`` / ``delegation_completed`` with
|
Consults are recorded as ``delegation_started`` / ``delegation_completed`` with
|
||||||
role ``consultant``. That is deliberate — L9 wants a "consult/delegation eval
|
role ``expert``. That is deliberate — L9 wants a "consult/delegation eval
|
||||||
set", and reusing the composite events gets consult evaluation for free rather
|
set", and reusing the composite events gets consult evaluation for free rather
|
||||||
than inventing a parallel record.
|
than inventing a parallel record.
|
||||||
"""
|
"""
|
||||||
@@ -35,7 +35,7 @@ from nisse_dev.canonical import (
|
|||||||
from nisse_dev.llm import LLMClient, LLMError, RoleMap
|
from nisse_dev.llm import LLMClient, LLMError, RoleMap
|
||||||
from nisse_dev.state import StateStore
|
from nisse_dev.state import StateStore
|
||||||
|
|
||||||
CONSULTANT_ROLE = "consultant"
|
EXPERT_ROLE = "expert"
|
||||||
DEFAULT_BUDGET = 3
|
DEFAULT_BUDGET = 3
|
||||||
|
|
||||||
SYSTEM_PROMPT = """You are a consultant answering one self-contained question.
|
SYSTEM_PROMPT = """You are a consultant answering one self-contained question.
|
||||||
@@ -111,8 +111,7 @@ class ExpertDesk:
|
|||||||
return sum(
|
return sum(
|
||||||
1
|
1
|
||||||
for e in self._store.read_events(after=since_event_id)
|
for e in self._store.read_events(after=since_event_id)
|
||||||
if isinstance(e.event, DelegationStarted)
|
if isinstance(e.event, DelegationStarted) and e.event.role == EXPERT_ROLE
|
||||||
and e.event.role == CONSULTANT_ROLE
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def ask(self, question: str, *, since_event_id: int = 0) -> ExpertAnswer:
|
def ask(self, question: str, *, since_event_id: int = 0) -> ExpertAnswer:
|
||||||
@@ -122,11 +121,11 @@ class ExpertDesk:
|
|||||||
f"consult budget of {self._budget} is spent for this task "
|
f"consult budget of {self._budget} is spent for this task "
|
||||||
f"({used} used); escalate or ask the operator instead"
|
f"({used} used); escalate or ask the operator instead"
|
||||||
)
|
)
|
||||||
model = self._roles.resolve(CONSULTANT_ROLE)
|
model = self._roles.resolve(EXPERT_ROLE)
|
||||||
self._store.append_event(
|
self._store.append_event(
|
||||||
DelegationStarted(
|
DelegationStarted(
|
||||||
brief=question,
|
brief=question,
|
||||||
role=CONSULTANT_ROLE,
|
role=EXPERT_ROLE,
|
||||||
model=model,
|
model=model,
|
||||||
caps={"consults": self._budget, "used": used},
|
caps={"consults": self._budget, "used": used},
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -109,10 +109,10 @@ class LLMClient(Protocol):
|
|||||||
class RoleMap:
|
class RoleMap:
|
||||||
executor: str
|
executor: str
|
||||||
planner: str
|
planner: str
|
||||||
delegate: str
|
helper: str
|
||||||
consultant: str
|
expert: str
|
||||||
|
|
||||||
ROLES: ClassVar[tuple[str, ...]] = ("executor", "planner", "delegate", "consultant")
|
ROLES: ClassVar[tuple[str, ...]] = ("executor", "planner", "helper", "expert")
|
||||||
|
|
||||||
def resolve(self, role: str) -> str:
|
def resolve(self, role: str) -> str:
|
||||||
if role not in self.ROLES:
|
if role not in self.ROLES:
|
||||||
|
|||||||
@@ -263,7 +263,7 @@ def test_a_tool_less_model_warns_but_still_boots(tmp_path: Path, monkeypatch) ->
|
|||||||
logged = stream.getvalue()
|
logged = stream.getvalue()
|
||||||
assert "model_tools_unsupported" in logged
|
assert "model_tools_unsupported" in logged
|
||||||
assert '"role": "executor"' in logged
|
assert '"role": "executor"' in logged
|
||||||
assert '"role": "consultant"' not in logged # ask_expert drives no tools
|
assert '"role": "expert"' not in logged # ask_expert drives no tools
|
||||||
|
|
||||||
|
|
||||||
def test_the_resolved_model_is_pinned_at_task_start(
|
def test_the_resolved_model_is_pinned_at_task_start(
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ from nisse_dev.planner import LLMPlanningAgent
|
|||||||
from nisse_dev.state import StateStore
|
from nisse_dev.state import StateStore
|
||||||
from nisse_dev.transport import InMemoryTransport
|
from nisse_dev.transport import InMemoryTransport
|
||||||
|
|
||||||
ROLES = RoleMap(executor="x", planner="y", delegate="y", consultant="y")
|
ROLES = RoleMap(executor="x", planner="y", helper="y", expert="y")
|
||||||
|
|
||||||
|
|
||||||
class FakeGitea:
|
class FakeGitea:
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ ALL_EVENTS = [
|
|||||||
GateOpened(gate="arch", reason="plan submitted (rev-1)"),
|
GateOpened(gate="arch", reason="plan submitted (rev-1)"),
|
||||||
DelegationStarted(
|
DelegationStarted(
|
||||||
brief="fix the flaky test",
|
brief="fix the flaky test",
|
||||||
role="delegate",
|
role="helper",
|
||||||
model="claude-opus-5",
|
model="claude-opus-5",
|
||||||
caps={"turns": 20, "tokens": 100000},
|
caps={"turns": 20, "tokens": 100000},
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -130,13 +130,13 @@ def test_validate_names_every_bad_role_at_once() -> None:
|
|||||||
{
|
{
|
||||||
"executor": "local-fast",
|
"executor": "local-fast",
|
||||||
"planner": "clause-opus-5",
|
"planner": "clause-opus-5",
|
||||||
"delegate": "remote-architecture",
|
"helper": "remote-architecture",
|
||||||
"consultant": "local-fast",
|
"expert": "local-fast",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
message = str(caught.value)
|
message = str(caught.value)
|
||||||
assert "planner='clause-opus-5'" in message
|
assert "planner='clause-opus-5'" in message
|
||||||
assert "delegate='remote-architecture'" in message
|
assert "helper='remote-architecture'" in message
|
||||||
assert "executor" not in message # the good ones are not noise
|
assert "executor" not in message # the good ones are not noise
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ COMPLETE: dict[str, Any] = {
|
|||||||
"roles": {
|
"roles": {
|
||||||
"executor": "claude-haiku-4-5",
|
"executor": "claude-haiku-4-5",
|
||||||
"planner": "claude-opus-5",
|
"planner": "claude-opus-5",
|
||||||
"delegate": "claude-opus-5",
|
"helper": "claude-opus-5",
|
||||||
"consultant": "claude-haiku-4-5",
|
"expert": "claude-haiku-4-5",
|
||||||
},
|
},
|
||||||
"matrix": {
|
"matrix": {
|
||||||
"homeserver": "https://matrix.test/",
|
"homeserver": "https://matrix.test/",
|
||||||
@@ -210,8 +210,8 @@ llm: {base_url: "https://litellm.test", token: "${TEST_LLM_TOKEN}"}
|
|||||||
roles:
|
roles:
|
||||||
executor: local-fast
|
executor: local-fast
|
||||||
planner: claude-opus-5
|
planner: claude-opus-5
|
||||||
delegate: claude-opus-5
|
helper: claude-opus-5
|
||||||
consultant: claude-haiku-4-5
|
expert: claude-haiku-4-5
|
||||||
matrix:
|
matrix:
|
||||||
homeserver: https://matrix.test
|
homeserver: https://matrix.test
|
||||||
user_id: "@nisse-dev-testing:test"
|
user_id: "@nisse-dev-testing:test"
|
||||||
|
|||||||
@@ -262,7 +262,7 @@ def test_the_executor_sees_only_its_milestone(tmp_path: Path) -> None:
|
|||||||
seen["system"] = system
|
seen["system"] = system
|
||||||
return [AssistantText(text="done", model=model)]
|
return [AssistantText(text="done", model=model)]
|
||||||
|
|
||||||
roles = RoleMap(executor="m", planner="m", delegate="m", consultant="m")
|
roles = RoleMap(executor="m", planner="m", helper="m", expert="m")
|
||||||
LLMExecutor(Client(), store, roles, Workspace()).run(1)
|
LLMExecutor(Client(), store, roles, Workspace()).run(1)
|
||||||
|
|
||||||
texts = [getattr(e, "text", None) for e in seen["events"]]
|
texts = [getattr(e, "text", None) for e in seen["events"]]
|
||||||
@@ -286,7 +286,7 @@ def test_a_request_that_cannot_fit_stops_the_milestone(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
inner = RecordingClient()
|
inner = RecordingClient()
|
||||||
client = BudgetedClient(inner, lambda m: (4096, 1024))
|
client = BudgetedClient(inner, lambda m: (4096, 1024))
|
||||||
roles = RoleMap(executor="m", planner="m", delegate="m", consultant="m")
|
roles = RoleMap(executor="m", planner="m", helper="m", expert="m")
|
||||||
outcome = LLMExecutor(client, store, roles, Workspace()).run(1)
|
outcome = LLMExecutor(client, store, roles, Workspace()).run(1)
|
||||||
|
|
||||||
assert outcome.status == "escalate"
|
assert outcome.status == "escalate"
|
||||||
|
|||||||
@@ -385,9 +385,7 @@ def test_clean_cut_after_a_tool_result_is_resumable(
|
|||||||
repo.checkout_new_branch("d")
|
repo.checkout_new_branch("d")
|
||||||
(repo.root / "partial.py").write_text("x")
|
(repo.root / "partial.py").write_text("x")
|
||||||
repo.commit_all("partial work")
|
repo.commit_all("partial work")
|
||||||
store.append_event(
|
store.append_event(DelegationStarted(brief="b", role="helper", model="m", caps={}))
|
||||||
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
||||||
)
|
|
||||||
store.append_event(ToolCall(call_id="c", tool="write_file", arguments={}))
|
store.append_event(ToolCall(call_id="c", tool="write_file", arguments={}))
|
||||||
store.append_event(ToolResult(call_id="c", result="wrote", success=True))
|
store.append_event(ToolResult(call_id="c", result="wrote", success=True))
|
||||||
intent = pending_delegation_intent(store)
|
intent = pending_delegation_intent(store)
|
||||||
@@ -400,9 +398,7 @@ def test_dangling_tool_call_gates(
|
|||||||
) -> None:
|
) -> None:
|
||||||
# The effect of the call is unknown and not verifiable by inspection.
|
# The effect of the call is unknown and not verifiable by inspection.
|
||||||
repo.checkout_new_branch("d")
|
repo.checkout_new_branch("d")
|
||||||
store.append_event(
|
store.append_event(DelegationStarted(brief="b", role="helper", model="m", caps={}))
|
||||||
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
||||||
)
|
|
||||||
store.append_event(ToolCall(call_id="c", tool="run_command", arguments={}))
|
store.append_event(ToolCall(call_id="c", tool="run_command", arguments={}))
|
||||||
intent = pending_delegation_intent(store)
|
intent = pending_delegation_intent(store)
|
||||||
|
|
||||||
@@ -412,9 +408,7 @@ def test_dangling_tool_call_gates(
|
|||||||
def test_missing_branch_means_nothing_started(
|
def test_missing_branch_means_nothing_started(
|
||||||
store: StateStore, ws: Workspace, repo: GitRepo
|
store: StateStore, ws: Workspace, repo: GitRepo
|
||||||
) -> None:
|
) -> None:
|
||||||
store.append_event(
|
store.append_event(DelegationStarted(brief="b", role="helper", model="m", caps={}))
|
||||||
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
||||||
)
|
|
||||||
store.append_event(ToolResult(call_id="c", result="ok", success=True))
|
store.append_event(ToolResult(call_id="c", result="ok", success=True))
|
||||||
intent = pending_delegation_intent(store, branch="never-created")
|
intent = pending_delegation_intent(store, branch="never-created")
|
||||||
|
|
||||||
@@ -425,9 +419,7 @@ def test_a_finished_delegation_settles_rather_than_resumes(
|
|||||||
store: StateStore, ws: Workspace, repo: GitRepo
|
store: StateStore, ws: Workspace, repo: GitRepo
|
||||||
) -> None:
|
) -> None:
|
||||||
repo.checkout_new_branch("d")
|
repo.checkout_new_branch("d")
|
||||||
store.append_event(
|
store.append_event(DelegationStarted(brief="b", role="helper", model="m", caps={}))
|
||||||
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
||||||
)
|
|
||||||
store.append_event(
|
store.append_event(
|
||||||
DelegationCompleted(
|
DelegationCompleted(
|
||||||
handback="done",
|
handback="done",
|
||||||
@@ -448,9 +440,7 @@ def test_resume_continues_the_same_delegation(
|
|||||||
# Interrupt after one turn, then resume: no second delegation_started,
|
# Interrupt after one turn, then resume: no second delegation_started,
|
||||||
# because L7 defines a session as the delegation, not the process.
|
# because L7 defines a session as the delegation, not the process.
|
||||||
repo.checkout_new_branch("d")
|
repo.checkout_new_branch("d")
|
||||||
store.append_event(
|
store.append_event(DelegationStarted(brief="b", role="helper", model="m", caps={}))
|
||||||
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
||||||
)
|
|
||||||
store.append_event(ToolResult(call_id="c", result="wrote a.py", success=True))
|
store.append_event(ToolResult(call_id="c", result="wrote a.py", success=True))
|
||||||
intent = pending_delegation_intent(store)
|
intent = pending_delegation_intent(store)
|
||||||
|
|
||||||
@@ -484,9 +474,7 @@ def test_recover_resumes_a_clean_cut_with_a_notice(
|
|||||||
store: StateStore, ws: Workspace, repo: GitRepo
|
store: StateStore, ws: Workspace, repo: GitRepo
|
||||||
) -> None:
|
) -> None:
|
||||||
repo.checkout_new_branch("d")
|
repo.checkout_new_branch("d")
|
||||||
store.append_event(
|
store.append_event(DelegationStarted(brief="b", role="helper", model="m", caps={}))
|
||||||
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
||||||
)
|
|
||||||
store.append_event(ToolResult(call_id="c", result="wrote a.py", success=True))
|
store.append_event(ToolResult(call_id="c", result="wrote a.py", success=True))
|
||||||
pending_delegation_intent(store)
|
pending_delegation_intent(store)
|
||||||
notices: list[str] = []
|
notices: list[str] = []
|
||||||
@@ -505,9 +493,7 @@ def test_recover_settles_a_finished_delegation_without_resuming(
|
|||||||
store: StateStore, ws: Workspace, repo: GitRepo
|
store: StateStore, ws: Workspace, repo: GitRepo
|
||||||
) -> None:
|
) -> None:
|
||||||
repo.checkout_new_branch("d")
|
repo.checkout_new_branch("d")
|
||||||
store.append_event(
|
store.append_event(DelegationStarted(brief="b", role="helper", model="m", caps={}))
|
||||||
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
||||||
)
|
|
||||||
store.append_event(
|
store.append_event(
|
||||||
DelegationCompleted(
|
DelegationCompleted(
|
||||||
handback="done",
|
handback="done",
|
||||||
@@ -539,9 +525,7 @@ def test_recover_leaves_an_unclean_cut_for_the_boot_gate(
|
|||||||
store: StateStore, ws: Workspace, repo: GitRepo
|
store: StateStore, ws: Workspace, repo: GitRepo
|
||||||
) -> None:
|
) -> None:
|
||||||
repo.checkout_new_branch("d")
|
repo.checkout_new_branch("d")
|
||||||
store.append_event(
|
store.append_event(DelegationStarted(brief="b", role="helper", model="m", caps={}))
|
||||||
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
||||||
)
|
|
||||||
store.append_event(ToolCall(call_id="c", tool="run_command", arguments={}))
|
store.append_event(ToolCall(call_id="c", tool="run_command", arguments={}))
|
||||||
pending_delegation_intent(store)
|
pending_delegation_intent(store)
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ def record_pair(
|
|||||||
*,
|
*,
|
||||||
brief: str = "add greet",
|
brief: str = "add greet",
|
||||||
model: str = "claude-haiku-4-5",
|
model: str = "claude-haiku-4-5",
|
||||||
role: str = "delegate",
|
role: str = "helper",
|
||||||
status: str = "handback",
|
status: str = "handback",
|
||||||
) -> int:
|
) -> int:
|
||||||
store.append_event(
|
store.append_event(
|
||||||
@@ -56,15 +56,13 @@ def test_unpaired_start_is_skipped(store: StateStore) -> None:
|
|||||||
# A crash mid-delegation leaves a start with no completion; a half-formed
|
# A crash mid-delegation leaves a start with no completion; a half-formed
|
||||||
# record would pollute the set L9 exists to make trustworthy.
|
# record would pollute the set L9 exists to make trustworthy.
|
||||||
store.append_event(
|
store.append_event(
|
||||||
DelegationStarted(brief="died", role="delegate", model="m", caps={})
|
DelegationStarted(brief="died", role="helper", model="m", caps={})
|
||||||
)
|
)
|
||||||
assert collect(store) == []
|
assert collect(store) == []
|
||||||
|
|
||||||
|
|
||||||
def test_unrelated_events_between_the_pair_are_ignored(store: StateStore) -> None:
|
def test_unrelated_events_between_the_pair_are_ignored(store: StateStore) -> None:
|
||||||
store.append_event(
|
store.append_event(DelegationStarted(brief="b", role="helper", model="m", caps={}))
|
||||||
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
||||||
)
|
|
||||||
store.append_event(AssistantText(text="working", model="m"))
|
store.append_event(AssistantText(text="working", model="m"))
|
||||||
store.append_event(
|
store.append_event(
|
||||||
DelegationCompleted(
|
DelegationCompleted(
|
||||||
@@ -97,10 +95,10 @@ def test_a_verdict_can_be_revised(store: StateStore) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_consults_and_delegations_share_the_set(store: StateStore) -> None:
|
def test_consults_and_delegations_share_the_set(store: StateStore) -> None:
|
||||||
record_pair(store, role="delegate")
|
record_pair(store, role="helper")
|
||||||
record_pair(store, role="consultant", brief="which approach?")
|
record_pair(store, role="expert", brief="which approach?")
|
||||||
roles = [r.role for r in collect(store)]
|
roles = [r.role for r in collect(store)]
|
||||||
assert roles == ["delegate", "consultant"]
|
assert roles == ["helper", "expert"]
|
||||||
|
|
||||||
|
|
||||||
def test_collect_can_start_after_a_marker(store: StateStore) -> None:
|
def test_collect_can_start_after_a_marker(store: StateStore) -> None:
|
||||||
|
|||||||
@@ -20,8 +20,8 @@ from nisse_dev.tools import Workspace
|
|||||||
ROLES = RoleMap(
|
ROLES = RoleMap(
|
||||||
executor="llama3.2",
|
executor="llama3.2",
|
||||||
planner="claude-haiku-4-5",
|
planner="claude-haiku-4-5",
|
||||||
delegate="claude-haiku-4-5",
|
helper="claude-haiku-4-5",
|
||||||
consultant="claude-haiku-4-5",
|
expert="claude-haiku-4-5",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -25,8 +25,8 @@ from nisse_dev.state import StateStore
|
|||||||
ROLES = RoleMap(
|
ROLES = RoleMap(
|
||||||
executor="llama3.2",
|
executor="llama3.2",
|
||||||
planner="haiku",
|
planner="haiku",
|
||||||
delegate="haiku",
|
helper="haiku",
|
||||||
consultant="claude-haiku-4-5",
|
expert="claude-haiku-4-5",
|
||||||
)
|
)
|
||||||
|
|
||||||
GOOD = '{"answer": "Use SQLite; it is enough here.", "confidence": "high", "caveats": "Assumes one writer."}'
|
GOOD = '{"answer": "Use SQLite; it is enough here.", "confidence": "high", "caveats": "Assumes one writer."}'
|
||||||
@@ -137,7 +137,7 @@ def test_consults_are_recorded_as_composite_events(store: StateStore) -> None:
|
|||||||
events = [e.event for e in store.read_events()]
|
events = [e.event for e in store.read_events()]
|
||||||
started = next(e for e in events if isinstance(e, DelegationStarted))
|
started = next(e for e in events if isinstance(e, DelegationStarted))
|
||||||
completed = next(e for e in events if isinstance(e, DelegationCompleted))
|
completed = next(e for e in events if isinstance(e, DelegationCompleted))
|
||||||
assert started.role == "consultant"
|
assert started.role == "expert"
|
||||||
assert started.brief == "q"
|
assert started.brief == "q"
|
||||||
assert completed.exit_status == "answered"
|
assert completed.exit_status == "answered"
|
||||||
|
|
||||||
|
|||||||
@@ -184,8 +184,8 @@ def test_malformed_response_raises_llm_error() -> None:
|
|||||||
ROLES = RoleMap(
|
ROLES = RoleMap(
|
||||||
executor="local/qwen",
|
executor="local/qwen",
|
||||||
planner="anthropic/claude-opus-5",
|
planner="anthropic/claude-opus-5",
|
||||||
delegate="anthropic/claude-opus-5",
|
helper="anthropic/claude-opus-5",
|
||||||
consultant="anthropic/claude-opus-5",
|
expert="anthropic/claude-opus-5",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -195,8 +195,8 @@ def test_role_map_resolves_and_pins() -> None:
|
|||||||
assert roles.pinned() == {
|
assert roles.pinned() == {
|
||||||
"executor": "local/qwen",
|
"executor": "local/qwen",
|
||||||
"planner": "anthropic/claude-opus-5",
|
"planner": "anthropic/claude-opus-5",
|
||||||
"delegate": "anthropic/claude-opus-5",
|
"helper": "anthropic/claude-opus-5",
|
||||||
"consultant": "anthropic/claude-opus-5",
|
"expert": "anthropic/claude-opus-5",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -29,8 +29,8 @@ from nisse_dev.transport import InMemoryTransport
|
|||||||
ROLES = RoleMap(
|
ROLES = RoleMap(
|
||||||
executor="local/qwen",
|
executor="local/qwen",
|
||||||
planner="anthropic/claude-opus-5",
|
planner="anthropic/claude-opus-5",
|
||||||
delegate="anthropic/claude-opus-5",
|
helper="anthropic/claude-opus-5",
|
||||||
consultant="anthropic/claude-opus-5",
|
expert="anthropic/claude-opus-5",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -27,8 +27,8 @@ from nisse_dev.transport import InMemoryTransport
|
|||||||
ROLES = RoleMap(
|
ROLES = RoleMap(
|
||||||
executor="llama3.2",
|
executor="llama3.2",
|
||||||
planner="haiku",
|
planner="haiku",
|
||||||
delegate="haiku",
|
helper="haiku",
|
||||||
consultant="haiku",
|
expert="haiku",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from nisse_dev.publish import PublishResult
|
|||||||
from nisse_dev.state import StateStore
|
from nisse_dev.state import StateStore
|
||||||
from nisse_dev.transport import InMemoryTransport
|
from nisse_dev.transport import InMemoryTransport
|
||||||
|
|
||||||
ROLES = RoleMap(executor="x", planner="y", delegate="y", consultant="y")
|
ROLES = RoleMap(executor="x", planner="y", helper="y", expert="y")
|
||||||
|
|
||||||
|
|
||||||
class ScriptedClient:
|
class ScriptedClient:
|
||||||
|
|||||||
@@ -31,8 +31,8 @@ from nisse_dev.worker import SerialWorker
|
|||||||
ROLES = RoleMap(
|
ROLES = RoleMap(
|
||||||
executor="llama3.2",
|
executor="llama3.2",
|
||||||
planner="claude-haiku-4-5",
|
planner="claude-haiku-4-5",
|
||||||
delegate="claude-haiku-4-5",
|
helper="claude-haiku-4-5",
|
||||||
consultant="claude-haiku-4-5",
|
expert="claude-haiku-4-5",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -23,8 +23,8 @@ llm:
|
|||||||
roles:
|
roles:
|
||||||
executor: claude-haiku-4-5
|
executor: claude-haiku-4-5
|
||||||
planner: claude-haiku-4-5
|
planner: claude-haiku-4-5
|
||||||
delegate: claude-haiku-4-5
|
helper: claude-haiku-4-5
|
||||||
consultant: claude-haiku-4-5
|
expert: claude-haiku-4-5
|
||||||
|
|
||||||
matrix:
|
matrix:
|
||||||
homeserver: https://matrix.dongar.net
|
homeserver: https://matrix.dongar.net
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ pytestmark = pytest.mark.integration
|
|||||||
def desk(tmp_path: Path, litellm: LiteLLMClient, live_model: str):
|
def desk(tmp_path: Path, litellm: LiteLLMClient, live_model: str):
|
||||||
def make(budget: int = 2) -> tuple[ExpertDesk, StateStore]:
|
def make(budget: int = 2) -> tuple[ExpertDesk, StateStore]:
|
||||||
model = live_model
|
model = live_model
|
||||||
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
roles = RoleMap(executor=model, planner=model, helper=model, expert=model)
|
||||||
store = StateStore(
|
store = StateStore(
|
||||||
tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())
|
tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())
|
||||||
)
|
)
|
||||||
@@ -43,7 +43,7 @@ def test_live_consult_parses_against_the_local_schema(desk) -> None:
|
|||||||
assert len(answer.answer) > 20
|
assert len(answer.answer) > 20
|
||||||
# The consult is captured for L9 without any extra bookkeeping.
|
# The consult is captured for L9 without any extra bookkeeping.
|
||||||
[record] = collect(store)
|
[record] = collect(store)
|
||||||
assert record.role == "consultant"
|
assert record.role == "expert"
|
||||||
assert record.exit_status == "answered"
|
assert record.exit_status == "answered"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -64,7 +64,7 @@ def test_plan_approve_execute_produces_a_real_file(
|
|||||||
# Both roles use the capable model here: the point is the whole gated
|
# Both roles use the capable model here: the point is the whole gated
|
||||||
# loop over a real proxy, not local-model competence (measured separately).
|
# loop over a real proxy, not local-model competence (measured separately).
|
||||||
model = live_model
|
model = live_model
|
||||||
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
roles = RoleMap(executor=model, planner=model, helper=model, expert=model)
|
||||||
client = litellm
|
client = litellm
|
||||||
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
||||||
workspace = Workspace(tmp_path / "work", timeout_s=60)
|
workspace = Workspace(tmp_path / "work", timeout_s=60)
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ def test_live_planning_turn_returns_a_tool_action(
|
|||||||
tmp_path: Path, litellm: LiteLLMClient, live_model: str
|
tmp_path: Path, litellm: LiteLLMClient, live_model: str
|
||||||
) -> None:
|
) -> None:
|
||||||
model = live_model
|
model = live_model
|
||||||
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
roles = RoleMap(executor=model, planner=model, helper=model, expert=model)
|
||||||
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
||||||
store.append_event(
|
store.append_event(
|
||||||
UserMessage(
|
UserMessage(
|
||||||
|
|||||||
Reference in new issue
Block a user