nisse/nisse-dev/src/nisse_dev/expert.py

"""`ask_expert` (architecture L9).

The one case delegation does not cover: a question needing no repo access —
"which of these two approaches is better", "is this API deprecated". A single
stateless call, a text answer, no session state, no diff application.

Two constraints from L9 shape the implementation:

**Structured output is contracted at the prompt level, not the API level.**
Vendors differ on JSON mode and schema enforcement, so the answer is parsed and
validated against a local schema, with one retry. This runs over LiteLLM's
*normalized* route, which L9 reserves for calls where fidelity is cheap.

**A per-task budget**, so a confused executor cannot spend twenty consults
circling. The count comes from the canonical log rather than a counter in
memory: it survives a restart, and it is observable from outside the model,
which is the same reasoning L9 applies to escalation triggers.

Consults are recorded as ``delegation_started`` / ``delegation_completed`` with
role ``expert``. That is deliberate — L9 wants a "consult/delegation eval
set", and reusing the composite events gets consult evaluation for free rather
than inventing a parallel record.
"""

import json
import re
from dataclasses import dataclass

from nisse_core.canonical import (
    DelegationCompleted,
    DelegationStarted,
    StoredEvent,
    UserMessage,
)
from nisse_core.llm import LLMClient, LLMError, RoleMap
from nisse_core.state import Transcript

EXPERT_ROLE = "expert"
DEFAULT_BUDGET = 3

SYSTEM_PROMPT = """You are a consultant answering one self-contained question.

You have no access to the caller's repository, so answer from the question
alone. If the question cannot be answered without seeing the code, say so
plainly in `answer` rather than guessing.

Reply with a single JSON object and nothing else:

{"answer": "...", "confidence": "high" | "medium" | "low", "caveats": "..."}

Keep `answer` direct and specific. Put what would change your mind, or what you
are assuming, in `caveats`."""

_RETRY_NUDGE = (
    "\n\nYour previous reply was not a single valid JSON object with the keys "
    "answer, confidence and caveats. Reply with only that object."
)
_CONFIDENCES = {"high", "medium", "low"}


ASK_EXPERT_TOOL: dict = {
    "type": "function",
    "function": {
        "name": "ask_expert",
        "description": (
            "Ask a stronger model one self-contained question that needs no "
            "access to this repository — a design trade-off, or how a library "
            "behaves. It cannot see your files. Use sparingly; the budget is "
            "small."
        ),
        "parameters": {
            "type": "object",
            "properties": {"question": {"type": "string"}},
            "required": ["question"],
        },
    },
}


class BudgetExhausted(RuntimeError):
    """Raised rather than silently answering: a caller that has burned its
    consult budget is circling, and should escalate or ask the operator."""


@dataclass(frozen=True)
class ExpertAnswer:
    answer: str
    confidence: str
    caveats: str

    def render(self) -> str:
        body = f"{self.answer}\n\n(confidence: {self.confidence})"
        return f"{body}\n\nCaveats: {self.caveats}" if self.caveats else body


class ExpertDesk:
    def __init__(
        self,
        client: LLMClient,
        roles: RoleMap,
        *,
        budget: int = DEFAULT_BUDGET,
    ) -> None:
        self._client = client
        self._roles = roles
        self._budget = budget

    def consults_used(self, transcript: Transcript, since_event_id: int = 0) -> int:
        return sum(
            1
            for e in transcript.read(after=since_event_id)
            if isinstance(e.event, DelegationStarted) and e.event.role == EXPERT_ROLE
        )

    def ask(
        self, transcript: Transcript, question: str, *, since_event_id: int = 0
    ) -> ExpertAnswer:
        """The consult is recorded in the work item's transcript, as the
        expert, and the budget counts that work item's consults (L9)."""
        used = self.consults_used(transcript, since_event_id)
        if used >= self._budget:
            raise BudgetExhausted(
                f"consult budget of {self._budget} is spent for this task "
                f"({used} used); escalate or ask the operator instead"
            )
        model = self._roles.resolve(EXPERT_ROLE)
        transcript.append(
            DelegationStarted(
                brief=question,
                role=EXPERT_ROLE,
                model=model,
                caps={"consults": self._budget, "used": used},
            )
        )
        answer, note = self._answer(question, model)
        transcript.append(
            DelegationCompleted(
                handback=answer.render() if answer else note,
                artifacts=[],
                tokens=0,
                duration_s=0.0,
                exit_status="answered" if answer else "failed",
            )
        )
        if answer is None:
            raise LLMError(f"ask_expert produced no usable answer: {note}")
        return answer

    def _answer(self, question: str, model: str) -> tuple[ExpertAnswer | None, str]:
        system = SYSTEM_PROMPT
        last = "no reply"
        for attempt in (1, 2):
            try:
                # Stateless by design (L9): the question travels as a
                # synthesised message that is never appended to the log, so a
                # consult neither depends on nor leaks repo context. It cannot
                # ride in the system prompt alone — the Messages API requires
                # at least one non-system message.
                produced = self._client.complete(
                    [
                        StoredEvent(
                            id=0,
                            ts="",
                            event=UserMessage(text=question, source="ask_expert"),
                        )
                    ],
                    tools=None,
                    model=model,
                    system=system,
                )
            except LLMError as exc:
                return None, str(exc)
            text = "\n".join(
                getattr(e, "text", "") for e in produced if hasattr(e, "text")
            ).strip()
            parsed = parse_answer(text)
            if parsed is not None:
                return parsed, "ok"
            last = text[:300] or "empty reply"
            if attempt == 1:
                system = SYSTEM_PROMPT + _RETRY_NUDGE
        return None, f"unparseable reply: {last}"


def parse_answer(text: str) -> ExpertAnswer | None:
    """Validated locally because the prompt-level contract is all we have.

    Tolerates a fenced code block, since models wrap JSON in one habitually,
    but not missing or malformed fields — those are a failed turn.
    """
    candidate = text.strip()
    fenced = re.search(r"```(?:json)?\s*(.*?)```", candidate, re.DOTALL)
    if fenced:
        candidate = fenced.group(1).strip()
    start, end = candidate.find("{"), candidate.rfind("}")
    if start == -1 or end <= start:
        return None
    try:
        data = json.loads(candidate[start : end + 1])
    except json.JSONDecodeError:
        return None
    if not isinstance(data, dict):
        return None
    answer = data.get("answer")
    confidence = data.get("confidence")
    if not isinstance(answer, str) or not answer.strip():
        return None
    if not isinstance(confidence, str) or confidence.lower() not in _CONFIDENCES:
        return None
    caveats = data.get("caveats")
    return ExpertAnswer(
        answer=answer.strip(),
        confidence=confidence.lower(),
        caveats=caveats.strip() if isinstance(caveats, str) else "",
    )