Gate live tests on one sentinel var each, fail on partial config
CI / checks (push) Successful in 16s
CI / checks (pull_request) Successful in 10s

Skipping now keys solely on NISSE_LLM_BASE_URL and
NISSE_MATRIX_HOMESERVER: if the sentinel is set, the live path is
expected to work and any other missing variable is an assertion
failure naming it, not a silent skip. A misnamed or misplaced CI
secret previously produced a green run with no live coverage.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dongarandClaude Opus 5 committed 2026-09-07 14:33:14 +00:00
1 parent 968347a6be
commit 10aa77899e
2 files changed
+28 -12

No files matched your search

+13 -6
View File
@@ -1,8 +1,9 @@
"""Live smoke test against the real LiteLLM proxy. """Live smoke test against the real LiteLLM proxy.
Skips unless NISSE_LLM_BASE_URL, NISSE_LLM_TOKEN, and NISSE_ROLE_EXECUTOR are NISSE_LLM_BASE_URL alone decides whether this runs: if it is set, the live
present (from the environment or the repo-root .env) — CI stays credential-free path is expected to work, and anything else missing is a *failure*, not a
by choice; this runs from the operator's machine. skip. A half-configured environment must not pass quietly as a skip — that is
how a broken CI credential hides.
""" """
import os import os
@@ -16,18 +17,24 @@ from nisse_dev.llm import LiteLLMClient
load_env_file(Path(__file__).resolve().parents[2] / ".env") load_env_file(Path(__file__).resolve().parents[2] / ".env")
REQUIRED = ("NISSE_LLM_BASE_URL", "NISSE_LLM_TOKEN", "NISSE_ROLE_EXECUTOR") SENTINEL = "NISSE_LLM_BASE_URL"
ALSO_REQUIRED = ("NISSE_LLM_TOKEN", "NISSE_ROLE_EXECUTOR")
pytestmark = [ pytestmark = [
pytest.mark.integration, pytest.mark.integration,
pytest.mark.skipif( pytest.mark.skipif(
not all(os.environ.get(var) for var in REQUIRED), not os.environ.get(SENTINEL),
reason=f"live LLM config absent; set {', '.join(REQUIRED)}", reason=f"no live LLM configured ({SENTINEL} unset)",
), ),
] ]
def test_executor_model_completes_via_litellm() -> None: def test_executor_model_completes_via_litellm() -> None:
missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)]
assert not missing, (
f"{SENTINEL} is set, so the live LLM path is expected to work, "
f"but these are missing: {', '.join(missing)}"
)
client = LiteLLMClient.from_env() client = LiteLLMClient.from_env()
events = [ events = [
StoredEvent( StoredEvent(
+15 -6
View File
@@ -1,6 +1,10 @@
"""Live smoke test against the real homeserver, bound strictly to the """Live smoke test against the real homeserver, bound strictly to the
dedicated test room (never the production room). Skips unless the Matrix env dedicated test room (never the production room).
vars, including NISSE_MATRIX_TEST_ROOM_ID, are present."""
NISSE_MATRIX_HOMESERVER alone decides whether this runs: if it is set, the
live path is expected to work, and anything else missing is a *failure*, not
a skip. A half-configured environment must not pass quietly as a skip — that
is how a broken CI credential hides."""
import os import os
import uuid import uuid
@@ -13,8 +17,8 @@ from nisse_dev.matrix import MatrixConfig, MatrixTransport
load_env_file(Path(__file__).resolve().parents[2] / ".env") load_env_file(Path(__file__).resolve().parents[2] / ".env")
REQUIRED = ( SENTINEL = "NISSE_MATRIX_HOMESERVER"
"NISSE_MATRIX_HOMESERVER", ALSO_REQUIRED = (
"NISSE_MATRIX_USER_ID", "NISSE_MATRIX_USER_ID",
"NISSE_MATRIX_TOKEN", "NISSE_MATRIX_TOKEN",
"NISSE_MATRIX_TEST_ROOM_ID", "NISSE_MATRIX_TEST_ROOM_ID",
@@ -23,13 +27,18 @@ REQUIRED = (
pytestmark = [ pytestmark = [
pytest.mark.integration, pytest.mark.integration,
pytest.mark.skipif( pytest.mark.skipif(
not all(os.environ.get(var) for var in REQUIRED), not os.environ.get(SENTINEL),
reason=f"live matrix config absent; set {', '.join(REQUIRED)}", reason=f"no live homeserver configured ({SENTINEL} unset)",
), ),
] ]
def test_send_lands_in_test_room() -> None: def test_send_lands_in_test_room() -> None:
missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)]
assert not missing, (
f"{SENTINEL} is set, so the live Matrix path is expected to work, "
f"but these are missing: {', '.join(missing)}"
)
config = MatrixConfig.from_env(room_var="NISSE_MATRIX_TEST_ROOM_ID") config = MatrixConfig.from_env(room_var="NISSE_MATRIX_TEST_ROOM_ID")
marker = f"transport smoke test {uuid.uuid4().hex}" marker = f"transport smoke test {uuid.uuid4().hex}"
transport = MatrixTransport(config) transport = MatrixTransport(config)