From 968347a6be4b3b127bad0c9ea67dbb20913c2eec Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Mon, 7 Sep 2026 14:28:11 +0000 Subject: [PATCH 1/2] Wire live-test credentials into CI from Actions secrets The integration step now receives the LLM and Matrix env vars from repo secrets, so the live smoke tests run in CI when the runner can reach the services; the skip-guards keep a missing secret or unreachable service as a skip, never a failure. -rs surfaces skip reasons in the CI log for diagnosis. Co-Authored-By: Claude Fable 5 --- .gitea/workflows/ci.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 10145d7..4bc502a 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -27,4 +27,15 @@ jobs: run: uv run pytest -m "not integration" - name: Integration tests - run: uv run pytest -m integration + # Live-path credentials come from repo Actions secrets; the tests + # skip-guard on these, so a missing secret or unreachable service + # degrades to "skipped", never a failure on unrelated work. + env: + NISSE_LLM_BASE_URL: ${{ secrets.NISSE_LLM_BASE_URL }} + NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }} + NISSE_ROLE_EXECUTOR: ${{ secrets.NISSE_ROLE_EXECUTOR }} + NISSE_MATRIX_HOMESERVER: ${{ secrets.NISSE_MATRIX_HOMESERVER }} + NISSE_MATRIX_USER_ID: ${{ secrets.NISSE_MATRIX_USER_ID }} + NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }} + NISSE_MATRIX_TEST_ROOM_ID: ${{ secrets.NISSE_MATRIX_TEST_ROOM_ID }} + run: uv run pytest -m integration -rs -- 2.54.0 From 10aa77899ee326e989340351d094faacecae1f87 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Mon, 7 Sep 2026 14:33:14 +0000 Subject: [PATCH 2/2] Gate live tests on one sentinel var each, fail on partial config Skipping now keys solely on NISSE_LLM_BASE_URL and NISSE_MATRIX_HOMESERVER: if the sentinel is set, the live path is expected to work and any other missing variable is an assertion failure naming it, not a silent skip. A misnamed or misplaced CI secret previously produced a green run with no live coverage. Co-Authored-By: Claude Opus 5 --- tests/integration/test_llm_live.py | 19 +++++++++++++------ tests/integration/test_matrix_live.py | 21 +++++++++++++++------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_llm_live.py b/tests/integration/test_llm_live.py index 86d5707..196487e 100644 --- a/tests/integration/test_llm_live.py +++ b/tests/integration/test_llm_live.py @@ -1,8 +1,9 @@ """Live smoke test against the real LiteLLM proxy. -Skips unless NISSE_LLM_BASE_URL, NISSE_LLM_TOKEN, and NISSE_ROLE_EXECUTOR are -present (from the environment or the repo-root .env) — CI stays credential-free -by choice; this runs from the operator's machine. +NISSE_LLM_BASE_URL alone decides whether this runs: if it is set, the live +path is expected to work, and anything else missing is a *failure*, not a +skip. A half-configured environment must not pass quietly as a skip — that is +how a broken CI credential hides. """ import os @@ -16,18 +17,24 @@ from nisse_dev.llm import LiteLLMClient load_env_file(Path(__file__).resolve().parents[2] / ".env") -REQUIRED = ("NISSE_LLM_BASE_URL", "NISSE_LLM_TOKEN", "NISSE_ROLE_EXECUTOR") +SENTINEL = "NISSE_LLM_BASE_URL" +ALSO_REQUIRED = ("NISSE_LLM_TOKEN", "NISSE_ROLE_EXECUTOR") pytestmark = [ pytest.mark.integration, pytest.mark.skipif( - not all(os.environ.get(var) for var in REQUIRED), - reason=f"live LLM config absent; set {', '.join(REQUIRED)}", + not os.environ.get(SENTINEL), + reason=f"no live LLM configured ({SENTINEL} unset)", ), ] def test_executor_model_completes_via_litellm() -> None: + missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)] + assert not missing, ( + f"{SENTINEL} is set, so the live LLM path is expected to work, " + f"but these are missing: {', '.join(missing)}" + ) client = LiteLLMClient.from_env() events = [ StoredEvent( diff --git a/tests/integration/test_matrix_live.py b/tests/integration/test_matrix_live.py index 26c5b52..e835d4b 100644 --- a/tests/integration/test_matrix_live.py +++ b/tests/integration/test_matrix_live.py @@ -1,6 +1,10 @@ """Live smoke test against the real homeserver, bound strictly to the -dedicated test room (never the production room). Skips unless the Matrix env -vars, including NISSE_MATRIX_TEST_ROOM_ID, are present.""" +dedicated test room (never the production room). + +NISSE_MATRIX_HOMESERVER alone decides whether this runs: if it is set, the +live path is expected to work, and anything else missing is a *failure*, not +a skip. A half-configured environment must not pass quietly as a skip — that +is how a broken CI credential hides.""" import os import uuid @@ -13,8 +17,8 @@ from nisse_dev.matrix import MatrixConfig, MatrixTransport load_env_file(Path(__file__).resolve().parents[2] / ".env") -REQUIRED = ( - "NISSE_MATRIX_HOMESERVER", +SENTINEL = "NISSE_MATRIX_HOMESERVER" +ALSO_REQUIRED = ( "NISSE_MATRIX_USER_ID", "NISSE_MATRIX_TOKEN", "NISSE_MATRIX_TEST_ROOM_ID", @@ -23,13 +27,18 @@ REQUIRED = ( pytestmark = [ pytest.mark.integration, pytest.mark.skipif( - not all(os.environ.get(var) for var in REQUIRED), - reason=f"live matrix config absent; set {', '.join(REQUIRED)}", + not os.environ.get(SENTINEL), + reason=f"no live homeserver configured ({SENTINEL} unset)", ), ] def test_send_lands_in_test_room() -> None: + missing = [var for var in ALSO_REQUIRED if not os.environ.get(var)] + assert not missing, ( + f"{SENTINEL} is set, so the live Matrix path is expected to work, " + f"but these are missing: {', '.join(missing)}" + ) config = MatrixConfig.from_env(room_var="NISSE_MATRIX_TEST_ROOM_ID") marker = f"transport smoke test {uuid.uuid4().hex}" transport = MatrixTransport(config) -- 2.54.0