From 181a51af9502214de4dda509ed1460971596821c Mon Sep 17 00:00:00 2001 From: engineer Date: Mon, 12 Oct 2026 08:36:51 +0100 Subject: [PATCH] chore(ci): run the real-HA config-flow tests and fail on silent skips tests/test_config_flow_real_ha.py skips itself at import time when homeassistant is absent, and a skipped file is indistinguishable from a passing one: the stub jobs end in "N passed, 1 skipped" and exit 0, and the drift sweep never reads it. That is how three real-HA-only facts stayed hidden while five gates were green. New real-ha-config-flow job installs the real stack on Home Assistant's interpreter floor (reusing HOMEASSISTANT_PYTHON_FLOOR from tests/ha_api_drift.py), runs the file with -rs, and then checks the pytest summary line itself: it must report passed tests and no skipped/failed/error line, because some pytest versions exit 0 for a file that skipped itself. The header comment now names both jobs that install Home Assistant. --- .gitea/workflows/test.yml | 113 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 109 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 03ea899..92028a8 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -7,10 +7,11 @@ # firmware captures live in tests/fixtures # (how to capture more: docs/RESEARCH_SHIELD.md §8). # -# The `ha-api-drift` job is the ONE place Home Assistant is installed: every -# other job validates our code against hand-written stubs, which stay green -# even when a call site no longer matches the real API. That is how three -# drifts reached master (t36 D1/D2/D3) — see tests/ha_api_drift.py. +# Home Assistant is installed in exactly two jobs: `ha-api-drift` (below) and +# `real-ha-config-flow`. Both exist because every other job validates our code +# against hand-written stubs, which stay green even when a call site no longer +# matches the real API. That is how three drifts reached master (t36 D1/D2/D3) — +# see tests/ha_api_drift.py. name: Test @@ -112,3 +113,107 @@ jobs: - name: HA API-drift sweep run: python3 -m pytest tests/test_ha_api_drift.py -q + + # The real-HA config-flow tests (tests/test_config_flow_real_ha.py) are the + # ONLY place the config flow meets the real homeassistant.data_entry_flow + # machinery — and that file SKIPS ITSELF at import time when homeassistant is + # absent (pytest.importorskip at module scope). A skipped file and a passing + # one are indistinguishable in the jobs above: `python3 -m pytest tests -q` + # ends in "N passed, 1 skipped" and still exits 0, and the drift sweep never + # touches this file. That is exactly how three real-HA-only facts stayed + # hidden while five gates were green: + # + # * a menu row id must equal its async_step_ method name (otherwise the + # real FlowManager raises when the row is dispatched), + # * the rendered step_id must equal the row id, and + # * async_show_menu takes no errors kwarg. + # + # So this job does two things the others cannot: it installs the real stack on + # Home Assistant's interpreter floor, and it FAILS LOUDLY if the tests are + # skipped instead of run. Do not "simplify" the guard at the bottom away — a + # test file that skips silently is indistinguishable from one that passes. + real-ha-config-flow: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Provision the interpreter Home Assistant needs + uses: actions/setup-python@v5 + with: + # Same floor as the ha-api-drift job: Home Assistant 2026.10.1 + # declares Requires-Python >= 3.14.2 (tests/ha_api_drift.py holds our + # copy of that number). This resolves the newest 3.14.x patch. + python-version: "3.14" + + - name: Ensure uv is available + run: | + set -euo pipefail + # uvx runs the exact command this gate was validated with locally; + # runner images are not guaranteed to ship it. + if ! command -v uvx >/dev/null 2>&1; then + python3 -m pip install --quiet uv \ + || python3 -m pip install --quiet --break-system-packages uv + fi + uvx --version + + - name: Check the interpreter satisfies Home Assistant's floor + run: | + set -euo pipefail + # The floor comes from tests/ha_api_drift.py — one source of truth for + # this guard and the metadata pin in tests/test_ha_api_drift.py. uvx + # resolves the same interpreter the test step will use, so a too-old + # interpreter fails here rather than halfway through the install. + uvx --python 3.14 python - <<'PY' + import sys + from pathlib import Path + + sys.path.insert(0, str(Path("tests").resolve())) + from ha_api_drift import HOMEASSISTANT_PYTHON_FLOOR as FLOOR # noqa: E402 + + needed = ".".join(str(part) for part in FLOOR) + if sys.version_info < FLOOR: + print( + f"Home Assistant needs Python >= {needed}, but this job runs " + f"{sys.version.split()[0]}." + ) + raise SystemExit(1) + print(f"interpreter ok: Python {sys.version.split()[0]} satisfies >= {needed}") + PY + + - name: Real-HA config-flow tests + run: | + set -euo pipefail + LOG=/tmp/real-ha-config-flow.log + # Verbatim the command this gate is validated with locally; -rs is what + # makes a silent skip visible in the log. + set +e + uvx --python 3.14 --with 'homeassistant>=2026.10' --with adb-shell \ + --with pytest-asyncio \ + pytest tests/test_config_flow_real_ha.py -q -rs 2>&1 | tee "$LOG" + rc=${PIPESTATUS[0]} + set -e + if [ "$rc" -ne 0 ]; then + echo "::error::the real-HA config-flow tests exited $rc" + exit "$rc" + fi + + # THE GUARD. Some pytest versions exit 0 for a file that skipped + # itself, so the exit code above is not enough. The last line of a -q + # run is the counts line ("26 passed in 1.6s" / "1 skipped in + # 0.01s"); it must report PASSED tests and nothing else. + summary=$(grep -v '^[[:space:]]*$' "$LOG" | tail -n 1) + echo "pytest summary: $summary" + case "$summary" in + [0-9]*" passed"*) ;; + *) + echo "::error::the real-HA config-flow tests did not run (summary: $summary)" + echo "The file skips itself when homeassistant/probatio is missing — a skip here is a FAILURE, not a pass." + exit 1 + ;; + esac + case "$summary" in + *skipped*|*failed*|*error*|*"no tests ran"*) + echo "::error::the real-HA config-flow run was not a clean pass (summary: $summary)" + exit 1 + ;; + esac