ha-nvidia-shield/tests/test_harness_scoping.py

"""tests/test_harness_scoping.py — the stub harness must not depend on file order (t44).

The failure this pins, as reported: a subset of test files in an order the suite was
not written for left three ``test_init.py::TestImportStep`` tests failing with
``ModuleNotFoundError: No module named 'voluptuous'``, while the full suite stayed
green. Two mechanisms produced it, and both are order-only:

1. ``entity_stublib.install()`` trusted a single sentinel
   (``homeassistant.components.media_player``). A file whose teardown popped
   ``voluptuous`` while leaving the sentinel behind turned the next file's
   ``install()`` into a no-op, so importing ``config_flow`` failed.
2. A file's teardown restored only the names it had listed. A module it loaded
   (``nsh.diagnostics``, ``nsh.media_player``) therefore survived into the next
   file still holding stub objects — an exception class, the redact helper — that
   had since been replaced, and comparisons against the new objects failed.

The fix keeps the t7 discipline (no stub leaks across files): ``install()`` checks
the whole face set and rebuilds when anything is missing, and every file's teardown
uses ``snapshot_modules()``/``restore_modules()``, which drops *everything* the file
created under the harness namespaces and puts back everything it replaced.

The in-process tests cover both mechanisms directly; the subprocess cases re-run the
reported orderings end to end, because that is where the failure was visible.
"""

import subprocess
import sys
from pathlib import Path

import entity_stublib
import pytest

REPO_ROOT = Path(__file__).resolve().parent.parent
TRACKED = ("homeassistant", "voluptuous", "nsh")


def _scoped_faces() -> list[str]:
    """The HA/voluptuous faces every file scopes on (the nsh.* names are modules)."""
    return [name for name in entity_stublib.ENTITIES_STUB_NAMES if not name.startswith("nsh.")]


def _run(*paths: str) -> subprocess.CompletedProcess:
    return subprocess.run(
        [
            sys.executable,
            "-m",
            "pytest",
            *paths,
            "-q",
            "--no-header",
            "-p",
            "no:cacheprovider",
        ],
        cwd=REPO_ROOT,
        capture_output=True,
        text=True,
        timeout=300,
    )


class TestStubFaceGuard:
    """Mechanism 1: a partially popped face set must be repaired, not trusted."""

    def test_install_provides_every_face_a_file_scopes_on(self):
        """A face listed for scoping but never installed is a file-order trap."""
        snapshot = entity_stublib.snapshot_modules()
        try:
            entity_stublib.install()

            missing = [name for name in _scoped_faces() if name not in sys.modules]

            assert missing == []
        finally:
            entity_stublib.restore_modules(snapshot)

    def test_a_complete_face_set_is_left_alone(self):
        """The guard must still be an idempotence guard, not a rebuild."""
        snapshot = entity_stublib.snapshot_modules()
        try:
            entity_stublib.install()
            before = {name: sys.modules[name] for name in _scoped_faces()}

            entity_stublib.install()

            assert {name: sys.modules[name] for name in _scoped_faces()} == before
        finally:
            entity_stublib.restore_modules(snapshot)

    def test_a_sentinel_left_behind_does_not_hide_a_missing_face(self):
        """The reported poisoning: media_player present, voluptuous gone."""
        snapshot = entity_stublib.snapshot_modules()
        try:
            entity_stublib.install()
            kept = sys.modules["homeassistant.components.media_player"]
            del sys.modules["voluptuous"]

            entity_stublib.install()

            assert "voluptuous" in sys.modules
            # the rebuild is wholesale, so the sentinel is a real face again
            assert sys.modules["homeassistant.components.media_player"] is not None
            assert kept is not None
        finally:
            entity_stublib.restore_modules(snapshot)


class TestTeardownLeaks:
    """Mechanism 2: nothing a file created or replaced may survive its teardown."""

    def test_teardown_removes_the_modules_a_file_loaded(self):
        snapshot = entity_stublib.snapshot_modules()
        try:
            entity_stublib.install()
            entity_stublib.load_entity_modules()
            created = [
                name
                for name in sys.modules
                if name.startswith("nsh.") and name not in snapshot
            ]
            assert created, "premise: loading the entity modules creates nsh.* modules"

            entity_stublib.restore_modules(snapshot)

            assert [name for name in sys.modules if name.startswith("nsh.")] == [
                name for name in snapshot if name.startswith("nsh.")
            ]
            assert {
                name: sys.modules.get(name)
                for name in snapshot
                if name.split(".", 1)[0] in TRACKED
            } == snapshot
        finally:
            entity_stublib.restore_modules(snapshot)

    def test_a_reloaded_module_holds_the_current_stub_objects(self):
        """The stale-object failure, exactly: diagnostics vs the redact helper."""
        snapshot = entity_stublib.snapshot_modules()
        try:
            entity_stublib.install()
            stale_module = entity_stublib.load_entity_modules()["diagnostics"]
            stale = stale_module.async_redact_data
            entity_stublib.restore_modules(snapshot)  # ...the file ends

            # ...the next file installs its own faces: a partial pop is how that
            # actually looked, and it is what forces the rebuild
            entity_stublib.install()
            del sys.modules["homeassistant.helpers.redact"]
            entity_stublib.install()
            fresh_modules = entity_stublib.load_entity_modules()

            current = sys.modules["homeassistant.helpers.redact"].async_redact_data
            assert stale is not current, "premise: the rebuild created a new helper"
            assert fresh_modules["diagnostics"].async_redact_data is current
            # ...and the stale module is not the one sitting in the import cache
            assert sys.modules["nsh.diagnostics"] is fresh_modules["diagnostics"]
        finally:
            entity_stublib.restore_modules(snapshot)


def test_every_file_that_loads_the_harness_scopes_it():
    """A missing teardown is invisible until the NEXT file runs — so check for it.

    This is the mistake the fix itself invites: a new test file calls
    load_entity_modules() for convenience and forgets the per-file scope, and the
    failure surfaces somewhere else entirely (t44 added exactly that file).
    """
    #: Files that exercise the harness in a CHILD process: they leave nothing in
    #: this process's sys.modules, so they need no teardown.
    child_process_only = {"test_module_coverage.py"}
    offenders = []
    for path in sorted((REPO_ROOT / "tests").glob("test_*.py")):
        if path.name in child_process_only:
            continue
        text = path.read_text(encoding="utf-8")
        loads = "load_entity_modules(" in text or "entity_stublib.install()" in text
        if loads and "snapshot_modules()" not in text:
            offenders.append(path.name)

    assert offenders == []


@pytest.mark.parametrize(
    "paths",
    [
        # the reported reproduction, cut to the two files that trigger it
        ("tests/test_standalone_setup.py", "tests/test_init.py::TestImportStep"),
        # the other failure mode: a file that loads the entity modules before a
        # file that compares them against freshly installed faces
        (
            "tests/test_standalone_setup.py",
            "tests/test_config_flow_protocol.py",
            "tests/test_init.py",
            "tests/test_command_routing.py::TestAdbCommandService",
            "tests/test_command_routing.py::TestFaithfulHaStubs",
            "tests/test_command_routing.py::TestServiceRegistryKey",
        ),
    ],
)
def test_arbitrary_file_orders_pass(paths):
    """Order is the thing under test, so it is executed, not reasoned about."""
    result = _run(*paths)

    assert result.returncode == 0, result.stdout[-3000:]