"""Guard: every CLI command GROUP has cli_vcr coverage or a reasoned exemption. The ``cli_vcr`` suite (``tests/integration/cli_vcr/``) is the project's behaviour net over the full CLI → Client → RPC path. A whole command *group* landing with no cli_vcr coverage — and no recorded reason — is how the net silently develops holes. This gate enumerates the top-level command groups of ``notebooklm.notebooklm_cli.cli`` and asserts each one is EITHER: 1. **Covered** — listed in :data:`GROUP_COVERAGE`, mapped to the ``tests/integration/cli_vcr/`` test file that exercises it (the file must exist), OR 2. **Exempt** — listed in :data:`COVERAGE_EXEMPT` with a one-line reason. This is group-level, not leaf-command-level: per-command enumeration would be noisy and unstable. A group is the right granularity — when a group is exercised at all, the per-command depth is the job of the re-record-safe assertion tiers (issue #1452), not this gate. Mapping is **explicit** rather than ``test_*.py`` filename inference, because the file names do not all match the group (``language`` → ``test_settings.py``, ``note`` → ``test_notes.py``). An explicit ``group → file`` map is honest and survives a rename better than a glob. The exempt set is a **shrink-only ratchet** (mirrors ``test_module_size_ratchet.py``): a group may only leave the exempt set by gaining real coverage — never by being added to dodge the gate. The two self-draining checks enforce this: * :func:`test_exempt_groups_have_no_cli_vcr_coverage` fails if an exempt group *does* have a ``GROUP_COVERAGE`` entry whose file exists — that group is now covered and MUST be removed from :data:`COVERAGE_EXEMPT` (the "newly-covered group must be removed" contract). * :func:`test_every_cli_group_is_classified` fails if a brand-new group appears that is in neither map — new groups start gated. Two exemption *reasons* exist, chosen by REALITY (verified by reading each command module for a ``NotebookLMClient`` / ``run_client_workflow`` RPC path): * ``"Phase-3: needs maintainer recording (#1452)"`` — the group hits the RPC API but has no cassettes yet; it flips on when a maintainer records (issue #1452 Phase 3). No group currently carries this reason (``research`` / ``auth`` drained out of it once their cassettes were wired up — see ``test_research.py`` / ``test_auth.py``); it is kept as a sanctioned reason for any future RPC-path group that lands before its cassette does. * ``"local-only, no RPC path"`` — the group never builds a client (filesystem / package-data only), so a VCR cassette would record nothing. These are permanent by design. The gate is GREEN on ``main`` today. Modelled on the ratchet lints in ``tests/_guardrails/`` (e.g. ``test_module_size_ratchet.py`` / ``test_no_module_shadowing.py``). """ from __future__ import annotations from pathlib import Path import click from notebooklm.notebooklm_cli import cli REPO_ROOT = Path(__file__).resolve().parents[2] CLI_VCR_DIR = REPO_ROOT / "tests" / "integration" / "cli_vcr" # Exemption reason strings (kept as named constants so the two reasons are # spelled identically everywhere and a typo can't fork them). _REASON_NEEDS_RECORDING = "Phase-3: needs maintainer recording (#1452)" _REASON_LOCAL_ONLY = "local-only, no RPC path" # Covered groups → the cli_vcr test file that exercises them. The file name does # not always match the group (the matcher keys on RPC shape, not the group # name), so the mapping is explicit. Each path is relative to ``CLI_VCR_DIR``. GROUP_COVERAGE: dict[str, str] = { "artifact": "test_artifacts.py", "auth": "test_auth.py", # `auth check`/`refresh` token-fetch paths over reused cassettes "download": "test_downloads.py", "generate": "test_generate.py", "label": "test_label.py", "language": "test_settings.py", # `language` commands live in test_settings.py "note": "test_notes.py", "profile": "test_profile.py", "research": "test_research.py", # `research status`/`wait` over reused poll cassettes "share": "test_share.py", "source": "test_sources.py", } # Groups with no cli_vcr coverage today → reason. Shrink-only: a group leaves # this map ONLY by gaining real coverage (move it to GROUP_COVERAGE). Verified # by reading each command module: ``agent``/``skill``/``mcp`` never touch the RPC # API (``agent show`` prints packaged prompt templates; ``skill`` reads/writes # skill files on disk; ``mcp install`` reads/writes the MCP client's config file # on disk), so a VCR cassette would capture nothing — permanently local-only. # (``research`` / ``auth`` left this map for GROUP_COVERAGE once their cassettes # were wired up; see the module docstring for the now-unused needs-recording reason.) COVERAGE_EXEMPT: dict[str, str] = { "agent": _REASON_LOCAL_ONLY, "skill": _REASON_LOCAL_ONLY, "mcp": _REASON_LOCAL_ONLY, } _VALID_REASONS = frozenset({_REASON_NEEDS_RECORDING, _REASON_LOCAL_ONLY}) def _cli_groups() -> set[str]: """The names of every top-level ``click.Group`` under ``cli``. Walks the live command tree so a newly-registered group is picked up automatically (and then fails :func:`test_every_cli_group_is_classified` until it is classified). """ return {name for name, cmd in cli.commands.items() if isinstance(cmd, click.Group)} def _covered_file_exists(group: str) -> bool: """True if ``group`` maps to a cli_vcr test file that exists on disk.""" rel = GROUP_COVERAGE.get(group) return rel is not None and (CLI_VCR_DIR / rel).is_file() def test_every_cli_group_is_classified() -> None: """Every CLI group must be covered OR exempt — a new group starts gated. A group in neither :data:`GROUP_COVERAGE` nor :data:`COVERAGE_EXEMPT` is an unclassified hole in the cli_vcr net. Add a cli_vcr test and map it in ``GROUP_COVERAGE``, or (only if it is RPC-less or genuinely needs recording) add it to ``COVERAGE_EXEMPT`` with a reason. """ classified = set(GROUP_COVERAGE) | set(COVERAGE_EXEMPT) unclassified = sorted(_cli_groups() - classified) assert unclassified == [], ( "CLI command group(s) have no cli_vcr coverage and no exemption " "(issue #1452 coverage gate). Add a cli_vcr test (and map it in " "GROUP_COVERAGE) or add a reasoned COVERAGE_EXEMPT entry:\n" + "\n".join(f" {g}" for g in unclassified) ) def test_covered_groups_have_an_existing_test_file() -> None: """Every :data:`GROUP_COVERAGE` entry must point at a real, existing group + file. A dangling mapping (the group was renamed/removed, or the test file was deleted) would silently claim coverage that no longer exists. """ groups = _cli_groups() broken = { group: rel for group, rel in GROUP_COVERAGE.items() if group not in groups or not (CLI_VCR_DIR / rel).is_file() } assert broken == {}, ( "GROUP_COVERAGE has entries whose group no longer exists or whose test " f"file is missing — fix or remove them (group -> file): {broken}" ) def test_exempt_groups_have_no_cli_vcr_coverage() -> None: """A newly-covered group must be REMOVED from :data:`COVERAGE_EXEMPT` (shrink-only). This is the ratchet: the moment an exempt group gains a real cli_vcr test (a ``GROUP_COVERAGE`` mapping to an existing file), it is no longer exempt and must move out of ``COVERAGE_EXEMPT``. Keeping a covered group in the exempt set would let coverage silently regress behind a stale exemption. """ regressed = sorted(g for g in COVERAGE_EXEMPT if _covered_file_exists(g)) assert regressed == [], ( "Group(s) now have cli_vcr coverage but are still in COVERAGE_EXEMPT " "(issue #1452 shrink-only ratchet). The exempt set may only shrink — " "remove each newly-covered group from COVERAGE_EXEMPT:\n" + "\n".join(f" {g}" for g in regressed) ) def test_exempt_and_covered_sets_are_disjoint() -> None: """No group may be both covered and exempt. Belt-and-braces against the two maps drifting out of sync — a group in both is contradictory (and would defeat the shrink-only check above). """ overlap = sorted(set(GROUP_COVERAGE) & set(COVERAGE_EXEMPT)) assert overlap == [], ( f"Group(s) appear in BOTH GROUP_COVERAGE and COVERAGE_EXEMPT: {overlap}. " "A covered group must not also be exempt." ) def test_every_exemption_has_a_known_reason() -> None: """Each :data:`COVERAGE_EXEMPT` reason must be one of the sanctioned strings. Forces every exemption into one of the two audited buckets (needs-recording vs local-only) so a free-text reason can't smuggle in an un-triaged hole. """ bad = {g: r for g, r in COVERAGE_EXEMPT.items() if r not in _VALID_REASONS} assert bad == {}, ( "COVERAGE_EXEMPT entries with an unrecognised reason — use one of " f"{sorted(_VALID_REASONS)} (group -> reason): {bad}" ) def test_exempt_groups_still_exist() -> None: """Every exempt group must still be a real CLI group (no stale entries). A removed/renamed group left in :data:`COVERAGE_EXEMPT` is dead weight that would mask a future re-introduction under the same name. """ stale = sorted(g for g in COVERAGE_EXEMPT if g not in _cli_groups()) assert stale == [], ( "COVERAGE_EXEMPT references group(s) that no longer exist on the CLI — " f"remove the stale entries: {stale}" )