/.pre-commit-hooks/check-readme-links.py

#!/usr/bin/env python3
"""Pre-commit hook: keep README.md claims verifiable against the repo.

README.md is the project storefront, and it drifts: files move, headings
get renamed, CLI entry points change, and example snippets keep naming
things that no longer exist. A 2026-07 audit found a benchmark command
that had never been runnable, a `reset` example missing its required
flag, and an MCP example querying a search engine that was never
implemented. This hook catches those drift classes at commit time, in
README.md, SECURITY.md, CONTRIBUTING.md, and every markdown file under
docs/:

1. Relative markdown links must point at files/directories that exist.
2. Anchor fragments (``docs/faq.md#some-heading`` or same-file
   ``#-benchmarks``) must match a real heading in the target file,
   using GitHub's slug rules. ``#L10``/``#L10-L20`` line anchors must
   be within the target file's line count.
3. ``python -m local_deep_research...`` examples must reference a real,
   runnable module: a ``<module>.py`` file or a package directory
   containing ``__main__.py`` under ``src/``.
4. ``engine="<name>"`` examples must name an engine registered in
   ``ENGINE_REGISTRY`` (parsed from engine_registry.py).

Links inside fenced code blocks are exempt from check 1/2 -- GitHub does
not render them, and they are usually example markup (e.g. runtime
``/static/...`` paths in an HTML snippet). Checks 3/4 deliberately DO
look inside code blocks; that is where the examples live.

External (http/https/mailto) links are not checked -- no network access
at commit time.
"""

import argparse
import re
import sys
from pathlib import Path
from urllib.parse import unquote, urlparse

REPO_ROOT = Path(__file__).resolve().parent.parent

# Fixed files whose internal references this hook validates; every
# markdown file under docs/ is added at runtime in main().
CHECKED_FILES = ["README.md", "SECURITY.md", "CONTRIBUTING.md"]

ENGINE_REGISTRY_PATH = Path(
    "src/local_deep_research/web_search_engines/engine_registry.py"
)

# Inline markdown links/images: [text](target) / ![alt](target "title").
# The target group stops at whitespace or ')' so optional "title" parts
# are excluded.
MARKDOWN_LINK = re.compile(r"!?\[[^\]]*\]\(\s*<?([^)<>\s]+)>?[^)]*\)")

# Relative href/src in inline HTML.
HTML_REF = re.compile(r"""(?:href|src)=["']([^"']+)["']""")

# `python -m some.module` (also matches python3).
PYTHON_M = re.compile(r"python3?\s+-m\s+([A-Za-z_][\w.]*)")

# engine="name" in example snippets.
ENGINE_KWARG = re.compile(r"""engine=["']([\w\-]+)["']""")

# GitHub file line anchors: #L10 or #L10-L20.
LINE_ANCHOR = re.compile(r"^L(\d+)(?:-L(\d+))?$")

# Registry entries: `    "name": EngineEntry(`.
REGISTRY_KEY = re.compile(r"^\s+\"([\w\-]+)\":\s*EngineEntry\(", re.M)

FENCE = re.compile(r"^\s*(```|~~~)")


def github_slug(heading: str) -> str:
    """Approximate GitHub's heading-to-anchor slug algorithm.

    Lowercase, strip markdown link syntax, drop everything that is not a
    letter/digit/space/hyphen/underscore (this removes emoji and other
    punctuation), then turn spaces into hyphens.
    """
    text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", heading).strip().lower()
    kept = [c for c in text if c.isalnum() or c in "-_ "]
    return "".join(kept).replace(" ", "-")


def iter_headings(md_text: str):
    """Yield ATX heading texts, skipping fenced code blocks."""
    in_fence = False
    for line in md_text.splitlines():
        if FENCE.match(line):
            in_fence = not in_fence
            continue
        if in_fence:
            continue
        match = re.match(r"^#{1,6}\s+(.*)$", line)
        if match:
            yield match.group(1)


def heading_slugs(md_text: str) -> set:
    return {github_slug(h) for h in iter_headings(md_text)}


def strip_fences(md_text: str) -> str:
    """Blank out fenced code blocks; GitHub renders no links inside them."""
    out, in_fence = [], False
    for line in md_text.splitlines():
        if FENCE.match(line):
            in_fence = not in_fence
            out.append("")
            continue
        out.append("" if in_fence else line)
    return "\n".join(out)


def check_anchor(target: Path, fragment: str) -> str:
    """Return an error message for a bad anchor, or '' if it resolves."""
    line_match = LINE_ANCHOR.match(fragment)
    if line_match:
        last = int(line_match.group(2) or line_match.group(1))
        lines = len(target.read_text(encoding="utf-8").splitlines())
        if last > lines:
            return (
                f"line anchor #{fragment} exceeds file length ({lines} lines)"
            )
        return ""
    if target.suffix.lower() not in (".md", ".markdown"):
        # GitHub only generates heading anchors for rendered markdown.
        return f"anchor #{fragment} on non-markdown file cannot be verified"
    slugs = heading_slugs(target.read_text(encoding="utf-8"))
    slug = fragment.lower()
    if slug in slugs:
        return ""
    # GitHub dedupes duplicate headings by appending -1, -2, ...
    base = re.sub(r"-\d+$", "", slug)
    if base != slug and base in slugs:
        return ""
    return f"no heading matches anchor #{fragment}"


def check_links(md_path: Path, text: str) -> list:
    errors = []
    targets = MARKDOWN_LINK.findall(text) + HTML_REF.findall(text)
    for raw_target in targets:
        parsed = urlparse(raw_target)
        if parsed.scheme or raw_target.startswith("//"):
            continue  # external; not checkable offline
        rel_path, fragment = unquote(parsed.path), parsed.fragment
        target = (md_path.parent / rel_path if rel_path else md_path).resolve()
        if not target.exists():
            errors.append(f"broken link: {raw_target} (file not found)")
            continue
        if fragment and target.is_file():
            anchor_error = check_anchor(target, fragment)
            if anchor_error:
                errors.append(f"broken link: {raw_target} ({anchor_error})")
    return errors


def check_python_modules(root: Path, text: str) -> list:
    errors = []
    for module in PYTHON_M.findall(text):
        if not module.startswith("local_deep_research"):
            continue
        base = root / "src" / Path(*module.split("."))
        if base.with_suffix(".py").is_file():
            continue
        if (base / "__main__.py").is_file():
            continue
        errors.append(
            f"`python -m {module}` is not runnable: expected "
            f"src/{'/'.join(module.split('.'))}.py or .../__main__.py"
        )
    return errors


def check_engine_names(root: Path, text: str) -> list:
    engines = ENGINE_KWARG.findall(text)
    if not engines:
        return []
    registry_file = root / ENGINE_REGISTRY_PATH
    if not registry_file.is_file():
        return [f"cannot verify engine names: {ENGINE_REGISTRY_PATH} missing"]
    registered = set(
        REGISTRY_KEY.findall(registry_file.read_text(encoding="utf-8"))
    )
    if not registered:
        # The registry format changed; failing loudly beats silently
        # skipping the check.
        return [
            f"could not parse any engine names from {ENGINE_REGISTRY_PATH}; "
            "update REGISTRY_KEY in this hook"
        ]
    return [
        f'engine="{name}" is not a registered search engine '
        f"(see {ENGINE_REGISTRY_PATH})"
        for name in engines
        if name not in registered
    ]


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--root",
        type=Path,
        default=REPO_ROOT,
        help="Repository root to check against (test seam)",
    )
    args = parser.parse_args()
    root = args.root.resolve()

    files = [root / name for name in CHECKED_FILES] + sorted(
        root.glob("docs/**/*.md")
    )

    all_errors = []
    for md_path in files:
        name = str(md_path.relative_to(root))
        if not md_path.is_file():
            all_errors.append((name, "file is missing"))
            continue
        text = md_path.read_text(encoding="utf-8")
        for error in (
            check_links(md_path, strip_fences(text))
            + check_python_modules(root, text)
            + check_engine_names(root, text)
        ):
            all_errors.append((name, error))

    if all_errors:
        print("❌ DOCUMENTATION REFERENCES SOMETHING THAT DOES NOT EXIST")
        print("=" * 60)
        for name, error in all_errors:
            print(f"📄 {name}: {error}")
        print("=" * 60)
        print("FIX: update the link/example to match the repo, or fix the")
        print("hook if GitHub's anchor rules are approximated incorrectly")
        print("(.pre-commit-hooks/check-readme-links.py). Links inside")
        print("fenced code blocks are already exempt.")
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())