#!/usr/bin/env python3 """Pre-commit hook: keep README.md claims verifiable against the repo. README.md is the project storefront, and it drifts: files move, headings get renamed, CLI entry points change, and example snippets keep naming things that no longer exist. A 2026-07 audit found a benchmark command that had never been runnable, a `reset` example missing its required flag, and an MCP example querying a search engine that was never implemented. This hook catches those drift classes at commit time, in README.md, SECURITY.md, CONTRIBUTING.md, and every markdown file under docs/: 1. Relative markdown links must point at files/directories that exist. 2. Anchor fragments (``docs/faq.md#some-heading`` or same-file ``#-benchmarks``) must match a real heading in the target file, using GitHub's slug rules. ``#L10``/``#L10-L20`` line anchors must be within the target file's line count. 3. ``python -m local_deep_research...`` examples must reference a real, runnable module: a ``.py`` file or a package directory containing ``__main__.py`` under ``src/``. 4. ``engine=""`` examples must name an engine registered in ``ENGINE_REGISTRY`` (parsed from engine_registry.py). Links inside fenced code blocks are exempt from check 1/2 -- GitHub does not render them, and they are usually example markup (e.g. runtime ``/static/...`` paths in an HTML snippet). Checks 3/4 deliberately DO look inside code blocks; that is where the examples live. External (http/https/mailto) links are not checked -- no network access at commit time. """ import argparse import re import sys from pathlib import Path from urllib.parse import unquote, urlparse REPO_ROOT = Path(__file__).resolve().parent.parent # Fixed files whose internal references this hook validates; every # markdown file under docs/ is added at runtime in main(). CHECKED_FILES = ["README.md", "SECURITY.md", "CONTRIBUTING.md"] ENGINE_REGISTRY_PATH = Path( "src/local_deep_research/web_search_engines/engine_registry.py" ) # Inline markdown links/images: [text](target) / ![alt](target "title"). # The target group stops at whitespace or ')' so optional "title" parts # are excluded. MARKDOWN_LINK = re.compile(r"!?\[[^\]]*\]\(\s*\s]+)>?[^)]*\)") # Relative href/src in inline HTML. HTML_REF = re.compile(r"""(?:href|src)=["']([^"']+)["']""") # `python -m some.module` (also matches python3). PYTHON_M = re.compile(r"python3?\s+-m\s+([A-Za-z_][\w.]*)") # engine="name" in example snippets. ENGINE_KWARG = re.compile(r"""engine=["']([\w\-]+)["']""") # GitHub file line anchors: #L10 or #L10-L20. LINE_ANCHOR = re.compile(r"^L(\d+)(?:-L(\d+))?$") # Registry entries: ` "name": EngineEntry(`. REGISTRY_KEY = re.compile(r"^\s+\"([\w\-]+)\":\s*EngineEntry\(", re.M) FENCE = re.compile(r"^\s*(```|~~~)") def github_slug(heading: str) -> str: """Approximate GitHub's heading-to-anchor slug algorithm. Lowercase, strip markdown link syntax, drop everything that is not a letter/digit/space/hyphen/underscore (this removes emoji and other punctuation), then turn spaces into hyphens. """ text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", heading).strip().lower() kept = [c for c in text if c.isalnum() or c in "-_ "] return "".join(kept).replace(" ", "-") def iter_headings(md_text: str): """Yield ATX heading texts, skipping fenced code blocks.""" in_fence = False for line in md_text.splitlines(): if FENCE.match(line): in_fence = not in_fence continue if in_fence: continue match = re.match(r"^#{1,6}\s+(.*)$", line) if match: yield match.group(1) def heading_slugs(md_text: str) -> set: return {github_slug(h) for h in iter_headings(md_text)} def strip_fences(md_text: str) -> str: """Blank out fenced code blocks; GitHub renders no links inside them.""" out, in_fence = [], False for line in md_text.splitlines(): if FENCE.match(line): in_fence = not in_fence out.append("") continue out.append("" if in_fence else line) return "\n".join(out) def check_anchor(target: Path, fragment: str) -> str: """Return an error message for a bad anchor, or '' if it resolves.""" line_match = LINE_ANCHOR.match(fragment) if line_match: last = int(line_match.group(2) or line_match.group(1)) lines = len(target.read_text(encoding="utf-8").splitlines()) if last > lines: return ( f"line anchor #{fragment} exceeds file length ({lines} lines)" ) return "" if target.suffix.lower() not in (".md", ".markdown"): # GitHub only generates heading anchors for rendered markdown. return f"anchor #{fragment} on non-markdown file cannot be verified" slugs = heading_slugs(target.read_text(encoding="utf-8")) slug = fragment.lower() if slug in slugs: return "" # GitHub dedupes duplicate headings by appending -1, -2, ... base = re.sub(r"-\d+$", "", slug) if base != slug and base in slugs: return "" return f"no heading matches anchor #{fragment}" def check_links(md_path: Path, text: str) -> list: errors = [] targets = MARKDOWN_LINK.findall(text) + HTML_REF.findall(text) for raw_target in targets: parsed = urlparse(raw_target) if parsed.scheme or raw_target.startswith("//"): continue # external; not checkable offline rel_path, fragment = unquote(parsed.path), parsed.fragment target = (md_path.parent / rel_path if rel_path else md_path).resolve() if not target.exists(): errors.append(f"broken link: {raw_target} (file not found)") continue if fragment and target.is_file(): anchor_error = check_anchor(target, fragment) if anchor_error: errors.append(f"broken link: {raw_target} ({anchor_error})") return errors def check_python_modules(root: Path, text: str) -> list: errors = [] for module in PYTHON_M.findall(text): if not module.startswith("local_deep_research"): continue base = root / "src" / Path(*module.split(".")) if base.with_suffix(".py").is_file(): continue if (base / "__main__.py").is_file(): continue errors.append( f"`python -m {module}` is not runnable: expected " f"src/{'/'.join(module.split('.'))}.py or .../__main__.py" ) return errors def check_engine_names(root: Path, text: str) -> list: engines = ENGINE_KWARG.findall(text) if not engines: return [] registry_file = root / ENGINE_REGISTRY_PATH if not registry_file.is_file(): return [f"cannot verify engine names: {ENGINE_REGISTRY_PATH} missing"] registered = set( REGISTRY_KEY.findall(registry_file.read_text(encoding="utf-8")) ) if not registered: # The registry format changed; failing loudly beats silently # skipping the check. return [ f"could not parse any engine names from {ENGINE_REGISTRY_PATH}; " "update REGISTRY_KEY in this hook" ] return [ f'engine="{name}" is not a registered search engine ' f"(see {ENGINE_REGISTRY_PATH})" for name in engines if name not in registered ] def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--root", type=Path, default=REPO_ROOT, help="Repository root to check against (test seam)", ) args = parser.parse_args() root = args.root.resolve() files = [root / name for name in CHECKED_FILES] + sorted( root.glob("docs/**/*.md") ) all_errors = [] for md_path in files: name = str(md_path.relative_to(root)) if not md_path.is_file(): all_errors.append((name, "file is missing")) continue text = md_path.read_text(encoding="utf-8") for error in ( check_links(md_path, strip_fences(text)) + check_python_modules(root, text) + check_engine_names(root, text) ): all_errors.append((name, error)) if all_errors: print("❌ DOCUMENTATION REFERENCES SOMETHING THAT DOES NOT EXIST") print("=" * 60) for name, error in all_errors: print(f"📄 {name}: {error}") print("=" * 60) print("FIX: update the link/example to match the repo, or fix the") print("hook if GitHub's anchor rules are approximated incorrectly") print("(.pre-commit-hooks/check-readme-links.py). Links inside") print("fenced code blocks are already exempt.") return 1 return 0 if __name__ == "__main__": sys.exit(main())