"""Benchmark tests for HTML content extraction pipeline.
Tests extraction quality across 200+ real-world pages from diverse domains.
Skipped in CI (requires network). Run manually with:
pytest tests/research_library/downloaders/test_extraction_benchmark.py -v -s
Results are printed as a comparison table with content length, boilerplate
count, and timing for each downloader mode.
"""
import time
from dataclasses import dataclass
from typing import Dict, List, Tuple
import pytest
from local_deep_research.content_fetcher import ContentFetcher
from local_deep_research.research_library.downloaders.html import HTMLDownloader
from local_deep_research.research_library.downloaders.playwright_html import (
AutoHTMLDownloader,
)
# Skip entire module in CI — these hit the network and need a browser.
# `integration` is the marker CI excludes via `-m 'not integration'`; `slow` is
# kept so the tests can be selected/deselected independently when run locally.
pytestmark = [pytest.mark.slow, pytest.mark.integration]
BOILERPLATE_KEYWORDS = [
"cookie",
"sign up",
"newsletter",
"skip to content",
"subscribe",
"accept all",
"privacy policy",
"terms of service",
"log in",
"sign in",
"add to cart",
"checkout",
"wishlist",
]
# ---------------------------------------------------------------------------
# URL corpus — 200+ pages across 17 categories
# Each entry: (label, url)
# Grouped by expected fetch behaviour: static-friendly vs JS-heavy
# ---------------------------------------------------------------------------
NEWS = [
("BBC front", "https://www.bbc.com/news"),
("Reuters", "https://www.reuters.com/"),
("NYTimes", "https://www.nytimes.com/"),
("Guardian", "https://www.theguardian.com/international"),
("CNN", "https://www.cnn.com/"),
("AP News", "https://apnews.com/"),
("Al Jazeera", "https://www.aljazeera.com/"),
("DW", "https://www.dw.com/en/"),
("France24", "https://www.france24.com/en/"),
("NBC News", "https://www.nbcnews.com/"),
("USA Today", "https://www.usatoday.com/"),
("Forbes", "https://www.forbes.com/"),
("Bloomberg", "https://www.bloomberg.com/"),
("Politico", "https://www.politico.com/"),
("HuffPost", "https://www.huffpost.com/"),
("ABC News", "https://abcnews.go.com/"),
("Time", "https://time.com/"),
("NPR", "https://www.npr.org/"),
("BBC Tech", "https://www.bbc.com/news/technology"),
("Ars Technica", "https://arstechnica.com/"),
]
TECH = [
("GitHub README", "https://github.com/AmiGandhi/WordPredict"),
("GH LDR", "https://github.com/LearningCircuit/local-deep-research"),
(
"GH Issue",
"https://github.com/LearningCircuit/local-deep-research/issues/1",
),
(
"StackOverflow",
"https://stackoverflow.com/questions/231767/what-does-the-yield-keyword-do-in-python",
),
("SO tagged", "https://stackoverflow.com/questions/tagged/python"),
("HackerNews", "https://news.ycombinator.com/item?id=40956541"),
("HN front", "https://news.ycombinator.com/"),
("TechCrunch", "https://techcrunch.com/"),
("Wired", "https://www.wired.com/"),
("The Verge", "https://www.theverge.com/"),
("ZDNet", "https://www.zdnet.com/"),
("Slashdot", "https://slashdot.org/"),
("Dev.to", "https://dev.to/"),
("Lobsters", "https://lobste.rs/"),
("InfoQ", "https://www.infoq.com/"),
]
REFERENCE = [
("Wiki EN", "https://en.wikipedia.org/wiki/Python_(programming_language)"),
("Wiki JA", "https://ja.wikipedia.org/wiki/Python"),
("Wiki DE", "https://de.wikipedia.org/wiki/Python_(Programmiersprache)"),
("Wiki FR", "https://fr.wikipedia.org/wiki/Intelligence_artificielle"),
("Wiki ZH", "https://zh.wikipedia.org/wiki/Python"),
(
"Wiki AR",
"https://ar.wikipedia.org/wiki/%D8%A8%D8%A7%D9%8A%D8%AB%D9%88%D9%86",
),
("Wiki ES", "https://es.wikipedia.org/wiki/Python"),
(
"Britannica",
"https://www.britannica.com/technology/artificial-intelligence",
),
("M-W Dict", "https://www.merriam-webster.com/dictionary/algorithm"),
("W3Schools", "https://www.w3schools.com/python/python_lists.asp"),
]
DOCS = [
("Python docs", "https://docs.python.org/3/library/asyncio.html"),
(
"MDN Docs",
"https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Functions",
),
(
"ReadTheDocs",
"https://requests.readthedocs.io/en/latest/user/quickstart/",
),
(
"Rust Book",
"https://doc.rust-lang.org/book/ch04-01-what-is-ownership.html",
),
("Go Docs", "https://go.dev/doc/effective_go"),
("Django Docs", "https://docs.djangoproject.com/en/5.0/topics/http/views/"),
("React Docs", "https://react.dev/learn"),
("Vue Docs", "https://vuejs.org/guide/introduction.html"),
(
"MS Learn",
"https://learn.microsoft.com/en-us/dotnet/csharp/tour-of-csharp/",
),
("Kubernetes", "https://kubernetes.io/docs/concepts/overview/"),
]
ACADEMIC = [
("ArXiv", "https://arxiv.org/abs/2301.07507"),
("ArXiv ICL", "https://arxiv.org/abs/2301.00234"),
("ArXiv LLM", "https://arxiv.org/abs/2303.08774"),
("ArXiv HTML", "https://arxiv.org/html/2301.07507"),
("PubMed", "https://pubmed.ncbi.nlm.nih.gov/37828879/"),
("PubMed CRISPR", "https://pubmed.ncbi.nlm.nih.gov/26553966/"),
("PubMed COVID", "https://pubmed.ncbi.nlm.nih.gov/32015507/"),
("PMC", "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7095418/"),
(
"SemScholar",
"https://www.semanticscholar.org/paper/Attention-Is-All-You-Need-Vaswani-Shazeer/204e3073870fae3d05bcbc2f6a8e263d9b72e776",
),
(
"SemScholar2",
"https://www.semanticscholar.org/paper/BERT%3A-Pre-training-of-Deep-Bidirectional-Devlin-Chang/df2b0e26d0599ce3e70df8a9da02e51594e0e992",
),
("OpenAlex", "https://openalex.org/works/W2741809807"),
("OpenAlex2", "https://openalex.org/works/W2963403868"),
("Nature", "https://www.nature.com/articles/s41586-024-07487-w"),
("Science", "https://www.science.org/"),
("ResearchGate", "https://www.researchgate.net/"),
("Springer", "https://link.springer.com/"),
("IEEE", "https://www.ieee.org/"),
("JSTOR", "https://www.jstor.org/"),
("ACM DL", "https://dl.acm.org/"),
("bioRxiv", "https://www.biorxiv.org/content/10.1101/2024.01.01.573841v1"),
]
GOVERNMENT = [
("WhiteHouse", "https://www.whitehouse.gov/"),
("NASA", "https://www.nasa.gov/"),
("NIH", "https://www.nih.gov/"),
("CDC", "https://www.cdc.gov/"),
("Europa EU", "https://europa.eu/"),
("WHO", "https://www.who.int/"),
("UN", "https://www.un.org/en/"),
("GOV UK", "https://www.gov.uk/"),
("USA.gov", "https://www.usa.gov/"),
("Data.gov", "https://data.gov/"),
]
EDUCATION = [
("MIT", "https://www.mit.edu/"),
("Stanford", "https://www.stanford.edu/"),
("Harvard", "https://www.harvard.edu/"),
("Berkeley", "https://www.berkeley.edu/"),
("Cornell", "https://www.cornell.edu/"),
("Oxford", "https://www.ox.ac.uk/"),
("Cambridge", "https://www.cam.ac.uk/"),
("Coursera", "https://www.coursera.org/"),
("Khan Academy", "https://www.khanacademy.org/"),
("edX", "https://www.edx.org/"),
]
SHOPPING = [
("Amazon", "https://www.amazon.com/"),
("eBay", "https://www.ebay.com/"),
("Walmart", "https://www.walmart.com/"),
("Target", "https://www.target.com/"),
("Etsy", "https://www.etsy.com/"),
("BestBuy", "https://www.bestbuy.com/"),
("Newegg", "https://www.newegg.com/"),
("IKEA", "https://www.ikea.com/"),
("HomeDepot", "https://www.homedepot.com/"),
("AliExpress", "https://www.aliexpress.com/"),
("Temu", "https://www.temu.com/"),
("Idealo", "https://www.idealo.de/"),
]
SOCIAL = [
("Reddit", "https://www.reddit.com/r/Python/"),
("Reddit old", "https://old.reddit.com/r/Python/"),
("LinkedIn", "https://www.linkedin.com/"),
("Pinterest", "https://www.pinterest.com/"),
("Quora", "https://www.quora.com/"),
("Tumblr", "https://www.tumblr.com/"),
]
ENTERTAINMENT = [
("YouTube", "https://www.youtube.com/"),
("IMDb", "https://www.imdb.com/title/tt0111161/"),
("Rotten Tom", "https://www.rottentomatoes.com/"),
("Spotify", "https://open.spotify.com/"),
("Goodreads", "https://www.goodreads.com/"),
("Letterboxd", "https://letterboxd.com/"),
]
FINANCE = [
("Yahoo Fin", "https://finance.yahoo.com/"),
("Investing", "https://www.investing.com/"),
("MarketWatch", "https://www.marketwatch.com/"),
("SeekingAlpha", "https://seekingalpha.com/"),
("CoinGecko", "https://www.coingecko.com/"),
("Investopedia", "https://www.investopedia.com/"),
]
PACKAGES = [
("PyPI justext", "https://pypi.org/project/justext/"),
("PyPI trafila", "https://pypi.org/project/trafilatura/"),
("npm readab", "https://www.npmjs.com/package/readability"),
("npm express", "https://www.npmjs.com/package/express"),
("crates serde", "https://crates.io/crates/serde"),
("RubyGems rails", "https://rubygems.org/gems/rails"),
("Docker Hub", "https://hub.docker.com/"),
]
INTERNATIONAL = [
("Baidu", "https://www.baidu.com/"),
("Yandex", "https://yandex.ru/"),
("Naver", "https://www.naver.com/"),
("Rakuten", "https://www.rakuten.co.jp/"),
("MercadoLibre", "https://www.mercadolibre.com/"),
("Allegro PL", "https://allegro.pl/"),
("Bol NL", "https://www.bol.com/"),
("Lemonde FR", "https://www.lemonde.fr/"),
("Spiegel DE", "https://www.spiegel.de/"),
("Corriere IT", "https://www.corriere.it/"),
]
BLOGS = [
(
"Medium",
"https://medium.com/@natassha6789/if-i-had-to-start-learning-data-science-again-how-would-i-do-it-78a02b1b56d2",
),
("Substack", "https://substack.com/"),
("S. Willison", "https://simonwillison.net/2024/Mar/8/gpt-4-barrier/"),
("WordPress", "https://wordpress.com/"),
("Ghost", "https://ghost.org/"),
("Hashnode", "https://hashnode.com/"),
]
HEALTH = [
("WebMD", "https://www.webmd.com/"),
("Mayo Clinic", "https://www.mayoclinic.org/"),
("Healthline", "https://www.healthline.com/"),
("MedNews", "https://www.medicalnewstoday.com/"),
("Cleveland", "https://my.clevelandclinic.org/"),
]
TRAVEL = [
("Booking", "https://www.booking.com/"),
("TripAdvisor", "https://www.tripadvisor.com/"),
("Airbnb", "https://www.airbnb.com/"),
("Expedia", "https://www.expedia.com/"),
("Lonely Planet", "https://www.lonelyplanet.com/"),
]
FOOD = [
("AllRecipes", "https://www.allrecipes.com/"),
("Epicurious", "https://www.epicurious.com/"),
("SeriousEats", "https://www.seriouseats.com/"),
("Bon Appetit", "https://www.bonappetit.com/"),
("Food Network", "https://www.foodnetwork.com/"),
]
MISC = [
("Craigslist", "https://www.craigslist.org/"),
("Archive.org", "https://archive.org/"),
("Wayback", "https://web.archive.org/"),
("Pastebin", "https://pastebin.com/"),
("Regex101", "https://regex101.com/"),
]
# Combine all categories
ALL_CATEGORIES: Dict[str, List[Tuple[str, str]]] = {
"news": NEWS,
"tech": TECH,
"reference": REFERENCE,
"docs": DOCS,
"academic": ACADEMIC,
"government": GOVERNMENT,
"education": EDUCATION,
"shopping": SHOPPING,
"social": SOCIAL,
"entertainment": ENTERTAINMENT,
"finance": FINANCE,
"packages": PACKAGES,
"international": INTERNATIONAL,
"blogs": BLOGS,
"health": HEALTH,
"travel": TRAVEL,
"food": FOOD,
"misc": MISC,
}
# Flat list of all pages
ALL_PAGES = []
for cat, pages in ALL_CATEGORIES.items():
for label, url in pages:
ALL_PAGES.append((cat, label, url))
# Pages expected to work with static fetch (no JS rendering needed)
STATIC_PAGES = []
for cat, label, url in ALL_PAGES:
if cat not in ("shopping", "social", "entertainment", "travel"):
STATIC_PAGES.append((label, url))
@dataclass
class FetchResult:
"""Result of a single fetch attempt."""
category: str = ""
page: str = ""
mode: str = ""
length: int = 0
boilerplate: int = 0
time_s: float = 0.0
sample: str = ""
def _count_boilerplate(data: bytes | None) -> int:
if not data:
return 0
text = data.decode("utf-8", errors="replace").lower()
return sum(1 for kw in BOILERPLATE_KEYWORDS if kw in text)
def _get_sample(data: bytes | None, n: int = 80) -> str:
if not data:
return "(no content)"
text = data.decode("utf-8", errors="replace")
idx = text.find("\n\n")
start = text[idx + 2 :] if idx > 0 else text
return start[:n].replace("\n", " ").strip()
def _run_downloader(
dl, category: str, name: str, url: str, mode: str
) -> FetchResult:
t0 = time.time()
try:
data = dl.download(url)
except Exception:
data = None
elapsed = time.time() - t0
return FetchResult(
category=category,
page=name,
mode=mode,
length=len(data) if data else 0,
boilerplate=_count_boilerplate(data),
time_s=round(elapsed, 2),
sample=_get_sample(data),
)
def _print_category_summary(results: list[FetchResult], mode: str):
"""Print per-category success rates."""
from collections import defaultdict
by_cat: Dict[str, list[FetchResult]] = defaultdict(list)
for r in results:
if r.mode == mode:
by_cat[r.category].append(r)
print(f"\n Per-category breakdown ({mode}):")
for cat in ALL_CATEGORIES:
cat_results = by_cat.get(cat, [])
if not cat_results:
continue
success = sum(1 for r in cat_results if r.length > 100)
total = len(cat_results)
avg_len = sum(r.length for r in cat_results if r.length > 100) // max(
success, 1
)
avg_bp = sum(r.boilerplate for r in cat_results) / max(total, 1)
bar = "█" * success + "░" * (total - success)
print(
f" {cat:<15} {bar} {success}/{total} "
f"avg_len={avg_len:>6} avg_bp={avg_bp:.1f}"
)
class TestExtractionBenchmark:
"""Benchmark extraction quality across downloader modes.
Tests 200+ real-world pages from 17 categories.
Not purely assertions-based — prints a comparison table for human review.
"""
@pytest.fixture(autouse=True)
def _setup(self):
self.static_dl = HTMLDownloader(timeout=20)
# Benchmark exercises the JS-rendering fallback path explicitly,
# so opt in regardless of the disable-by-default ctor.
self.auto_dl = AutoHTMLDownloader(timeout=20, enable_js_rendering=True)
yield
self.static_dl.close()
self.auto_dl.close()
def test_static_pages_return_content(self):
"""Static downloader should extract content from most static pages."""
failures = []
for name, url in STATIC_PAGES[:30]: # Test first 30 for speed
result = _run_downloader(self.static_dl, "", name, url, "static")
if result.length < 100:
failures.append(f"{name}: {result.length} chars")
# Allow up to 20% failure rate (bot protection, geo-blocks, etc.)
max_failures = len(STATIC_PAGES[:30]) * 0.2
assert len(failures) <= max_failures, (
f"Static downloader failed on too many pages "
f"({len(failures)}/{len(STATIC_PAGES[:30])}): "
f"{', '.join(failures[:10])}"
)
def test_full_benchmark(self):
"""Full benchmark across all pages with Auto downloader.
Prints detailed results table grouped by category.
"""
results: list[FetchResult] = []
total = len(ALL_PAGES)
print(f"\n{'=' * 90}")
print(
f" EXTRACTION BENCHMARK: {total} pages across "
f"{len(ALL_CATEGORIES)} categories"
)
print(f"{'=' * 90}")
for i, (cat, name, url) in enumerate(ALL_PAGES):
r = _run_downloader(self.auto_dl, cat, name, url, "Auto")
results.append(r)
# Progress indicator
status = "✓" if r.length > 100 else "✗"
print(
f" [{i + 1:>3}/{total}] {status} {cat:<14} "
f"{name:<16} {r.length:>7} chars "
f"{r.time_s:>5.1f}s bp={r.boilerplate}"
)
# Summary table by category
print(f"\n{'=' * 90}")
print(" SUMMARY")
print(f"{'=' * 90}")
total_success = sum(1 for r in results if r.length > 100)
total_chars = sum(r.length for r in results)
total_bp = sum(r.boilerplate for r in results)
avg_time = sum(r.time_s for r in results) / len(results)
print(
f"\n Overall: {total_success}/{total} pages extracted "
f"({total_success / total * 100:.0f}%)"
)
print(f" Total chars: {total_chars:,}")
print(f" Total boilerplate hits: {total_bp}")
print(f" Avg time per page: {avg_time:.1f}s")
_print_category_summary(results, "Auto")
# Show failures
failures = [r for r in results if r.length <= 100]
if failures:
print(f"\n Failed pages ({len(failures)}):")
for r in failures:
print(
f" {r.category:<14} {r.page:<16} "
f"{r.length} chars {r.sample[:60]}"
)
# Show high-boilerplate pages (potential quality issues)
high_bp = [r for r in results if r.boilerplate >= 3 and r.length > 100]
if high_bp:
print(f"\n High boilerplate pages ({len(high_bp)}):")
for r in sorted(high_bp, key=lambda x: -x.boilerplate):
print(
f" {r.category:<14} {r.page:<16} "
f"bp={r.boilerplate} {r.length} chars"
)
# Soft assertion: at least 60% of pages should extract content
# (many sites have bot protection, geo-blocks, paywalls)
assert total_success >= total * 0.6, (
f"Too many failures: {total_success}/{total} "
f"({total_success / total * 100:.0f}%) — expected at least 60%"
)
# URLs that ContentFetcher should route to specialized downloaders
CONTENT_FETCHER_URLS = [
("ArXiv abs", "https://arxiv.org/abs/2301.07507", "arXiv"),
("ArXiv GPT-4", "https://arxiv.org/abs/2303.08774", "arXiv"),
("PubMed", "https://pubmed.ncbi.nlm.nih.gov/37828879/", "PubMed"),
("PubMed CRISPR", "https://pubmed.ncbi.nlm.nih.gov/26553966/", "PubMed"),
("PubMed COVID", "https://pubmed.ncbi.nlm.nih.gov/32015507/", "PubMed"),
("PMC", "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7095418/", "PMC"),
(
"SemScholar",
"https://www.semanticscholar.org/paper/Attention-Is-All-You-Need-Vaswani-Shazeer/204e3073870fae3d05bcbc2f6a8e263d9b72e776",
"Semantic Scholar",
),
("OpenAlex", "https://openalex.org/works/W2741809807", "OpenAlex"),
(
"bioRxiv",
"https://www.biorxiv.org/content/10.1101/2024.01.01.573841v1",
"bioRxiv",
),
(
"HTML page",
"https://en.wikipedia.org/wiki/Python_(programming_language)",
"Web Page",
),
]
class TestContentFetcherRouting:
"""Test that ContentFetcher routes academic URLs to specialized downloaders.
Verifies the new fetch_batch path used by FullSearchResults actually
works end-to-end with real URLs.
"""
def test_academic_urls_via_content_fetcher(self):
"""ContentFetcher.fetch_batch returns content for academic URLs."""
urls = [url for _, url, _ in CONTENT_FETCHER_URLS]
with ContentFetcher(timeout=30) as fetcher:
results = fetcher.fetch_batch(urls)
print(f"\n{'=' * 90}")
print(" CONTENT FETCHER ROUTING BENCHMARK")
print(f"{'=' * 90}")
failures = []
for label, url, expected_source in CONTENT_FETCHER_URLS:
content = results.get(url)
length = len(content) if content else 0
sample = (
content[:80].replace("\n", " ") if content else "(no content)"
)
# Check URL classification
info = fetcher.get_url_info(url)
detected = info["source_name"]
status = "✓" if length > 50 else "✗"
match = "✓" if detected == expected_source else "✗"
print(
f" {status} {label:<16} "
f"route={match} {detected:<18} "
f"{length:>7} chars {sample[:50]}"
)
if length <= 50:
failures.append(f"{label} ({detected}): {length} chars")
print(
f"\n Results: {len(CONTENT_FETCHER_URLS) - len(failures)}"
f"/{len(CONTENT_FETCHER_URLS)} URLs returned content"
)
if failures:
print(f" Failures: {', '.join(failures)}")
# At least 60% should work (some may be rate-limited or paywalled)
max_failures = len(CONTENT_FETCHER_URLS) * 0.4
assert len(failures) <= max_failures, (
f"ContentFetcher failed on too many URLs: "
f"{len(failures)}/{len(CONTENT_FETCHER_URLS)}: "
f"{', '.join(failures)}"
)