import glob import os from typing import Dict, List, Optional, Tuple from tqdm import tqdm from olmocr.data.renderpdf import render_pdf_to_base64webp from .tests import BasePDFTest def _filter_by_max_reports( test_results_by_candidate: Dict[str, Dict[str, Dict[int, List[Tuple[BasePDFTest, bool, str]]]]], test_to_jsonl: Dict[str, str], max_reports: int, ) -> Dict[str, Dict[str, Dict[int, List[Tuple[BasePDFTest, bool, str]]]]]: """Filter test results to include at most max_reports unique PDFs per .jsonl file.""" filtered = {} for candidate, pdf_results in test_results_by_candidate.items(): # Track which unique PDFs we've included per jsonl file jsonl_pdfs: Dict[str, set] = {} filtered_pdfs: Dict[str, Dict[int, List[Tuple[BasePDFTest, bool, str]]]] = {} for pdf_name in sorted(pdf_results.keys()): pages = pdf_results[pdf_name] for page in sorted(pages.keys()): for test_tuple in pages[page]: test, passed, explanation = test_tuple jsonl_file = test_to_jsonl.get(test.id, "unknown") jsonl_pdfs.setdefault(jsonl_file, set()) # If this PDF is new for this jsonl file, check the limit if pdf_name not in jsonl_pdfs[jsonl_file]: if len(jsonl_pdfs[jsonl_file]) >= max_reports: continue jsonl_pdfs[jsonl_file].add(pdf_name) filtered_pdfs.setdefault(pdf_name, {}).setdefault(page, []).append(test_tuple) filtered[candidate] = filtered_pdfs return filtered def generate_html_report( test_results_by_candidate: Dict[str, Dict[str, Dict[int, List[Tuple[BasePDFTest, bool, str]]]]], pdf_folder: str, output_file: str, max_reports: Optional[int] = None, test_to_jsonl: Optional[Dict[str, str]] = None, ) -> None: """ Generate a simple static HTML report of test results. Args: test_results_by_candidate: Dictionary mapping candidate name to dictionary mapping PDF name to dictionary mapping page number to list of (test, passed, explanation) tuples. pdf_folder: Path to the folder containing PDF files. output_file: Path to the output HTML file. max_reports: If set, limit to at most N tests per .jsonl file in the report. test_to_jsonl: Dictionary mapping test IDs to their source jsonl filenames. """ # If max_reports is set, filter test_results_by_candidate to limit per jsonl file if max_reports is not None and test_to_jsonl is not None: test_results_by_candidate = _filter_by_max_reports(test_results_by_candidate, test_to_jsonl, max_reports) candidates = list(test_results_by_candidate.keys()) # Create HTML report html = """ OLMOCR Bench Test Report

OLMOCR Bench Test Report

""" # Process all candidates print("Generating test report...") for candidate in candidates: html += f"

Candidate: {candidate}

\n" # Get all PDFs for this candidate all_pdfs = sorted(test_results_by_candidate[candidate].keys()) for pdf_name in tqdm(all_pdfs, desc=f"Processing {candidate}"): pages = sorted(test_results_by_candidate[candidate][pdf_name].keys()) for page in pages: # Get tests for this PDF page tests = test_results_by_candidate[candidate][pdf_name][page] for test, passed, explanation in tests: result_class = "pass" if passed else "fail" status_text = "PASSED" if passed else "FAILED" status_class = "pass-status" if passed else "fail-status" # Begin test block html += f"""

Test ID: {test.id} {status_text}

PDF: {pdf_name} | Page: {page} | Type: {test.type}

""" # Add test details based on type test_type = getattr(test, "type", "").lower() if test_type == "present" and hasattr(test, "text"): text = getattr(test, "text", "") html += f"""

Text to find: "{text}"

\n""" elif test_type == "absent" and hasattr(test, "text"): text = getattr(test, "text", "") html += f"""

Text should not appear: "{text}"

\n""" elif test_type == "order" and hasattr(test, "before") and hasattr(test, "after"): before = getattr(test, "before", "") after = getattr(test, "after", "") html += f"""

Text order: "{before}" should appear before "{after}"

\n""" elif test_type == "table": if hasattr(test, "cell"): cell = getattr(test, "cell", "") html += f"""

Table cell: "{cell}"

\n""" if hasattr(test, "up") and getattr(test, "up", None): up = getattr(test, "up") html += f"""

Above: "{up}"

\n""" if hasattr(test, "down") and getattr(test, "down", None): down = getattr(test, "down") html += f"""

Below: "{down}"

\n""" if hasattr(test, "left") and getattr(test, "left", None): left = getattr(test, "left") html += f"""

Left: "{left}"

\n""" if hasattr(test, "right") and getattr(test, "right", None): right = getattr(test, "right") html += f"""

Right: "{right}"

\n""" elif test_type == "math" and hasattr(test, "math"): math = getattr(test, "math", "") html += f"""

Math equation: {math}

\n""" elif test_type == "format" and hasattr(test, "text"): text = getattr(test, "text", "") fmt = getattr(test, "format", "") html += f"""

Text: "{text}" should be formatted as {fmt}

\n""" elif test_type == "footnote" and hasattr(test, "marker"): marker = getattr(test, "marker", "") html += f"""

Footnote marker: {marker}

\n""" before = getattr(test, "appears_before_marker", None) after = getattr(test, "appears_after_marker", None) if before: html += f"""

Text before marker: "{before}"

\n""" if after: html += f"""

Text after marker: "{after}"

\n""" elif test_type == "baseline": max_length = getattr(test, "max_length", None) if max_length is not None: html += f"""

Baseline check: max length {max_length} (blank page check)

\n""" else: html += f"""

Baseline check: non-blank, no repeats, valid characters

\n""" html += """
\n""" # Add explanation for failed tests if not passed: html += f"""
Explanation: {explanation}
\n""" # Render PDF page pdf_path = os.path.join(pdf_folder, pdf_name) try: html += """

PDF Render:

\n""" image_data = render_pdf_to_base64webp(pdf_path, page, 1024) html += f""" PDF Page {page}\n""" except Exception as e: html += f"""

Error rendering PDF: {str(e)}

\n""" # Get the Markdown content for this page md_content = None try: md_base = os.path.splitext(pdf_name)[0] md_files = list(glob.glob(os.path.join(os.path.dirname(pdf_folder), candidate, f"{md_base}_pg{page}_repeat*.md"))) if md_files: md_file_path = md_files[0] # Use the first repeat as an example with open(md_file_path, "r", encoding="utf-8") as f: md_content = f.read() except Exception as e: md_content = f"Error loading Markdown content: {str(e)}" if md_content: html += """

Markdown Content:

\n""" html += f"""
{md_content}
\n""" # End test block html += """
\n""" # Add separator between pages html += """
\n""" # Close HTML html += """ """ with open(output_file, "w", encoding="utf-8") as f: f.write(html) print(f"Simple HTML report generated: {output_file}")