""" Comprehensive edge case tests for citation formatter to ensure robust handling of various citation scenarios and malformed inputs. """ from local_deep_research.text_optimization.citation_formatter import ( CitationFormatter, CitationMode, ) class TestCitationEdgeCases: """Test edge cases and potential bugs in citation formatting.""" def test_multiple_citations_same_sentence(self): """Test multiple citations in the same sentence.""" content = """# Research Report Recent advances in AI [1] [2] have shown that transformers [3] [4] [5] are effective. ## Sources [1] Attention Is All You Need URL: https://arxiv.org/abs/1706.03762 [2] BERT Paper URL: https://arxiv.org/abs/1810.04805 [3] GPT-3 Paper URL: https://openai.com/research/gpt-3 [4] T5 Paper URL: https://arxiv.org/abs/1910.10683 [5] RoBERTa Paper URL: https://arxiv.org/abs/1907.11692""" formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) # Should have correct number of citations assert result.count("[arxiv.org]") == 4 # sources 1, 2, 4, 5 assert result.count("[openai.com]") == 1 # source 3 # Original numbers should not appear in body body_text = result.split("## Sources")[0] for i in range(1, 6): assert f"[{i}]" not in body_text def test_citations_with_punctuation(self): """Test citations followed by various punctuation marks.""" content = """# Research Report This is proven [1]. Also shown [2], and demonstrated [3]; furthermore [4]: see [5]! ## Sources [1] First Paper URL: https://example.com/1 [2] Second Paper URL: https://example.com/2 [3] Third Paper URL: https://example.com/3 [4] Fourth Paper URL: https://example.com/4 [5] Fifth Paper URL: https://example.com/5""" formatter = CitationFormatter(mode=CitationMode.DOMAIN_ID_HYPERLINKS) result = formatter.format_document(content) # All citations should be properly formatted assert "[[example.com-1]](https://example.com/1)." in result assert "[[example.com-2]](https://example.com/2)," in result assert "[[example.com-3]](https://example.com/3);" in result assert "[[example.com-4]](https://example.com/4):" in result assert "[[example.com-5]](https://example.com/5)!" in result def test_missing_url_in_source(self): """Test handling of sources without URLs.""" content = """# Research Report This cites a book [1] and a paper [2]. ## Sources [1] Deep Learning Book by Goodfellow et al. [2] Neural Networks Paper URL: https://nature.com/article""" formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) # Source with URL should be formatted assert "[nature.com]" in result # Source without URL should remain as-is assert "[1]" in result assert "Deep Learning Book" in result def test_malformed_source_section(self): """Test handling of malformed source sections.""" content = """# Research Report Citation here [1] and here [2]. ## Sources [1] Paper One Some description URL: https://arxiv.org/abs/1234.5678 [2] Paper Two URL: https://nature.com/123 Additional text [3] Not cited in text URL: https://unused.com/paper""" formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) # The formatter's regex only captures URLs that are on the immediate next line # after the title line, starting with "URL:" # Source [1] has extra description line, so URL is not captured assert "[1]" in result # Citation remains unchanged assert "[arxiv.org]" not in result # Source [2] has inline URL which won't be parsed as a URL assert "[2]" in result # Remains unchanged assert "[nature.com]" not in result # Source [3] is properly formatted and would work if cited body = result.split("## Sources")[0] assert "[3]" not in body # Not cited in text def test_duplicate_urls_different_sources(self): """Test handling of multiple sources with the same URL.""" content = """# Research Report First reference [1] and second reference [2] to same paper. ## Sources [1] Original Paper Title URL: https://arxiv.org/abs/1706.03762 [2] Same Paper Different Title URL: https://arxiv.org/abs/1706.03762""" formatter = CitationFormatter( mode=CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS ) result = formatter.format_document(content) # Both should be formatted with IDs assert "[arxiv.org-1]" in result assert "[arxiv.org-2]" in result def test_citations_in_headers_and_lists(self): """Test citations in markdown headers and lists.""" content = """# Research Report [1] ## Introduction with Citation [2] - First point [3] - Second point with reference [4] - Nested item [5] ### Subsection [6] ## Sources [1] Header Paper URL: https://example.com/1 [2] Intro Paper URL: https://example.com/2 [3] List Item One URL: https://example.com/3 [4] List Item Two URL: https://example.com/4 [5] Nested Paper URL: https://example.com/5 [6] Subsection Paper URL: https://example.com/6""" formatter = CitationFormatter(mode=CitationMode.DOMAIN_ID_HYPERLINKS) result = formatter.format_document(content) # All citations should be formatted, even in headers and lists for i in range(1, 7): assert f"[example.com-{i}]" in result assert f"[{i}]" not in result.split("## Sources")[0] def test_citations_in_code_blocks(self): """Test that citations inside code blocks are not formatted.""" content = """# Research Report This is a citation [1]. ```python # This [1] should not be formatted def cite(ref): return f"[{ref}]" # [2] in string ``` Another citation [2]. ## Sources [1] First Paper URL: https://arxiv.org/abs/1111.1111 [2] Second Paper URL: https://nature.com/2222""" formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) # Citations outside code blocks should be formatted assert "[[arxiv.org]](https://arxiv.org/abs/1111.1111)" in result assert "[[nature.com]](https://nature.com/2222)" in result # Current implementation formats citations even in code blocks # This is a known limitation - code blocks are not preserved assert ( "# This [[arxiv.org]](https://arxiv.org/abs/1111.1111) should not be formatted" in result ) assert ( 'return f"[{ref}]" # [[nature.com]](https://nature.com/2222) in string' in result ) def test_non_sequential_citations(self): """Test non-sequential citation numbers.""" content = """# Research Report Citations [5] and [1] and [10]. ## Sources [1] First Paper URL: https://first.com/paper [5] Fifth Paper URL: https://fifth.com/paper [10] Tenth Paper URL: https://tenth.com/paper""" formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) # All citations should be formatted correctly assert "[first.com]" in result assert "[fifth.com]" in result assert "[tenth.com]" in result # Original numbers should not remain body = result.split("## Sources")[0] assert "[1]" not in body assert "[5]" not in body assert "[10]" not in body def test_citations_with_special_urls(self): """Test citations with special characters in URLs.""" content = """# Research Report Various sources [1] [2] [3] [4]. ## Sources [1] DOI Paper URL: https://doi.org/10.1234/journal.2023.12345 [2] Paper with Query URL: https://example.com/search?q=machine+learning&filter=2023 [3] Paper with Fragment URL: https://example.com/paper#section-3.2 [4] Paper with Port URL: https://repository.edu:8080/papers/123""" formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) # All domains should be extracted correctly assert "[doi.org]" in result assert "[example.com]" in result # Port numbers are included in domain extraction assert "[repository.edu:8080]" in result def test_empty_and_whitespace_handling(self): """Test handling of empty content and various whitespace.""" # Empty content formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS) assert formatter.format_document("") == "" # Only whitespace assert formatter.format_document(" \n\n \t") == " \n\n \t" # No sources section content = "This has a citation [1] but no sources." assert formatter.format_document(content) == content def test_case_sensitivity_in_sources_header(self): """Test various cases for Sources section header.""" test_cases = [ "## Sources", "## SOURCES", "## sources", "## Sources:", "##Sources", "## References", # Should not match ] for header in test_cases[:4]: content = f"""Citation [1]. {header} [1] Paper URL: https://example.com/1""" formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) if "References" not in header: assert "[example.com]" in result else: assert "[1]" in result # Should remain unchanged def test_unicode_and_international_domains(self): """Test handling of unicode and international domains.""" content = """# Research Report Chinese research [1], Russian study [2], and emoji domain [3]. ## Sources [1] 中文论文 URL: https://研究.中国/paper [2] Русская статья URL: https://наука.рф/article [3] Modern Paper URL: https://👍.ws/research""" formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) # Should handle international domains assert "[研究.中国]" in result or "[xn--" in result # Punycode assert "[наука.рф]" in result or "[xn--" in result assert "[👍.ws]" in result or "[xn--" in result def test_very_long_urls(self): """Test handling of very long URLs.""" long_path = "a" * 500 content = f"""# Research Report Long URL citation [1]. ## Sources [1] Paper with Long URL URL: https://example.com/{long_path}""" formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) # Should handle long URLs without issues assert "[example.com]" in result assert long_path in result # URL should be preserved in sources def test_multiple_source_sections(self): """Test document with multiple sections that look like sources.""" content = """# Research Report Main citation [1]. ## Sources [1] Real Source URL: https://real.com/paper ## Other Sources (not formatted) [2] This should not be formatted URL: https://notformatted.com Back to text [2]. ## Sources Again This section also has [3] which shouldn't format.""" formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) # Only first Sources section should be used assert "[real.com]" in result assert "[2]" in result # Should remain as-is assert "[notformatted.com]" not in result