/tests/library/download_management/test_failure_classifier_extended.py

"""
Extended Tests for Failure Classifier System

Phase 16: Download Management Deep Coverage - Failure Classification Tests
Tests failure classification, domain-specific logic, and pattern matching.
"""

from datetime import datetime, timedelta, UTC

from local_deep_research.library.download_management.failure_classifier import (
    PermanentFailure,
    TemporaryFailure,
    RateLimitFailure,
    FailureClassifier,
)


class TestFailureClassification:
    """Tests for HTTP status code and error type classification"""

    def test_classify_http_404_not_found(self):
        """Test 404 status code is classified as permanent failure"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="http_error",
            status_code=404,
            url="https://example.com/resource.pdf",
        )

        assert isinstance(failure, PermanentFailure)
        assert failure.error_type == "not_found"
        assert "404" in failure.message
        assert failure.is_permanent() is True

    def test_classify_http_403_forbidden(self):
        """Test 403 status code is classified as permanent failure"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="http_error",
            status_code=403,
            url="https://example.com/forbidden.pdf",
        )

        assert isinstance(failure, PermanentFailure)
        assert failure.error_type == "forbidden"
        assert "403" in failure.message
        assert failure.is_permanent() is True

    def test_classify_http_429_rate_limit(self):
        """Test 429 status code is classified as rate limit failure"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="http_error",
            status_code=429,
            url="https://arxiv.org/pdf/2301.00001.pdf",
        )

        assert isinstance(failure, RateLimitFailure)
        assert failure.error_type == "rate_limited"
        assert failure.is_permanent() is False
        assert failure.domain == "arxiv.org"

    def test_classify_http_500_server_error(self):
        """Test 500 status code is classified as temporary (unknown error)"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="http_error",
            status_code=500,
            url="https://example.com/error.pdf",
        )

        # 500 doesn't have a specific handler, falls through to unknown
        assert isinstance(failure, TemporaryFailure)
        assert failure.is_permanent() is False

    def test_classify_http_503_service_unavailable(self):
        """Test 503 status code is classified as temporary server error"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="http_error",
            status_code=503,
            url="https://example.com/unavailable.pdf",
        )

        assert isinstance(failure, TemporaryFailure)
        assert failure.error_type == "server_error"
        assert "503" in failure.message
        assert failure.is_permanent() is False
        # Should have 1-hour cooldown
        assert failure.retry_after == timedelta(hours=1)

    def test_classify_timeout_error(self):
        """Test timeout errors are classified as temporary failures"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="TimeoutError",
            url="https://example.com/slow.pdf",
        )

        assert isinstance(failure, TemporaryFailure)
        assert failure.error_type == "timeout"
        assert failure.is_permanent() is False
        assert failure.retry_after == timedelta(minutes=30)

    def test_classify_connection_refused(self):
        """Test connection errors are classified as temporary failures"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="ConnectionError",
            url="https://example.com/unreachable.pdf",
            details="Connection refused",
        )

        assert isinstance(failure, TemporaryFailure)
        assert failure.error_type == "network_error"
        assert failure.is_permanent() is False
        assert failure.retry_after == timedelta(minutes=5)

    def test_classify_dns_resolution_failure(self):
        """Test DNS resolution failures are classified as network errors"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="NetworkError",
            url="https://nonexistent.invalid/file.pdf",
            details="DNS resolution failed",
        )

        assert isinstance(failure, TemporaryFailure)
        assert failure.error_type == "network_error"
        assert failure.is_permanent() is False

    def test_classify_ssl_certificate_error(self):
        """Test SSL errors fall to unknown temporary failure"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="SSLError",
            url="https://expired-cert.example.com/file.pdf",
            details="SSL certificate verification failed",
        )

        # SSL errors don't have specific handling, fall to unknown
        assert isinstance(failure, TemporaryFailure)
        assert failure.is_permanent() is False

    def test_classify_arxiv_recaptcha_detection(self):
        """Test arXiv reCAPTCHA detection results in 3-day cooldown"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="ArxivError",
            url="https://arxiv.org/pdf/2301.00001.pdf",
            details="reCAPTCHA protection detected",
        )

        assert isinstance(failure, TemporaryFailure)
        assert failure.error_type == "recaptcha_protection"
        assert failure.retry_after == timedelta(days=3)
        assert failure.is_permanent() is False

    def test_classify_incompatible_format_pdf(self):
        """Test incompatible PDF format is classified as permanent"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="ArxivError",
            url="https://arxiv.org/abs/2301.00001",
            details="not a PDF file",
        )

        assert isinstance(failure, PermanentFailure)
        assert failure.error_type == "incompatible_format"
        assert failure.is_permanent() is True

    def test_classify_incompatible_format_html(self):
        """Test HTML instead of PDF is classified as permanent"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="ArxivError",
            url="https://arxiv.org/abs/2301.00001",
            details="Received HTML content instead of PDF",
        )

        assert isinstance(failure, PermanentFailure)
        assert failure.error_type == "incompatible_format"
        assert failure.is_permanent() is True

    def test_classify_permanent_failure_detection(self):
        """Test 410 Gone is classified as permanent"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="http_error",
            status_code=410,
            url="https://example.com/removed.pdf",
        )

        assert isinstance(failure, PermanentFailure)
        assert failure.error_type == "gone"
        assert "410" in failure.message
        assert failure.is_permanent() is True

    def test_classify_temporary_failure_detection(self):
        """Test temporary failures are properly detected"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="temporary_issue",
            url="https://example.com/temp.pdf",
        )

        assert isinstance(failure, TemporaryFailure)
        assert failure.is_permanent() is False
        assert failure.retry_after is not None

    def test_classify_unknown_error_fallback(self):
        """Test unknown errors default to temporary 1-hour cooldown"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="WeirdUnknownError",
            url="https://example.com/mystery.pdf",
            details="Something unusual happened",
        )

        assert isinstance(failure, TemporaryFailure)
        assert failure.error_type == "unknown_error"
        assert failure.retry_after == timedelta(hours=1)
        assert failure.is_permanent() is False


class TestDomainSpecificLogic:
    """Tests for domain-specific cooldown and handling logic"""

    def test_domain_cooldown_arxiv(self):
        """Test arXiv-specific rate limit cooldown of 6 hours"""
        failure = RateLimitFailure("arxiv.org")

        assert failure.domain == "arxiv.org"
        assert failure.retry_after == timedelta(hours=6)
        assert "arxiv.org" in failure.message

    def test_domain_cooldown_semantic_scholar(self):
        """Test Semantic Scholar rate limit cooldown of 4 hours"""
        failure = RateLimitFailure("semanticscholar.org")

        assert failure.domain == "semanticscholar.org"
        assert failure.retry_after == timedelta(hours=4)

    def test_domain_cooldown_pubmed(self):
        """Test PubMed rate limit cooldown of 2 hours"""
        failure = RateLimitFailure("pubmed.ncbi.nlm.nih.gov")

        assert failure.domain == "pubmed.ncbi.nlm.nih.gov"
        assert failure.retry_after == timedelta(hours=2)

    def test_domain_cooldown_ieee(self):
        """Test IEEE (unknown domain) gets default 1-hour cooldown"""
        failure = RateLimitFailure("ieeexplore.ieee.org")

        assert failure.domain == "ieeexplore.ieee.org"
        # IEEE not in specific domains, falls to default
        assert failure.retry_after == timedelta(hours=1)

    def test_domain_cooldown_springer(self):
        """Test Springer (unknown domain) gets default cooldown"""
        failure = RateLimitFailure("link.springer.com")

        assert failure.domain == "link.springer.com"
        assert failure.retry_after == timedelta(hours=1)

    def test_domain_cooldown_default(self):
        """Test unknown domains get default 1-hour cooldown"""
        failure = RateLimitFailure("unknown-domain.com")

        assert failure.domain == "unknown-domain.com"
        assert failure.retry_after == timedelta(hours=1)

    def test_rate_limit_failure_initialization(self):
        """Test RateLimitFailure initializes with proper attributes"""
        failure = RateLimitFailure("biorxiv.org", "Too many requests")

        assert failure.error_type == "rate_limited"
        assert failure.domain == "biorxiv.org"
        assert "biorxiv.org" in failure.message
        assert "Too many requests" in failure.message
        assert failure.retry_after == timedelta(hours=6)

    def test_pattern_matching_null_values(self):
        """Test classifier handles None/null values gracefully"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="",
            status_code=None,
            url="",
            details="",
        )

        # Should default to unknown temporary failure
        assert isinstance(failure, TemporaryFailure)
        assert failure.is_permanent() is False

    def test_pattern_matching_mixed_patterns(self):
        """Test classifier with mixed patterns in details"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="arxiv_download_error",
            url="https://arxiv.org/pdf/test.pdf",
            details="Captcha required - please solve the reCAPTCHA",
        )

        assert isinstance(failure, TemporaryFailure)
        assert failure.error_type == "recaptcha_protection"
        assert failure.retry_after == timedelta(days=3)

    def test_custom_domain_handling(self):
        """Test custom domain with details in RateLimitFailure"""
        failure = RateLimitFailure("researchgate.net", "Account rate limited")

        assert failure.domain == "researchgate.net"
        assert failure.retry_after == timedelta(hours=12)
        assert "Account rate limited" in failure.message

    def test_failure_message_extraction(self):
        """Test failure message is properly set"""
        failure = PermanentFailure("test_error", "This is a test message")

        assert failure.message == "This is a test message"
        assert failure.error_type == "test_error"

    def test_failure_type_enumeration(self):
        """Test various failure types can be created"""
        # Permanent types
        perm1 = PermanentFailure("not_found", "Not found")
        perm2 = PermanentFailure("forbidden", "Forbidden")
        perm3 = PermanentFailure("gone", "Gone")
        perm4 = PermanentFailure("incompatible_format", "Wrong format")

        assert all(f.is_permanent() for f in [perm1, perm2, perm3, perm4])

        # Temporary types
        temp1 = TemporaryFailure("timeout", "Timeout", timedelta(minutes=30))
        temp2 = TemporaryFailure(
            "network_error", "Network", timedelta(minutes=5)
        )

        assert not any(f.is_permanent() for f in [temp1, temp2])

    def test_retry_eligible_classification(self):
        """Test correct identification of retry-eligible failures"""
        # audit: PUNCHLIST reviewed 2026-05 — issue resolved by prior PR (recommendation: use_zero_cooldown_instead).
        temp_failure = TemporaryFailure(
            "temp", "Temporary", timedelta(seconds=1)
        )
        perm_failure = PermanentFailure("perm", "Permanent")

        # Permanent should always be permanent; temporary should not
        assert temp_failure.is_permanent() is False
        assert perm_failure.is_permanent() is True

    def test_permanent_failure_reasons(self):
        """Test permanent failure has expected attributes"""
        failure = PermanentFailure("test_permanent", "Test permanent failure")

        assert failure.retry_after is None
        assert failure.is_permanent() is True


class TestBaseFailureMethods:
    """Tests for BaseFailure base class methods"""

    def test_is_permanent_for_permanent_failure(self):
        """Test is_permanent returns True for PermanentFailure"""
        failure = PermanentFailure("test", "Test")
        assert failure.is_permanent() is True

    def test_is_permanent_for_temporary_failure(self):
        """Test is_permanent returns False for TemporaryFailure"""
        failure = TemporaryFailure("test", "Test", timedelta(hours=1))
        assert failure.is_permanent() is False

    def test_created_at_timestamp(self):
        """Test created_at is set to current UTC time"""
        before = datetime.now(UTC)
        failure = TemporaryFailure("test", "Test", timedelta(hours=1))
        after = datetime.now(UTC)

        assert before <= failure.created_at <= after


class TestClassifyFromException:
    """Tests for classify_from_exception method"""

    def test_classify_from_timeout_exception(self):
        """Test classification from TimeoutError exception"""
        classifier = FailureClassifier()
        exc = TimeoutError("Connection timed out")

        failure = classifier.classify_from_exception(exc, "https://example.com")

        assert isinstance(failure, TemporaryFailure)
        assert failure.error_type == "timeout"

    def test_classify_from_connection_exception(self):
        """Test classification from ConnectionError exception"""
        classifier = FailureClassifier()
        exc = ConnectionError("Connection refused")

        failure = classifier.classify_from_exception(exc, "https://example.com")

        assert isinstance(failure, TemporaryFailure)
        assert failure.error_type == "network_error"

    def test_classify_from_generic_exception(self):
        """Test classification from generic Exception"""
        classifier = FailureClassifier()
        exc = Exception("Something went wrong")

        failure = classifier.classify_from_exception(exc, "https://example.com")

        assert isinstance(failure, TemporaryFailure)
        assert failure.is_permanent() is False

    def test_classify_from_exception_with_empty_url(self):
        """Test classify_from_exception with empty URL"""
        classifier = FailureClassifier()
        exc = ValueError("Invalid data")

        failure = classifier.classify_from_exception(exc, "")

        assert isinstance(failure, TemporaryFailure)
        assert failure.is_permanent() is False

    def test_classify_from_exception_extracts_message(self):
        """Test exception message is included in details"""
        classifier = FailureClassifier()
        exc = RuntimeError("Specific runtime error message")

        failure = classifier.classify_from_exception(exc, "https://example.com")

        # The exception message should influence classification
        assert failure.message is not None


class TestEdgeCases:
    """Tests for edge cases and boundary conditions"""

    def test_case_insensitive_error_matching(self):
        """Test error matching is case-insensitive"""
        classifier = FailureClassifier()

        # Test uppercase TIMEOUT
        failure1 = classifier.classify_failure(
            error_type="TIMEOUT",
            url="https://example.com",
        )

        # Test mixed case TimeOut
        failure2 = classifier.classify_failure(
            error_type="TimeOut",
            url="https://example.com",
        )

        assert failure1.error_type == "timeout"
        assert failure2.error_type == "timeout"

    def test_arxiv_pattern_in_details(self):
        """Test arXiv detection from details field"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="download_error",
            url="https://example.com",
            details="Error from arxiv - captcha detected",
        )

        assert failure.error_type == "recaptcha_protection"

    def test_url_domain_extraction_for_rate_limit(self):
        """Test domain is correctly extracted from URL for rate limits"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="http_error",
            status_code=429,
            url="https://api.semanticscholar.org/v1/paper/123",
        )

        assert isinstance(failure, RateLimitFailure)
        assert failure.domain == "api.semanticscholar.org"

    def test_url_with_port_number(self):
        """Test URL with port number is handled"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="http_error",
            status_code=429,
            url="https://localhost:8080/api/download",
        )

        assert isinstance(failure, RateLimitFailure)
        assert failure.domain == "localhost:8080"

    def test_malformed_url_handling(self):
        """Test malformed URLs don't cause crashes"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="http_error",
            status_code=429,
            url="not-a-valid-url",
        )

        # Should still create a failure object
        assert isinstance(failure, RateLimitFailure)

    def test_very_long_details_string(self):
        """Test handling of very long details strings"""
        classifier = FailureClassifier()
        long_details = "x" * 10000

        failure = classifier.classify_failure(
            error_type="error",
            url="https://example.com",
            details=long_details,
        )

        assert isinstance(failure, TemporaryFailure)

    def test_unicode_in_error_message(self):
        """Test unicode characters in error messages"""
        classifier = FailureClassifier()
        failure = classifier.classify_failure(
            error_type="error",
            url="https://example.com",
            details="Error: \u4e2d\u6587\u6d4b\u8bd5 (Chinese test)",
        )

        assert isinstance(failure, TemporaryFailure)

    def test_html_pdf_exclusion(self):
        """Test HTML is flagged but application/pdf is allowed"""
        classifier = FailureClassifier()

        # This should NOT be classified as incompatible (has application/pdf)
        failure1 = classifier.classify_failure(
            error_type="ArxivError",
            url="https://arxiv.org/pdf/test.pdf",
            details="Content-Type: html with application/pdf fallback",
        )

        # failure1 should not be permanent incompatible format
        assert (
            failure1.error_type != "incompatible_format"
            or not failure1.is_permanent()
        )

        # This SHOULD be classified as incompatible (no application/pdf)
        failure2 = classifier.classify_failure(
            error_type="ArxivError",
            url="https://arxiv.org/abs/test",
            details="Received HTML instead of expected format",
        )

        assert isinstance(failure2, PermanentFailure)
        assert failure2.error_type == "incompatible_format"