/tests/journal_quality/test_downloader.py

"""Tests for journal_data_downloader — fetch from OpenAlex/DOAJ APIs."""

import gzip
import json
from unittest.mock import patch

import pytest

from local_deep_research.journal_quality.downloader import (
    JOURNAL_DATA_VERSION,
    download_journal_data,
    ensure_journal_data,
    get_journal_data_status,
)


@pytest.fixture
def tmp_data_dir(tmp_path):
    """Override journal data directory to a temp path."""
    with patch(
        "local_deep_research.journal_quality.downloader._get_data_dir",
        return_value=tmp_path,
    ):
        yield tmp_path


def _write_openalex(data_dir):
    """Create a minimal openalex file."""
    with gzip.open(data_dir / "openalex_sources.json.gz", "wt") as f:
        json.dump({"s": {"S1": {"n": "Nature", "t": "j", "h": 1000}}}, f)


class TestGetJournalDataStatus:
    def test_no_files(self, tmp_data_dir):
        status = get_journal_data_status()
        assert status["available"] is False
        assert status["version"] is None
        assert status["needs_update"] is True

    def test_files_present(self, tmp_data_dir):
        _write_openalex(tmp_data_dir)
        (tmp_data_dir / "doaj_journals.json").write_text("{}")
        (tmp_data_dir / "version.json").write_text(
            json.dumps({"version": JOURNAL_DATA_VERSION})
        )
        status = get_journal_data_status()
        assert status["available"] is True
        assert status["needs_update"] is False

    def test_outdated_version(self, tmp_data_dir):
        _write_openalex(tmp_data_dir)
        (tmp_data_dir / "version.json").write_text(
            json.dumps({"version": "v0"})
        )
        status = get_journal_data_status()
        assert status["needs_update"] is True


class TestDownloadJournalData:
    def test_already_up_to_date(self, tmp_data_dir):
        _write_openalex(tmp_data_dir)
        (tmp_data_dir / "doaj_journals.json").write_text("{}")
        (tmp_data_dir / "version.json").write_text(
            json.dumps({"version": JOURNAL_DATA_VERSION})
        )
        success, msg = download_journal_data()
        assert success is True
        assert "up to date" in msg

    def test_concurrent_download_blocked(self, tmp_data_dir):
        # Simulate an in-flight download by writing the *current* PID
        # into the sentinel. The runtime liveness check sees our own
        # PID → returns alive → download_journal_data bows out without
        # reclaiming (the "don't nuke self" guard).
        #
        # An empty sentinel (old behavior) would be treated as orphan
        # debris, reclaimed, and the test would fall through to real
        # network calls — which is exactly what the PID stamp prevents.
        import os

        sentinel = tmp_data_dir / ".downloading"
        sentinel.write_text(str(os.getpid()))
        success, msg = download_journal_data()
        assert success is False
        assert "already in progress" in msg

    @patch("local_deep_research.journal_quality.downloader._fetch_institutions")
    @patch(
        "local_deep_research.journal_quality.downloader"
        "._fetch_jabref_abbreviations"
    )
    @patch("local_deep_research.journal_quality.downloader._fetch_predatory")
    @patch(
        "local_deep_research.journal_quality.downloader._fetch_openalex_sources"
    )
    @patch(
        "local_deep_research.journal_quality.downloader._fetch_doaj_journals"
    )
    @patch("local_deep_research.journal_quality.db.build_db")
    def test_successful_fetch(
        self,
        mock_build_db,
        mock_doaj,
        mock_openalex,
        mock_pred,
        mock_jabref,
        mock_inst,
        tmp_data_dir,
    ):
        mock_openalex.return_value = 100
        mock_doaj.return_value = 50
        mock_pred.return_value = 0
        mock_jabref.return_value = 0
        mock_inst.return_value = 0
        # build_db runs after all fetches succeed; mock it so the happy-path
        # test doesn't depend on a real build against the minimal fixture.
        mock_build_db.return_value = None
        # Need the openalex file to exist after mock runs
        _write_openalex(tmp_data_dir)

        success, msg = download_journal_data(force=True)
        assert success is True
        assert "100 OpenAlex" in msg
        assert "50 DOAJ" in msg
        mock_openalex.assert_called_once()
        mock_doaj.assert_called_once()

    @patch("local_deep_research.journal_quality.downloader._fetch_institutions")
    @patch(
        "local_deep_research.journal_quality.downloader"
        "._fetch_jabref_abbreviations"
    )
    @patch("local_deep_research.journal_quality.downloader._fetch_predatory")
    @patch(
        "local_deep_research.journal_quality.downloader._fetch_openalex_sources"
    )
    @patch(
        "local_deep_research.journal_quality.downloader._fetch_doaj_journals"
    )
    def test_openalex_failure(
        self,
        mock_doaj,
        mock_openalex,
        mock_pred,
        mock_jabref,
        mock_inst,
        tmp_data_dir,
    ):
        """If the required OpenAlex source fails, the whole download
        returns failure — even though the other four sources now run
        in parallel and may succeed.
        """
        mock_openalex.side_effect = Exception("Network error")
        # Keep optional sources cheap so the parallel pool finishes
        # quickly; we're only exercising the required-source-failure
        # path.
        mock_doaj.return_value = 0
        mock_pred.return_value = 0
        mock_jabref.return_value = 0
        mock_inst.return_value = 0
        success, msg = download_journal_data(force=True)
        assert success is False
        assert "Failed" in msg

    @patch("local_deep_research.journal_quality.downloader._fetch_institutions")
    @patch(
        "local_deep_research.journal_quality.downloader"
        "._fetch_jabref_abbreviations"
    )
    @patch("local_deep_research.journal_quality.downloader._fetch_predatory")
    @patch(
        "local_deep_research.journal_quality.downloader._fetch_openalex_sources"
    )
    @patch(
        "local_deep_research.journal_quality.downloader._fetch_doaj_journals"
    )
    @patch("local_deep_research.journal_quality.db.build_db")
    def test_build_db_failure_returns_false(
        self,
        mock_build_db,
        mock_doaj,
        mock_openalex,
        mock_pred,
        mock_jabref,
        mock_inst,
        tmp_data_dir,
    ):
        """If fetches succeed but build_db raises, caller must see success=False.

        Previously this case silently returned (True, 'Fetched ...') while the
        DB was never built, leading the dashboard to show a green toast with
        no DB on disk.
        """
        mock_openalex.return_value = 100
        mock_doaj.return_value = 50
        mock_pred.return_value = 0
        mock_jabref.return_value = 0
        mock_inst.return_value = 0
        mock_build_db.side_effect = RuntimeError("synthetic build failure")
        _write_openalex(tmp_data_dir)

        success, msg = download_journal_data(force=True)
        assert success is False
        assert "DB build failed" in msg
        # Exception message is not surfaced to callers (info-disclosure hardening
        # in commit da803376d); only the exception class name is included.
        assert "RuntimeError" in msg

    @patch("local_deep_research.journal_quality.downloader._fetch_institutions")
    @patch(
        "local_deep_research.journal_quality.downloader"
        "._fetch_jabref_abbreviations"
    )
    @patch("local_deep_research.journal_quality.downloader._fetch_predatory")
    @patch(
        "local_deep_research.journal_quality.downloader._fetch_openalex_sources"
    )
    @patch(
        "local_deep_research.journal_quality.downloader._fetch_doaj_journals"
    )
    @patch("local_deep_research.journal_quality.db.build_db")
    def test_build_db_schema_drift_error_surfaced(
        self,
        mock_build_db,
        mock_doaj,
        mock_openalex,
        mock_pred,
        mock_jabref,
        mock_inst,
        tmp_data_dir,
    ):
        """SchemaDriftError messages are developer-authored and safe to
        surface verbatim, so operators can see *which* upstream field
        drifted. Only the generic-exception path gets the class-name
        scrubbing; drift-errors pass through.
        """
        from local_deep_research.journal_quality.data_sources.openalex import (
            SchemaDriftError,
        )

        mock_openalex.return_value = 100
        mock_doaj.return_value = 50
        mock_pred.return_value = 0
        mock_jabref.return_value = 0
        mock_inst.return_value = 0
        drift_msg = (
            "OpenAlex snapshot appears to have renamed a required field: "
            "h_index present in journal sample=False"
        )
        mock_build_db.side_effect = SchemaDriftError(drift_msg)
        _write_openalex(tmp_data_dir)

        success, msg = download_journal_data(force=True)
        assert success is False
        assert "DB build failed" in msg
        # Full drift message is surfaced — operators need to see which
        # field drifted to act on it.
        assert "h_index present in journal sample=False" in msg
        # The class name path is only used for non-SchemaDriftError.
        assert "SchemaDriftError" not in msg


class TestDownloadStateCountsLifecycle:
    """Verify `_download_state["counts"]` is populated only on success and
    cleared on every other return path, so the `/api/journal-data/download`
    route (and any other reader of `get_download_state()`) cannot observe
    stale counts from a prior run.
    """

    def _get_counts(self):
        from local_deep_research.journal_quality.downloader import (
            get_download_state,
        )

        return get_download_state().get("counts")

    def test_counts_none_on_already_up_to_date(self, tmp_data_dir):
        _write_openalex(tmp_data_dir)
        (tmp_data_dir / "doaj_journals.json").write_text("{}")
        (tmp_data_dir / "version.json").write_text(
            json.dumps({"version": JOURNAL_DATA_VERSION})
        )
        success, _ = download_journal_data()
        assert success is True
        assert self._get_counts() is None

    def test_counts_none_on_disk_space_failure(self, tmp_data_dir):
        class _LowDisk:
            total = 10 * 1024**3
            used = total - 500 * 1024**2
            free = 500 * 1024**2

        with patch("shutil.disk_usage", return_value=_LowDisk()):
            success, _ = download_journal_data(force=True)
        assert success is False
        assert self._get_counts() is None

    def test_counts_none_on_concurrent_download_block(self, tmp_data_dir):
        import os

        (tmp_data_dir / ".downloading").write_text(str(os.getpid()))
        success, _ = download_journal_data()
        assert success is False
        assert self._get_counts() is None

    @patch("local_deep_research.journal_quality.downloader._fetch_institutions")
    @patch(
        "local_deep_research.journal_quality.downloader"
        "._fetch_jabref_abbreviations"
    )
    @patch("local_deep_research.journal_quality.downloader._fetch_predatory")
    @patch(
        "local_deep_research.journal_quality.downloader._fetch_openalex_sources"
    )
    @patch(
        "local_deep_research.journal_quality.downloader._fetch_doaj_journals"
    )
    @patch("local_deep_research.journal_quality.db.build_db")
    def test_counts_populated_on_success_then_cleared_on_up_to_date(
        self,
        mock_build_db,
        mock_doaj,
        mock_openalex,
        mock_pred,
        mock_jabref,
        mock_inst,
        tmp_data_dir,
    ):
        """The key anti-stale-leak scenario: a successful download
        populates counts, then a subsequent "already up to date" call
        must invalidate them so the route cannot render last-run's
        numbers as if they came from the new (non-existent) fetch.
        """
        mock_openalex.return_value = 100
        mock_doaj.return_value = 50
        mock_pred.return_value = 0
        mock_jabref.return_value = 0
        mock_inst.return_value = 0
        mock_build_db.return_value = None
        _write_openalex(tmp_data_dir)

        success, _ = download_journal_data(force=True)
        assert success is True
        counts = self._get_counts()
        assert counts is not None
        assert counts["openalex"] == 100
        assert counts["doaj"] == 50

        # Stamp the version.json that the real success path would have
        # written, so the next call takes the "already up to date" branch.
        (tmp_data_dir / "doaj_journals.json").write_text("{}")
        (tmp_data_dir / "version.json").write_text(
            json.dumps({"version": JOURNAL_DATA_VERSION})
        )
        success, _ = download_journal_data()
        assert success is True
        assert self._get_counts() is None


class TestEnsureJournalData:
    def test_files_in_user_dir(self, tmp_data_dir):
        _write_openalex(tmp_data_dir)
        data_dir, available = ensure_journal_data(auto_download=False)
        assert available is True
        assert data_dir == tmp_data_dir

    def test_missing_no_download(self, tmp_data_dir):
        # The user data dir (tmp_data_dir) has no files.
        # Also need to prevent the package dir fallback from finding
        # bundled files in a dev install.
        # Verify status reports unavailable when user dir is empty
        status = get_journal_data_status()
        assert status["available"] is False


class TestEnsureJournalDataCache:
    """Verify the thundering-herd guard on ensure_journal_data.

    When 30+ filter workers call this concurrently during a search and
    the data files aren't yet on disk, only one should reach the real
    ``download_journal_data`` call; the rest should hit the 30-second
    cooldown cache and return the previous result without adding log
    spam or racing for the sentinel.
    """

    def test_second_call_within_ttl_uses_cache(self, tmp_data_dir):
        import local_deep_research.journal_quality.downloader as dl

        # Clear any stale module-level cache from previous tests.
        dl._ensure_cache = None

        with patch.object(dl, "download_journal_data") as mock_dl:
            mock_dl.return_value = (False, "network error")
            # First call: reaches download_journal_data, gets the
            # negative result, caches it.
            first = ensure_journal_data(auto_download=True)
            assert first == (None, False)
            assert mock_dl.call_count == 1

            # Rapid second call (well within 30 s TTL) must NOT
            # re-invoke download_journal_data — that's the whole
            # point of the herd guard.
            second = ensure_journal_data(auto_download=True)
            assert second == (None, False)
            assert mock_dl.call_count == 1