""" Tests for LibraryRAGService. """ from unittest.mock import Mock, MagicMock from pathlib import Path from langchain_core.documents import Document as LangchainDocument class TestLibraryRAGServiceInit: """Tests for LibraryRAGService initialization.""" def test_init_with_default_parameters(self, mocker): """Initializes with default parameters.""" # Mock database session mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock settings manager mock_settings_manager = Mock() mock_settings_manager.get_settings_snapshot.return_value = { "search.tool": "searxng" } mocker.patch( "local_deep_research.settings.manager.SettingsManager", return_value=mock_settings_manager, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.LocalEmbeddingManager", return_value=mock_embedding_manager, ) # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService(username="test_user") assert service.username == "test_user" assert service.embedding_model == "all-MiniLM-L6-v2" assert service.embedding_provider == "sentence_transformers" assert service.chunk_size == 1000 assert service.chunk_overlap == 200 assert service.distance_metric == "cosine" assert service.normalize_vectors is True def test_init_with_custom_parameters(self, mocker): """Initializes with custom parameters.""" # Mock database session mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock settings manager mock_settings_manager = Mock() mock_settings_manager.get_settings_snapshot.return_value = { "search.tool": "searxng" } mocker.patch( "local_deep_research.settings.manager.SettingsManager", return_value=mock_settings_manager, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.LocalEmbeddingManager", return_value=mock_embedding_manager, ) # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_model="custom-model", chunk_size=500, chunk_overlap=100, distance_metric="l2", ) assert service.embedding_model == "custom-model" assert service.chunk_size == 500 assert service.chunk_overlap == 100 assert service.distance_metric == "l2" def test_init_with_provided_embedding_manager(self, mocker): """Uses provided embedding manager instead of creating new one.""" # Mock database session mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) # Provided embedding manager mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) assert service.embedding_manager == mock_embedding_manager class TestLibraryRAGServiceIndexHash: """Tests for index hash generation.""" def test_get_index_hash_deterministic(self, mocker): """Index hash is deterministic for same inputs.""" # Create service with minimal mocking mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) hash1 = service._get_index_hash("collection_123", "model-a", "type-b") hash2 = service._get_index_hash("collection_123", "model-a", "type-b") assert hash1 == hash2 def test_get_index_hash_different_for_different_inputs(self, mocker): """Index hash differs for different inputs.""" # Create service with minimal mocking mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) hash1 = service._get_index_hash("collection_123", "model-a", "type-b") hash2 = service._get_index_hash("collection_456", "model-a", "type-b") hash3 = service._get_index_hash("collection_123", "model-x", "type-b") assert hash1 != hash2 assert hash1 != hash3 assert hash2 != hash3 class TestLibraryRAGServiceIndexPath: """Tests for index path generation.""" def test_get_index_path_returns_path(self, mocker): """Returns valid Path for index.""" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) path = service._get_index_path("abc123hash") assert isinstance(path, Path) assert path.suffix == ".faiss" assert "abc123hash" in str(path) class TestLibraryRAGServiceIndexDocument: """Tests for document indexing.""" def test_index_document_not_found(self, mocker): """Returns error when document not found.""" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mock_session.query.return_value.filter_by.return_value.first.return_value = None mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) result = service.index_document("nonexistent-doc", "collection-123") assert result["status"] == "error" assert "not found" in result["error"].lower() def test_index_document_no_text_content(self, mocker): """Returns error when document has no text content.""" # Mock document with no text mock_doc = Mock() mock_doc.id = "doc-123" mock_doc.text_content = None mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock ensure_in_collection to return a non-indexed DocumentCollection mock_dc = MagicMock(indexed=False, chunk_count=0) mocker.patch( "local_deep_research.research_library.services.library_rag_service.ensure_in_collection", return_value=mock_dc, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) result = service.index_document("doc-123", "collection-123") assert result["status"] == "error" assert "no text content" in result["error"].lower() def test_index_document_already_indexed_skips(self, mocker): """Skips indexing when document already indexed.""" # Mock document mock_doc = Mock() mock_doc.id = "doc-123" mock_doc.text_content = "Some text content" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock ensure_in_collection to return an already-indexed DocumentCollection mock_doc_collection = MagicMock(indexed=True, chunk_count=5) mocker.patch( "local_deep_research.research_library.services.library_rag_service.ensure_in_collection", return_value=mock_doc_collection, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) result = service.index_document( "doc-123", "collection-123", force_reindex=False ) assert result["status"] == "skipped" assert result["chunk_count"] == 5 class TestLibraryRAGServiceGetRAGStats: """Tests for RAG statistics.""" def test_get_rag_stats_returns_dict(self, mocker): """Returns dictionary with RAG statistics.""" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) # Mock query results mock_session.query.return_value.filter_by.return_value.count.return_value = 10 mock_session.query.return_value.filter_by.return_value.first.return_value = None mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock get_default_library_id mocker.patch( "local_deep_research.database.library_init.get_default_library_id", return_value="default-lib-id", ) # Mock embedding manager mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) stats = service.get_rag_stats("collection-123") assert isinstance(stats, dict) assert "total_documents" in stats assert "indexed_documents" in stats assert "unindexed_documents" in stats assert "total_chunks" in stats assert "chunk_size" in stats assert "chunk_overlap" in stats class TestLibraryRAGServiceRemoveDocument: """Tests for removing document from RAG.""" def test_remove_document_not_in_collection(self, mocker): """Returns error when document not in collection.""" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mock_session.query.return_value.filter_by.return_value.first.return_value = None mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() mock_embedding_manager._delete_chunks_from_db = Mock(return_value=0) # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) result = service.remove_document_from_rag("doc-123", "collection-123") assert result["status"] == "error" assert "not found" in result["error"].lower() class TestLibraryRAGServiceLoadOrCreateFaissIndex: """Tests for FAISS index loading/creation.""" def test_load_or_create_faiss_index_creates_new(self, mocker): """Creates new FAISS index when none exists.""" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mock_session.query.return_value.filter_by.return_value.first.return_value = None mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock embedding manager with dimension mock_embedding_manager = Mock() mock_embeddings = Mock() mock_embeddings.embed_query.return_value = [ 0.1 ] * 384 # 384-dim embedding mock_embedding_manager.embeddings = mock_embeddings # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mock_integrity.verify_file.return_value = (False, "File not found") mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) # Mock RAGIndex creation mock_rag_index = Mock() mock_rag_index.index_path = "/tmp/test.faiss" mock_rag_index.embedding_dimension = 384 mock_rag_index.id = "rag-idx-123" # Mock FAISS mock_faiss = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FAISS", return_value=mock_faiss, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) # Mock _get_or_create_rag_index service._get_or_create_rag_index = Mock(return_value=mock_rag_index) # Mock Path.exists to return False (no existing index) mocker.patch("pathlib.Path.exists", return_value=False) result = service.load_or_create_faiss_index("collection-123") assert result is not None class TestLibraryRAGServiceIndexBatch: """Tests for batch document indexing.""" def test_index_documents_batch_returns_dict(self, mocker): """Returns dictionary with results per document.""" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) # Mock documents mock_doc = Mock() mock_doc.id = "doc-123" mock_doc.text_content = "Some content" mock_doc.title = "Test Doc" mock_session.query.return_value.filter.return_value.all.return_value = [ mock_doc ] mock_session.query.return_value.filter.return_value.first.return_value = None mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) # Mock index_document to return success service.index_document = Mock( return_value={"status": "success", "chunk_count": 5} ) result = service.index_documents_batch( [("doc-123", "Test Doc")], "collection-123" ) assert isinstance(result, dict) assert "doc-123" in result class TestLoadOrCreateFaissIndexEdgeCases: """Additional tests for FAISS index loading/creation edge cases.""" def test_load_existing_faiss_index(self, mocker): """Loads existing FAISS index from disk when available.""" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) # Mock RAG index record mock_rag_index = Mock() mock_rag_index.id = "rag-idx-123" mock_rag_index.index_path = "/tmp/test.faiss" mock_rag_index.embedding_dimension = 384 mock_rag_index.collection_id = "collection-123" mock_session.query.return_value.filter_by.return_value.first.return_value = mock_rag_index mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embeddings = Mock() mock_embeddings.embed_query.return_value = [0.1] * 384 mock_embedding_manager.embeddings = mock_embeddings # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager - file exists and is valid mock_integrity = Mock() mock_integrity.verify_file.return_value = (True, "Valid") mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) # Mock Path.exists to return True mocker.patch("pathlib.Path.exists", return_value=True) # Mock the safe loader (replaces FAISS.load_local's dangerous # deserialization) mock_faiss_index = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.safe_load_faiss", return_value=mock_faiss_index, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) result = service.load_or_create_faiss_index("collection-123") # Should return the index loaded via the safe loader (not a fresh one) assert result is mock_faiss_index def test_load_or_create_handles_corrupted_index(self, mocker): """Creates new index when existing one is corrupted.""" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) # Mock RAG index that will fail integrity check mock_rag_index = Mock() mock_rag_index.id = "rag-idx-123" mock_rag_index.index_path = "/tmp/test.faiss" mock_rag_index.embedding_dimension = 384 mock_session.query.return_value.filter_by.return_value.first.return_value = mock_rag_index mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embeddings = Mock() mock_embeddings.embed_query.return_value = [0.1] * 384 mock_embedding_manager.embeddings = mock_embeddings # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager - file is corrupted mock_integrity = Mock() mock_integrity.verify_file.return_value = (False, "Hash mismatch") mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) # Corrupted index path: SUT quarantines the bad file and falls # through to building a fresh FAISS index — must not return None. result = service.load_or_create_faiss_index("collection-123") assert result is not None def test_load_index_with_different_embedding_dimension(self, mocker): """Handles dimension mismatch between index and current embeddings.""" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) # Mock RAG index with different dimension mock_rag_index = Mock() mock_rag_index.id = "rag-idx-123" mock_rag_index.index_path = "/tmp/test.faiss" mock_rag_index.embedding_dimension = 768 # Different from current mock_session.query.return_value.filter_by.return_value.first.return_value = mock_rag_index mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock embedding manager with 384 dim mock_embedding_manager = Mock() mock_embeddings = Mock() mock_embeddings.embed_query.return_value = [0.1] * 384 mock_embedding_manager.embeddings = mock_embeddings # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mock_integrity.verify_file.return_value = (True, "Valid") mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) # Dimension mismatch: SUT deletes the stale index, updates the # RAGIndex row, and rebuilds — must return a fresh FAISS index. result = service.load_or_create_faiss_index("collection-123") assert result is not None def test_create_index_with_normalize_vectors(self, mocker): """Creates index with vector normalization enabled.""" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mock_session.query.return_value.filter_by.return_value.first.return_value = None mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Mock embedding manager mock_embedding_manager = Mock() mock_embeddings = Mock() mock_embeddings.embed_query.return_value = [0.1] * 384 mock_embedding_manager.embeddings = mock_embeddings # Mock text splitter mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) # Mock integrity manager mock_integrity = Mock() mock_integrity.verify_file.return_value = (False, "No file") mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, normalize_vectors=True, ) assert service.normalize_vectors is True class TestIndexAllDocuments: """Tests for index_all_documents method.""" def test_index_all_documents_method_exists(self, mocker): """Verifies index_all_documents method exists on service.""" from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) assert hasattr(LibraryRAGService, "index_all_documents") assert callable(getattr(LibraryRAGService, "index_all_documents", None)) def test_index_all_documents_signature(self, mocker): """Verifies index_all_documents has expected parameters.""" import inspect from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) sig = inspect.signature(LibraryRAGService.index_all_documents) params = list(sig.parameters.keys()) # Should have self and collection_id at minimum assert "self" in params assert "collection_id" in params def test_index_all_documents_returns_dict(self, mocker): """Verifies index_all_documents returns a dictionary.""" from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) # Method should exist and be callable assert callable(LibraryRAGService.index_all_documents) def test_index_all_documents_accepts_collection_id(self, mocker): """Verifies index_all_documents accepts collection_id parameter.""" import inspect from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) sig = inspect.signature(LibraryRAGService.index_all_documents) params = list(sig.parameters.keys()) assert "collection_id" in params def test_index_all_documents_accepts_force_reindex(self, mocker): """Verifies index_all_documents accepts force_reindex parameter.""" import inspect from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) sig = inspect.signature(LibraryRAGService.index_all_documents) params = list(sig.parameters.keys()) # force_reindex should be a parameter assert "force_reindex" in params or len(params) > 2 class TestRemoveCollectionFromIndex: """Tests for remove_collection_from_index method.""" def test_remove_collection_from_index_method_exists(self, mocker): """Verifies remove_collection_from_index method exists.""" from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) assert hasattr(LibraryRAGService, "remove_collection_from_index") assert callable( getattr(LibraryRAGService, "remove_collection_from_index", None) ) def test_remove_collection_from_index_signature(self, mocker): """Verifies remove_collection_from_index has expected parameters.""" import inspect from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) sig = inspect.signature(LibraryRAGService.remove_collection_from_index) params = list(sig.parameters.keys()) # Should have self and collection_name at minimum assert "self" in params assert "collection_name" in params def test_remove_collection_from_index_returns_dict(self, mocker): """Verifies remove_collection_from_index returns a dictionary.""" from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) # Method should exist and be callable assert callable(LibraryRAGService.remove_collection_from_index) def test_remove_collection_from_index_accepts_collection_name(self, mocker): """Verifies remove_collection_from_index accepts collection_name.""" import inspect from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) sig = inspect.signature(LibraryRAGService.remove_collection_from_index) params = list(sig.parameters.keys()) assert "collection_name" in params def test_remove_collection_has_return_type(self, mocker): """Verifies remove_collection_from_index is properly defined.""" from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) # Method should have docstring or be properly documented method = LibraryRAGService.remove_collection_from_index assert method is not None class TestLoadOrCreateFaissIndexCollectionId: """Tests that load_or_create_faiss_index receives the correct collection_id. Fixes a pre-existing bug where index_user_document and remove_collection_from_index called load_or_create_faiss_index() without the required collection_id argument. """ def _create_service(self, mocker): """Create a minimally mocked LibraryRAGService.""" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() mock_embedding_manager.embeddings.embed_query.return_value = [0.1] * 384 mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) return service def test_index_user_document_passes_extracted_collection_id(self, mocker): """index_user_document extracts collection_id from collection_name.""" service = self._create_service(mocker) # Create mock user document mock_user_doc = Mock() mock_user_doc.id = "user-doc-123" mock_user_doc.text_content = ( "Enough content for indexing test purposes." ) mock_user_doc.filename = "test.pdf" mock_user_doc.file_type = "pdf" mock_user_doc.file_size = 1024 # Mock FAISS index as None to trigger load service.faiss_index = None mock_faiss_index = MagicMock() mock_faiss_index.docstore = Mock() mock_faiss_index.docstore._dict = {} service.load_or_create_faiss_index = Mock(return_value=mock_faiss_index) # Mock _store_chunks_to_db service.embedding_manager._store_chunks_to_db = Mock( return_value=["chunk-1", "chunk-2"] ) # Mock text_splitter from langchain_core.documents import Document as LangchainDocument mock_chunks = [ LangchainDocument(page_content="chunk 1"), LangchainDocument(page_content="chunk 2"), ] service.text_splitter.split_documents.return_value = mock_chunks # Mock rag_index_record for save path service.rag_index_record = None collection_name = "collection_abc-uuid-123" service.index_user_document(mock_user_doc, collection_name) # Verify collection_id was correctly extracted and passed service.load_or_create_faiss_index.assert_called_once_with( "abc-uuid-123" ) def test_remove_collection_passes_extracted_collection_id(self, mocker): """remove_collection_from_index extracts collection_id from collection_name.""" service = self._create_service(mocker) # Mock DB session to return chunks — patch at source since method does local import mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mock_chunk = Mock() mock_chunk.id = "chunk-1" mock_session.query.return_value.filter_by.return_value.all.return_value = [ mock_chunk ] mocker.patch( "local_deep_research.database.session_context.get_user_db_session", return_value=mock_session, ) # Mock FAISS index as None to trigger load service.faiss_index = None mock_faiss_index = MagicMock() service.load_or_create_faiss_index = Mock(return_value=mock_faiss_index) collection_name = "collection_xyz-uuid-456" service.remove_collection_from_index(collection_name) # Verify collection_id was correctly extracted and passed service.load_or_create_faiss_index.assert_called_once_with( "xyz-uuid-456" ) class TestDeduplicateChunks: """Tests for _deduplicate_chunks static method. This covers the bug fix where duplicate chunk IDs from _store_chunks_to_db() caused FAISS add_documents() to reject the batch with 'Duplicate ids found in the ids list'. """ def test_no_duplicates_passes_all_through(self): """All chunks pass through when there are no duplicates.""" from langchain_core.documents import Document as LangchainDocument from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) chunks = [ LangchainDocument(page_content="chunk 1"), LangchainDocument(page_content="chunk 2"), LangchainDocument(page_content="chunk 3"), ] ids = ["id-1", "id-2", "id-3"] result_chunks, result_ids = LibraryRAGService._deduplicate_chunks( chunks, ids ) assert len(result_chunks) == 3 assert result_ids == ["id-1", "id-2", "id-3"] def test_batch_duplicates_are_removed(self): """Duplicate IDs within a batch keep only the first occurrence.""" from langchain_core.documents import Document as LangchainDocument from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) chunks = [ LangchainDocument(page_content="header text"), LangchainDocument(page_content="unique content"), LangchainDocument( page_content="header text" ), # same hash -> same ID ] ids = ["id-dup", "id-unique", "id-dup"] result_chunks, result_ids = LibraryRAGService._deduplicate_chunks( chunks, ids ) assert len(result_chunks) == 2 assert result_ids == ["id-dup", "id-unique"] assert result_chunks[0].page_content == "header text" assert result_chunks[1].page_content == "unique content" def test_existing_ids_are_excluded(self): """Chunks with IDs already in the FAISS index are filtered out.""" from langchain_core.documents import Document as LangchainDocument from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) chunks = [ LangchainDocument(page_content="already indexed"), LangchainDocument(page_content="new content"), ] ids = ["id-old", "id-new"] existing = {"id-old"} result_chunks, result_ids = LibraryRAGService._deduplicate_chunks( chunks, ids, existing_ids=existing ) assert len(result_chunks) == 1 assert result_ids == ["id-new"] def test_batch_duplicates_and_existing_ids_combined(self): """Both batch duplicates and existing IDs are filtered.""" from langchain_core.documents import Document as LangchainDocument from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) chunks = [ LangchainDocument(page_content="old"), LangchainDocument(page_content="new A"), LangchainDocument(page_content="new A copy"), # dup of new A LangchainDocument(page_content="new B"), ] ids = ["id-old", "id-a", "id-a", "id-b"] existing = {"id-old"} result_chunks, result_ids = LibraryRAGService._deduplicate_chunks( chunks, ids, existing_ids=existing ) assert len(result_chunks) == 2 assert result_ids == ["id-a", "id-b"] def test_all_duplicates_returns_empty(self): """Returns empty lists when all chunks are duplicates.""" from langchain_core.documents import Document as LangchainDocument from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) chunks = [ LangchainDocument(page_content="same"), LangchainDocument(page_content="same"), ] ids = ["id-x", "id-x"] existing = {"id-x"} result_chunks, result_ids = LibraryRAGService._deduplicate_chunks( chunks, ids, existing_ids=existing ) assert len(result_chunks) == 0 assert len(result_ids) == 0 def test_none_existing_ids_skips_existing_check(self): """When existing_ids is None (force_reindex), only batch dedup applies.""" from langchain_core.documents import Document as LangchainDocument from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) chunks = [ LangchainDocument(page_content="A"), LangchainDocument(page_content="B"), LangchainDocument(page_content="A copy"), ] ids = ["id-1", "id-2", "id-1"] result_chunks, result_ids = LibraryRAGService._deduplicate_chunks( chunks, ids, existing_ids=None ) assert len(result_chunks) == 2 assert result_ids == ["id-1", "id-2"] def _create_rag_service(mocker): """Helper to create a minimally-mocked LibraryRAGService.""" mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) mock_embedding_manager = Mock() mock_embedding_manager.embeddings = Mock() mock_embedding_manager.embeddings.embed_query.return_value = [0.1] * 384 mock_splitter = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_text_splitter", return_value=mock_splitter, ) mock_integrity = Mock() mocker.patch( "local_deep_research.research_library.services.library_rag_service.FileIntegrityManager", return_value=mock_integrity, ) from local_deep_research.research_library.services.library_rag_service import ( LibraryRAGService, ) service = LibraryRAGService( username="test_user", embedding_manager=mock_embedding_manager, ) return service class TestForceReindexDedup: """Tests for force_reindex old_chunk_ids deduplication.""" def test_force_reindex_old_chunk_ids_deduplicated_index_document( self, mocker ): """Verify deletion path in index_document uses unique IDs.""" service = _create_rag_service(mocker) # Mock document lookup mock_doc = Mock() mock_doc.id = "doc-1" mock_doc.text_content = "Hello world test content" mock_doc.title = "Test" mock_doc.filename = "test.txt" mock_doc.original_url = "http://example.com" mock_doc.authors = None mock_doc.published_date = None mock_doc.doi = None mock_doc.arxiv_id = None mock_doc_collection = Mock() mock_doc_collection.indexed = True mock_doc_collection.chunk_count = 2 mock_session = MagicMock() mock_session.__enter__ = Mock(return_value=mock_session) mock_session.__exit__ = Mock(return_value=False) # First query: Document lookup, second: DocumentCollection mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc mock_session.query.return_value.filter_by.return_value.all.return_value = [ mock_doc_collection ] mocker.patch( "local_deep_research.research_library.services.library_rag_service.get_user_db_session", return_value=mock_session, ) # Chunks with duplicate IDs (the root cause of #2182) chunks = [ LangchainDocument(page_content="chunk A"), LangchainDocument(page_content="chunk B"), LangchainDocument(page_content="chunk A dup"), ] service.text_splitter.split_documents.return_value = chunks # _store_chunks_to_db returns IDs with duplicates duplicate_ids = ["id-1", "id-2", "id-1"] service.embedding_manager._store_chunks_to_db.return_value = ( duplicate_ids ) # Setup FAISS mock with existing IDs mock_faiss = Mock() mock_docstore = Mock() mock_docstore._dict = {"id-1": "doc", "id-2": "doc"} mock_faiss.docstore = mock_docstore service.faiss_index = mock_faiss mock_rag_index = Mock() mock_rag_index.index_path = "/tmp/test.faiss" mock_rag_index.id = "rag-1" service.rag_index_record = mock_rag_index service.index_document("doc-1", "coll-1", force_reindex=True) # Verify delete was called with deduplicated IDs delete_call = mock_faiss.delete.call_args[0][0] assert len(delete_call) == len(set(delete_call)), ( "old_chunk_ids passed to FAISS.delete() must not contain duplicates" ) def test_force_reindex_old_chunk_ids_deduplicated_index_user_document( self, mocker ): """Verify deletion path in index_user_document uses unique IDs.""" service = _create_rag_service(mocker) # Mock user document mock_user_doc = Mock() mock_user_doc.id = "udoc-1" mock_user_doc.filename = "test.txt" mock_user_doc.text_content = "Some content here for testing" mock_user_doc.file_type = "txt" mock_user_doc.file_size = 100 chunks = [ LangchainDocument(page_content="chunk A"), LangchainDocument(page_content="chunk B"), LangchainDocument(page_content="chunk A dup"), ] service.text_splitter.split_documents.return_value = chunks # _store_chunks_to_db returns IDs with duplicates duplicate_ids = ["id-1", "id-2", "id-1"] service.embedding_manager._store_chunks_to_db.return_value = ( duplicate_ids ) # Setup FAISS mock mock_faiss = Mock() mock_docstore = Mock() mock_docstore._dict = {"id-1": "doc", "id-2": "doc"} mock_faiss.docstore = mock_docstore service.faiss_index = mock_faiss mock_rag_index = Mock() mock_rag_index.index_path = "/tmp/test.faiss" mock_rag_index.id = "rag-1" service.rag_index_record = mock_rag_index service.index_user_document( mock_user_doc, "collection_123", force_reindex=True ) # Verify delete was called with deduplicated IDs delete_call = mock_faiss.delete.call_args[0][0] assert len(delete_call) == len(set(delete_call)), ( "old_chunk_ids passed to FAISS.delete() must not contain duplicates" )