"""Tests for llm_config module.""" from unittest.mock import AsyncMock, MagicMock, patch from local_deep_research.config.llm_config import ( get_selected_llm_provider, wrap_llm_without_think_tags, get_llm, ) class TestGetSelectedLlmProvider: """Tests for get_selected_llm_provider function.""" def test_returns_provider_from_settings(self): """Should return provider from settings.""" with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value="anthropic", ): result = get_selected_llm_provider() assert result == "anthropic" def test_returns_lowercase(self): """Should return lowercase provider.""" with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value="OPENAI", ): result = get_selected_llm_provider() assert result == "openai" def test_defaults_to_ollama(self): """Should default to ollama.""" with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value="ollama", ) as mock: get_selected_llm_provider() # Check default is ollama mock.assert_called_with( "llm.provider", "ollama", settings_snapshot=None ) class TestWrapLlmWithoutThinkTags: """Tests for wrap_llm_without_think_tags function.""" def test_returns_wrapper_instance(self): """Should return a wrapper instance.""" mock_llm = MagicMock() with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): result = wrap_llm_without_think_tags(mock_llm) assert hasattr(result, "invoke") assert hasattr(result, "base_llm") def test_wrapper_invoke_calls_base_llm(self): """Should call base LLM on invoke.""" mock_llm = MagicMock() mock_response = MagicMock() mock_response.content = "test response" mock_llm.invoke.return_value = mock_response with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): wrapper = wrap_llm_without_think_tags(mock_llm) wrapper.invoke("test prompt") mock_llm.invoke.assert_called_with("test prompt") def test_wrapper_removes_think_tags(self): """Should remove think tags from response.""" mock_llm = MagicMock() mock_response = MagicMock() mock_response.content = "internalvisible" mock_llm.invoke.return_value = mock_response with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): with patch( "local_deep_research.config.llm_config.remove_think_tags", return_value="visible", ) as mock_remove: wrapper = wrap_llm_without_think_tags(mock_llm) wrapper.invoke("test") mock_remove.assert_called_with("internalvisible") def test_wrapper_preserves_nonstring_content(self): """Non-string content (e.g. provider content-block lists) must pass through unchanged instead of raising TypeError in remove_think_tags.""" mock_llm = MagicMock() mock_response = MagicMock() blocks = [{"type": "text", "text": "visible"}] mock_response.content = blocks mock_llm.invoke.return_value = mock_response with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): wrapper = wrap_llm_without_think_tags(mock_llm) result = wrapper.invoke("test") assert result.content == blocks def test_wrapper_preserves_none_content(self): """None content must pass through unchanged rather than raising.""" mock_llm = MagicMock() mock_response = MagicMock() mock_response.content = None mock_llm.invoke.return_value = mock_response with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): wrapper = wrap_llm_without_think_tags(mock_llm) result = wrapper.invoke("test") assert result.content is None def test_wrapper_delegates_attributes(self): """Should delegate attribute access to base LLM.""" mock_llm = MagicMock() mock_llm.model_name = "gpt-4" with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): wrapper = wrap_llm_without_think_tags(mock_llm) assert wrapper.model_name == "gpt-4" def test_applies_rate_limiting_when_enabled(self): """Should apply rate limiting when enabled in settings.""" mock_llm = MagicMock() mock_wrapped = MagicMock() with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=True, ): # Patch at source location since it's imported inside the function with patch( "local_deep_research.web_search_engines.rate_limiting.llm.create_rate_limited_llm_wrapper", return_value=mock_wrapped, ) as mock_create: wrap_llm_without_think_tags(mock_llm, provider="openai") mock_create.assert_called_with(mock_llm, "openai") # -- bind_tools regression tests (issue #4804) ------------------------- # # ``create_agent()`` calls ``model.bind_tools(tools)`` and uses the # return value as the model that runs inside the agent loop. Without # a ``bind_tools`` override on ``ProcessingLLMWrapper``, Python falls # through ``__getattr__`` to ``self.base_llm.bind_tools(...)`` and the # agent loop runs on an unwrapped ``BaseChatModel`` — which means # ``_normalize_response`` never strips ``…`` and reasoning-mode # models (Qwen 3.x, deepseek-r1, etc.) leak raw ```` text into # the conversation history. On the next LLM turn, strict # OpenAI-compatible providers reject the request with # ``Failed to parse input at pos 0: …``. The override keeps # the bound model wrapped, so the stripper stays in the loop. def test_bind_tools_returns_wrapper_not_raw_base_llm(self): """``bind_tools`` must return a ``ProcessingLLMWrapper`` instance, not the unwrapped bound model. Guards against future regressions that reintroduce the bypass by deleting this override (issue #4804).""" mock_llm = MagicMock() mock_bound = MagicMock() mock_llm.bind_tools.return_value = mock_bound with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): wrapper = wrap_llm_without_think_tags(mock_llm) bound_wrapper = wrapper.bind_tools([lambda: None]) assert hasattr(bound_wrapper, "base_llm") assert bound_wrapper.base_llm is mock_bound # The wrapped object must NOT be the raw base LLM (or its # bound child) — it must be the wrapper that runs # _normalize_response. assert bound_wrapper is not mock_bound assert bound_wrapper is not mock_llm def test_bind_tools_forwards_tools_and_kwargs_to_base_llm(self): """Tools and provider-specific kwargs must be forwarded verbatim to ``base_llm.bind_tools``.""" mock_llm = MagicMock() mock_bound = MagicMock() mock_llm.bind_tools.return_value = mock_bound def sentinel_tool(): return None with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): wrapper = wrap_llm_without_think_tags(mock_llm) wrapper.bind_tools(sentinel_tool, tool_choice="auto", parallel=True) mock_llm.bind_tools.assert_called_once_with( sentinel_tool, tool_choice="auto", parallel=True ) def test_bound_wrapper_strips_think_tags_from_invoke(self): """After ``bind_tools``, ``invoke()`` on the bound wrapper must still strip ```` from the model response. This is the exact path ``create_agent()`` uses — if it strips, the ``BadRequestError`` from issue #4804 cannot recur.""" mock_llm = MagicMock() mock_bound = MagicMock() mock_llm.bind_tools.return_value = mock_bound mock_response = MagicMock() mock_response.content = "reasoning trace…visible answer" mock_bound.invoke.return_value = mock_response with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): with patch( "local_deep_research.config.llm_config.remove_think_tags", wraps=lambda text: text.replace( "reasoning trace…", "" ), ) as mock_remove: wrapper = wrap_llm_without_think_tags(mock_llm) bound_wrapper = wrapper.bind_tools([lambda: None]) bound_wrapper.invoke("test") mock_bound.invoke.assert_called_once_with("test") mock_remove.assert_called_once_with( "reasoning trace…visible answer" ) # The bound model is what the agent calls — the original # base LLM must NOT receive the invoke (otherwise the # bypass is back). assert not mock_llm.invoke.called def test_bound_wrapper_delegates_attributes_to_bound_model(self): """Attributes on the bound model (e.g. ``model_name``) must still be reachable through the wrapped binding, so LangChain internals that introspect the model see the bound model's metadata.""" mock_llm = MagicMock() mock_bound = MagicMock() mock_bound.model_name = "qwen3:8b" mock_llm.bind_tools.return_value = mock_bound with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): wrapper = wrap_llm_without_think_tags(mock_llm) bound_wrapper = wrapper.bind_tools([lambda: None]) assert bound_wrapper.model_name == "qwen3:8b" def test_bound_wrapper_ainvoke_strips_think_tags(self): """Async counterpart of ``test_bound_wrapper_strips_think_tags_from_invoke``. ``ainvoke`` is the path async LangGraph nodes use; it must also pass through ``_normalize_response`` after the re-wrap.""" import asyncio mock_llm = MagicMock() mock_bound = MagicMock() mock_llm.bind_tools.return_value = mock_bound mock_response = MagicMock() mock_response.content = "ok" mock_bound.ainvoke = AsyncMock(return_value=mock_response) with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): with patch( "local_deep_research.config.llm_config.remove_think_tags", wraps=lambda text: text.replace("", ""), ): wrapper = wrap_llm_without_think_tags(mock_llm) bound_wrapper = wrapper.bind_tools([lambda: None]) result = asyncio.run(bound_wrapper.ainvoke("test")) mock_bound.ainvoke.assert_called_once_with("test") # Must actually strip on the async path. Without the bind_tools # override this fails: the raw bound model is returned and its # ainvoke never runs _normalize_response, so content keeps the # block. Regression guard for #4804. assert result is mock_response assert result.content == "ok" class TestBindToolsCreateAgentIntegration: """End-to-end guard for #4804. The unit tests above exercise ``ProcessingLLMWrapper.bind_tools`` in isolation. This class drives a REAL ``langchain.agents.create_agent`` over the think-stripping wrapper and asserts the agent's final message has ```` stripped — the actual path that regressed. Without the ``bind_tools`` override, ``create_agent`` binds tools to the unwrapped base model and the ```` block leaks into the agent's output. """ @staticmethod def _build_agent(): from langchain.agents import create_agent from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.messages import AIMessage from langchain_core.outputs import ChatGeneration, ChatResult from langchain_core.tools import tool class _FakeThinkModel(BaseChatModel): """Reasoning-style model: emits a ```` block and no tool calls, so ``create_agent`` runs exactly one model turn and stops.""" @property def _llm_type(self) -> str: return "fake-think" def _generate( self, messages, stop=None, run_manager=None, **kwargs ): message = AIMessage( content="internal reasoningFINAL_ANSWER" ) return ChatResult(generations=[ChatGeneration(message=message)]) def bind_tools(self, tools, **kwargs): # The fake never emits tool_calls, so returning self is enough # for create_agent to bind tools and run a single turn. return self @tool def noop(query: str) -> str: """No-op tool so the agent has a non-empty tool list.""" return query # Patch the settings lookup so wrap_llm_without_think_tags doesn't try # to read rate-limit settings from a (nonexistent) settings context. with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): wrapped = wrap_llm_without_think_tags(_FakeThinkModel()) return create_agent(model=wrapped, tools=[noop]) def test_create_agent_invoke_strips_think_tags(self): agent = self._build_agent() result = agent.invoke({"messages": [{"role": "user", "content": "hi"}]}) final = result["messages"][-1] assert "" not in final.content assert final.content == "FINAL_ANSWER" def test_create_agent_ainvoke_strips_think_tags(self): import asyncio agent = self._build_agent() result = asyncio.run( agent.ainvoke({"messages": [{"role": "user", "content": "hi"}]}) ) final = result["messages"][-1] assert "" not in final.content assert final.content == "FINAL_ANSWER" class TestGetLlm: """Tests for get_llm function. Many historical tests in this class mocked ``is_llm_registered=False`` to force the procedural ``if/elif`` chain in ``get_llm`` (lines ~405-707 prior to the dead-code deletion). That chain has been removed; equivalent live-path coverage now lives in ``tests/llm_providers/implementations/test_*_provider.py``. The remaining tests below exercise the registered-LLM branch and the coordinator surface (provider normalization, model-name validation, final guards). The deleted tests are replaced 1:1 by their class-path equivalents — see commit history for the mapping. """ def test_uses_custom_registered_llm(self): """Should use custom LLM when registered.""" # Import BaseChatModel for proper spec from langchain_core.language_models import BaseChatModel mock_llm = MagicMock(spec=BaseChatModel) with patch( "local_deep_research.config.llm_config.is_llm_registered", return_value=True, ): with patch( "local_deep_research.config.llm_config.get_llm_from_registry", return_value=mock_llm, ): with patch( "local_deep_research.config.llm_config.wrap_llm_without_think_tags", return_value=mock_llm, ): with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value="custom_provider", ): result = get_llm( provider="custom_provider", settings_snapshot={"search.tool": "searxng"}, ) assert result is mock_llm def test_invalid_provider_raises_error(self): """Should raise ValueError for invalid provider.""" import pytest with patch( "local_deep_research.config.llm_config.is_llm_registered", return_value=False, ): with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot" ) as mock_get: mock_get.side_effect = lambda key, default=None, **kwargs: { "llm.model": "test-model", "llm.temperature": 0.7, "llm.provider": "invalid_provider", }.get(key, default) with pytest.raises(ValueError, match="Invalid provider"): get_llm(settings_snapshot={"search.tool": "searxng"}) def test_raises_when_model_setting_empty(self): """get_llm() must raise ValueError when llm.model is empty string.""" import pytest with patch( "local_deep_research.config.llm_config.is_llm_registered", return_value=False, ): with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot" ) as mock_get: mock_get.side_effect = lambda key, default=None, **kwargs: { "llm.model": "", "llm.temperature": 0.7, "llm.provider": "ollama", }.get(key, default) with pytest.raises( ValueError, match="LLM model not configured" ): get_llm() def test_raises_when_model_setting_whitespace_only(self): """get_llm() must raise ValueError when llm.model is whitespace.""" import pytest with patch( "local_deep_research.config.llm_config.is_llm_registered", return_value=False, ): with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot" ) as mock_get: mock_get.side_effect = lambda key, default=None, **kwargs: { "llm.model": " ", "llm.temperature": 0.7, "llm.provider": "ollama", }.get(key, default) with pytest.raises( ValueError, match="LLM model not configured" ): get_llm() def test_raises_when_model_setting_missing_returns_empty_default(self): """get_llm() must raise when snapshot returns empty default.""" import pytest with patch( "local_deep_research.config.llm_config.is_llm_registered", return_value=False, ): with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot" ) as mock_get: # Snapshot has neither llm.model nor a custom default; # function falls back to its own "" default. mock_get.side_effect = lambda key, default=None, **kwargs: { "llm.temperature": 0.7, "llm.provider": "ollama", }.get(key, default) with pytest.raises( ValueError, match="LLM model not configured" ): get_llm() def test_custom_factory_function_is_called(self): """Should call factory function for custom registered LLM.""" from langchain_core.language_models import BaseChatModel mock_llm = MagicMock(spec=BaseChatModel) mock_factory = MagicMock(return_value=mock_llm) with patch( "local_deep_research.config.llm_config.is_llm_registered", return_value=True, ): with patch( "local_deep_research.config.llm_config.get_llm_from_registry", return_value=mock_factory, ): with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot" ) as mock_get: mock_get.side_effect = lambda key, default=None, **kwargs: { "llm.model": "custom-model", "llm.temperature": 0.5, "llm.provider": "custom_provider", "rate_limiting.llm_enabled": False, }.get(key, default) get_llm( model_name="custom-model", temperature=0.5, provider="custom_provider", settings_snapshot={"search.tool": "searxng"}, ) mock_factory.assert_called_once() call_kwargs = mock_factory.call_args.kwargs assert call_kwargs["model_name"] == "custom-model" assert call_kwargs["temperature"] == 0.5 def test_custom_factory_with_invalid_signature_raises(self): """Should raise TypeError when factory has invalid signature.""" import pytest def bad_factory(): return MagicMock() with patch( "local_deep_research.config.llm_config.is_llm_registered", return_value=True, ): with patch( "local_deep_research.config.llm_config.get_llm_from_registry", return_value=bad_factory, ): with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot" ) as mock_get: mock_get.side_effect = lambda key, default=None, **kwargs: { "llm.model": "model", "llm.temperature": 0.7, "llm.provider": "bad_factory", "rate_limiting.llm_enabled": False, }.get(key, default) with pytest.raises(TypeError, match="invalid signature"): get_llm( provider="bad_factory", settings_snapshot={"search.tool": "searxng"}, ) def test_custom_factory_returning_non_basechatmodel_raises(self): """Should raise ValueError when factory returns non-BaseChatModel.""" import pytest def bad_factory( model_name=None, temperature=None, settings_snapshot=None ): return "not a model" with patch( "local_deep_research.config.llm_config.is_llm_registered", return_value=True, ): with patch( "local_deep_research.config.llm_config.get_llm_from_registry", return_value=bad_factory, ): with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot" ) as mock_get: mock_get.side_effect = lambda key, default=None, **kwargs: { "llm.model": "model", "llm.temperature": 0.7, "llm.provider": "bad_factory", "rate_limiting.llm_enabled": False, }.get(key, default) with pytest.raises( ValueError, match="must return a BaseChatModel" ): get_llm( provider="bad_factory", settings_snapshot={"search.tool": "searxng"}, ) class TestWrapperStringResponse: """Tests for wrapper handling string responses.""" def test_wrapper_handles_string_response(self): """Should handle string response from LLM.""" mock_llm = MagicMock() mock_llm.invoke.return_value = "thoughtanswer" with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): wrapper = wrap_llm_without_think_tags(mock_llm) result = wrapper.invoke("test") # A bare-string return is normalized into a message (so callers can # always use .content); think tags are still removed. assert not isinstance(result, str) assert "answer" in result.content assert "" not in result.content def test_wrapper_handles_invoke_exception(self): """Should propagate exceptions from LLM invoke.""" import pytest mock_llm = MagicMock() mock_llm.invoke.side_effect = RuntimeError("LLM error") with patch( "local_deep_research.config.llm_config.get_setting_from_snapshot", return_value=False, ): wrapper = wrap_llm_without_think_tags(mock_llm) with pytest.raises(RuntimeError, match="LLM error"): wrapper.invoke("test") class TestImportTimeAutoDiscovery: """Guard the import-time provider auto-discovery contract. ``llm_config`` imports ``discover_providers`` (``# noqa: F401``) purely for its import-time side effect: registering every built-in provider. Since get_llm() has no fallback construction path, removing that import would leave the registry empty and break every dispatch. This runs in a fresh interpreter so the assertion genuinely exercises import-time registration rather than registrations left over from earlier tests. """ def test_importing_llm_config_registers_builtin_providers(self): import subprocess import sys code = ( "from local_deep_research.config import llm_config # noqa: F401\n" "from local_deep_research.llm import is_llm_registered\n" "assert is_llm_registered('openai'), 'openai not registered'\n" "assert is_llm_registered('anthropic'), 'anthropic not registered'\n" "print('OK')\n" ) result = subprocess.run( [sys.executable, "-c", code], capture_output=True, text=True, ) assert result.returncode == 0, ( f"import-time registration failed:\n" f"stdout={result.stdout}\nstderr={result.stderr}" ) assert "OK" in result.stdout class TestDiscoveredProviderOptions: """The live UI provider-dropdown path (replaces the removed get_available_providers()): get_discovered_provider_options() enumerates all discovered provider classes; get_available_discovered_provider_options() filters that set by ProviderClass.is_available(settings_snapshot).""" def test_discovered_options_shape_and_core_providers(self): from local_deep_research.llm.providers import ( get_discovered_provider_options, ) options = get_discovered_provider_options() assert isinstance(options, list) and options for opt in options: assert "value" in opt and "label" in opt values = {opt["value"].lower() for opt in options} # Core built-in providers must always be discovered. The local # providers (llamacpp, lmstudio) are included so they can't silently # drop out of auto-discovery: the model-provider dropdown is derived # from this set, and #4594 removed the hardcoded LLAMACPP fallback that # would otherwise have masked such a regression. for provider in ( "openai", "anthropic", "ollama", "llamacpp", "lmstudio", ): assert provider in values def test_available_options_is_filtered_subset(self): from local_deep_research.llm.providers import ( get_discovered_provider_options, get_available_discovered_provider_options, ) all_values = { o["value"].lower() for o in get_discovered_provider_options() } # With no settings snapshot, no API keys / reachable local servers # are configured, so the filtered set is a (here empty) subset. available = get_available_discovered_provider_options(None) assert isinstance(available, list) available_values = {o["value"].lower() for o in available} assert available_values <= all_values