""" Tests for OpenAI-Compatible Endpoint provider. These tests ensure that custom OpenAI-compatible endpoints work correctly, especially with private IP addresses commonly used in Docker/self-hosted setups. REGRESSION TESTS: These tests were added to prevent regression of the issue where custom endpoints with private IPs (172.x, 10.x, 192.168.x) stopped working in v1.3.10+ due to improper URL handling. """ import pytest from unittest.mock import Mock, patch from local_deep_research.llm.providers.implementations.custom_openai_endpoint import ( CustomOpenAIEndpointProvider, ) class TestCustomOpenAIEndpointProviderMetadata: """Tests for CustomOpenAIEndpointProvider class attributes.""" def test_provider_name(self): """Provider name is correctly set.""" assert ( CustomOpenAIEndpointProvider.provider_name == "OpenAI-Compatible Endpoint" ) def test_provider_key(self): """Provider key is correctly set for auto-discovery.""" assert CustomOpenAIEndpointProvider.provider_key == "OPENAI_ENDPOINT" def test_url_setting(self): """URL setting key is defined for configurable endpoint.""" assert ( CustomOpenAIEndpointProvider.url_setting == "llm.openai_endpoint.url" ) def test_api_key_setting(self): """API key setting is defined.""" assert ( CustomOpenAIEndpointProvider.api_key_setting == "llm.openai_endpoint.api_key" ) def test_does_not_require_auth_for_models(self): """Custom endpoints don't require auth for listing models. Many self-hosted servers (vLLM, Ollama) don't require authentication. """ assert CustomOpenAIEndpointProvider.requires_auth_for_models() is False class TestCustomOpenAIEndpointListModels: """Tests for model listing functionality.""" @pytest.fixture def mock_openai_client(self): """Create a mock OpenAI client with model list response.""" client = Mock() mock_model1 = Mock() mock_model1.id = "llama-3-70b" mock_model2 = Mock() mock_model2.id = "mistral-7b" models_response = Mock() models_response.data = [mock_model1, mock_model2] client.models.list.return_value = models_response return client def test_list_models_with_private_ip(self, mock_openai_client): """Can list models from private IP endpoints. REGRESSION TEST: This is the exact scenario that broke in v1.3.10+ """ with patch("openai.OpenAI") as mock_openai: mock_openai.return_value = mock_openai_client # Simulate a private Docker network IP private_url = "http://172.19.0.5:8000/v1" result = CustomOpenAIEndpointProvider.list_models_for_api( api_key=None, # Many custom endpoints don't need API key base_url=private_url, ) # Verify client was created with correct URL mock_openai.assert_called_once() call_kwargs = mock_openai.call_args[1] assert call_kwargs["base_url"] == private_url assert call_kwargs["api_key"] == "dummy-key-for-models-list" # Verify models were returned assert len(result) == 2 assert result[0]["value"] == "llama-3-70b" def test_list_models_various_private_ips(self, mock_openai_client): """Works with all common private IP ranges. REGRESSION TEST: Ensures all private IP formats work. """ test_cases = [ ("http://10.0.0.100:8000/v1", "Class A private (10.x)"), ("http://172.16.0.50:5000/v1", "Class B private (172.16.x)"), ("http://172.19.0.5:8000/v1", "Docker bridge network"), ("http://172.31.255.255:8000/v1", "Class B private max"), ("http://192.168.1.100:11434/v1", "Class C private"), ("http://localhost:8000/v1", "localhost hostname"), ("http://127.0.0.1:8000/v1", "loopback IP"), ("http://[::1]:8000/v1", "IPv6 loopback"), ] for url, description in test_cases: with patch("openai.OpenAI") as mock_openai: mock_openai.return_value = mock_openai_client result = CustomOpenAIEndpointProvider.list_models_for_api( base_url=url ) call_kwargs = mock_openai.call_args[1] assert call_kwargs["base_url"] == url, ( f"Failed for {description}: URL not passed correctly" ) assert len(result) == 2, f"Failed for {description}" def test_list_models_with_url_containing_v1_suffix( self, mock_openai_client ): """Handles URLs that already have /v1 suffix. REGRESSION TEST: Previously the code was appending /v1 to URLs that already had it, causing double /v1/v1 paths. """ with patch("openai.OpenAI") as mock_openai: mock_openai.return_value = mock_openai_client url_with_v1 = "http://172.19.0.5:8000/v1" CustomOpenAIEndpointProvider.list_models_for_api( base_url=url_with_v1 ) call_kwargs = mock_openai.call_args[1] # Should NOT have /v1/v1 assert "/v1/v1" not in call_kwargs["base_url"] assert call_kwargs["base_url"] == url_with_v1 def test_list_models_without_auth(self, mock_openai_client): """Can list models without providing API key. Custom endpoints often don't require authentication. """ with patch("openai.OpenAI") as mock_openai: mock_openai.return_value = mock_openai_client # No API key provided result = CustomOpenAIEndpointProvider.list_models_for_api( api_key=None, base_url="http://localhost:8000/v1" ) # Should use dummy key and still work call_kwargs = mock_openai.call_args[1] assert call_kwargs["api_key"] == "dummy-key-for-models-list" assert len(result) == 2 class TestCustomOpenAIEndpointCreateLLM: """Tests for LLM creation.""" def test_create_llm_uses_custom_url_from_settings(self): """Uses URL from settings when creating LLM.""" def mock_get_setting_side_effect(key, default=None, *args, **kwargs): settings_map = { "llm.openai_endpoint.url": "http://172.19.0.5:8000/v1", "llm.openai_endpoint.api_key": "test-api-key", } return settings_map.get(key, default) with patch( "local_deep_research.llm.providers.implementations.custom_openai_endpoint.get_setting_from_snapshot" ) as mock_get_setting: mock_get_setting.side_effect = mock_get_setting_side_effect with patch( "local_deep_research.config.thread_settings.get_setting_from_snapshot" ) as mock_base_get_setting: mock_base_get_setting.side_effect = mock_get_setting_side_effect with patch( "local_deep_research.llm.providers.openai_base.ChatOpenAI" ) as mock_chat_openai: mock_llm = Mock() mock_chat_openai.return_value = mock_llm result = CustomOpenAIEndpointProvider.create_llm( model_name="test-model" ) assert result is mock_llm call_kwargs = mock_chat_openai.call_args[1] assert "172.19.0.5" in call_kwargs["base_url"] @staticmethod def _create_llm_with_settings(settings_map): def side_effect(key, default=None, *args, **kwargs): return settings_map.get(key, default) with patch( "local_deep_research.llm.providers.implementations.custom_openai_endpoint.get_setting_from_snapshot" ) as mock_get_setting: mock_get_setting.side_effect = side_effect with patch( "local_deep_research.config.thread_settings.get_setting_from_snapshot" ) as mock_base_get_setting: mock_base_get_setting.side_effect = side_effect with patch( "local_deep_research.llm.providers.openai_base.ChatOpenAI" ) as mock_chat_openai: mock_chat_openai.return_value = Mock() CustomOpenAIEndpointProvider.create_llm( model_name="test-model" ) return mock_chat_openai.call_args[1] def test_create_llm_stream_usage_enabled(self): """stream_usage setting on -> passed to ChatOpenAI.""" call_kwargs = self._create_llm_with_settings( { "llm.openai_endpoint.url": "http://localhost:1234/v1", "llm.openai_endpoint.api_key": "test-api-key", "llm.openai_endpoint.stream_usage": True, } ) assert call_kwargs["stream_usage"] is True def test_create_llm_stream_usage_off_by_default(self): """Without the setting, stream_usage is not sent at all — some OpenAI-compatible gateways 400 on unknown stream_options.""" call_kwargs = self._create_llm_with_settings( { "llm.openai_endpoint.url": "http://localhost:1234/v1", "llm.openai_endpoint.api_key": "test-api-key", } ) assert "stream_usage" not in call_kwargs def test_create_llm_stream_usage_explicit_false_not_sent(self): """An explicit false behaves like the default: parameter omitted.""" call_kwargs = self._create_llm_with_settings( { "llm.openai_endpoint.url": "http://localhost:1234/v1", "llm.openai_endpoint.api_key": "test-api-key", "llm.openai_endpoint.stream_usage": False, } ) assert "stream_usage" not in call_kwargs class TestCustomOpenAIEndpointAPISignature: """Tests for API method signatures. These tests ensure the API contract is maintained to prevent regressions. """ def test_list_models_for_api_accepts_base_url(self): """list_models_for_api accepts base_url parameter. REGRESSION TEST: The method must accept base_url directly, not through settings_snapshot indirection. """ import inspect sig = inspect.signature( CustomOpenAIEndpointProvider.list_models_for_api ) params = list(sig.parameters.keys()) assert "api_key" in params, "api_key parameter is required" assert "base_url" in params, "base_url parameter is required" # Should NOT have settings_snapshot - that was the problematic pattern assert "settings_snapshot" not in params, ( "settings_snapshot should not be a parameter - use base_url directly" ) def test_list_models_for_api_base_url_is_second_param(self): """base_url is the second parameter for easy calling. This ensures callers can use positional args: list_models_for_api(key, url) """ import inspect sig = inspect.signature( CustomOpenAIEndpointProvider.list_models_for_api ) params = list(sig.parameters.keys()) assert params[0] == "api_key" assert params[1] == "base_url" class TestCustomOpenAIEndpointIntegration: """Integration tests that verify the full flow works.""" @pytest.fixture def mock_openai_client(self): """Create a mock OpenAI client.""" client = Mock() mock_model = Mock() mock_model.id = "test-model" models_response = Mock() models_response.data = [mock_model] client.models.list.return_value = models_response return client def test_end_to_end_model_listing_with_private_ip(self, mock_openai_client): """Full end-to-end test simulating settings_routes.py flow. REGRESSION TEST: This simulates exactly what happens when the settings API fetches models for a custom endpoint. """ # This simulates what settings_routes.py does: # 1. Get URL from session/settings # 2. Get API key from session/settings # 3. Call list_models_for_api with both custom_url = "http://172.19.0.5:8000/v1" api_key = "" # Often empty for self-hosted with patch("openai.OpenAI") as mock_openai: mock_openai.return_value = mock_openai_client # This is exactly how settings_routes.py calls it now result = CustomOpenAIEndpointProvider.list_models_for_api( api_key if api_key else None, custom_url if custom_url else None, ) # Verify the URL was used correctly mock_openai.assert_called_once() call_kwargs = mock_openai.call_args[1] assert call_kwargs["base_url"] == custom_url # Verify models came back assert len(result) == 1 assert result[0]["value"] == "test-model" class TestCustomOpenAIEndpointIsAvailable: """Tests for CustomOpenAIEndpointProvider.is_available(). The provider should be available when EITHER an API key OR a non-default custom URL is configured. """ def test_available_with_api_key_only(self): """Available when API key is set, even without custom URL.""" snapshot = { "llm.openai_endpoint.api_key": "my-secret-key", } assert CustomOpenAIEndpointProvider.is_available(snapshot) is True def test_available_with_custom_url_only(self): """Available when custom URL is set, even without API key.""" snapshot = { "llm.openai_endpoint.url": "http://192.168.1.100:8000/v1", } assert CustomOpenAIEndpointProvider.is_available(snapshot) is True def test_available_with_both(self): """Available when both API key and custom URL are set.""" snapshot = { "llm.openai_endpoint.api_key": "my-key", "llm.openai_endpoint.url": "http://10.0.0.5:8000/v1", } assert CustomOpenAIEndpointProvider.is_available(snapshot) is True def test_not_available_with_neither(self): """Not available when neither API key nor custom URL is set.""" snapshot = {} assert CustomOpenAIEndpointProvider.is_available(snapshot) is False def test_not_available_with_default_url_no_key(self): """Not available when URL equals the default OpenAI endpoint.""" snapshot = { "llm.openai_endpoint.url": "https://api.openai.com/v1", } assert CustomOpenAIEndpointProvider.is_available(snapshot) is False def test_not_available_with_empty_key(self): """Not available when API key is empty string.""" snapshot = { "llm.openai_endpoint.api_key": "", } assert CustomOpenAIEndpointProvider.is_available(snapshot) is False def test_not_available_with_whitespace_key(self): """Not available when API key is whitespace only.""" snapshot = { "llm.openai_endpoint.api_key": " ", } assert CustomOpenAIEndpointProvider.is_available(snapshot) is False def test_available_with_private_ip_url(self): """Available when URL points to a private IP (Docker, LAN, etc.).""" snapshot = { "llm.openai_endpoint.url": "http://172.19.0.5:8000/v1", } assert CustomOpenAIEndpointProvider.is_available(snapshot) is True def test_not_available_with_none_snapshot(self): """Not available when settings_snapshot is None (no thread-local).""" assert CustomOpenAIEndpointProvider.is_available(None) is False class TestProviderDiscoveryAttributes: """Tests ensuring provider has required attributes for auto-discovery.""" def test_provider_has_required_attributes_for_discovery(self): """Provider has all attributes needed for auto-discovery.""" required_attrs = [ "provider_name", "provider_key", "company_name", "api_key_setting", "url_setting", "default_base_url", "default_model", ] for attr in required_attrs: assert hasattr(CustomOpenAIEndpointProvider, attr), ( f"Missing required attribute: {attr}" )