"""Unit tests for LLM capability helpers in app.llm.""" from unittest.mock import AsyncMock, MagicMock, patch import pytest from app.llm import _appears_truncated, _get_retry_temperature, _supports_temperature # --------------------------------------------------------------------------- # _supports_temperature # --------------------------------------------------------------------------- class TestSupportsTemperature: """Tests for _supports_temperature().""" def test_none_temperature_returns_true(self): """When temperature is None, the caller isn't setting a value — allow.""" assert _supports_temperature("gpt-4", None) is True def test_ollama_always_true(self): """Ollama models support temperature even when not in registry.""" assert _supports_temperature("ollama/llama3", 0.7) is True assert _supports_temperature("ollama_chat/llama3", 0.7) is True @patch("app.llm.litellm.get_model_info") def test_openai_gpt4_supports_temperature(self, mock_get_model_info): """GPT-4 has temperature in supported_openai_params.""" mock_get_model_info.return_value = { "supported_openai_params": ["temperature", "max_tokens", "top_p"] } assert _supports_temperature("gpt-4", 0.7) is True @patch("app.llm.litellm.get_model_info") def test_model_without_temperature_param(self, mock_get_model_info): """Model registry omits temperature → not supported.""" mock_get_model_info.return_value = { "supported_openai_params": ["max_tokens"] } assert _supports_temperature("some-model", 0.7) is False @patch("app.llm.litellm.get_model_info") def test_opus4_deprecated_temperature(self, mock_get_model_info): """Anthropic Opus 4.x deprecated temperature entirely.""" mock_get_model_info.return_value = { "supported_openai_params": ["temperature", "max_tokens"] } assert _supports_temperature("anthropic/claude-opus-4-7", 0.7) is False # Also check with temperature=1 — still deprecated assert _supports_temperature("anthropic/claude-opus-4-7", 1.0) is False @patch("app.llm.litellm.get_model_info") def test_kimi_k26_only_allows_one(self, mock_get_model_info): """Moonshot kimi-k2.6 only allows temperature=1.""" mock_get_model_info.return_value = { "supported_openai_params": ["temperature", "max_tokens"] } assert _supports_temperature("openai/kimi-k2.6", 0.7) is False assert _supports_temperature("openai/kimi-k2.6", 1.0) is True @patch("app.llm.litellm.get_model_info") def test_model_not_in_registry(self, mock_get_model_info): """Unknown model not in registry — be conservative, skip temperature.""" mock_get_model_info.side_effect = Exception("model not found") assert _supports_temperature("unknown-vendor/model", 0.7) is False @patch("app.llm.litellm.get_model_info") def test_case_insensitive_model_name(self, mock_get_model_info): """Provider-specific checks are case-insensitive.""" mock_get_model_info.return_value = { "supported_openai_params": ["temperature", "max_tokens"] } assert _supports_temperature("Anthropic/Claude-Opus-4-7", 0.7) is False assert _supports_temperature("OPENAI/KIMI-K2.6", 0.7) is False assert _supports_temperature("openai/KIMI-K2.6", 1.0) is True # --------------------------------------------------------------------------- # _get_retry_temperature # --------------------------------------------------------------------------- class TestGetRetryTemperature: """Tests for _get_retry_temperature().""" @patch("app.llm.litellm.get_model_info") def test_openai_progression(self, mock_get_model_info): """Standard retry temperature progression for supported models.""" mock_get_model_info.return_value = { "supported_openai_params": ["temperature", "max_tokens"] } assert _get_retry_temperature("gpt-4", 0) == 0.1 assert _get_retry_temperature("gpt-4", 1) == 0.3 assert _get_retry_temperature("gpt-4", 2) == 0.5 assert _get_retry_temperature("gpt-4", 3) == 0.7 assert _get_retry_temperature("gpt-4", 10) == 0.7 # clamped @patch("app.llm.litellm.get_model_info") def test_opus4_returns_none(self, mock_get_model_info): """Opus 4 doesn't support temperature → None on all retries.""" mock_get_model_info.return_value = { "supported_openai_params": ["temperature", "max_tokens"] } assert _get_retry_temperature("anthropic/claude-opus-4-7", 0) is None assert _get_retry_temperature("anthropic/claude-opus-4-7", 3) is None @patch("app.llm.litellm.get_model_info") def test_kimi_k26_returns_one(self, mock_get_model_info): """Kimi K2.6 only allows temperature=1 → always 1.0.""" mock_get_model_info.return_value = { "supported_openai_params": ["temperature", "max_tokens"] } assert _get_retry_temperature("openai/kimi-k2.6", 0) == 1.0 assert _get_retry_temperature("openai/kimi-k2.6", 1) == 1.0 assert _get_retry_temperature("openai/kimi-k2.6", 5) == 1.0 @patch("app.llm.litellm.get_model_info") def test_custom_base_temp(self, mock_get_model_info): """Custom base_temp is respected for supported models.""" mock_get_model_info.return_value = { "supported_openai_params": ["temperature", "max_tokens"] } assert _get_retry_temperature("gpt-4", 0, base_temp=0.2) == 0.2 assert _get_retry_temperature("gpt-4", 1, base_temp=0.2) == 0.3 # --------------------------------------------------------------------------- # _appears_truncated # --------------------------------------------------------------------------- class TestAppearsTruncated: """Tests for _appears_truncated() with schema_type awareness.""" # --- resume schema --- def test_resume_empty_work_experience(self): """Empty workExperience array in resume structure is suspicious.""" data = { "personalInfo": {"name": "John"}, "workExperience": [], "education": [{"degree": "BS"}], "skills": ["Python"], } assert _appears_truncated(data, schema_type="resume") is True def test_resume_empty_education(self): """Empty education array in resume structure is suspicious.""" data = { "personalInfo": {"name": "John"}, "workExperience": [{"title": "Dev"}], "education": [], "skills": ["Python"], } assert _appears_truncated(data, schema_type="resume") is True def test_resume_empty_skills(self): """Empty skills array in resume structure is suspicious.""" data = { "personalInfo": {"name": "John"}, "workExperience": [{"title": "Dev"}], "education": [{"degree": "BS"}], "skills": [], } assert _appears_truncated(data, schema_type="resume") is True def test_resume_valid(self): """Well-formed resume with all sections present is not truncated.""" data = { "personalInfo": {"name": "John"}, "workExperience": [{"title": "Dev"}], "education": [{"degree": "BS"}], "skills": ["Python"], } assert _appears_truncated(data, schema_type="resume") is False def test_resume_missing_fields_not_empty(self): """Missing fields are not the same as empty arrays — not flagged.""" data = { "personalInfo": {"name": "John"}, "workExperience": [{"title": "Dev"}], # education and skills omitted } assert _appears_truncated(data, schema_type="resume") is False # --- enrichment schema --- def test_enrichment_missing_keys(self): """Missing required keys in enrichment output is suspicious.""" data = {"analysis_summary": "Good resume"} assert _appears_truncated(data, schema_type="enrichment") is True def test_enrichment_empty_arrays(self): """Empty items_to_enrich and questions are valid (resume already strong).""" data = { "items_to_enrich": [], "questions": [], "analysis_summary": "Already strong", } assert _appears_truncated(data, schema_type="enrichment") is False def test_enrichment_populated(self): """Populated enrichment output is not truncated.""" data = { "items_to_enrich": [{"item_id": "exp_0"}], "questions": [{"question_id": "q_0"}], "analysis_summary": "Needs work", } assert _appears_truncated(data, schema_type="enrichment") is False # --- diff schema --- def test_diff_empty_changes(self): """Empty changes array in diff output is valid (no changes needed).""" data = {"changes": [], "strategy_notes": "No changes needed"} assert _appears_truncated(data, schema_type="diff") is False def test_diff_populated(self): """Populated diff output is not truncated.""" data = {"changes": [{"path": "summary", "action": "replace"}]} assert _appears_truncated(data, schema_type="diff") is False # --- keywords schema --- def test_keywords_empty(self): """Empty keyword lists are valid (sparse job description).""" data = {"required_skills": [], "preferred_skills": [], "keywords": []} assert _appears_truncated(data, schema_type="keywords") is False # --- default / unknown schema --- def test_default_schema_acts_like_resume(self): """Default schema_type behaves like 'resume' for backwards compatibility.""" data = {"workExperience": [], "education": [{"degree": "BS"}]} assert _appears_truncated(data) is True def test_unknown_schema_no_heuristics(self): """Unknown schema types have no truncation heuristics.""" data = {"anything": []} assert _appears_truncated(data, schema_type="custom") is False # --------------------------------------------------------------------------- # complete_json JSON mode fallback # --------------------------------------------------------------------------- class TestCompleteJsonFallback: """Tests for JSON mode fallback in complete_json().""" @pytest.mark.asyncio @patch("app.llm.get_router") @patch("app.llm.get_model_name") @patch("app.llm._supports_json_mode") async def test_json_mode_fallback_on_parse_error( self, mock_supports_json, mock_get_name, mock_get_router ): """When JSON mode returns invalid JSON, fallback to prompt-only mode. First call: JSON mode enabled → returns malformed JSON (trailing comma) → _extract_json succeeds → json.loads fails → JSONDecodeError Second call: JSON mode disabled → returns valid JSON → success """ mock_supports_json.return_value = True mock_get_name.return_value = "openrouter/openai/gpt-5.4" # First response: balanced braces but trailing comma → json.loads fails bad_choice = MagicMock() bad_choice.message.content = '{"items_to_enrich": [], "questions": [],}' bad_response = MagicMock() bad_response.choices = [bad_choice] # Second response: valid JSON without JSON mode good_choice = MagicMock() good_choice.message.content = '{"items_to_enrich": [], "questions": [], "analysis_summary": "ok"}' good_response = MagicMock() good_response.choices = [good_choice] router = MagicMock() router.acompletion = AsyncMock(side_effect=[bad_response, good_response]) config = MagicMock() config.provider = "openrouter" config.reasoning_effort = None mock_get_router.return_value = (router, config) from app.llm import complete_json result = await complete_json( prompt="Test prompt", schema_type="enrichment", retries=2, ) assert result == { "items_to_enrich": [], "questions": [], "analysis_summary": "ok", } # Verify JSON mode was used on first call but not second calls = router.acompletion.call_args_list assert calls[0].kwargs.get("response_format") == {"type": "json_object"} assert "response_format" not in calls[1].kwargs @pytest.mark.asyncio @patch("app.llm.get_router") @patch("app.llm.get_model_name") @patch("app.llm._supports_json_mode") async def test_json_mode_fallback_on_response_format_rejection( self, mock_supports_json, mock_get_name, mock_get_router ): """Issue #857: an OpenAI-compatible server (e.g. LM Studio) rejects ``response_format={"type": "json_object"}`` with a 400. First call: JSON mode enabled → server raises ``BadRequestError`` ("'response_format.type' must be 'json_schema' or 'text'"). Second call: JSON mode disabled → returns valid JSON → success. Before the fix the 400 was re-raised immediately (the existing fallback only handled malformed JSON, not rejection of the parameter itself), so the wizard turn failed with a 500. """ import litellm mock_supports_json.return_value = True mock_get_name.return_value = "openai/gemma-4-e2b" # First call raises the exact LM Studio rejection over the wire. rejection = litellm.BadRequestError( "OpenAIException - Error code: 400 - " "{'error': \"'response_format.type' must be 'json_schema' or 'text'\"}", model="openai/gemma-4-e2b", llm_provider="openai", ) good_choice = MagicMock() good_choice.message.content = '{"answer": "ok"}' good_response = MagicMock() good_response.choices = [good_choice] router = MagicMock() router.acompletion = AsyncMock(side_effect=[rejection, good_response]) config = MagicMock() config.provider = "openai_compatible" config.reasoning_effort = None mock_get_router.return_value = (router, config) from app.llm import complete_json result = await complete_json( prompt="Test prompt", schema_type="resume", retries=2, ) assert result == {"answer": "ok"} # JSON mode was sent on the first (rejected) call, dropped on the retry. calls = router.acompletion.call_args_list assert calls[0].kwargs.get("response_format") == {"type": "json_object"} assert "response_format" not in calls[1].kwargs @pytest.mark.asyncio @patch("app.llm.get_router") @patch("app.llm.get_model_name") @patch("app.llm._supports_json_mode") async def test_json_mode_fallback_on_varied_rejection_wording( self, mock_supports_json, mock_get_name, mock_get_router ): """The fallback must trigger across provider wording, not just LM Studio's. Guards against narrowing the heuristic so much that a genuine response_format rejection phrased as "not supported" is missed (which would re-introduce issue #857 for that provider). """ import litellm mock_supports_json.return_value = True mock_get_name.return_value = "openai/some-local-model" rejection = litellm.BadRequestError( "OpenAIException - Error code: 400 - " "{'error': 'response_format json_object is not supported by this model'}", model="openai/some-local-model", llm_provider="openai", ) good_choice = MagicMock() good_choice.message.content = '{"answer": "ok"}' good_response = MagicMock() good_response.choices = [good_choice] router = MagicMock() router.acompletion = AsyncMock(side_effect=[rejection, good_response]) config = MagicMock() config.provider = "openai_compatible" config.reasoning_effort = None mock_get_router.return_value = (router, config) from app.llm import complete_json result = await complete_json( prompt="Test prompt", schema_type="resume", retries=2 ) assert result == {"answer": "ok"} assert "response_format" not in router.acompletion.call_args_list[1].kwargs @pytest.mark.asyncio @patch("app.llm.get_router") @patch("app.llm.get_model_name") @patch("app.llm._supports_json_mode") async def test_unrelated_bad_request_is_not_swallowed( self, mock_supports_json, mock_get_name, mock_get_router ): """A 400 unrelated to response_format must still propagate, not retry. Uses a context-length error that *also names* response_format — the false-positive case raised in review (cubic/Kilo). Dropping JSON mode would not help, so the fallback must NOT fire and the error must surface. """ import litellm mock_supports_json.return_value = True mock_get_name.return_value = "openai/gpt-4o" rejection = litellm.BadRequestError( "OpenAIException - Error code: 400 - {'error': 'maximum context " "length exceeded while using response_format=json_object'}", model="openai/gpt-4o", llm_provider="openai", ) router = MagicMock() router.acompletion = AsyncMock(side_effect=rejection) config = MagicMock() config.provider = "openai" config.reasoning_effort = None mock_get_router.return_value = (router, config) from app.llm import complete_json with pytest.raises(litellm.BadRequestError): await complete_json(prompt="Test prompt", schema_type="resume", retries=2) # No retry: an unrelated 400 fails fast (Router already handles retries). assert router.acompletion.await_count == 1 # --------------------------------------------------------------------------- # complete() dynamic timeout # --------------------------------------------------------------------------- class TestCompleteDynamicTimeout: """Tests for complete() using _calculate_timeout().""" @pytest.mark.asyncio @patch("app.llm.get_router") @patch("app.llm.get_model_name") @patch("app.llm._calculate_timeout") @patch("app.llm._supports_temperature") async def test_uses_calculate_timeout( self, mock_supports_temp, mock_calc_timeout, mock_get_name, mock_get_router ): """complete() passes provider and max_tokens to _calculate_timeout.""" mock_supports_temp.return_value = True mock_calc_timeout.return_value = 180 mock_get_name.return_value = "deepseek/deepseek-chat" choice = MagicMock() choice.message.content = "Hello" response = MagicMock() response.choices = [choice] router = MagicMock() router.acompletion = AsyncMock(return_value=response) config = MagicMock() config.provider = "deepseek" mock_get_router.return_value = (router, config) from app.llm import complete await complete(prompt="Hi", max_tokens=8192) mock_calc_timeout.assert_called_once_with("completion", 8192, "deepseek") router.acompletion.assert_awaited_once() assert router.acompletion.call_args.kwargs["timeout"] == 180