learningcircuit--local-deep-research
7a0da7932b
OSV-Scanner (Scheduled) / scan-scheduled (push) Failing after 0s
Create Release / test-gate (push) Has been cancelled
Create Release / release-gate (push) Has been cancelled
Create Release / ci-gate (push) Has been cancelled
Create Release / version-check (push) Has been cancelled
Create Release / e2e-test-gate (push) Has been cancelled
Create Release / responsive-test-gate (push) Has been cancelled
Create Release / compat-test-gate (push) Has been cancelled
Create Release / compose-integration-gate (push) Has been cancelled
Create Release / vulture-gate (push) Has been cancelled
Create Release / build (push) Has been cancelled
Create Release / provenance (push) Has been cancelled
Create Release / prerelease-docker (push) Has been cancelled
Create Release / publish-docker (push) Has been cancelled
Create Release / create-release (push) Has been cancelled
Create Release / cleanup-changelog (push) Has been cancelled
Create Release / trigger-pypi (push) Has been cancelled
Create Release / monitor-pypi (push) Has been cancelled
Create Release / Clean up orphan prerelease tags and signatures (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-form] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-metrics] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-workflow] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-core] (push) Has been cancelled
CodeQL Advanced / Analyze (javascript-typescript) (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [history-news] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [library] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [link-analytics] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-core] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-lifecycle] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [error-benchmark] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-pages] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) (push) Has been cancelled
Docker Tests (Consolidated) / Accessibility Tests (push) Has been cancelled
Docker Tests (Consolidated) / LLM Unit Tests (push) Has been cancelled
Docker Tests (Consolidated) / LLM Example Tests (push) Has been cancelled
Docker Tests (Consolidated) / Production Image Smoke Test (push) Has been cancelled
Docker Tests (Consolidated) / Infrastructure Tests (push) Has been cancelled
OSSF Scorecard / OSSF Security Scorecard Analysis (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [mobile] (push) Has been cancelled
Backwards Compatibility / Verify Encryption Constants (push) Has been cancelled
Backwards Compatibility / PyPI Version Compatibility (push) Has been cancelled
Backwards Compatibility / Database Migration Tests (push) Has been cancelled
CodeQL Advanced / Analyze (python) (push) Has been cancelled
Docker Tests (Consolidated) / detect-changes (push) Has been cancelled
Docker Tests (Consolidated) / Build Test Image (push) Has been cancelled
Docker Tests (Consolidated) / All Pytest Tests + Coverage (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [accessibility] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [api-crud] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-login] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-pages] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-register] (push) Has been cancelled
562 行
17 KiB
Python
562 行
17 KiB
Python
"""
|
|
Edge case and error handling tests for strategies.
|
|
|
|
Tests unusual inputs, error conditions, and boundary cases.
|
|
"""
|
|
|
|
import pytest
|
|
from loguru import logger
|
|
from unittest.mock import Mock
|
|
|
|
|
|
# Working strategies for testing
|
|
WORKING_STRATEGIES = [
|
|
"source-based",
|
|
"rapid",
|
|
"parallel",
|
|
"iterdrag",
|
|
"news",
|
|
"focused-iteration",
|
|
]
|
|
|
|
|
|
class TestEmptyInputs:
|
|
"""Test strategy behavior with empty inputs."""
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_empty_query(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_mock_search,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test strategy with empty query string."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=strategy_mock_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
# Empty query
|
|
result = strategy.analyze_topic("")
|
|
|
|
assert isinstance(result, dict)
|
|
logger.info(f"{strategy_name} with empty query: {list(result.keys())}")
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_whitespace_only_query(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_mock_search,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test strategy with whitespace-only query."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=strategy_mock_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
result = strategy.analyze_topic(" \t\n ")
|
|
|
|
assert isinstance(result, dict)
|
|
|
|
|
|
class TestLongInputs:
|
|
"""Test strategy behavior with very long inputs."""
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_very_long_query(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_mock_search,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test strategy with very long query."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=strategy_mock_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
# Very long query (1000+ characters)
|
|
long_query = "What is " + "very " * 200 + "important?"
|
|
|
|
result = strategy.analyze_topic(long_query)
|
|
|
|
assert isinstance(result, dict)
|
|
logger.info(f"{strategy_name} handled {len(long_query)} char query")
|
|
|
|
|
|
class TestSpecialCharacters:
|
|
"""Test strategy behavior with special characters in input."""
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_unicode_query(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_mock_search,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test strategy with unicode characters."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=strategy_mock_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
unicode_query = "What is 日本語 and 中文 and العربية?"
|
|
|
|
result = strategy.analyze_topic(unicode_query)
|
|
|
|
assert isinstance(result, dict)
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_query_with_special_chars(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_mock_search,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test strategy with special characters."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=strategy_mock_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
special_query = "What about <script>alert('xss')</script> and $PATH?"
|
|
|
|
result = strategy.analyze_topic(special_query)
|
|
|
|
assert isinstance(result, dict)
|
|
|
|
|
|
class TestLLMErrors:
|
|
"""Test strategy behavior when LLM returns errors or unusual responses."""
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_llm_returns_empty_response(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_search,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test when LLM returns empty response."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
mock_llm = Mock()
|
|
mock_llm.invoke = Mock(return_value=Mock(content=""))
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=mock_llm,
|
|
search=strategy_mock_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
try:
|
|
result = strategy.analyze_topic("Test query")
|
|
assert isinstance(result, dict)
|
|
except Exception as e:
|
|
logger.warning(
|
|
f"{strategy_name} failed with empty LLM response: {e}"
|
|
)
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_llm_returns_none(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_search,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test when LLM returns None."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
mock_llm = Mock()
|
|
mock_llm.invoke = Mock(return_value=None)
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=mock_llm,
|
|
search=strategy_mock_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
try:
|
|
result = strategy.analyze_topic("Test query")
|
|
# May succeed or fail - we're testing it doesn't crash badly
|
|
if result:
|
|
assert isinstance(result, dict)
|
|
except AttributeError:
|
|
# Expected when trying to access .content on None
|
|
pass
|
|
except Exception as e:
|
|
logger.warning(
|
|
f"{strategy_name} with None LLM response: {type(e).__name__}"
|
|
)
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:2])
|
|
def test_llm_raises_exception(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_search,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test when LLM raises an exception."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
mock_llm = Mock()
|
|
mock_llm.invoke = Mock(side_effect=Exception("LLM API Error"))
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=mock_llm,
|
|
search=strategy_mock_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
# Strategy should handle or propagate the exception
|
|
try:
|
|
result = strategy.analyze_topic("Test query")
|
|
# If it succeeds, should return dict with error
|
|
if isinstance(result, dict) and "error" in result:
|
|
logger.info(
|
|
f"{strategy_name} returned error: {result['error']}"
|
|
)
|
|
except Exception as e:
|
|
logger.info(
|
|
f"{strategy_name} propagated exception: {type(e).__name__}"
|
|
)
|
|
|
|
|
|
class TestSearchErrors:
|
|
"""Test strategy behavior when search returns errors."""
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_search_returns_empty_list(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test when search returns empty list."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
empty_search = Mock()
|
|
empty_search.run = Mock(return_value=[])
|
|
empty_search.include_full_content = True
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=empty_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
result = strategy.analyze_topic("Test query")
|
|
|
|
assert isinstance(result, dict)
|
|
logger.info(f"{strategy_name} with empty search: {list(result.keys())}")
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_search_returns_none(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test when search returns None."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
none_search = Mock()
|
|
none_search.run = Mock(return_value=None)
|
|
none_search.include_full_content = True
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=none_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
try:
|
|
result = strategy.analyze_topic("Test query")
|
|
assert isinstance(result, dict)
|
|
except TypeError:
|
|
# May fail when trying to iterate None
|
|
pass
|
|
except Exception as e:
|
|
logger.warning(
|
|
f"{strategy_name} with None search: {type(e).__name__}"
|
|
)
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_search_raises_exception(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test when search raises exception."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
error_search = Mock()
|
|
error_search.run = Mock(side_effect=Exception("Search API Error"))
|
|
error_search.include_full_content = True
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=error_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
try:
|
|
result = strategy.analyze_topic("Test query")
|
|
# If handled, should return dict
|
|
assert isinstance(result, dict)
|
|
except Exception as e:
|
|
logger.info(
|
|
f"{strategy_name} propagated search error: {type(e).__name__}"
|
|
)
|
|
|
|
|
|
class TestMalformedSearchResults:
|
|
"""Test strategy behavior with malformed search results."""
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_search_missing_fields(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test with search results missing expected fields."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
malformed_search = Mock()
|
|
malformed_search.run = Mock(
|
|
return_value=[
|
|
{}, # Empty dict
|
|
{"title": "Only title"}, # Missing link, snippet
|
|
{"link": "https://example.com"}, # Missing title, snippet
|
|
]
|
|
)
|
|
malformed_search.include_full_content = True
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=malformed_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
result = strategy.analyze_topic("Test query")
|
|
|
|
assert isinstance(result, dict)
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_search_with_none_values(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test with search results containing None values."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
none_values_search = Mock()
|
|
none_values_search.run = Mock(
|
|
return_value=[
|
|
{"title": None, "link": None, "snippet": None},
|
|
{
|
|
"title": "Valid",
|
|
"link": "https://example.com",
|
|
"snippet": "Content",
|
|
},
|
|
]
|
|
)
|
|
none_values_search.include_full_content = True
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=none_values_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
result = strategy.analyze_topic("Test query")
|
|
|
|
assert isinstance(result, dict)
|
|
|
|
|
|
class TestProgressCallback:
|
|
"""Test progress callback behavior."""
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_callback_receives_valid_progress(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_mock_search,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test that callback receives valid progress values."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=strategy_mock_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
progress_values = []
|
|
|
|
def track_progress(message, progress, data):
|
|
progress_values.append(progress)
|
|
|
|
strategy.set_progress_callback(track_progress)
|
|
strategy.analyze_topic("Test query")
|
|
|
|
# Progress values should be non-negative (None is valid for optional updates)
|
|
for p in progress_values:
|
|
assert p is None or p >= 0, f"Negative progress value: {p}"
|
|
|
|
# Document any values exceeding 100 as warnings for later analysis
|
|
over_100 = [p for p in progress_values if p is not None and p > 100]
|
|
if over_100:
|
|
logger.warning(
|
|
f"{strategy_name} reported progress > 100: {over_100}. "
|
|
"This is a strategy bug that should be investigated."
|
|
)
|
|
|
|
logger.info(f"{strategy_name} progress values: {progress_values}")
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_callback_exception_doesnt_crash_strategy(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_mock_search,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test that exception in callback doesn't crash strategy."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=strategy_mock_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
def bad_callback(message, progress, data):
|
|
raise Exception("Callback error!")
|
|
|
|
strategy.set_progress_callback(bad_callback)
|
|
|
|
# Should not crash (strategy may catch callback exceptions)
|
|
try:
|
|
result = strategy.analyze_topic("Test query")
|
|
assert isinstance(result, dict)
|
|
except Exception as e:
|
|
logger.warning(
|
|
f"{strategy_name} callback exception propagated: {type(e).__name__}"
|
|
)
|
|
|
|
|
|
class TestMultipleAnalyzeCalls:
|
|
"""Test calling analyze_topic multiple times on same strategy."""
|
|
|
|
@pytest.mark.parametrize("strategy_name", WORKING_STRATEGIES[:3])
|
|
def test_multiple_calls_same_strategy(
|
|
self,
|
|
strategy_name,
|
|
strategy_mock_llm,
|
|
strategy_mock_search,
|
|
strategy_settings_snapshot,
|
|
):
|
|
"""Test multiple analyze_topic calls on same strategy instance."""
|
|
from local_deep_research.search_system_factory import (
|
|
create_strategy,
|
|
)
|
|
|
|
strategy = create_strategy(
|
|
strategy_name=strategy_name,
|
|
model=strategy_mock_llm,
|
|
search=strategy_mock_search,
|
|
settings_snapshot=strategy_settings_snapshot,
|
|
)
|
|
|
|
# First call
|
|
result1 = strategy.analyze_topic("Query 1")
|
|
links_after_1 = len(strategy.all_links_of_system)
|
|
|
|
# Second call
|
|
result2 = strategy.analyze_topic("Query 2")
|
|
links_after_2 = len(strategy.all_links_of_system)
|
|
|
|
# Third call
|
|
result3 = strategy.analyze_topic("Query 3")
|
|
links_after_3 = len(strategy.all_links_of_system)
|
|
|
|
assert isinstance(result1, dict)
|
|
assert isinstance(result2, dict)
|
|
assert isinstance(result3, dict)
|
|
|
|
logger.info(
|
|
f"{strategy_name} links: {links_after_1} -> {links_after_2} -> {links_after_3}"
|
|
)
|