learningcircuit--local-deep-research
7a0da7932b
OSV-Scanner (Scheduled) / scan-scheduled (push) Failing after 0s
Create Release / test-gate (push) Has been cancelled
Create Release / release-gate (push) Has been cancelled
Create Release / ci-gate (push) Has been cancelled
Create Release / version-check (push) Has been cancelled
Create Release / e2e-test-gate (push) Has been cancelled
Create Release / responsive-test-gate (push) Has been cancelled
Create Release / compat-test-gate (push) Has been cancelled
Create Release / compose-integration-gate (push) Has been cancelled
Create Release / vulture-gate (push) Has been cancelled
Create Release / build (push) Has been cancelled
Create Release / provenance (push) Has been cancelled
Create Release / prerelease-docker (push) Has been cancelled
Create Release / publish-docker (push) Has been cancelled
Create Release / create-release (push) Has been cancelled
Create Release / cleanup-changelog (push) Has been cancelled
Create Release / trigger-pypi (push) Has been cancelled
Create Release / monitor-pypi (push) Has been cancelled
Create Release / Clean up orphan prerelease tags and signatures (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-form] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-metrics] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-workflow] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-core] (push) Has been cancelled
CodeQL Advanced / Analyze (javascript-typescript) (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [history-news] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [library] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [link-analytics] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-core] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-lifecycle] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [error-benchmark] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-pages] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) (push) Has been cancelled
Docker Tests (Consolidated) / Accessibility Tests (push) Has been cancelled
Docker Tests (Consolidated) / LLM Unit Tests (push) Has been cancelled
Docker Tests (Consolidated) / LLM Example Tests (push) Has been cancelled
Docker Tests (Consolidated) / Production Image Smoke Test (push) Has been cancelled
Docker Tests (Consolidated) / Infrastructure Tests (push) Has been cancelled
OSSF Scorecard / OSSF Security Scorecard Analysis (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [mobile] (push) Has been cancelled
Backwards Compatibility / Verify Encryption Constants (push) Has been cancelled
Backwards Compatibility / PyPI Version Compatibility (push) Has been cancelled
Backwards Compatibility / Database Migration Tests (push) Has been cancelled
CodeQL Advanced / Analyze (python) (push) Has been cancelled
Docker Tests (Consolidated) / detect-changes (push) Has been cancelled
Docker Tests (Consolidated) / Build Test Image (push) Has been cancelled
Docker Tests (Consolidated) / All Pytest Tests + Coverage (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [accessibility] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [api-crud] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-login] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-pages] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-register] (push) Has been cancelled
264 行
8.4 KiB
Python
264 行
8.4 KiB
Python
"""
|
|
Tests for XBenchDeepSearchDataset pure logic methods.
|
|
|
|
Existing tests: none — this file is the first test coverage.
|
|
|
|
Covers:
|
|
- xor_decrypt: symmetric encryption, empty data, key cycling, single byte
|
|
- get_dataset_info: expected keys and values
|
|
- get_default_dataset_path: exact string
|
|
- process_example: dict enrichment, shallow copy, evaluation criteria weights
|
|
"""
|
|
|
|
import base64
|
|
|
|
|
|
def _cls():
|
|
"""Import and return the XBenchDeepSearchDataset class."""
|
|
from local_deep_research.benchmarks.datasets.xbench_deepsearch import (
|
|
XBenchDeepSearchDataset,
|
|
)
|
|
|
|
return XBenchDeepSearchDataset
|
|
|
|
|
|
class TestXorDecrypt:
|
|
"""Tests for xor_decrypt static method."""
|
|
|
|
def test_symmetry(self):
|
|
"""XOR decrypt is symmetric: decrypt(encrypt(x)) == x."""
|
|
cls = _cls()
|
|
plaintext = b"Hello, World!"
|
|
key = "secret"
|
|
encrypted = cls.xor_decrypt(plaintext, key)
|
|
decrypted = cls.xor_decrypt(encrypted, key)
|
|
assert decrypted == plaintext
|
|
|
|
def test_known_values(self):
|
|
"""Known plaintext/key produces expected ciphertext."""
|
|
cls = _cls()
|
|
# XOR of b'A' (0x41) with b'k' (0x6B) = 0x2A = b'*'
|
|
result = cls.xor_decrypt(b"A", "k")
|
|
assert result == bytes([0x41 ^ 0x6B])
|
|
|
|
def test_empty_data(self):
|
|
"""Empty data returns empty bytes."""
|
|
cls = _cls()
|
|
result = cls.xor_decrypt(b"", "anykey")
|
|
assert result == b""
|
|
|
|
def test_key_cycling(self):
|
|
"""Key cycles through for data longer than key."""
|
|
cls = _cls()
|
|
data = b"ABCD"
|
|
key = "ab"
|
|
# A^a, B^b, C^a, D^b
|
|
expected = bytes(
|
|
[
|
|
ord("A") ^ ord("a"),
|
|
ord("B") ^ ord("b"),
|
|
ord("C") ^ ord("a"),
|
|
ord("D") ^ ord("b"),
|
|
]
|
|
)
|
|
result = cls.xor_decrypt(data, key)
|
|
assert result == expected
|
|
|
|
def test_single_byte_key(self):
|
|
"""Single byte key applied to every byte."""
|
|
cls = _cls()
|
|
data = b"HELLO"
|
|
key = "X"
|
|
expected = bytes([b ^ ord("X") for b in data])
|
|
result = cls.xor_decrypt(data, key)
|
|
assert result == expected
|
|
|
|
def test_all_zeros(self):
|
|
"""XOR with all-zero data returns key bytes repeated."""
|
|
cls = _cls()
|
|
data = bytes(5) # 5 zero bytes
|
|
key = "AB"
|
|
# 0^A, 0^B, 0^A, 0^B, 0^A
|
|
expected = bytes([ord("A"), ord("B"), ord("A"), ord("B"), ord("A")])
|
|
result = cls.xor_decrypt(data, key)
|
|
assert result == expected
|
|
|
|
def test_unicode_key(self):
|
|
"""Unicode key is encoded as UTF-8."""
|
|
cls = _cls()
|
|
data = b"\x00\x00\x00"
|
|
key = "\u00e9" # é = 0xC3 0xA9 in UTF-8 (2 bytes)
|
|
key_bytes = key.encode("utf-8")
|
|
expected = bytes(
|
|
[
|
|
0x00 ^ key_bytes[0],
|
|
0x00 ^ key_bytes[1],
|
|
0x00 ^ key_bytes[0],
|
|
]
|
|
)
|
|
result = cls.xor_decrypt(data, key)
|
|
assert result == expected
|
|
|
|
def test_roundtrip_with_base64(self):
|
|
"""Full roundtrip: plaintext -> encrypt -> base64 -> decode -> decrypt."""
|
|
cls = _cls()
|
|
plaintext = b"This is a test question for benchmarking"
|
|
key = "canary_key_123"
|
|
encrypted = cls.xor_decrypt(plaintext, key)
|
|
encoded = base64.b64encode(encrypted)
|
|
decoded = base64.b64decode(encoded)
|
|
decrypted = cls.xor_decrypt(decoded, key)
|
|
assert decrypted == plaintext
|
|
|
|
|
|
class TestGetDatasetInfo:
|
|
"""Tests for get_dataset_info class method."""
|
|
|
|
def test_returns_dict(self):
|
|
"""Returns a dictionary."""
|
|
cls = _cls()
|
|
info = cls.get_dataset_info()
|
|
assert isinstance(info, dict)
|
|
|
|
def test_has_required_keys(self):
|
|
"""Contains id, name, description, and url."""
|
|
cls = _cls()
|
|
info = cls.get_dataset_info()
|
|
assert "id" in info
|
|
assert "name" in info
|
|
assert "description" in info
|
|
assert "url" in info
|
|
|
|
def test_id_value(self):
|
|
"""Dataset ID is 'xbench_deepsearch'."""
|
|
cls = _cls()
|
|
info = cls.get_dataset_info()
|
|
assert info["id"] == "xbench_deepsearch"
|
|
|
|
def test_name_value(self):
|
|
"""Dataset name is 'xbench-DeepSearch'."""
|
|
cls = _cls()
|
|
info = cls.get_dataset_info()
|
|
assert info["name"] == "xbench-DeepSearch"
|
|
|
|
def test_url_is_huggingface(self):
|
|
"""URL points to Hugging Face."""
|
|
cls = _cls()
|
|
info = cls.get_dataset_info()
|
|
assert "huggingface.co" in info["url"]
|
|
|
|
def test_callable_on_class_and_instance(self):
|
|
"""Can be called on both class and instance."""
|
|
cls = _cls()
|
|
class_result = cls.get_dataset_info()
|
|
instance_result = cls().get_dataset_info()
|
|
assert class_result == instance_result
|
|
|
|
|
|
class TestGetDefaultDatasetPath:
|
|
"""Tests for get_default_dataset_path class method."""
|
|
|
|
def test_returns_string(self):
|
|
"""Returns a string."""
|
|
cls = _cls()
|
|
path = cls.get_default_dataset_path()
|
|
assert isinstance(path, str)
|
|
|
|
def test_exact_value(self):
|
|
"""Returns the expected HuggingFace path."""
|
|
cls = _cls()
|
|
path = cls.get_default_dataset_path()
|
|
assert path == "xbench/DeepSearch"
|
|
|
|
|
|
class TestProcessExample:
|
|
"""Tests for process_example instance method."""
|
|
|
|
def test_adds_requires_deep_search(self):
|
|
"""Adds requires_deep_search=True."""
|
|
cls = _cls()
|
|
instance = cls()
|
|
result = instance.process_example({"id": "q1", "problem": "test"})
|
|
assert result["requires_deep_search"] is True
|
|
|
|
def test_adds_expected_iterations(self):
|
|
"""Adds expected_iterations=4."""
|
|
cls = _cls()
|
|
instance = cls()
|
|
result = instance.process_example({"id": "q1"})
|
|
assert result["expected_iterations"] == 4
|
|
|
|
def test_adds_evaluation_criteria(self):
|
|
"""Adds evaluation_criteria dict."""
|
|
cls = _cls()
|
|
instance = cls()
|
|
result = instance.process_example({"id": "q1"})
|
|
criteria = result["evaluation_criteria"]
|
|
assert "accuracy" in criteria
|
|
assert "completeness" in criteria
|
|
assert "reasoning" in criteria
|
|
assert "sources" in criteria
|
|
|
|
def test_evaluation_criteria_weights_sum_to_one(self):
|
|
"""Evaluation criteria weights sum to 1.0."""
|
|
cls = _cls()
|
|
instance = cls()
|
|
result = instance.process_example({"id": "q1"})
|
|
total = sum(result["evaluation_criteria"].values())
|
|
assert abs(total - 1.0) < 1e-9
|
|
|
|
def test_preserves_original_fields(self):
|
|
"""Original example fields are preserved."""
|
|
cls = _cls()
|
|
instance = cls()
|
|
original = {"id": "q42", "problem": "What is X?", "answer": "Y"}
|
|
result = instance.process_example(original)
|
|
assert result["id"] == "q42"
|
|
assert result["problem"] == "What is X?"
|
|
assert result["answer"] == "Y"
|
|
|
|
def test_does_not_modify_original(self):
|
|
"""Original dict is not modified (shallow copy)."""
|
|
cls = _cls()
|
|
instance = cls()
|
|
original = {"id": "q1", "problem": "test"}
|
|
instance.process_example(original)
|
|
assert "requires_deep_search" not in original
|
|
assert "evaluation_criteria" not in original
|
|
|
|
def test_empty_input(self):
|
|
"""Empty dict gets all enrichment fields."""
|
|
cls = _cls()
|
|
instance = cls()
|
|
result = instance.process_example({})
|
|
assert result["requires_deep_search"] is True
|
|
assert result["expected_iterations"] == 4
|
|
assert "evaluation_criteria" in result
|
|
|
|
def test_existing_field_overwritten(self):
|
|
"""If input has 'requires_deep_search', it gets overwritten."""
|
|
cls = _cls()
|
|
instance = cls()
|
|
result = instance.process_example({"requires_deep_search": False})
|
|
assert result["requires_deep_search"] is True
|
|
|
|
|
|
class TestXBenchInheritance:
|
|
"""Tests for XBenchDeepSearchDataset class structure."""
|
|
|
|
def test_inherits_from_benchmark_dataset(self):
|
|
"""Inherits from BenchmarkDataset base class."""
|
|
cls = _cls()
|
|
from local_deep_research.benchmarks.datasets.base import (
|
|
BenchmarkDataset,
|
|
)
|
|
|
|
assert issubclass(cls, BenchmarkDataset)
|
|
|
|
def test_instantiation(self):
|
|
"""Can be instantiated without arguments."""
|
|
cls = _cls()
|
|
instance = cls()
|
|
assert instance is not None
|
|
assert instance.dataset_path == "xbench/DeepSearch"
|