feat(plugin-eval): implement Layer 3 Monte Carlo simulation with statistical analysis
Adds MonteCarloAnalyzer with SimResult/MonteCarloConfig dataclasses, run_simulation async helper, and _compute_statistics using Wilson score CI, bootstrap CI, Clopper-Pearson CI, and coefficient of variation. Wires MC layer into evaluate_skill for Depth.DEEP and Depth.THOROUGH runs (50 and 100 runs respectively).
这个提交包含在:
@@ -0,0 +1,45 @@
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from plugin_eval.layers.monte_carlo import MonteCarloAnalyzer, MonteCarloConfig, SimResult
|
||||
|
||||
|
||||
class TestSimResult:
|
||||
def test_sim_result(self):
|
||||
sr = SimResult(activated=True, quality_score=0.8, tokens=2500, duration_ms=1200)
|
||||
assert sr.activated is True
|
||||
assert sr.errored is False
|
||||
|
||||
|
||||
class TestMonteCarloAnalyzer:
|
||||
@pytest.mark.asyncio
|
||||
@patch("plugin_eval.layers.monte_carlo.run_simulation")
|
||||
async def test_run_with_mocked_sims(self, mock_sim, sample_skill_dir: Path):
|
||||
mock_sim.return_value = SimResult(
|
||||
activated=True, quality_score=0.82, tokens=2800, duration_ms=1500
|
||||
)
|
||||
config = MonteCarloConfig(n_runs=10, concurrency=2)
|
||||
analyzer = MonteCarloAnalyzer(config)
|
||||
result = await analyzer.analyze_skill(sample_skill_dir)
|
||||
assert result.layer == "monte_carlo"
|
||||
assert result.score > 0
|
||||
assert "triggering" in result.sub_scores
|
||||
assert "output_consistency" in result.sub_scores
|
||||
assert "failure_rate" in result.sub_scores
|
||||
|
||||
def test_statistical_analysis(self):
|
||||
"""Test the statistical analysis on pre-computed sim results."""
|
||||
analyzer = MonteCarloAnalyzer(MonteCarloConfig(n_runs=50))
|
||||
results = [
|
||||
SimResult(activated=True, quality_score=0.8 + i * 0.002, tokens=2500, duration_ms=1200)
|
||||
for i in range(48)
|
||||
] + [
|
||||
SimResult(activated=False, quality_score=0.0, tokens=500, duration_ms=200, errored=True),
|
||||
SimResult(activated=True, quality_score=0.75, tokens=8000, duration_ms=5000),
|
||||
]
|
||||
stats = analyzer._compute_statistics(results)
|
||||
assert stats["triggering"]["activation_rate"] == pytest.approx(0.98)
|
||||
assert stats["failure_rate"]["p_fail"] == pytest.approx(0.02)
|
||||
assert stats["output_consistency"]["cv"] < 0.15
|
||||
在新工单中引用