hkuds--vibe-trading
wehub-resource-sync
241 行
8.8 KiB
Python
241 行
8.8 KiB
Python
"""Tests for backtest validation module.
|
|
|
|
Validates:
|
|
- Monte Carlo permutation test: p-value, output structure
|
|
- Bootstrap Sharpe CI: confidence interval bounds, prob_positive
|
|
- Walk-Forward analysis: window splitting, consistency metrics
|
|
- run_validation dispatcher
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
import pytest
|
|
|
|
from backtest.models import TradeRecord
|
|
from backtest.validation import (
|
|
bootstrap_sharpe_ci,
|
|
monte_carlo_test,
|
|
run_validation,
|
|
walk_forward_analysis,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_trades(pnls: list[float], start: str = "2025-01-01") -> list[TradeRecord]:
|
|
"""Create TradeRecord list from PnL values."""
|
|
trades = []
|
|
base = pd.Timestamp(start)
|
|
for i, pnl in enumerate(pnls):
|
|
entry = base + pd.Timedelta(days=i * 2)
|
|
exit_ = entry + pd.Timedelta(days=1)
|
|
trades.append(TradeRecord(
|
|
symbol="TEST",
|
|
direction=1,
|
|
entry_price=100.0,
|
|
exit_price=100.0 + pnl / 10,
|
|
entry_time=entry,
|
|
exit_time=exit_,
|
|
size=10.0,
|
|
leverage=1.0,
|
|
pnl=pnl,
|
|
pnl_pct=pnl / 1000 * 100,
|
|
exit_reason="signal",
|
|
holding_bars=1,
|
|
commission=0.0,
|
|
))
|
|
return trades
|
|
|
|
|
|
def _make_equity(n: int = 100, drift: float = 0.001, seed: int = 42) -> pd.Series:
|
|
"""Create a synthetic equity curve."""
|
|
rng = np.random.default_rng(seed)
|
|
returns = rng.normal(drift, 0.02, n)
|
|
prices = 1_000_000 * np.cumprod(1 + returns)
|
|
dates = pd.bdate_range("2025-01-01", periods=n)
|
|
return pd.Series(prices, index=dates)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Monte Carlo Permutation Test
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestMonteCarlo:
|
|
def test_output_structure(self) -> None:
|
|
trades = _make_trades([100, -50, 200, -30, 150, -80, 120, -40, 90, -20])
|
|
result = monte_carlo_test(trades, 1_000_000, n_simulations=100)
|
|
assert "actual_sharpe" in result
|
|
assert "p_value_sharpe" in result
|
|
assert "p_value_max_dd" in result
|
|
assert "n_simulations" in result
|
|
assert result["n_simulations"] == 100
|
|
assert result["n_trades"] == 10
|
|
|
|
def test_p_value_range(self) -> None:
|
|
trades = _make_trades([100, -50, 200, -30, 150])
|
|
result = monte_carlo_test(trades, 1_000_000, n_simulations=200)
|
|
assert 0.0 <= result["p_value_sharpe"] <= 1.0
|
|
assert 0.0 <= result["p_value_max_dd"] <= 1.0
|
|
|
|
def test_strong_strategy_low_p_value(self) -> None:
|
|
"""A consistently profitable strategy should have low p-value."""
|
|
trades = _make_trades([100, 200, 150, 180, 120, 90, 110, 130, 160, 140])
|
|
result = monte_carlo_test(trades, 1_000_000, n_simulations=500, seed=42)
|
|
# All trades profitable → hard to beat by shuffling (already optimal)
|
|
# p-value should be moderate (shuffling can't make it worse when all positive)
|
|
assert result["actual_sharpe"] > 0
|
|
|
|
def test_too_few_trades(self) -> None:
|
|
trades = _make_trades([100, -50])
|
|
result = monte_carlo_test(trades, 1_000_000)
|
|
assert "error" in result
|
|
|
|
def test_reproducibility(self) -> None:
|
|
trades = _make_trades([100, -50, 200, -30, 150, -80])
|
|
r1 = monte_carlo_test(trades, 1_000_000, n_simulations=100, seed=42)
|
|
r2 = monte_carlo_test(trades, 1_000_000, n_simulations=100, seed=42)
|
|
assert r1["p_value_sharpe"] == r2["p_value_sharpe"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Bootstrap Sharpe CI
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestBootstrapSharpe:
|
|
def test_output_structure(self) -> None:
|
|
eq = _make_equity(100)
|
|
result = bootstrap_sharpe_ci(eq, n_bootstrap=100)
|
|
assert "observed_sharpe" in result
|
|
assert "ci_lower" in result
|
|
assert "ci_upper" in result
|
|
assert "prob_positive" in result
|
|
assert "confidence" in result
|
|
assert result["confidence"] == 0.95
|
|
|
|
def test_ci_contains_observed(self) -> None:
|
|
"""The observed Sharpe should usually fall within the CI."""
|
|
eq = _make_equity(200, drift=0.001)
|
|
result = bootstrap_sharpe_ci(eq, n_bootstrap=500)
|
|
# Not guaranteed, but very likely for 95% CI
|
|
assert result["ci_lower"] <= result["ci_upper"]
|
|
|
|
def test_positive_drift_mostly_positive(self) -> None:
|
|
"""Equity with positive drift should have high prob_positive."""
|
|
eq = _make_equity(200, drift=0.003, seed=123)
|
|
result = bootstrap_sharpe_ci(eq, n_bootstrap=500)
|
|
assert result["prob_positive"] > 0.5
|
|
|
|
def test_too_few_observations(self) -> None:
|
|
eq = pd.Series([100, 101, 102], index=pd.bdate_range("2025-01-01", periods=3))
|
|
result = bootstrap_sharpe_ci(eq, n_bootstrap=100)
|
|
assert "error" in result
|
|
|
|
def test_reproducibility(self) -> None:
|
|
eq = _make_equity(50)
|
|
r1 = bootstrap_sharpe_ci(eq, n_bootstrap=100, seed=42)
|
|
r2 = bootstrap_sharpe_ci(eq, n_bootstrap=100, seed=42)
|
|
assert r1["ci_lower"] == r2["ci_lower"]
|
|
|
|
def test_custom_confidence(self) -> None:
|
|
eq = _make_equity(100)
|
|
r90 = bootstrap_sharpe_ci(eq, confidence=0.90, n_bootstrap=200)
|
|
r99 = bootstrap_sharpe_ci(eq, confidence=0.99, n_bootstrap=200)
|
|
# 99% CI should be wider than 90% CI
|
|
width_90 = r90["ci_upper"] - r90["ci_lower"]
|
|
width_99 = r99["ci_upper"] - r99["ci_lower"]
|
|
assert width_99 >= width_90
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Walk-Forward Analysis
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestWalkForward:
|
|
def test_output_structure(self) -> None:
|
|
eq = _make_equity(100)
|
|
trades = _make_trades([100, -50] * 10)
|
|
result = walk_forward_analysis(eq, trades, n_windows=4)
|
|
assert result["n_windows"] == 4
|
|
assert len(result["windows"]) == 4
|
|
assert "consistency_rate" in result
|
|
assert "return_mean" in result
|
|
assert "sharpe_mean" in result
|
|
|
|
def test_window_fields(self) -> None:
|
|
eq = _make_equity(100)
|
|
trades = _make_trades([100, -50] * 10)
|
|
result = walk_forward_analysis(eq, trades, n_windows=5)
|
|
w = result["windows"][0]
|
|
assert "window" in w
|
|
assert "start" in w
|
|
assert "end" in w
|
|
assert "return" in w
|
|
assert "sharpe" in w
|
|
assert "max_dd" in w
|
|
assert "trades" in w
|
|
assert "win_rate" in w
|
|
|
|
def test_consistency_rate(self) -> None:
|
|
"""Equity with positive drift should have high consistency."""
|
|
eq = _make_equity(200, drift=0.003)
|
|
trades = _make_trades([100] * 50)
|
|
result = walk_forward_analysis(eq, trades, n_windows=5)
|
|
assert result["consistency_rate"] > 0.5
|
|
|
|
def test_windows_cover_full_range(self) -> None:
|
|
eq = _make_equity(100)
|
|
trades = _make_trades([100] * 10)
|
|
result = walk_forward_analysis(eq, trades, n_windows=5)
|
|
first_start = result["windows"][0]["start"]
|
|
last_end = result["windows"][-1]["end"]
|
|
assert first_start == str(eq.index[0].date())
|
|
assert last_end == str(eq.index[-1].date())
|
|
|
|
def test_too_few_bars(self) -> None:
|
|
eq = pd.Series([100, 101], index=pd.bdate_range("2025-01-01", periods=2))
|
|
result = walk_forward_analysis(eq, [], n_windows=5)
|
|
assert "error" in result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# run_validation dispatcher
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestRunValidation:
|
|
def test_empty_config_returns_empty(self) -> None:
|
|
eq = _make_equity(50)
|
|
result = run_validation({}, eq, [], 1_000_000)
|
|
assert result == {}
|
|
|
|
def test_all_three(self) -> None:
|
|
eq = _make_equity(100)
|
|
trades = _make_trades([100, -50, 200, -30, 150])
|
|
config = {
|
|
"validation": {
|
|
"monte_carlo": {"n_simulations": 50},
|
|
"bootstrap": {"n_bootstrap": 50},
|
|
"walk_forward": {"n_windows": 3},
|
|
}
|
|
}
|
|
result = run_validation(config, eq, trades, 1_000_000)
|
|
assert "monte_carlo" in result
|
|
assert "bootstrap" in result
|
|
assert "walk_forward" in result
|
|
|
|
def test_single_tool(self) -> None:
|
|
eq = _make_equity(100)
|
|
trades = _make_trades([100, -50, 200])
|
|
config = {"validation": {"bootstrap": {"n_bootstrap": 50}}}
|
|
result = run_validation(config, eq, trades, 1_000_000)
|
|
assert "bootstrap" in result
|
|
assert "monte_carlo" not in result
|