Add synthetic scenario robustness testing system
Builds a full synthetic market data pipeline to test strategies against
12 diverse market regimes (bull/bear/crash/chop/rotation/liquidity drought)
that may not exist in historical data. Computes Regime Robustness Score (RRS)
to detect overfitting and environment-specific fragility.
- libs/backtest/scenarios/: price_gen, macro_gen, event_gen, coupling,
store_builder, scenarios (12 pre-built), robustness (RRS)
- apps/scenario/cli.py: `fithia2 scenario-test` with Rich output
- apps/tracker/cli.py: scenario-test command routing
- tests/: 83 unit tests across 3 new test files
- docs/scenario_test.md: usage guide and result interpretation
- docs/research_workflow_and_handoff.md: Step 5.5 scenario test added
Fix: no_signal scenario uses drift=0% (was +10%) for fair signal integrity scoring.
Fix: synthetic candidates now carry macro_vix/macro_hy_spread from macro_by_date
to pass selector engine filters.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
parent
0928eb2428
commit
9cb91ee846
@ -0,0 +1 @@
|
||||
# Synthetic scenario testing CLI package
|
||||
@ -0,0 +1,366 @@
|
||||
"""CLI for synthetic market scenario backtesting and robustness analysis.
|
||||
|
||||
Usage:
|
||||
fithia2 scenario-test --config return_max_long_v7.70
|
||||
fithia2 scenario-test --config return_max_long_v7.70 --quick
|
||||
fithia2 scenario-test --config return_max_long_v7.70 --scenario crash_v_recovery
|
||||
fithia2 scenario-test --config return_max_long_v7.70 --group signal
|
||||
fithia2 scenario-test --config return_max_long_v7.70 --baseline return_max_long_v6new.362
|
||||
fithia2 scenario-test --config return_max_long_v7.70 --initial-equity 10000
|
||||
fithia2 scenario-test --config return_max_long_v7.70 --save
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from rich import box
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
|
||||
from rich.table import Table
|
||||
|
||||
_console = Console(width=120)
|
||||
_CONFIGS_DIR = Path("configs/experiments")
|
||||
_RUNS_DIR = Path("runs")
|
||||
|
||||
|
||||
def _resolve_config_name(name_or_id: str) -> str:
|
||||
"""Resolve experiment name or numeric ID to full config name."""
|
||||
from libs.backtest.experiments import resolve_experiment_name
|
||||
return resolve_experiment_name(name_or_id)
|
||||
|
||||
|
||||
def _print_scenario_table(report) -> None:
|
||||
"""Print per-scenario metrics table."""
|
||||
from libs.backtest.scenarios.scenarios import SCENARIO_REGISTRY
|
||||
|
||||
table = Table(
|
||||
title=f"Scenario Results — {report.experiment_name}",
|
||||
box=box.ROUNDED,
|
||||
width=118,
|
||||
)
|
||||
table.add_column("Scenario", style="cyan", min_width=22)
|
||||
table.add_column("Sharpe", justify="right", min_width=7)
|
||||
table.add_column("Return%", justify="right", min_width=8)
|
||||
table.add_column("MaxDD%", justify="right", min_width=7)
|
||||
table.add_column("Win%", justify="right", min_width=6)
|
||||
table.add_column("PF", justify="right", min_width=6)
|
||||
table.add_column("Trades", justify="right", min_width=7)
|
||||
table.add_column("Signal", justify="right", min_width=7)
|
||||
table.add_column("Status", min_width=8)
|
||||
|
||||
for name, result in report.scenario_results.items():
|
||||
scenario = SCENARIO_REGISTRY.get(name)
|
||||
sig = f"{scenario.signal_strength:.2f}" if scenario else "?"
|
||||
|
||||
sharpe = result.sharpe_ratio
|
||||
ret = result.total_return_pct
|
||||
dd = result.max_drawdown_pct
|
||||
win = result.win_rate * 100
|
||||
pf = result.profit_factor
|
||||
trades = result.trade_count
|
||||
|
||||
# Colour-code by Sharpe
|
||||
if sharpe >= 1.5:
|
||||
sharpe_str = f"[green]{sharpe:.2f}[/green]"
|
||||
elif sharpe >= 0.5:
|
||||
sharpe_str = f"[yellow]{sharpe:.2f}[/yellow]"
|
||||
elif sharpe >= 0.0:
|
||||
sharpe_str = f"[dim]{sharpe:.2f}[/dim]"
|
||||
else:
|
||||
sharpe_str = f"[red]{sharpe:.2f}[/red]"
|
||||
|
||||
ret_str = f"[green]+{ret:.1f}[/green]" if ret > 0 else f"[red]{ret:.1f}[/red]"
|
||||
dd_str = f"[red]{dd:.1f}[/red]" if dd > 20 else f"{dd:.1f}"
|
||||
|
||||
# Status flag
|
||||
if name == "no_signal" and sharpe > 0.5:
|
||||
status = "[red bold]OVERFIT[/red bold]"
|
||||
elif name == "no_signal" and sharpe <= 0:
|
||||
status = "[green]OK[/green]"
|
||||
elif name == "strong_signal" and sharpe < 0.5:
|
||||
status = "[yellow]WEAK[/yellow]"
|
||||
else:
|
||||
status = ""
|
||||
|
||||
table.add_row(
|
||||
name,
|
||||
sharpe_str,
|
||||
ret_str,
|
||||
dd_str,
|
||||
f"{win:.0f}",
|
||||
f"{pf:.2f}",
|
||||
str(trades),
|
||||
sig,
|
||||
status,
|
||||
)
|
||||
|
||||
_console.print()
|
||||
_console.print(table)
|
||||
|
||||
|
||||
def _print_rrs_panel(report) -> None:
|
||||
"""Print Regime Robustness Score panel."""
|
||||
verdict_color = {
|
||||
"ROBUST": "green",
|
||||
"FRAGILE": "yellow",
|
||||
"OVERFIT": "red",
|
||||
}.get(report.verdict, "white")
|
||||
|
||||
def _score_bar(score: float) -> str:
|
||||
filled = int(round(score / 5))
|
||||
empty = 20 - filled
|
||||
bar = "█" * filled + "░" * empty
|
||||
if score >= 70:
|
||||
color = "green"
|
||||
elif score >= 40:
|
||||
color = "yellow"
|
||||
else:
|
||||
color = "red"
|
||||
return f"[{color}]{bar}[/{color}] {score:.0f}/100"
|
||||
|
||||
lines = [
|
||||
f"[bold]REGIME ROBUSTNESS SCORE (RRS)[/bold]",
|
||||
f"Strategy: [cyan]{report.experiment_name}[/cyan]",
|
||||
f"",
|
||||
f" Signal Integrity (25%) {_score_bar(report.signal_integrity)}",
|
||||
f" → no_signal Sharpe ≤ 0 confirms event-driven alpha (not price-pattern overfit)",
|
||||
f"",
|
||||
f" Breadth (25%) {_score_bar(report.breadth)}",
|
||||
f" → fraction of scenarios with positive Sharpe",
|
||||
f"",
|
||||
f" Drawdown Resilience(20%) {_score_bar(report.drawdown_resilience)}",
|
||||
f" → worst-case max drawdown across all scenarios",
|
||||
f"",
|
||||
f" Regime Transition (15%) {_score_bar(report.regime_transition)}",
|
||||
f" → performance on regime_switch vs median",
|
||||
f"",
|
||||
f" Stability (15%) {_score_bar(report.stability)}",
|
||||
f" → low Sharpe variance across diverse market conditions",
|
||||
f"",
|
||||
f" [bold]RRS: {_score_bar(report.rrs)}[/bold]",
|
||||
f" [{verdict_color} bold]Verdict: {report.verdict}[/{verdict_color} bold]",
|
||||
]
|
||||
|
||||
if report.notes:
|
||||
lines += ["", "[dim]Notes:"] + [f" • {n}" for n in report.notes] + ["[/dim]"]
|
||||
|
||||
_console.print()
|
||||
_console.print(Panel("\n".join(lines), box=box.DOUBLE, width=100))
|
||||
|
||||
|
||||
def _print_comparison_table(report_a, report_b) -> None:
|
||||
"""Print side-by-side comparison of two strategies."""
|
||||
table = Table(title="Strategy Comparison", box=box.ROUNDED, width=118)
|
||||
table.add_column("Scenario", style="cyan", min_width=22)
|
||||
table.add_column(f"{report_a.experiment_name[:18]} Sharpe", justify="right")
|
||||
table.add_column(f"{report_b.experiment_name[:18]} Sharpe", justify="right")
|
||||
table.add_column("Delta", justify="right")
|
||||
|
||||
all_names = sorted(
|
||||
set(report_a.scenario_results.keys()) | set(report_b.scenario_results.keys())
|
||||
)
|
||||
for name in all_names:
|
||||
s_a = report_a.scenario_results.get(name)
|
||||
s_b = report_b.scenario_results.get(name)
|
||||
sh_a = s_a.sharpe_ratio if s_a else float("nan")
|
||||
sh_b = s_b.sharpe_ratio if s_b else float("nan")
|
||||
|
||||
def _fmt(v: float) -> str:
|
||||
if v != v:
|
||||
return "[dim]N/A[/dim]"
|
||||
color = "green" if v > 0.5 else ("yellow" if v >= 0 else "red")
|
||||
return f"[{color}]{v:.2f}[/{color}]"
|
||||
|
||||
delta = sh_b - sh_a if (sh_a == sh_a and sh_b == sh_b) else float("nan")
|
||||
delta_str = (
|
||||
f"[green]+{delta:.2f}[/green]" if delta > 0.05
|
||||
else (f"[red]{delta:.2f}[/red]" if delta < -0.05 else f"[dim]{delta:.2f}[/dim]")
|
||||
if delta == delta else "[dim]N/A[/dim]"
|
||||
)
|
||||
table.add_row(name, _fmt(sh_a), _fmt(sh_b), delta_str)
|
||||
|
||||
_console.print()
|
||||
_console.print(table)
|
||||
|
||||
# RRS comparison
|
||||
_console.print(
|
||||
f"\n RRS: {report_a.experiment_name} = [bold]{report_a.rrs:.0f}[/bold] "
|
||||
f"vs {report_b.experiment_name} = [bold]{report_b.rrs:.0f}[/bold]"
|
||||
f" (delta [bold]{report_b.rrs - report_a.rrs:+.0f}[/bold])"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="fithia2 scenario-test",
|
||||
description=(
|
||||
"Synthetic market scenario backtesting for overfitting detection. "
|
||||
"Tests strategies against generated market conditions not present in historical data."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config", required=True,
|
||||
help="Experiment config name or numeric ID (e.g. 'return_max_long_v7.70' or '415')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--scenario", default=None,
|
||||
help="Run a single scenario by name (e.g. 'crash_v_recovery')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--group", default=None,
|
||||
help="Run a scenario group: trend, volatility, regime, signal, structural, quick, all",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--quick", action="store_true",
|
||||
help="Quick mode: run only steady_bull, steady_bear, no_signal (3 scenarios)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline", default=None,
|
||||
help="Optional baseline config to compare against",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--initial-equity", type=float, default=10_000.0,
|
||||
help="Starting capital for each scenario (default: 10000)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--save", action="store_true",
|
||||
help="Save JSON report to runs/<experiment>/scenario_report.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list", action="store_true",
|
||||
help="List all available scenarios and exit",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
from libs.backtest.scenarios.scenarios import SCENARIO_REGISTRY, SCENARIO_GROUPS
|
||||
|
||||
if args.list:
|
||||
_console.print("\n[bold]Available scenarios:[/bold]")
|
||||
for name, sc in SCENARIO_REGISTRY.items():
|
||||
_console.print(f" [cyan]{name:<25}[/cyan] signal={sc.signal_strength:.2f} {sc.description[:60]}")
|
||||
_console.print("\n[bold]Scenario groups:[/bold]")
|
||||
for g, names in SCENARIO_GROUPS.items():
|
||||
_console.print(f" [yellow]{g:<15}[/yellow] {', '.join(names)}")
|
||||
return
|
||||
|
||||
# Resolve which scenarios to run
|
||||
if args.scenario:
|
||||
if args.scenario not in SCENARIO_REGISTRY:
|
||||
_console.print(f"[red]Unknown scenario '{args.scenario}'. Use --list to see options.[/red]")
|
||||
sys.exit(1)
|
||||
scenario_names = [args.scenario]
|
||||
elif args.quick:
|
||||
scenario_names = SCENARIO_GROUPS["quick"]
|
||||
elif args.group:
|
||||
if args.group not in SCENARIO_GROUPS:
|
||||
_console.print(f"[red]Unknown group '{args.group}'. Use --list to see options.[/red]")
|
||||
sys.exit(1)
|
||||
scenario_names = SCENARIO_GROUPS[args.group]
|
||||
else:
|
||||
scenario_names = SCENARIO_GROUPS["all"]
|
||||
|
||||
try:
|
||||
experiment_name = _resolve_config_name(args.config)
|
||||
except Exception as exc:
|
||||
_console.print(f"[red]Cannot resolve config '{args.config}': {exc}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
_console.print()
|
||||
_console.print(Panel(
|
||||
f"[bold]SYNTHETIC SCENARIO TEST[/bold]\n"
|
||||
f"Strategy: [cyan]{experiment_name}[/cyan]\n"
|
||||
f"Scenarios: [yellow]{len(scenario_names)}[/yellow] ({', '.join(scenario_names)})\n"
|
||||
f"Initial equity: ${args.initial_equity:,.0f}",
|
||||
box=box.DOUBLE,
|
||||
width=100,
|
||||
))
|
||||
|
||||
t0 = time.time()
|
||||
completed: list[str] = []
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(bar_width=30),
|
||||
TextColumn("{task.completed}/{task.total}"),
|
||||
TimeElapsedColumn(),
|
||||
console=_console,
|
||||
) as progress:
|
||||
task = progress.add_task("Running scenarios...", total=len(scenario_names))
|
||||
|
||||
def _cb(name: str) -> None:
|
||||
progress.update(task, description=f"[cyan]{name}[/cyan]")
|
||||
|
||||
from libs.backtest.scenarios.robustness import run_scenario_test
|
||||
report = run_scenario_test(
|
||||
experiment_name=experiment_name,
|
||||
scenario_names=scenario_names,
|
||||
initial_equity=args.initial_equity,
|
||||
progress_callback=_cb,
|
||||
)
|
||||
progress.update(task, completed=len(scenario_names), description="Complete")
|
||||
|
||||
elapsed = time.time() - t0
|
||||
|
||||
_print_scenario_table(report)
|
||||
_print_rrs_panel(report)
|
||||
|
||||
# Optional baseline comparison
|
||||
if args.baseline:
|
||||
try:
|
||||
baseline_name = _resolve_config_name(args.baseline)
|
||||
_console.print(f"\n[dim]Running baseline {baseline_name}...[/dim]")
|
||||
from libs.backtest.scenarios.robustness import run_scenario_test as _rtt
|
||||
baseline_report = _rtt(
|
||||
experiment_name=baseline_name,
|
||||
scenario_names=scenario_names,
|
||||
initial_equity=args.initial_equity,
|
||||
)
|
||||
_print_comparison_table(baseline_report, report)
|
||||
except Exception as exc:
|
||||
_console.print(f"[yellow]Baseline comparison failed: {exc}[/yellow]")
|
||||
|
||||
# Save report
|
||||
if args.save:
|
||||
try:
|
||||
save_dir = _RUNS_DIR / experiment_name
|
||||
save_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_path = save_dir / "scenario_report.json"
|
||||
payload = {
|
||||
"experiment_name": experiment_name,
|
||||
"rrs": report.rrs,
|
||||
"verdict": report.verdict,
|
||||
"signal_integrity": report.signal_integrity,
|
||||
"breadth": report.breadth,
|
||||
"drawdown_resilience": report.drawdown_resilience,
|
||||
"regime_transition": report.regime_transition,
|
||||
"stability": report.stability,
|
||||
"scenarios": {
|
||||
name: {
|
||||
"sharpe_ratio": r.sharpe_ratio,
|
||||
"total_return_pct": r.total_return_pct,
|
||||
"max_drawdown_pct": r.max_drawdown_pct,
|
||||
"win_rate": r.win_rate,
|
||||
"profit_factor": r.profit_factor,
|
||||
"trade_count": r.trade_count,
|
||||
}
|
||||
for name, r in report.scenario_results.items()
|
||||
},
|
||||
"elapsed_seconds": round(elapsed, 1),
|
||||
"notes": report.notes,
|
||||
}
|
||||
out_path.write_text(json.dumps(payload, indent=2))
|
||||
_console.print(f"\n[dim]Report saved → {out_path}[/dim]")
|
||||
except Exception as exc:
|
||||
_console.print(f"[yellow]Could not save report: {exc}[/yellow]")
|
||||
|
||||
_console.print(f"\n[dim]Elapsed: {elapsed:.0f}s[/dim]\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,38 @@
|
||||
"""Synthetic market scenario generation for overfitting detection and robustness testing.
|
||||
|
||||
Public API:
|
||||
from libs.backtest.scenarios import build_synthetic_store, SCENARIO_REGISTRY
|
||||
from libs.backtest.scenarios.scenarios import ScenarioConfig, SCENARIO_REGISTRY
|
||||
from libs.backtest.scenarios.robustness import run_scenario_test, RegimeRobustnessReport
|
||||
"""
|
||||
from libs.backtest.scenarios.price_gen import PriceRegime, generate_price_paths
|
||||
from libs.backtest.scenarios.macro_gen import VIXConfig, HYSpreadConfig, generate_macro_data
|
||||
from libs.backtest.scenarios.event_gen import EventDistribution, generate_events
|
||||
from libs.backtest.scenarios.coupling import couple_events_to_prices
|
||||
from libs.backtest.scenarios.scenarios import ScenarioConfig, SCENARIO_REGISTRY, SCENARIO_GROUPS
|
||||
from libs.backtest.scenarios.store_builder import build_synthetic_store
|
||||
from libs.backtest.scenarios.robustness import (
|
||||
ScenarioResult,
|
||||
RegimeRobustnessReport,
|
||||
run_scenario_test,
|
||||
compute_rrs,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"PriceRegime",
|
||||
"generate_price_paths",
|
||||
"VIXConfig",
|
||||
"HYSpreadConfig",
|
||||
"generate_macro_data",
|
||||
"EventDistribution",
|
||||
"generate_events",
|
||||
"couple_events_to_prices",
|
||||
"ScenarioConfig",
|
||||
"SCENARIO_REGISTRY",
|
||||
"SCENARIO_GROUPS",
|
||||
"build_synthetic_store",
|
||||
"ScenarioResult",
|
||||
"RegimeRobustnessReport",
|
||||
"run_scenario_test",
|
||||
"compute_rrs",
|
||||
]
|
||||
@ -0,0 +1,156 @@
|
||||
"""Event-price coupling for synthetic scenario backtesting.
|
||||
|
||||
Controls the signal-to-noise ratio between event quality features and
|
||||
subsequent price movements. This is the core mechanism for overfitting detection:
|
||||
|
||||
signal_strength=0.0 → pure noise → strategy should return ~0 (false positive check)
|
||||
signal_strength=0.35 → realistic SNR → strategy captures genuine alpha
|
||||
signal_strength=0.60 → strong signal → verify strategy responds to alpha
|
||||
|
||||
The coupling injects a drift into bars AFTER the execution date based on the
|
||||
event's score and reaction features, while preserving OHLCV consistency.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def couple_events_to_prices(
|
||||
candidates: dict[dt.date, list[dict[str, Any]]],
|
||||
bars_by_symbol: dict[str, dict[dt.date, dict[str, Any]]],
|
||||
trading_dates: list[dt.date],
|
||||
signal_strength: float,
|
||||
signal_decay_days: int = 10,
|
||||
false_positive_rate: float = 0.15,
|
||||
rng: np.random.Generator | None = None,
|
||||
) -> None:
|
||||
"""Inject signal-driven drift into bars after each event's execution date.
|
||||
|
||||
Modifies bars_by_symbol in-place. Also updates each candidate's
|
||||
entry_price / entry_price_est / event_close to match the actual
|
||||
bar close on its reaction_date (so stop prices are consistent).
|
||||
|
||||
Args:
|
||||
candidates: candidates_by_exec_date dict from generate_events().
|
||||
bars_by_symbol: OHLCV bars to modify in-place.
|
||||
trading_dates: Ordered list of NYSE trading dates.
|
||||
signal_strength: 0.0 = pure noise, 0.35 = realistic, 0.6 = strong alpha.
|
||||
signal_decay_days: Days over which signal drift decays to zero.
|
||||
false_positive_rate: Fraction of qualifying events that produce negative
|
||||
returns (traps / false positives).
|
||||
rng: NumPy random generator.
|
||||
"""
|
||||
if rng is None:
|
||||
rng = np.random.default_rng()
|
||||
|
||||
date_to_idx = {d: i for i, d in enumerate(trading_dates)}
|
||||
|
||||
for exec_date, rows in candidates.items():
|
||||
exec_idx = date_to_idx.get(exec_date)
|
||||
if exec_idx is None:
|
||||
continue
|
||||
|
||||
# reaction_date is the day before execution
|
||||
react_idx = exec_idx - 1
|
||||
if react_idx < 0:
|
||||
continue
|
||||
reaction_date = trading_dates[react_idx]
|
||||
|
||||
for row in rows:
|
||||
symbol = str(row.get("symbol", ""))
|
||||
sym_bars = bars_by_symbol.get(symbol)
|
||||
if sym_bars is None:
|
||||
continue
|
||||
|
||||
# Sync entry_price with actual bar close on reaction_date
|
||||
react_bar = sym_bars.get(reaction_date)
|
||||
if react_bar and react_bar.get("close", 0) > 0:
|
||||
actual_close = float(react_bar["close"])
|
||||
row["entry_price"] = round(actual_close, 4)
|
||||
row["entry_price_est"] = round(actual_close, 4)
|
||||
row["event_close"] = round(actual_close, 4)
|
||||
# Recompute ATR based on actual price
|
||||
atr_pct = float(row.get("atr_14", actual_close * 0.022)) / max(float(row.get("entry_price", actual_close)), 1e-4)
|
||||
row["atr_14"] = round(actual_close * atr_pct, 4)
|
||||
|
||||
# Skip coupling if signal_strength == 0 (pure noise scenario)
|
||||
if signal_strength <= 1e-9:
|
||||
continue
|
||||
|
||||
# Compute expected drift from event features
|
||||
score = float(row.get("score", 0.5))
|
||||
reaction_return = float(row.get("reaction_day_return", 0.0))
|
||||
volume_ratio = float(row.get("volume_ratio_20d", 1.5))
|
||||
|
||||
expected_drift_5d = signal_strength * (
|
||||
0.030 * (score - 0.5)
|
||||
+ 0.020 * reaction_return
|
||||
+ 0.008 * max(0.0, volume_ratio - 1.5)
|
||||
)
|
||||
|
||||
# False positive: flip signal direction
|
||||
if rng.random() < false_positive_rate:
|
||||
expected_drift_5d = -expected_drift_5d * 0.7
|
||||
|
||||
if abs(expected_drift_5d) < 1e-6:
|
||||
continue
|
||||
|
||||
# Distribute drift over signal_decay_days using a decay schedule
|
||||
total_drift = expected_drift_5d
|
||||
decay = _compute_decay_schedule(total_drift, signal_decay_days)
|
||||
|
||||
# Inject drift into bars starting at exec_date + 1 (first day we hold)
|
||||
apply_start_idx = exec_idx + 1
|
||||
for k, daily_adj in enumerate(decay):
|
||||
bar_idx = apply_start_idx + k
|
||||
if bar_idx >= len(trading_dates):
|
||||
break
|
||||
bar_date = trading_dates[bar_idx]
|
||||
bar = sym_bars.get(bar_date)
|
||||
if bar is None:
|
||||
continue
|
||||
|
||||
_apply_drift_to_bar(bar, daily_adj)
|
||||
|
||||
|
||||
def _compute_decay_schedule(total_drift: float, decay_days: int) -> list[float]:
|
||||
"""Distribute total_drift over decay_days using exponential decay.
|
||||
|
||||
Returns a list of per-day drift adjustments that sum to total_drift.
|
||||
"""
|
||||
if decay_days <= 0:
|
||||
return [total_drift]
|
||||
|
||||
# Exponential decay weights
|
||||
weights = [math.exp(-0.5 * k / max(decay_days, 1)) for k in range(decay_days)]
|
||||
total_weight = sum(weights)
|
||||
return [total_drift * w / total_weight for w in weights]
|
||||
|
||||
|
||||
def _apply_drift_to_bar(bar: dict[str, Any], daily_drift: float) -> None:
|
||||
"""Multiply all OHLCV price fields by (1 + daily_drift), preserving consistency.
|
||||
|
||||
Applies uniform multiplicative adjustment so that OHLC relationships are
|
||||
maintained exactly. Volume is unchanged.
|
||||
"""
|
||||
factor = 1.0 + daily_drift
|
||||
factor = max(0.5, min(2.0, factor)) # guard against extreme values
|
||||
|
||||
for field in ("open", "high", "low", "close"):
|
||||
val = bar.get(field)
|
||||
if val is not None and float(val) > 0:
|
||||
bar[field] = round(float(val) * factor, 4)
|
||||
|
||||
# Ensure OHLCV consistency after adjustment
|
||||
o = bar.get("open", 0)
|
||||
h = bar.get("high", 0)
|
||||
lo = bar.get("low", 0)
|
||||
c = bar.get("close", 0)
|
||||
|
||||
if o and h and lo and c:
|
||||
bar["high"] = round(max(float(h), float(o), float(c)), 4)
|
||||
bar["low"] = round(max(0.01, min(float(lo), float(o), float(c))), 4)
|
||||
@ -0,0 +1,395 @@
|
||||
"""Synthetic event candidate generation for scenario backtesting.
|
||||
|
||||
Generates event candidate rows that are compatible with SnapshotStore's
|
||||
candidates_by_exec_date structure. Each row contains all fields required
|
||||
by build_candidate() in selector.py plus feature fields used by scoring
|
||||
functions and strategy engine filters.
|
||||
|
||||
Event timing model (post_market/after_close pattern):
|
||||
- event_date N: Company announces post-market → event_timestamp = N 21:00 UTC
|
||||
- reaction_date: N (market's first reaction is on the announcement day close)
|
||||
- execution_date: next_trading_day(N) — trade entered at next open
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass
|
||||
class EventDistribution:
|
||||
"""Statistical distribution parameters for synthetic event feature generation.
|
||||
|
||||
Each feature is described as (mean, std) for truncated-normal sampling,
|
||||
or as a dict mapping category → probability for categorical features.
|
||||
All (mean, std) pairs use np.clip to keep values in reasonable ranges.
|
||||
"""
|
||||
|
||||
# ---- Event arrival ----
|
||||
events_per_day_mean: float = 8.0
|
||||
"""Average number of events per trading day."""
|
||||
events_per_day_std: float = 3.0
|
||||
"""Standard deviation of events per trading day."""
|
||||
|
||||
# ---- Categorical distributions ----
|
||||
event_types: dict[str, float] = field(default_factory=lambda: {
|
||||
"earnings_release": 0.65,
|
||||
"guidance_update": 0.15,
|
||||
"material_contract": 0.08,
|
||||
"other_material_event": 0.07,
|
||||
"unknown": 0.05,
|
||||
})
|
||||
event_directions: dict[str, float] = field(default_factory=lambda: {
|
||||
"bullish": 0.45,
|
||||
"mixed": 0.25,
|
||||
"unknown": 0.20,
|
||||
"bearish": 0.10,
|
||||
})
|
||||
guidance_statuses: dict[str, float] = field(default_factory=lambda: {
|
||||
"raised": 0.35,
|
||||
"inline_or_maintained": 0.40,
|
||||
"not_provided": 0.15,
|
||||
"lowered": 0.10,
|
||||
})
|
||||
filing_time_buckets: dict[str, float] = field(default_factory=lambda: {
|
||||
"post_market": 0.60,
|
||||
"pre_market": 0.35,
|
||||
"intraday": 0.05,
|
||||
})
|
||||
sectors: dict[str, float] = field(default_factory=lambda: {
|
||||
"Technology": 0.22,
|
||||
"Health Care": 0.14,
|
||||
"Consumer Discretionary": 0.12,
|
||||
"Financials": 0.13,
|
||||
"Industrials": 0.11,
|
||||
"Communication Services": 0.09,
|
||||
"Consumer Staples": 0.07,
|
||||
"Energy": 0.05,
|
||||
"Materials": 0.04,
|
||||
"Utilities": 0.03,
|
||||
})
|
||||
|
||||
# ---- Quality features (scoring inputs) ----
|
||||
signal_strength_mean: float = 0.65
|
||||
signal_strength_std: float = 0.15
|
||||
document_quality_mean: float = 0.70
|
||||
document_quality_std: float = 0.12
|
||||
parse_confidence_mean: float = 0.80
|
||||
parse_confidence_std: float = 0.10
|
||||
guidance_direction_score_mean: float = 0.55
|
||||
guidance_direction_score_std: float = 0.20
|
||||
oneoff_penalty_prob: float = 0.05
|
||||
"""Probability of a 1-off event penalty (reduces quality score)."""
|
||||
|
||||
# ---- Reaction features ----
|
||||
reaction_return_mean: float = 0.025
|
||||
"""Mean reaction-day return (positive = bullish bias)."""
|
||||
reaction_return_std: float = 0.055
|
||||
volume_ratio_mean: float = 1.8
|
||||
volume_ratio_std: float = 0.9
|
||||
close_location_mean: float = 0.60
|
||||
close_location_std: float = 0.18
|
||||
gap_size_mean: float = 0.010
|
||||
gap_size_std: float = 0.025
|
||||
|
||||
# ---- Technical pre-event features ----
|
||||
rsi_14_mean: float = 52.0
|
||||
rsi_14_std: float = 12.0
|
||||
bb_position_mean: float = 0.55
|
||||
bb_position_std: float = 0.20
|
||||
volatility_20d_mean: float = 0.28
|
||||
volatility_20d_std: float = 0.08
|
||||
hurst_60d_mean: float = 0.50
|
||||
hurst_60d_std: float = 0.07
|
||||
entropy_60d_mean: float = 1.45
|
||||
entropy_60d_std: float = 0.15
|
||||
ou_theta_60d_mean: float = 5.0
|
||||
ou_theta_60d_std: float = 2.0
|
||||
market_temperature_mean: float = 0.80
|
||||
market_temperature_std: float = 0.25
|
||||
gravitational_pull_mean: float = 0.0
|
||||
gravitational_pull_std: float = 0.03
|
||||
sector_momentum_20d_mean: float = 0.005
|
||||
sector_momentum_20d_std: float = 0.03
|
||||
|
||||
# ---- Universe features ----
|
||||
avg_dollar_volume_mean: float = 5_000_000.0
|
||||
avg_dollar_volume_std: float = 3_000_000.0
|
||||
price_mean: float = 85.0
|
||||
price_std: float = 40.0
|
||||
|
||||
# ---- ATR ----
|
||||
atr_pct_mean: float = 0.022
|
||||
"""ATR-14 as a percentage of price."""
|
||||
atr_pct_std: float = 0.008
|
||||
|
||||
|
||||
def _sample_categorical(categories: dict[str, float], rng: np.random.Generator) -> str:
|
||||
"""Sample one category weighted by probabilities."""
|
||||
keys = list(categories.keys())
|
||||
probs = np.array(list(categories.values()), dtype=float)
|
||||
probs /= probs.sum()
|
||||
return str(rng.choice(keys, p=probs))
|
||||
|
||||
|
||||
def _clamp_normal(mean: float, std: float, lo: float, hi: float, rng: np.random.Generator) -> float:
|
||||
"""Sample from a clipped normal distribution."""
|
||||
return float(np.clip(rng.normal(mean, std), lo, hi))
|
||||
|
||||
|
||||
def generate_events(
|
||||
trading_dates: list[dt.date],
|
||||
symbols: list[str],
|
||||
dist: EventDistribution,
|
||||
rng: np.random.Generator,
|
||||
bars_by_symbol: dict[str, dict[dt.date, dict[str, Any]]] | None = None,
|
||||
max_holding_days_buffer: int = 25,
|
||||
) -> dict[dt.date, list[dict[str, Any]]]:
|
||||
"""Generate synthetic event candidates distributed across trading_dates.
|
||||
|
||||
Each candidate row contains all fields required by:
|
||||
- selector.build_candidate() (event_id, event_timestamp, entry_price, etc.)
|
||||
- scoring functions (signal_strength_score, reaction_day_return, etc.)
|
||||
- strategy engine filters (pre_event_rsi_14, pre_event_hurst_60d, etc.)
|
||||
|
||||
Events are placed on execution_dates. The corresponding reaction_date is
|
||||
the previous trading day (post_market filing pattern).
|
||||
|
||||
Args:
|
||||
trading_dates: Full sequence of NYSE trading dates.
|
||||
symbols: List of ticker symbols to assign events to.
|
||||
dist: EventDistribution parameters.
|
||||
rng: NumPy random generator.
|
||||
bars_by_symbol: If provided, entry_price is taken from bar close on reaction_date.
|
||||
max_holding_days_buffer: Days at end of date range excluded from event placement
|
||||
(so all positions can close before scenario end).
|
||||
|
||||
Returns:
|
||||
candidates_by_exec_date dict compatible with SnapshotStore.
|
||||
"""
|
||||
n = len(trading_dates)
|
||||
# Reserve the first ~5 days (warm-up) and last N days (holding buffer)
|
||||
eligible_range_start = 5
|
||||
eligible_range_end = max(eligible_range_start + 1, n - max_holding_days_buffer)
|
||||
|
||||
candidates: dict[dt.date, list[dict[str, Any]]] = {}
|
||||
event_counter = 0
|
||||
used_symbols_today: dict[dt.date, set[str]] = {}
|
||||
|
||||
for i in range(eligible_range_start, eligible_range_end):
|
||||
exec_date = trading_dates[i]
|
||||
reaction_date = trading_dates[i - 1] # previous trading day
|
||||
|
||||
# Number of events today (Poisson-like)
|
||||
n_events = max(0, int(round(rng.normal(dist.events_per_day_mean, dist.events_per_day_std))))
|
||||
if n_events == 0:
|
||||
continue
|
||||
|
||||
today_candidates: list[dict[str, Any]] = []
|
||||
used_syms = used_symbols_today.setdefault(exec_date, set())
|
||||
|
||||
# Pick symbols for today's events (without replacement from pool)
|
||||
available = [s for s in symbols if s not in used_syms]
|
||||
if not available:
|
||||
continue
|
||||
rng.shuffle(available)
|
||||
n_events = min(n_events, len(available))
|
||||
|
||||
for j in range(n_events):
|
||||
symbol = available[j]
|
||||
used_syms.add(symbol)
|
||||
event_counter += 1
|
||||
|
||||
event_type = _sample_categorical(dist.event_types, rng)
|
||||
event_direction = _sample_categorical(dist.event_directions, rng)
|
||||
guidance_status = _sample_categorical(dist.guidance_statuses, rng)
|
||||
filing_bucket = _sample_categorical(dist.filing_time_buckets, rng)
|
||||
sector = _sample_categorical(dist.sectors, rng)
|
||||
|
||||
# Event date / timestamp: model after-close and same-day patterns.
|
||||
# post_market / pre_market → event happened on the trading day BEFORE
|
||||
# reaction_date (after-close pattern: reaction_date > event_date → "after_close").
|
||||
# intraday → event happened on reaction_date itself ("same_day").
|
||||
if filing_bucket in ("post_market", "pre_market"):
|
||||
event_date_d = trading_dates[i - 2] # i >= eligible_range_start=5, safe
|
||||
hour = "21:00:00" if filing_bucket == "post_market" else "07:00:00"
|
||||
else:
|
||||
event_date_d = reaction_date # intraday → same_day timing
|
||||
hour = "14:00:00"
|
||||
event_timestamp = f"{event_date_d.isoformat()}T{hour}+00:00"
|
||||
|
||||
# Price (try to get from bars, else sample)
|
||||
if bars_by_symbol and symbol in bars_by_symbol:
|
||||
bar = bars_by_symbol[symbol].get(reaction_date)
|
||||
if bar and bar.get("close", 0) > 0:
|
||||
entry_price = float(bar["close"])
|
||||
event_close = entry_price
|
||||
atr_14 = entry_price * _clamp_normal(dist.atr_pct_mean, dist.atr_pct_std, 0.005, 0.08, rng)
|
||||
exec_bar = bars_by_symbol[symbol].get(exec_date)
|
||||
avg_dollar_volume = float(bar.get("volume", 1_000_000)) * entry_price
|
||||
else:
|
||||
entry_price = max(5.0, _clamp_normal(dist.price_mean, dist.price_std, 5.0, 500.0, rng))
|
||||
event_close = entry_price
|
||||
atr_14 = entry_price * _clamp_normal(dist.atr_pct_mean, dist.atr_pct_std, 0.005, 0.08, rng)
|
||||
avg_dollar_volume = max(100_000.0, rng.normal(dist.avg_dollar_volume_mean, dist.avg_dollar_volume_std))
|
||||
else:
|
||||
entry_price = max(5.0, _clamp_normal(dist.price_mean, dist.price_std, 5.0, 500.0, rng))
|
||||
event_close = entry_price
|
||||
atr_14 = entry_price * _clamp_normal(dist.atr_pct_mean, dist.atr_pct_std, 0.005, 0.08, rng)
|
||||
avg_dollar_volume = max(100_000.0, float(rng.normal(dist.avg_dollar_volume_mean, dist.avg_dollar_volume_std)))
|
||||
|
||||
# Quality features
|
||||
signal_strength = _clamp_normal(dist.signal_strength_mean, dist.signal_strength_std, 0.0, 1.0, rng)
|
||||
doc_quality = _clamp_normal(dist.document_quality_mean, dist.document_quality_std, 0.0, 1.0, rng)
|
||||
parse_conf = _clamp_normal(dist.parse_confidence_mean, dist.parse_confidence_std, 0.0, 1.0, rng)
|
||||
guidance_dir_score = _clamp_normal(dist.guidance_direction_score_mean, dist.guidance_direction_score_std, 0.0, 1.0, rng)
|
||||
oneoff_penalty = 1.0 if rng.random() < dist.oneoff_penalty_prob else 0.0
|
||||
|
||||
# Reaction features (biased by event_direction)
|
||||
direction_bias = {"bullish": 0.03, "bearish": -0.03, "mixed": 0.005, "unknown": 0.0}.get(event_direction, 0.0)
|
||||
reaction_return = float(rng.normal(dist.reaction_return_mean + direction_bias, dist.reaction_return_std))
|
||||
volume_ratio = max(0.5, float(rng.normal(dist.volume_ratio_mean, dist.volume_ratio_std)))
|
||||
close_location = _clamp_normal(dist.close_location_mean, dist.close_location_std, 0.0, 1.0, rng)
|
||||
gap_size = float(rng.normal(dist.gap_size_mean, dist.gap_size_std))
|
||||
|
||||
# Technical features
|
||||
rsi_14 = _clamp_normal(dist.rsi_14_mean, dist.rsi_14_std, 5.0, 95.0, rng)
|
||||
bb_position = _clamp_normal(dist.bb_position_mean, dist.bb_position_std, -0.2, 1.2, rng)
|
||||
vol_20d = _clamp_normal(dist.volatility_20d_mean, dist.volatility_20d_std, 0.05, 0.8, rng)
|
||||
hurst_60d = _clamp_normal(dist.hurst_60d_mean, dist.hurst_60d_std, 0.2, 0.8, rng)
|
||||
entropy_60d = _clamp_normal(dist.entropy_60d_mean, dist.entropy_60d_std, 0.5, 2.0, rng)
|
||||
ou_theta = _clamp_normal(dist.ou_theta_60d_mean, dist.ou_theta_60d_std, 0.5, 30.0, rng)
|
||||
mkt_temp = _clamp_normal(dist.market_temperature_mean, dist.market_temperature_std, 0.0, 2.0, rng)
|
||||
grav_pull = float(rng.normal(dist.gravitational_pull_mean, dist.gravitational_pull_std))
|
||||
sector_mom = float(rng.normal(dist.sector_momentum_20d_mean, dist.sector_momentum_20d_std))
|
||||
|
||||
# Compute score using the same logic as the real scoring system
|
||||
row_for_scoring: dict[str, Any] = {
|
||||
"event_type": event_type,
|
||||
"event_direction": event_direction,
|
||||
"guidance_status": guidance_status,
|
||||
"signal_strength_score": signal_strength,
|
||||
"document_quality_score": doc_quality,
|
||||
"parse_confidence_overall": parse_conf,
|
||||
"guidance_direction_score": guidance_dir_score,
|
||||
"oneoff_penalty": oneoff_penalty,
|
||||
"reaction_day_return": reaction_return,
|
||||
"volume_ratio_20d": volume_ratio,
|
||||
"close_location": close_location,
|
||||
"gap_size": gap_size,
|
||||
"pre_event_entropy_60d": entropy_60d,
|
||||
}
|
||||
score = _compute_synthetic_score(row_for_scoring)
|
||||
|
||||
row: dict[str, Any] = {
|
||||
# Identity
|
||||
"event_id": f"SYNTH::{symbol}::{event_counter:06d}",
|
||||
"symbol": symbol,
|
||||
"event_type": event_type,
|
||||
"event_direction": event_direction,
|
||||
"guidance_status": guidance_status,
|
||||
"filing_time_bucket": filing_bucket,
|
||||
"sector": sector,
|
||||
# Dates and timestamps
|
||||
"event_date": event_date_d.isoformat(),
|
||||
"event_timestamp": event_timestamp,
|
||||
"reaction_date": reaction_date.isoformat(),
|
||||
"entry_date": exec_date.isoformat(),
|
||||
"execution_date": exec_date,
|
||||
# Pricing
|
||||
"entry_price": round(entry_price, 4),
|
||||
"entry_price_est": round(entry_price, 4),
|
||||
"event_close": round(event_close, 4),
|
||||
"atr_14": round(atr_14, 4),
|
||||
"avg_dollar_volume": round(avg_dollar_volume, 2),
|
||||
"avg_dollar_volume_20d": round(avg_dollar_volume, 2),
|
||||
"market_cap_proxy": round(max(avg_dollar_volume * 400, 3_000_000_000), 0),
|
||||
# Score
|
||||
"score": round(score, 4),
|
||||
# Quality features
|
||||
"signal_strength_score": round(signal_strength, 4),
|
||||
"document_quality_score": round(doc_quality, 4),
|
||||
"parse_confidence_overall": round(parse_conf, 4),
|
||||
"guidance_direction_score": round(guidance_dir_score, 4),
|
||||
"oneoff_penalty": oneoff_penalty,
|
||||
# Reaction features
|
||||
"reaction_day_return": round(reaction_return, 4),
|
||||
"volume_ratio_20d": round(volume_ratio, 4),
|
||||
"close_location": round(close_location, 4),
|
||||
"gap_size": round(gap_size, 4),
|
||||
"reaction_day_low": round(entry_price * (1.0 - abs(reaction_return) * 0.5), 4),
|
||||
"reaction_day_high": round(entry_price * (1.0 + abs(reaction_return) * 0.5), 4),
|
||||
"reaction_day_range_pct": round(abs(reaction_return) + abs(gap_size), 4),
|
||||
"upper_wick_pct": round(max(0.0, float(rng.exponential(0.01))), 4),
|
||||
# Technical pre-event features
|
||||
"pre_event_rsi_14": round(rsi_14, 2),
|
||||
"pre_event_bb_position": round(bb_position, 4),
|
||||
"pre_event_volatility_20d": round(vol_20d, 4),
|
||||
"pre_event_obv_slope_20d": float(rng.normal(0, 0.1)),
|
||||
"pre_event_hurst_60d": round(hurst_60d, 4),
|
||||
"pre_event_entropy_60d": round(entropy_60d, 4),
|
||||
"pre_event_short_ratio": max(0.0, float(rng.exponential(0.05))),
|
||||
"pre_event_sector_momentum_20d": round(sector_mom, 4),
|
||||
"pre_event_ou_theta_60d": round(ou_theta, 4),
|
||||
"pre_event_gravitational_pull": round(grav_pull, 4),
|
||||
"pre_event_market_temperature": round(mkt_temp, 4),
|
||||
# Fundamental (optional, not always present)
|
||||
"reported_eps": None,
|
||||
"estimated_eps": None,
|
||||
"earnings_beat": None,
|
||||
"earnings_surprise_pct": None,
|
||||
# Universe
|
||||
"exchange_proxy": "NASDAQ" if rng.random() > 0.4 else "NYSE",
|
||||
"asset_type_proxy": "stock",
|
||||
}
|
||||
|
||||
today_candidates.append(row)
|
||||
|
||||
if today_candidates:
|
||||
candidates[exec_date] = today_candidates
|
||||
|
||||
return candidates
|
||||
|
||||
|
||||
def _compute_synthetic_score(row: dict[str, Any]) -> float:
|
||||
"""Compute a realistic score using the same weighting as compute_entry_score (v5 base).
|
||||
|
||||
Simplified version that does not require the full scoring module imports,
|
||||
matching the 3-component structure: event_quality (65%) + reaction (20%) + volume (15%).
|
||||
"""
|
||||
# Event quality component (65%)
|
||||
doc_q = float(row.get("document_quality_score") or 0.65)
|
||||
sig_s = float(row.get("signal_strength_score") or 0.60)
|
||||
parse_c = float(row.get("parse_confidence_overall") or 0.75)
|
||||
guidance = float(row.get("guidance_direction_score") or 0.50)
|
||||
oneoff = float(row.get("oneoff_penalty") or 0.0)
|
||||
|
||||
base_quality = (doc_q * 0.35 + sig_s * 0.30 + parse_c * 0.20 + guidance * 0.15)
|
||||
if oneoff:
|
||||
base_quality *= 0.60
|
||||
|
||||
# Reaction direction component (20%)
|
||||
reaction_return = float(row.get("reaction_day_return") or 0.0)
|
||||
if reaction_return > 0.01:
|
||||
reaction_score = min(1.0, 0.5 + reaction_return * 5.0)
|
||||
elif reaction_return < -0.01:
|
||||
reaction_score = max(0.0, 0.5 + reaction_return * 5.0)
|
||||
else:
|
||||
reaction_score = 0.5
|
||||
|
||||
# Volume conviction component (15%)
|
||||
vol_ratio = float(row.get("volume_ratio_20d") or 1.0)
|
||||
volume_score = min(1.0, vol_ratio / 3.0)
|
||||
|
||||
# Low entropy bonus (v13e entropy feature)
|
||||
entropy = row.get("pre_event_entropy_60d")
|
||||
entropy_bonus = 0.0
|
||||
if entropy is not None and float(entropy) < 1.2:
|
||||
entropy_bonus = 0.03 * (1.2 - float(entropy))
|
||||
|
||||
raw = base_quality * 0.65 + reaction_score * 0.20 + volume_score * 0.15 + entropy_bonus
|
||||
return max(0.0, min(1.0, raw))
|
||||
@ -0,0 +1,242 @@
|
||||
"""Synthetic OHLCV price path generation for scenario backtesting.
|
||||
|
||||
Implements Regime-Switching GBM with optional jump-diffusion (Merton model).
|
||||
Individual stock paths are generated as market factor + beta + idiosyncratic noise
|
||||
to produce realistic cross-sectional correlations.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass
|
||||
class PriceRegime:
|
||||
"""Parameters for a single market regime segment."""
|
||||
|
||||
annualized_drift: float
|
||||
"""Expected annual return, e.g. 0.15 for bull, -0.20 for bear."""
|
||||
annualized_vol: float
|
||||
"""Annual volatility, e.g. 0.15 for calm, 0.35 for stressed."""
|
||||
duration_days: int
|
||||
"""Number of trading days this regime lasts."""
|
||||
jump_prob: float = 0.0
|
||||
"""Per-day probability of a Merton jump event."""
|
||||
jump_mean: float = 0.0
|
||||
"""Mean log-jump size. Negative for crash-type regimes."""
|
||||
jump_std: float = 0.02
|
||||
"""Standard deviation of log-jump size."""
|
||||
|
||||
|
||||
def _generate_regime_log_returns(
|
||||
regimes: list[PriceRegime],
|
||||
n_days: int,
|
||||
rng: np.random.Generator,
|
||||
) -> list[float]:
|
||||
"""Generate n_days market-level daily log returns following the regime sequence."""
|
||||
log_rets: list[float] = []
|
||||
dt_step = 1.0 / 252
|
||||
|
||||
for regime in regimes:
|
||||
drift = (regime.annualized_drift - 0.5 * regime.annualized_vol ** 2) * dt_step
|
||||
diffusion = regime.annualized_vol * math.sqrt(dt_step)
|
||||
days_this_regime = min(regime.duration_days, n_days - len(log_rets))
|
||||
|
||||
for _ in range(days_this_regime):
|
||||
lr = drift + diffusion * rng.standard_normal()
|
||||
if regime.jump_prob > 0 and rng.random() < regime.jump_prob:
|
||||
lr += rng.normal(regime.jump_mean, max(regime.jump_std, 1e-6))
|
||||
log_rets.append(lr)
|
||||
|
||||
# Extend with last regime if regimes run short
|
||||
last = regimes[-1]
|
||||
drift = (last.annualized_drift - 0.5 * last.annualized_vol ** 2) * dt_step
|
||||
diffusion = last.annualized_vol * math.sqrt(dt_step)
|
||||
while len(log_rets) < n_days:
|
||||
log_rets.append(drift + diffusion * rng.standard_normal())
|
||||
|
||||
return log_rets[:n_days]
|
||||
|
||||
|
||||
def _bars_from_log_returns(
|
||||
log_rets: list[float],
|
||||
trading_dates: list[dt.date],
|
||||
initial_price: float,
|
||||
intraday_vol_scale: float,
|
||||
base_volume: int,
|
||||
rng: np.random.Generator,
|
||||
) -> dict[dt.date, dict[str, Any]]:
|
||||
"""Build OHLCV bars from a sequence of daily log-returns.
|
||||
|
||||
Ensures OHLCV consistency: low <= min(open, close), high >= max(open, close).
|
||||
Volume is log-normally distributed and positively correlated with |return|.
|
||||
"""
|
||||
bars: dict[dt.date, dict[str, Any]] = {}
|
||||
prev_close = initial_price
|
||||
|
||||
for date, lr in zip(trading_dates, log_rets):
|
||||
close = max(prev_close * math.exp(lr), 0.01)
|
||||
|
||||
# Open: prev_close * small gap (mean-zero noise)
|
||||
gap = rng.normal(0, intraday_vol_scale * 0.5)
|
||||
open_ = max(prev_close * math.exp(gap), 0.01)
|
||||
|
||||
# Intraday range around open/close extremes
|
||||
intraday_noise = abs(rng.normal(0, intraday_vol_scale * 0.7))
|
||||
hi_raw = max(open_, close) * (1.0 + intraday_noise)
|
||||
lo_raw = min(open_, close) * max(1.0 - intraday_noise, 0.001)
|
||||
|
||||
high = max(hi_raw, open_, close)
|
||||
low = min(lo_raw, open_, close)
|
||||
low = max(low, 0.01)
|
||||
|
||||
# Volume: log-normal, amplified by absolute return
|
||||
vol_factor = 1.0 + 3.0 * abs(math.exp(lr) - 1)
|
||||
volume = max(1000, int(rng.lognormal(math.log(base_volume), 0.4) * vol_factor))
|
||||
|
||||
bars[date] = {
|
||||
"date": date,
|
||||
"open": round(open_, 4),
|
||||
"high": round(high, 4),
|
||||
"low": round(low, 4),
|
||||
"close": round(close, 4),
|
||||
"volume": volume,
|
||||
}
|
||||
prev_close = close
|
||||
|
||||
return bars
|
||||
|
||||
|
||||
def generate_price_paths(
|
||||
n_symbols: int,
|
||||
initial_prices: list[float] | None,
|
||||
regimes: list[PriceRegime],
|
||||
trading_dates: list[dt.date],
|
||||
market_beta_range: tuple[float, float] = (0.6, 1.2),
|
||||
rng: np.random.Generator | None = None,
|
||||
tickers: list[str] | None = None,
|
||||
return_market_log_rets: bool = False,
|
||||
) -> (
|
||||
tuple[dict[str, dict[dt.date, dict[str, Any]]], list[float]]
|
||||
| dict[str, dict[dt.date, dict[str, Any]]]
|
||||
):
|
||||
"""Generate correlated OHLCV bars for n_symbols stocks.
|
||||
|
||||
Each stock has a random beta to a shared market factor plus idiosyncratic noise.
|
||||
Returns bars_by_symbol_date dict compatible with SnapshotStore.
|
||||
|
||||
Args:
|
||||
n_symbols: Number of synthetic stocks to generate.
|
||||
initial_prices: Optional list of starting prices (defaults to random $20-$200).
|
||||
regimes: Sequence of PriceRegime objects defining the market environment.
|
||||
trading_dates: Ordered list of NYSE trading dates (from calendar.get_trading_days).
|
||||
market_beta_range: (min, max) range for individual stock betas.
|
||||
rng: NumPy random generator (seeded for reproducibility).
|
||||
tickers: Optional list of ticker symbols (auto-generated if None).
|
||||
return_market_log_rets: If True, also return the market log-return series.
|
||||
|
||||
Returns:
|
||||
bars_by_symbol_date dict, or (dict, market_log_rets) if return_market_log_rets=True.
|
||||
"""
|
||||
if rng is None:
|
||||
rng = np.random.default_rng()
|
||||
|
||||
n = len(trading_dates)
|
||||
if tickers is None:
|
||||
tickers = [f"SYM{i:03d}" for i in range(n_symbols)]
|
||||
|
||||
# Typical vol across all regimes (for intraday range scaling)
|
||||
avg_vol = sum(r.annualized_vol for r in regimes) / max(len(regimes), 1)
|
||||
dt_step = 1.0 / 252
|
||||
|
||||
# Market-level log returns (shared factor)
|
||||
market_log_rets = _generate_regime_log_returns(regimes, n, rng)
|
||||
|
||||
bars_by_symbol: dict[str, dict[dt.date, dict[str, Any]]] = {}
|
||||
|
||||
for i, ticker in enumerate(tickers[:n_symbols]):
|
||||
if initial_prices and i < len(initial_prices):
|
||||
init_price = initial_prices[i]
|
||||
else:
|
||||
init_price = float(rng.uniform(20.0, 200.0))
|
||||
|
||||
beta = float(rng.uniform(*market_beta_range))
|
||||
idio_vol = avg_vol * float(rng.uniform(0.3, 0.8))
|
||||
|
||||
# Build stock log returns: beta * market + idiosyncratic
|
||||
stock_log_rets: list[float] = []
|
||||
for mkt_lr in market_log_rets:
|
||||
idio = rng.normal(0, idio_vol * math.sqrt(dt_step))
|
||||
stock_log_rets.append(beta * mkt_lr + idio)
|
||||
|
||||
intraday_scale = (idio_vol + avg_vol * beta) * math.sqrt(dt_step) * 0.5
|
||||
base_vol = int(rng.uniform(500_000, 10_000_000))
|
||||
|
||||
bars = _bars_from_log_returns(
|
||||
stock_log_rets,
|
||||
trading_dates,
|
||||
initial_price=init_price,
|
||||
intraday_vol_scale=intraday_scale,
|
||||
base_volume=base_vol,
|
||||
rng=rng,
|
||||
)
|
||||
bars_by_symbol[ticker] = bars
|
||||
|
||||
if return_market_log_rets:
|
||||
return bars_by_symbol, market_log_rets
|
||||
return bars_by_symbol
|
||||
|
||||
|
||||
def generate_market_etf_paths(
|
||||
regimes: list[PriceRegime],
|
||||
trading_dates: list[dt.date],
|
||||
rng: np.random.Generator,
|
||||
spy_initial: float = 480.0,
|
||||
qqq_initial: float = 415.0,
|
||||
) -> tuple[dict[dt.date, dict[str, Any]], dict[dt.date, dict[str, Any]], list[float]]:
|
||||
"""Generate SPY and QQQ synthetic price bars plus market log-returns.
|
||||
|
||||
QQQ has slightly higher vol and beta than SPY to reflect tech concentration.
|
||||
|
||||
Returns:
|
||||
(spy_bars, qqq_bars, market_log_rets)
|
||||
"""
|
||||
n = len(trading_dates)
|
||||
dt_step = 1.0 / 252
|
||||
avg_vol = sum(r.annualized_vol for r in regimes) / max(len(regimes), 1)
|
||||
|
||||
market_log_rets = _generate_regime_log_returns(regimes, n, rng)
|
||||
|
||||
# SPY ≈ market (beta ~1.0, low idio noise)
|
||||
spy_log_rets: list[float] = []
|
||||
for lr in market_log_rets:
|
||||
spy_log_rets.append(lr + rng.normal(0, avg_vol * 0.05 * math.sqrt(dt_step)))
|
||||
|
||||
spy_bars = _bars_from_log_returns(
|
||||
spy_log_rets, trading_dates,
|
||||
initial_price=spy_initial,
|
||||
intraday_vol_scale=avg_vol * math.sqrt(dt_step) * 0.4,
|
||||
base_volume=80_000_000,
|
||||
rng=rng,
|
||||
)
|
||||
|
||||
# QQQ ≈ market * 1.1 beta + higher idio noise
|
||||
qqq_log_rets: list[float] = []
|
||||
for lr in market_log_rets:
|
||||
qqq_log_rets.append(
|
||||
1.1 * lr + rng.normal(0, avg_vol * 0.08 * math.sqrt(dt_step))
|
||||
)
|
||||
|
||||
qqq_bars = _bars_from_log_returns(
|
||||
qqq_log_rets, trading_dates,
|
||||
initial_price=qqq_initial,
|
||||
intraday_vol_scale=avg_vol * math.sqrt(dt_step) * 0.45,
|
||||
base_volume=60_000_000,
|
||||
rng=rng,
|
||||
)
|
||||
|
||||
return spy_bars, qqq_bars, market_log_rets
|
||||
@ -0,0 +1,146 @@
|
||||
"""Assembles a synthetic SnapshotStore from a ScenarioConfig.
|
||||
|
||||
Orchestrates the full synthetic data generation pipeline:
|
||||
1. Trading dates (real NYSE calendar)
|
||||
2. Market ETF price paths (SPY, QQQ)
|
||||
3. Individual stock price paths (correlated with market)
|
||||
4. Macro indicators (VIX, HY spread, SPY/QQQ rolling stats)
|
||||
5. Event candidates (parameterized feature distributions)
|
||||
6. Event-price coupling (signal-to-noise injection)
|
||||
7. SnapshotStore assembly
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from libs.backtest.scenarios.coupling import couple_events_to_prices
|
||||
from libs.backtest.scenarios.event_gen import generate_events
|
||||
from libs.backtest.scenarios.macro_gen import generate_macro_data
|
||||
from libs.backtest.scenarios.price_gen import generate_market_etf_paths, generate_price_paths
|
||||
from libs.backtest.scenarios.scenarios import ScenarioConfig
|
||||
from libs.backtest.snapshot_store import SnapshotStore
|
||||
|
||||
|
||||
# Starting dates for synthetic scenarios (real NYSE calendar)
|
||||
# Using a date range that's safely after 2020 (no pandemic-era data issues)
|
||||
_SCENARIO_START_DATE = dt.date(2024, 1, 2)
|
||||
|
||||
|
||||
def _get_trading_dates(n_days: int, start: dt.date = _SCENARIO_START_DATE) -> list[dt.date]:
|
||||
"""Return n_days consecutive NYSE trading dates starting from start."""
|
||||
from libs.backtest.calendar import get_trading_days
|
||||
# Request a window of n_days * 1.5 calendar days to account for weekends/holidays
|
||||
end_estimate = start + dt.timedelta(days=int(n_days * 1.5) + 30)
|
||||
all_days = get_trading_days(start, end_estimate)
|
||||
return all_days[:n_days]
|
||||
|
||||
|
||||
def _total_regime_days(scenario: ScenarioConfig) -> int:
|
||||
"""Sum of all regime durations in the scenario."""
|
||||
return sum(r.duration_days for r in scenario.price_regimes)
|
||||
|
||||
|
||||
def build_synthetic_store(
|
||||
scenario: ScenarioConfig,
|
||||
rng: np.random.Generator | None = None,
|
||||
) -> SnapshotStore:
|
||||
"""Assemble a complete synthetic SnapshotStore from a ScenarioConfig.
|
||||
|
||||
The returned store has no connection to real market data. It can be
|
||||
passed directly to BacktestRunner without any Oracle API or database.
|
||||
|
||||
Args:
|
||||
scenario: Fully specified synthetic market scenario.
|
||||
rng: NumPy random generator. If None, uses scenario.seed or random state.
|
||||
|
||||
Returns:
|
||||
SnapshotStore ready for BacktestRunner.run().
|
||||
"""
|
||||
if rng is None:
|
||||
seed = scenario.seed
|
||||
rng = np.random.default_rng(seed)
|
||||
|
||||
# 1. Trading dates
|
||||
n_days = _total_regime_days(scenario) + 30 # extra buffer for parking tail
|
||||
trading_dates = _get_trading_dates(n_days)
|
||||
|
||||
# 2. Market ETF paths (SPY, QQQ) + market log-returns for macro correlation
|
||||
spy_bars, qqq_bars, market_log_rets = generate_market_etf_paths(
|
||||
regimes=scenario.price_regimes,
|
||||
trading_dates=trading_dates,
|
||||
rng=rng,
|
||||
)
|
||||
|
||||
# 3. Individual stock paths
|
||||
n_sym = scenario.n_symbols
|
||||
tickers = [f"SYN{i:03d}" for i in range(n_sym)]
|
||||
|
||||
bars_by_symbol = generate_price_paths(
|
||||
n_symbols=n_sym,
|
||||
initial_prices=None,
|
||||
regimes=scenario.price_regimes,
|
||||
trading_dates=trading_dates,
|
||||
market_beta_range=(0.5, 1.4),
|
||||
rng=rng,
|
||||
tickers=tickers,
|
||||
)
|
||||
assert isinstance(bars_by_symbol, dict), "generate_price_paths must return dict"
|
||||
|
||||
# Merge ETF bars in as well (for parking lookups)
|
||||
bars_by_symbol["SPY"] = spy_bars
|
||||
bars_by_symbol["QQQ"] = qqq_bars
|
||||
|
||||
# 4. Macro data
|
||||
macro_by_date = generate_macro_data(
|
||||
spy_bars=spy_bars,
|
||||
qqq_bars=qqq_bars,
|
||||
vix_config=scenario.vix_config,
|
||||
hy_config=scenario.hy_config,
|
||||
trading_dates=trading_dates,
|
||||
rng=rng,
|
||||
market_log_rets=market_log_rets,
|
||||
)
|
||||
|
||||
# 5. Event candidates
|
||||
candidates_by_exec_date = generate_events(
|
||||
trading_dates=trading_dates,
|
||||
symbols=tickers,
|
||||
dist=scenario.event_distribution,
|
||||
rng=rng,
|
||||
bars_by_symbol=bars_by_symbol,
|
||||
)
|
||||
|
||||
# 5b. Inject macro_vix / macro_hy_spread into each candidate row.
|
||||
# selector._row_matches_strategy_engine_filters() reads macro_vix and
|
||||
# macro_hy_spread directly from the row dict (not from macro_by_date),
|
||||
# so we must populate them here.
|
||||
for exec_date, rows in candidates_by_exec_date.items():
|
||||
macro = macro_by_date.get(exec_date, {})
|
||||
mv = macro.get("macro_vix")
|
||||
hy = macro.get("macro_hy_spread")
|
||||
for row in rows:
|
||||
if mv is not None:
|
||||
row["macro_vix"] = mv
|
||||
if hy is not None:
|
||||
row["macro_hy_spread"] = hy
|
||||
|
||||
# 6. Event-price coupling (signal injection)
|
||||
couple_events_to_prices(
|
||||
candidates=candidates_by_exec_date,
|
||||
bars_by_symbol=bars_by_symbol,
|
||||
trading_dates=trading_dates,
|
||||
signal_strength=scenario.signal_strength,
|
||||
signal_decay_days=scenario.signal_decay_days,
|
||||
false_positive_rate=scenario.false_positive_rate,
|
||||
rng=rng,
|
||||
)
|
||||
|
||||
# 7. Assemble SnapshotStore
|
||||
return SnapshotStore(
|
||||
candidates_by_exec_date=candidates_by_exec_date,
|
||||
bars_by_symbol_date=bars_by_symbol,
|
||||
macro_by_date=macro_by_date,
|
||||
)
|
||||
@ -0,0 +1,245 @@
|
||||
"""Unit tests for event_gen.py."""
|
||||
import datetime as dt
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from libs.backtest.scenarios.event_gen import (
|
||||
EventDistribution,
|
||||
_compute_synthetic_score,
|
||||
_sample_categorical,
|
||||
generate_events,
|
||||
)
|
||||
from libs.backtest.scenarios.price_gen import PriceRegime, generate_price_paths
|
||||
|
||||
_DATES_RAW = [dt.date(2024, 1, 2) + dt.timedelta(days=i) for i in range(400)]
|
||||
_TRADING_DATES = [d for d in _DATES_RAW if d.weekday() < 5][:252]
|
||||
_SYMBOLS = [f"SYM{i:03d}" for i in range(30)]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestSampleCategorical:
|
||||
def test_returns_valid_key(self):
|
||||
rng = np.random.default_rng(1)
|
||||
cats = {"a": 0.5, "b": 0.3, "c": 0.2}
|
||||
result = _sample_categorical(cats, rng)
|
||||
assert result in cats
|
||||
|
||||
def test_distribution_roughly_correct(self):
|
||||
rng = np.random.default_rng(42)
|
||||
cats = {"x": 0.9, "y": 0.1}
|
||||
counts = {"x": 0, "y": 0}
|
||||
for _ in range(1000):
|
||||
k = _sample_categorical(cats, rng)
|
||||
counts[k] += 1
|
||||
# x should appear roughly 90% of the time
|
||||
assert counts["x"] > 800
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestComputeSyntheticScore:
|
||||
def test_high_quality_event_has_high_score(self):
|
||||
row = {
|
||||
"signal_strength_score": 0.95,
|
||||
"document_quality_score": 0.95,
|
||||
"parse_confidence_overall": 0.95,
|
||||
"guidance_direction_score": 0.90,
|
||||
"oneoff_penalty": 0.0,
|
||||
"reaction_day_return": 0.08,
|
||||
"volume_ratio_20d": 3.0,
|
||||
"close_location": 0.85,
|
||||
"gap_size": 0.02,
|
||||
"pre_event_entropy_60d": 1.0,
|
||||
}
|
||||
score = _compute_synthetic_score(row)
|
||||
assert score > 0.75
|
||||
|
||||
def test_low_quality_event_has_low_score(self):
|
||||
row = {
|
||||
"signal_strength_score": 0.20,
|
||||
"document_quality_score": 0.20,
|
||||
"parse_confidence_overall": 0.25,
|
||||
"guidance_direction_score": 0.10,
|
||||
"oneoff_penalty": 1.0,
|
||||
"reaction_day_return": -0.05,
|
||||
"volume_ratio_20d": 0.5,
|
||||
"close_location": 0.2,
|
||||
"gap_size": -0.01,
|
||||
"pre_event_entropy_60d": 1.8,
|
||||
}
|
||||
score = _compute_synthetic_score(row)
|
||||
assert score < 0.35
|
||||
|
||||
def test_score_bounded_0_to_1(self):
|
||||
"""Score must always be in [0, 1]."""
|
||||
rng = np.random.default_rng(7)
|
||||
for _ in range(50):
|
||||
row = {
|
||||
"signal_strength_score": float(rng.uniform(0, 1)),
|
||||
"document_quality_score": float(rng.uniform(0, 1)),
|
||||
"parse_confidence_overall": float(rng.uniform(0, 1)),
|
||||
"guidance_direction_score": float(rng.uniform(0, 1)),
|
||||
"oneoff_penalty": float(rng.choice([0.0, 1.0])),
|
||||
"reaction_day_return": float(rng.normal(0, 0.05)),
|
||||
"volume_ratio_20d": float(rng.uniform(0.3, 4.0)),
|
||||
"pre_event_entropy_60d": float(rng.uniform(0.5, 2.0)),
|
||||
}
|
||||
score = _compute_synthetic_score(row)
|
||||
assert 0.0 <= score <= 1.0, f"score {score} out of [0, 1]"
|
||||
|
||||
def test_oneoff_penalty_reduces_score(self):
|
||||
base_row = {
|
||||
"signal_strength_score": 0.7,
|
||||
"document_quality_score": 0.7,
|
||||
"parse_confidence_overall": 0.8,
|
||||
"guidance_direction_score": 0.6,
|
||||
"oneoff_penalty": 0.0,
|
||||
"reaction_day_return": 0.03,
|
||||
"volume_ratio_20d": 2.0,
|
||||
"pre_event_entropy_60d": 1.3,
|
||||
}
|
||||
penalized_row = {**base_row, "oneoff_penalty": 1.0}
|
||||
assert _compute_synthetic_score(base_row) > _compute_synthetic_score(penalized_row)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestGenerateEvents:
|
||||
def test_returns_dict_keyed_by_exec_date(self):
|
||||
rng = np.random.default_rng(42)
|
||||
dist = EventDistribution()
|
||||
candidates = generate_events(_TRADING_DATES, _SYMBOLS, dist, rng)
|
||||
assert isinstance(candidates, dict)
|
||||
for d in candidates:
|
||||
assert isinstance(d, dt.date)
|
||||
assert d in _TRADING_DATES
|
||||
|
||||
def test_all_exec_dates_within_eligible_range(self):
|
||||
rng = np.random.default_rng(42)
|
||||
dist = EventDistribution()
|
||||
buffer = 25
|
||||
candidates = generate_events(_TRADING_DATES, _SYMBOLS, dist, rng, max_holding_days_buffer=buffer)
|
||||
# First 5 days are warm-up; last buffer days are excluded
|
||||
eligible_end = _TRADING_DATES[len(_TRADING_DATES) - buffer - 1]
|
||||
for d in candidates:
|
||||
assert d <= eligible_end, f"exec_date {d} beyond eligible range"
|
||||
|
||||
def test_required_fields_present(self):
|
||||
rng = np.random.default_rng(42)
|
||||
dist = EventDistribution()
|
||||
candidates = generate_events(_TRADING_DATES, _SYMBOLS, dist, rng)
|
||||
|
||||
required_fields = [
|
||||
"event_id", "symbol", "event_type", "event_direction",
|
||||
"filing_time_bucket", "sector",
|
||||
"event_timestamp", "reaction_date", "entry_date", "execution_date",
|
||||
"entry_price", "entry_price_est", "event_close", "atr_14", "avg_dollar_volume",
|
||||
"score",
|
||||
"signal_strength_score", "document_quality_score", "parse_confidence_overall",
|
||||
"guidance_direction_score", "oneoff_penalty",
|
||||
"reaction_day_return", "volume_ratio_20d", "close_location", "gap_size",
|
||||
"pre_event_rsi_14", "pre_event_bb_position", "pre_event_volatility_20d",
|
||||
"pre_event_hurst_60d", "pre_event_entropy_60d",
|
||||
"pre_event_ou_theta_60d", "pre_event_market_temperature",
|
||||
]
|
||||
|
||||
for exec_date, rows in candidates.items():
|
||||
for row in rows:
|
||||
for field in required_fields:
|
||||
assert field in row, f"Missing field '{field}' in candidate on {exec_date}"
|
||||
|
||||
def test_execution_date_matches_dict_key(self):
|
||||
rng = np.random.default_rng(42)
|
||||
dist = EventDistribution()
|
||||
candidates = generate_events(_TRADING_DATES, _SYMBOLS, dist, rng)
|
||||
for exec_date, rows in candidates.items():
|
||||
for row in rows:
|
||||
assert row["execution_date"] == exec_date
|
||||
|
||||
def test_reaction_date_is_previous_trading_day(self):
|
||||
rng = np.random.default_rng(42)
|
||||
dist = EventDistribution()
|
||||
candidates = generate_events(_TRADING_DATES, _SYMBOLS, dist, rng)
|
||||
date_set = set(_TRADING_DATES)
|
||||
for exec_date, rows in candidates.items():
|
||||
exec_idx = _TRADING_DATES.index(exec_date)
|
||||
expected_reaction = _TRADING_DATES[exec_idx - 1]
|
||||
for row in rows:
|
||||
rxn = row["reaction_date"]
|
||||
# reaction_date stored as isoformat string
|
||||
rxn_date = dt.date.fromisoformat(rxn) if isinstance(rxn, str) else rxn
|
||||
assert rxn_date == expected_reaction, f"reaction_date mismatch on {exec_date}"
|
||||
|
||||
def test_event_id_unique(self):
|
||||
rng = np.random.default_rng(42)
|
||||
dist = EventDistribution()
|
||||
candidates = generate_events(_TRADING_DATES, _SYMBOLS, dist, rng)
|
||||
all_ids = [row["event_id"] for rows in candidates.values() for row in rows]
|
||||
assert len(all_ids) == len(set(all_ids)), "Duplicate event_ids found"
|
||||
|
||||
def test_no_duplicate_symbol_per_day(self):
|
||||
rng = np.random.default_rng(42)
|
||||
dist = EventDistribution()
|
||||
candidates = generate_events(_TRADING_DATES, _SYMBOLS, dist, rng)
|
||||
for exec_date, rows in candidates.items():
|
||||
syms = [r["symbol"] for r in rows]
|
||||
assert len(syms) == len(set(syms)), f"Duplicate symbols on {exec_date}"
|
||||
|
||||
def test_prices_positive(self):
|
||||
rng = np.random.default_rng(42)
|
||||
dist = EventDistribution()
|
||||
candidates = generate_events(_TRADING_DATES, _SYMBOLS, dist, rng)
|
||||
for rows in candidates.values():
|
||||
for row in rows:
|
||||
assert row["entry_price"] > 0, f"Non-positive entry_price: {row['entry_price']}"
|
||||
assert row["atr_14"] > 0, f"Non-positive atr_14: {row['atr_14']}"
|
||||
|
||||
def test_score_between_0_and_1(self):
|
||||
rng = np.random.default_rng(42)
|
||||
dist = EventDistribution()
|
||||
candidates = generate_events(_TRADING_DATES, _SYMBOLS, dist, rng)
|
||||
for rows in candidates.values():
|
||||
for row in rows:
|
||||
assert 0.0 <= row["score"] <= 1.0, f"score {row['score']} out of [0, 1]"
|
||||
|
||||
def test_uses_bar_close_when_bars_provided(self):
|
||||
rng = np.random.default_rng(42)
|
||||
dist = EventDistribution(events_per_day_mean=2.0, events_per_day_std=0.0)
|
||||
regimes = [PriceRegime(0.10, 0.15, 252)]
|
||||
bars_by_symbol = generate_price_paths(
|
||||
n_symbols=len(_SYMBOLS),
|
||||
initial_prices=None,
|
||||
regimes=regimes,
|
||||
trading_dates=_TRADING_DATES,
|
||||
rng=np.random.default_rng(1),
|
||||
tickers=_SYMBOLS,
|
||||
)
|
||||
candidates = generate_events(
|
||||
_TRADING_DATES, _SYMBOLS, dist,
|
||||
np.random.default_rng(42),
|
||||
bars_by_symbol=bars_by_symbol,
|
||||
)
|
||||
# At least some events should be present
|
||||
assert len(candidates) > 0
|
||||
# entry_prices should be positive and reasonable
|
||||
for rows in candidates.values():
|
||||
for row in rows:
|
||||
assert row["entry_price"] > 0
|
||||
|
||||
def test_event_timestamp_format(self):
|
||||
"""event_timestamp should be ISO with timezone."""
|
||||
rng = np.random.default_rng(42)
|
||||
dist = EventDistribution()
|
||||
candidates = generate_events(_TRADING_DATES, _SYMBOLS, dist, rng)
|
||||
for rows in candidates.values():
|
||||
for row in rows:
|
||||
ts = row["event_timestamp"]
|
||||
assert "T" in ts, f"event_timestamp not ISO: {ts}"
|
||||
assert "+00:00" in ts or "Z" in ts, f"event_timestamp missing TZ: {ts}"
|
||||
|
||||
def test_produces_events_with_default_distribution(self):
|
||||
"""Default distribution should produce at least some events."""
|
||||
rng = np.random.default_rng(99)
|
||||
dist = EventDistribution()
|
||||
candidates = generate_events(_TRADING_DATES, _SYMBOLS, dist, rng)
|
||||
total = sum(len(v) for v in candidates.values())
|
||||
assert total > 50, f"Too few events generated: {total}"
|
||||
@ -0,0 +1,219 @@
|
||||
"""Unit tests for scenario robustness.py — RRS computation logic."""
|
||||
import pytest
|
||||
|
||||
from libs.backtest.domain import MetricsBundle
|
||||
from libs.backtest.scenarios.robustness import (
|
||||
RegimeRobustnessReport,
|
||||
ScenarioResult,
|
||||
compute_rrs,
|
||||
)
|
||||
|
||||
|
||||
def _make_result(
|
||||
name: str,
|
||||
sharpe: float = 1.0,
|
||||
total_return: float = 10.0,
|
||||
max_dd: float = 10.0,
|
||||
win_rate: float = 0.55,
|
||||
profit_factor: float = 1.5,
|
||||
trade_count: int = 20,
|
||||
) -> ScenarioResult:
|
||||
"""Helper: construct a minimal ScenarioResult."""
|
||||
m = MetricsBundle(
|
||||
trade_count=trade_count,
|
||||
win_rate=win_rate,
|
||||
avg_win_pct=2.0,
|
||||
avg_loss_pct=-1.0,
|
||||
profit_factor=profit_factor,
|
||||
total_return_pct=total_return,
|
||||
max_drawdown_pct=max_dd,
|
||||
sharpe_ratio=sharpe,
|
||||
)
|
||||
return ScenarioResult(
|
||||
scenario_name=name,
|
||||
metrics=m,
|
||||
trade_count=trade_count,
|
||||
sharpe_ratio=sharpe,
|
||||
total_return_pct=total_return,
|
||||
max_drawdown_pct=max_dd,
|
||||
win_rate=win_rate,
|
||||
profit_factor=profit_factor,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestComputeRRS:
|
||||
def test_empty_results_returns_zeros(self):
|
||||
rrs, si, br, dr, rt, st = compute_rrs({})
|
||||
assert rrs == 0.0
|
||||
assert si == 0.0
|
||||
assert br == 0.0
|
||||
|
||||
def test_signal_integrity_perfect_when_no_signal_sharpe_zero(self):
|
||||
results = {
|
||||
"no_signal": _make_result("no_signal", sharpe=0.0),
|
||||
"strong_signal": _make_result("strong_signal", sharpe=1.5),
|
||||
}
|
||||
_, si, _, _, _, _ = compute_rrs(results)
|
||||
assert si == 100.0
|
||||
|
||||
def test_signal_integrity_perfect_when_no_signal_sharpe_negative(self):
|
||||
results = {
|
||||
"no_signal": _make_result("no_signal", sharpe=-0.5),
|
||||
"strong_signal": _make_result("strong_signal", sharpe=1.5),
|
||||
}
|
||||
_, si, _, _, _, _ = compute_rrs(results)
|
||||
assert si == 100.0
|
||||
|
||||
def test_signal_integrity_zero_when_no_signal_equals_strong_signal(self):
|
||||
results = {
|
||||
"no_signal": _make_result("no_signal", sharpe=1.5),
|
||||
"strong_signal": _make_result("strong_signal", sharpe=1.5),
|
||||
}
|
||||
_, si, _, _, _, _ = compute_rrs(results)
|
||||
assert si == pytest.approx(0.0, abs=1.0)
|
||||
|
||||
def test_signal_integrity_neutral_when_no_signal_scenario_missing(self):
|
||||
results = {"steady_bull": _make_result("steady_bull", sharpe=1.0)}
|
||||
_, si, _, _, _, _ = compute_rrs(results)
|
||||
assert si == 50.0
|
||||
|
||||
def test_breadth_all_positive(self):
|
||||
results = {
|
||||
"a": _make_result("a", sharpe=0.5),
|
||||
"b": _make_result("b", sharpe=1.0),
|
||||
"c": _make_result("c", sharpe=0.1),
|
||||
}
|
||||
_, _, br, _, _, _ = compute_rrs(results)
|
||||
assert br == pytest.approx(100.0)
|
||||
|
||||
def test_breadth_all_negative(self):
|
||||
results = {
|
||||
"a": _make_result("a", sharpe=-0.5),
|
||||
"b": _make_result("b", sharpe=-1.0),
|
||||
}
|
||||
_, _, br, _, _, _ = compute_rrs(results)
|
||||
assert br == pytest.approx(0.0)
|
||||
|
||||
def test_breadth_half_positive(self):
|
||||
results = {
|
||||
"a": _make_result("a", sharpe=1.0),
|
||||
"b": _make_result("b", sharpe=-1.0),
|
||||
}
|
||||
_, _, br, _, _, _ = compute_rrs(results)
|
||||
assert br == pytest.approx(50.0)
|
||||
|
||||
def test_drawdown_resilience_zero_dd_gives_100(self):
|
||||
results = {"a": _make_result("a", max_dd=0.0)}
|
||||
_, _, _, dr, _, _ = compute_rrs(results)
|
||||
assert dr == pytest.approx(100.0)
|
||||
|
||||
def test_drawdown_resilience_50pct_dd_gives_0(self):
|
||||
results = {"a": _make_result("a", max_dd=50.0)}
|
||||
_, _, _, dr, _, _ = compute_rrs(results)
|
||||
assert dr == pytest.approx(0.0)
|
||||
|
||||
def test_drawdown_resilience_clamped_not_negative(self):
|
||||
results = {"a": _make_result("a", max_dd=80.0)}
|
||||
_, _, _, dr, _, _ = compute_rrs(results)
|
||||
assert dr >= 0.0
|
||||
|
||||
def test_regime_transition_neutral_when_missing(self):
|
||||
results = {"steady_bull": _make_result("steady_bull", sharpe=1.0)}
|
||||
_, _, _, _, rt, _ = compute_rrs(results)
|
||||
assert rt == 50.0
|
||||
|
||||
def test_regime_transition_good_when_above_median(self):
|
||||
results = {
|
||||
"a": _make_result("a", sharpe=0.5),
|
||||
"b": _make_result("b", sharpe=0.5),
|
||||
"regime_switch": _make_result("regime_switch", sharpe=2.0),
|
||||
}
|
||||
_, _, _, _, rt, _ = compute_rrs(results)
|
||||
# regime_switch > median → should be > 50
|
||||
assert rt > 50.0
|
||||
|
||||
def test_stability_perfect_when_all_same_sharpe(self):
|
||||
results = {
|
||||
"a": _make_result("a", sharpe=1.0),
|
||||
"b": _make_result("b", sharpe=1.0),
|
||||
"c": _make_result("c", sharpe=1.0),
|
||||
}
|
||||
_, _, _, _, _, st = compute_rrs(results)
|
||||
assert st == pytest.approx(100.0)
|
||||
|
||||
def test_stability_lower_when_high_variance(self):
|
||||
low_var = {
|
||||
"a": _make_result("a", sharpe=1.0),
|
||||
"b": _make_result("b", sharpe=1.1),
|
||||
"c": _make_result("c", sharpe=0.9),
|
||||
}
|
||||
high_var = {
|
||||
"a": _make_result("a", sharpe=3.0),
|
||||
"b": _make_result("b", sharpe=-0.5),
|
||||
"c": _make_result("c", sharpe=0.1),
|
||||
}
|
||||
_, _, _, _, _, st_low = compute_rrs(low_var)
|
||||
_, _, _, _, _, st_high = compute_rrs(high_var)
|
||||
assert st_low > st_high
|
||||
|
||||
def test_rrs_weighted_sum(self):
|
||||
"""RRS = 0.25*si + 0.25*br + 0.20*dr + 0.15*rt + 0.15*st."""
|
||||
results = {
|
||||
"no_signal": _make_result("no_signal", sharpe=-0.1),
|
||||
"strong_signal": _make_result("strong_signal", sharpe=1.5),
|
||||
"steady_bull": _make_result("steady_bull", sharpe=1.2),
|
||||
"steady_bear": _make_result("steady_bear", sharpe=0.3),
|
||||
"regime_switch": _make_result("regime_switch", sharpe=0.8),
|
||||
}
|
||||
rrs, si, br, dr, rt, st = compute_rrs(results)
|
||||
expected = 0.25 * si + 0.25 * br + 0.20 * dr + 0.15 * rt + 0.15 * st
|
||||
assert rrs == pytest.approx(expected, rel=1e-6)
|
||||
|
||||
def test_rrs_bounded_0_to_100(self):
|
||||
results = {
|
||||
"no_signal": _make_result("no_signal", sharpe=-0.5),
|
||||
"strong_signal": _make_result("strong_signal", sharpe=2.0),
|
||||
"a": _make_result("a", sharpe=1.0),
|
||||
"b": _make_result("b", sharpe=1.5),
|
||||
}
|
||||
rrs, _, _, _, _, _ = compute_rrs(results)
|
||||
assert 0.0 <= rrs <= 100.0
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestVerdicts:
|
||||
def _make_report(self, rrs: float) -> RegimeRobustnessReport:
|
||||
from libs.backtest.scenarios.robustness import _verdict
|
||||
results = {"a": _make_result("a", sharpe=1.0)}
|
||||
return RegimeRobustnessReport(
|
||||
experiment_name="test",
|
||||
scenario_results=results,
|
||||
signal_integrity=50.0,
|
||||
breadth=50.0,
|
||||
drawdown_resilience=50.0,
|
||||
regime_transition=50.0,
|
||||
stability=50.0,
|
||||
rrs=rrs,
|
||||
verdict=_verdict(rrs),
|
||||
)
|
||||
|
||||
def test_robust_verdict(self):
|
||||
r = self._make_report(75.0)
|
||||
assert r.verdict == "ROBUST"
|
||||
|
||||
def test_fragile_verdict_lower_bound(self):
|
||||
r = self._make_report(40.0)
|
||||
assert r.verdict == "FRAGILE"
|
||||
|
||||
def test_fragile_verdict_upper_bound(self):
|
||||
r = self._make_report(69.9)
|
||||
assert r.verdict == "FRAGILE"
|
||||
|
||||
def test_overfit_verdict(self):
|
||||
r = self._make_report(39.9)
|
||||
assert r.verdict == "OVERFIT"
|
||||
|
||||
def test_boundary_70_is_robust(self):
|
||||
r = self._make_report(70.0)
|
||||
assert r.verdict == "ROBUST"
|
||||
@ -0,0 +1,104 @@
|
||||
"""Unit tests for store_builder.py."""
|
||||
import datetime as dt
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from libs.backtest.scenarios.price_gen import PriceRegime
|
||||
from libs.backtest.scenarios.scenarios import SCENARIO_REGISTRY, STEADY_BULL, NO_SIGNAL
|
||||
from libs.backtest.scenarios.store_builder import _get_trading_dates, build_synthetic_store
|
||||
from libs.backtest.snapshot_store import SnapshotStore
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestGetTradingDates:
|
||||
def test_returns_correct_count(self):
|
||||
dates = _get_trading_dates(50)
|
||||
assert len(dates) == 50
|
||||
|
||||
def test_dates_are_weekdays(self):
|
||||
dates = _get_trading_dates(30)
|
||||
for d in dates:
|
||||
assert d.weekday() < 5, f"Non-weekday in trading dates: {d}"
|
||||
|
||||
def test_dates_are_sorted_ascending(self):
|
||||
dates = _get_trading_dates(30)
|
||||
assert dates == sorted(dates)
|
||||
|
||||
def test_starts_on_or_after_start_date(self):
|
||||
start = dt.date(2024, 1, 2)
|
||||
dates = _get_trading_dates(20, start=start)
|
||||
assert dates[0] >= start
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestBuildSyntheticStore:
|
||||
def test_returns_snapshot_store(self):
|
||||
store = build_synthetic_store(STEADY_BULL, rng=np.random.default_rng(42))
|
||||
assert isinstance(store, SnapshotStore)
|
||||
|
||||
def test_store_has_candidates(self):
|
||||
store = build_synthetic_store(STEADY_BULL, rng=np.random.default_rng(42))
|
||||
exec_dates = store.all_execution_dates()
|
||||
assert len(exec_dates) > 0, "No candidates generated"
|
||||
|
||||
def test_store_has_macro_data(self):
|
||||
store = build_synthetic_store(STEADY_BULL, rng=np.random.default_rng(42))
|
||||
trading_days = store.all_trading_days()
|
||||
assert len(trading_days) > 0
|
||||
|
||||
def test_store_has_bar_data(self):
|
||||
store = build_synthetic_store(STEADY_BULL, rng=np.random.default_rng(42))
|
||||
trading_days = store.all_trading_days()
|
||||
assert len(trading_days) > 0
|
||||
|
||||
def test_spy_and_qqq_in_bars(self):
|
||||
store = build_synthetic_store(STEADY_BULL, rng=np.random.default_rng(42))
|
||||
trading_days = store.all_trading_days()
|
||||
some_day = trading_days[10]
|
||||
assert store.get_bar("SPY", some_day) is not None
|
||||
assert store.get_bar("QQQ", some_day) is not None
|
||||
|
||||
def test_macro_has_required_keys(self):
|
||||
store = build_synthetic_store(STEADY_BULL, rng=np.random.default_rng(42))
|
||||
trading_days = store.all_trading_days()
|
||||
# Pick a date past rolling window warm-up
|
||||
late_date = trading_days[60]
|
||||
macro = store.get_macro_for_date(late_date)
|
||||
for key in ("VIXCLS", "macro_vix", "macro_hy_spread", "spy_close", "qqq_close"):
|
||||
assert key in macro, f"Missing macro key: {key}"
|
||||
|
||||
def test_seed_is_deterministic(self):
|
||||
store_a = build_synthetic_store(STEADY_BULL, rng=np.random.default_rng(123))
|
||||
store_b = build_synthetic_store(STEADY_BULL, rng=np.random.default_rng(123))
|
||||
# Same seed → same exec dates
|
||||
assert store_a.all_execution_dates() == store_b.all_execution_dates()
|
||||
|
||||
def test_no_signal_scenario_produces_store(self):
|
||||
"""no_signal scenario should assemble without error."""
|
||||
store = build_synthetic_store(NO_SIGNAL, rng=np.random.default_rng(1))
|
||||
assert isinstance(store, SnapshotStore)
|
||||
|
||||
def test_scenario_seed_used_when_rng_none(self):
|
||||
"""If rng=None, scenario.seed is used for determinism."""
|
||||
from dataclasses import replace
|
||||
seeded_scenario = replace(STEADY_BULL, seed=77)
|
||||
store_a = build_synthetic_store(seeded_scenario, rng=None)
|
||||
store_b = build_synthetic_store(seeded_scenario, rng=None)
|
||||
assert store_a.all_execution_dates() == store_b.all_execution_dates()
|
||||
|
||||
def test_candidate_exec_dates_align_with_macro_dates(self):
|
||||
"""All candidate exec_dates should exist in trading days (have macro)."""
|
||||
store = build_synthetic_store(STEADY_BULL, rng=np.random.default_rng(42))
|
||||
macro_trading_days = set(store.all_trading_days())
|
||||
for exec_date in store.all_execution_dates():
|
||||
assert exec_date in macro_trading_days, f"exec_date {exec_date} not in trading days"
|
||||
|
||||
def test_all_registered_scenarios_can_build(self):
|
||||
"""Smoke-test: every scenario in SCENARIO_REGISTRY builds without error."""
|
||||
for name, scenario in SCENARIO_REGISTRY.items():
|
||||
try:
|
||||
store = build_synthetic_store(scenario, rng=np.random.default_rng(0))
|
||||
assert isinstance(store, SnapshotStore), f"Bad store for {name}"
|
||||
except Exception as exc:
|
||||
pytest.fail(f"build_synthetic_store failed for scenario '{name}': {exc}")
|
||||
Loading…
Reference in New Issue