fithia2/libs/backtest/selector.py

"""Build, rank and filter Candidate objects from raw Parquet row dicts."""
from __future__ import annotations

import datetime as dt
import math
from typing import Any
from zoneinfo import ZoneInfo

from libs.backtest.domain import (
    Candidate,
    EventTypeProfile,
    SignalConfig,
    StrategyEngineConfig,
    UniverseConfig,
)
from libs.common.logging import get_logger

logger = get_logger(__name__)
_UTC = ZoneInfo("UTC")


def build_candidate(
    row: dict[str, Any],
    strategy_engine: StrategyEngineConfig | None = None,
) -> Candidate | None:
    """Build a Candidate from a raw Parquet row dict.

    Returns None (logged as skip) if:
    - event_timestamp is null/missing
    - entry_price_est is null/zero
    - execution_date is null/missing
    """
    event_id = row.get("event_id", "")

    # Strict: no silent substitution for event_timestamp
    raw_ts = row.get("event_timestamp")
    if raw_ts is None:
        logger.warning("skip_candidate_no_timestamp", event_id=event_id)
        return None

    # Normalise to timezone-aware datetime
    if isinstance(raw_ts, str):
        try:
            event_timestamp = dt.datetime.fromisoformat(raw_ts)
        except ValueError:
            logger.warning("skip_candidate_bad_timestamp", event_id=event_id, raw=raw_ts)
            return None
    elif isinstance(raw_ts, dt.datetime):
        event_timestamp = raw_ts
    else:
        logger.warning("skip_candidate_unknown_timestamp_type", event_id=event_id)
        return None

    if event_timestamp.tzinfo is None:
        event_timestamp = event_timestamp.replace(tzinfo=_UTC)

    # reaction_date
    raw_react = row.get("reaction_date")
    reaction_date = _parse_date(raw_react)

    # execution_date (mapped from Parquet entry_date) or reaction date for close-entry engines
    raw_exec_date = row.get("execution_date") or row.get("entry_date")
    execution_date = _parse_date(raw_exec_date)
    if execution_date is None:
        if strategy_engine and strategy_engine.entry_timing_policy == "reaction_close":
            execution_date = reaction_date
        else:
            logger.warning("skip_candidate_no_exec_date", event_id=event_id)
            return None

    if reaction_date is None:
        reaction_date = execution_date

    event_date = _parse_date(row.get("event_date")) or event_timestamp.date()
    timing_class = _classify_timing_class(event_date, reaction_date)

    trade_direction = _resolve_trade_direction(row, strategy_engine=strategy_engine)
    if strategy_engine is not None and not _matches_strategy_engine(
        row=row,
        strategy_engine=strategy_engine,
        event_type=str(row.get("event_type", "")),
        timing_class=timing_class,
        trade_direction=trade_direction,
    ):
        return None

    if strategy_engine and strategy_engine.entry_timing_policy == "reaction_close":
        execution_date = reaction_date
        entry_price_est = row.get("event_close") or row.get("entry_price_est")
        if not entry_price_est:
            logger.debug(
                "skip_candidate_no_event_close",
                event_id=event_id,
                engine_id=strategy_engine.engine_id,
            )
            return None
    else:
        entry_price_est = row.get("entry_price") or row.get("entry_price_est")

    if not entry_price_est:
        logger.warning("skip_candidate_no_entry_price", event_id=event_id)
        return None
    entry_price_est = float(entry_price_est)
    if entry_price_est <= 0:
        logger.warning("skip_candidate_zero_entry_price", event_id=event_id)
        return None

    score = float(row.get("score", 0.0))
    avg_dollar_volume = float(row.get("avg_dollar_volume", 0.0))
    atr_14_raw = row.get("atr_14")
    atr_14 = float(atr_14_raw) if atr_14_raw is not None else None
    event_direction = str(row.get("event_direction", "")).lower()
    guidance_status = str(row.get("guidance_status", "")).lower()
    close_location_raw = row.get("close_location")
    close_location = float(close_location_raw) if close_location_raw is not None else None
    gap_size_raw = row.get("gap_size")
    gap_size = float(gap_size_raw) if gap_size_raw is not None else None
    is_unknown_inline = (
        event_direction == "unknown"
        and guidance_status == "inline_or_maintained"
    )
    is_mixed_inline = (
        event_direction == "mixed"
        and guidance_status == "inline_or_maintained"
    )

    engine_early_failure_close_below_entry_and_reaction_close = None
    engine_early_failure_no_progress_days = None
    engine_early_failure_no_progress_r = None
    engine_early_failure_no_progress_fraction = None
    engine_dynamic_hold_checkpoints = None
    engine_dynamic_hold_extend_day = None
    engine_dynamic_hold_extend_r = None
    engine_dynamic_hold_extend_to = None
    engine_veto_oneoff_penalty = None
    engine_allow_oneoff_downsizing = None
    engine_oneoff_downsize_floor = None
    engine_veto_parse_confidence_min = None
    engine_allow_unknown_direction = None
    if strategy_engine is not None:
        engine_early_failure_close_below_entry_and_reaction_close = (
            strategy_engine.early_failure_close_below_entry_and_reaction_close_override
        )
        engine_early_failure_no_progress_days = (
            strategy_engine.early_failure_no_progress_days_override
        )
        engine_early_failure_no_progress_r = (
            strategy_engine.early_failure_no_progress_r_override
        )
        engine_early_failure_no_progress_fraction = (
            strategy_engine.early_failure_no_progress_fraction_override
        )
        engine_dynamic_hold_checkpoints = strategy_engine.dynamic_hold_checkpoints_override
        engine_dynamic_hold_extend_day = strategy_engine.dynamic_hold_extend_day_override
        engine_dynamic_hold_extend_r = strategy_engine.dynamic_hold_extend_r_override
        engine_dynamic_hold_extend_to = strategy_engine.dynamic_hold_extend_to_override
        engine_veto_oneoff_penalty = strategy_engine.veto_oneoff_penalty_override
        engine_allow_oneoff_downsizing = strategy_engine.allow_oneoff_downsizing_override
        engine_oneoff_downsize_floor = strategy_engine.oneoff_downsize_floor_override
        engine_veto_parse_confidence_min = (
            strategy_engine.veto_parse_confidence_min_override
        )
        if strategy_engine.event_directions and "unknown" in strategy_engine.event_directions:
            engine_allow_unknown_direction = True
        apply_unknown_inline_override = is_unknown_inline
        if apply_unknown_inline_override:
            if (
                strategy_engine.unknown_inline_exit_close_location_min is not None
                and (
                    close_location is None
                    or close_location < strategy_engine.unknown_inline_exit_close_location_min
                )
            ):
                apply_unknown_inline_override = False
            if (
                strategy_engine.unknown_inline_exit_gap_size_max is not None
                and (
                    gap_size is None
                    or gap_size > strategy_engine.unknown_inline_exit_gap_size_max
                )
            ):
                apply_unknown_inline_override = False
        if apply_unknown_inline_override:
            if (
                strategy_engine.unknown_inline_early_failure_close_below_entry_and_reaction_close_override
                is not None
            ):
                engine_early_failure_close_below_entry_and_reaction_close = (
                    strategy_engine.unknown_inline_early_failure_close_below_entry_and_reaction_close_override
                )
            if strategy_engine.unknown_inline_early_failure_no_progress_days_override is not None:
                engine_early_failure_no_progress_days = (
                    strategy_engine.unknown_inline_early_failure_no_progress_days_override
                )
            if strategy_engine.unknown_inline_early_failure_no_progress_r_override is not None:
                engine_early_failure_no_progress_r = (
                    strategy_engine.unknown_inline_early_failure_no_progress_r_override
                )
            if strategy_engine.unknown_inline_early_failure_no_progress_fraction_override is not None:
                engine_early_failure_no_progress_fraction = (
                    strategy_engine.unknown_inline_early_failure_no_progress_fraction_override
                )
        if is_mixed_inline:
            if (
                strategy_engine.mixed_inline_early_failure_close_below_entry_and_reaction_close_override
                is not None
            ):
                engine_early_failure_close_below_entry_and_reaction_close = (
                    strategy_engine.mixed_inline_early_failure_close_below_entry_and_reaction_close_override
                )
            if strategy_engine.mixed_inline_early_failure_no_progress_days_override is not None:
                engine_early_failure_no_progress_days = (
                    strategy_engine.mixed_inline_early_failure_no_progress_days_override
                )
            if strategy_engine.mixed_inline_early_failure_no_progress_r_override is not None:
                engine_early_failure_no_progress_r = (
                    strategy_engine.mixed_inline_early_failure_no_progress_r_override
                )
            if strategy_engine.mixed_inline_early_failure_no_progress_fraction_override is not None:
                engine_early_failure_no_progress_fraction = (
                    strategy_engine.mixed_inline_early_failure_no_progress_fraction_override
                )

    # Classify score bucket
    score_bucket = _classify_score_bucket(score)

    return Candidate(
        event_id=event_id,
        symbol=str(row.get("symbol", row.get("ticker", ""))),
        issuer_id=row.get("issuer_id"),
        score=score,
        sector=str(row.get("sector") or "UNKNOWN"),
        event_type=str(row.get("event_type", "")),
        event_timestamp=event_timestamp,
        event_date=event_date,
        filing_time_bucket=str(row.get("filing_time_bucket", "unknown")),
        timing_class=timing_class,
        reaction_date=reaction_date,
        execution_date=execution_date,
        entry_price_est=entry_price_est,
        avg_dollar_volume=avg_dollar_volume,
        atr_14=atr_14,
        score_bucket=score_bucket,
        engine_id=strategy_engine.engine_id if strategy_engine else "default",
        entry_timing_policy=(
            strategy_engine.entry_timing_policy
            if strategy_engine
            else "next_open"
        ),
        shadow_only=strategy_engine.shadow_only if strategy_engine else False,
        engine_min_entry_price=(
            strategy_engine.min_entry_price_override
            if strategy_engine
            else None
        ),
        engine_max_entry_price=(
            strategy_engine.max_entry_price_override
            if strategy_engine
            else None
        ),
        engine_max_holding_days=(
            strategy_engine.max_holding_days
            if strategy_engine
            else None
        ),
        engine_max_positions_per_sector=(
            strategy_engine.max_positions_per_sector_override
            if strategy_engine
            else None
        ),
        engine_max_position_value_pct=(
            strategy_engine.max_position_value_pct_override
            if strategy_engine
            else None
        ),
        engine_max_adv_fraction=(
            strategy_engine.max_adv_fraction_override
            if strategy_engine
            else None
        ),
        engine_risk_budget_pct=(
            strategy_engine.engine_risk_budget_pct
            if strategy_engine
            else 1.0
        ),
        engine_per_trade_risk_pct=(
            strategy_engine.per_trade_risk_pct_override
            if strategy_engine
            else None
        ),
        engine_target_atr_multiplier=(
            strategy_engine.target_atr_multiplier_override
            if strategy_engine
            else None
        ),
        engine_stop_atr_multiplier=(
            strategy_engine.stop_atr_multiplier_override
            if strategy_engine
            else None
        ),
        engine_target_1_r=(
            strategy_engine.target_1_r_override
            if strategy_engine
            else None
        ),
        engine_target_1_fraction=(
            strategy_engine.target_1_fraction_override
            if strategy_engine
            else None
        ),
        engine_trailing_model=(
            strategy_engine.trailing_model_override
            if strategy_engine
            else None
        ),
        engine_trailing_warmup_days=(
            strategy_engine.trailing_warmup_days_override
            if strategy_engine
            else None
        ),
        engine_use_reaction_day_low_stop=(
            strategy_engine.use_reaction_day_low_stop_override
            if strategy_engine
            else None
        ),
        engine_early_failure_close_below_entry_and_reaction_close=(
            engine_early_failure_close_below_entry_and_reaction_close
        ),
        engine_early_failure_no_progress_days=engine_early_failure_no_progress_days,
        engine_early_failure_no_progress_r=engine_early_failure_no_progress_r,
        engine_early_failure_no_progress_fraction=engine_early_failure_no_progress_fraction,
        engine_dynamic_hold_checkpoints=engine_dynamic_hold_checkpoints,
        engine_dynamic_hold_extend_day=engine_dynamic_hold_extend_day,
        engine_dynamic_hold_extend_r=engine_dynamic_hold_extend_r,
        engine_dynamic_hold_extend_to=engine_dynamic_hold_extend_to,
        engine_veto_oneoff_penalty=engine_veto_oneoff_penalty,
        engine_allow_oneoff_downsizing=engine_allow_oneoff_downsizing,
        engine_oneoff_downsize_floor=engine_oneoff_downsize_floor,
        engine_veto_parse_confidence_min=engine_veto_parse_confidence_min,
        engine_allow_unknown_direction=engine_allow_unknown_direction,
        engine_next_open_gap_cap_pct=(
            strategy_engine.next_open_gap_cap_pct
            if strategy_engine
            else None
        ),
        engine_add_on_max_count=(
            strategy_engine.add_on_max_count
            if strategy_engine
            else None
        ),
        engine_add_on_size_fraction=(
            strategy_engine.add_on_size_fraction
            if strategy_engine
            else None
        ),
        trade_direction=trade_direction,
        parent_position_id=row.get("parent_position_id"),
        is_add_on=bool(row.get("is_add_on", False)),
        forced_shares=(
            int(row["forced_shares"])
            if row.get("forced_shares") not in (None, "")
            else None
        ),
        features={k: v for k, v in row.items() if k not in _RESERVED_KEYS},
    )


_RESERVED_KEYS = {
    "event_id", "symbol", "ticker", "issuer_id", "score", "sector",
    "event_type", "event_timestamp", "event_date", "filing_time_bucket", "reaction_date",
    "entry_date", "execution_date", "entry_price", "entry_price_est",
    "event_close", "avg_dollar_volume", "atr_14", "score_bucket", "trade_direction",
    "parent_position_id", "is_add_on", "forced_shares",
}


def _parse_date(raw: Any) -> dt.date | None:
    if isinstance(raw, dt.datetime):
        return raw.date()
    if isinstance(raw, dt.date):
        return raw
    if isinstance(raw, str):
        try:
            return dt.date.fromisoformat(raw)
        except ValueError:
            return None
    return None


def _classify_timing_class(event_date: dt.date | None, reaction_date: dt.date) -> str:
    if event_date is None:
        return "unknown"
    if reaction_date == event_date:
        return "same_day"
    if reaction_date > event_date:
        return "after_close"
    return "unknown"


def _resolve_trade_direction(
    row: dict[str, Any],
    strategy_engine: StrategyEngineConfig | None = None,
) -> str:
    forced_direction = (
        str(strategy_engine.forced_trade_direction_override).lower()
        if strategy_engine and strategy_engine.forced_trade_direction_override
        else ""
    )
    if forced_direction in {"long", "short"}:
        return forced_direction

    raw_direction = str(row.get("trade_direction", "")).lower()
    if raw_direction in {"long", "short"}:
        return raw_direction

    reaction = row.get("reaction_day_return")
    if reaction is not None:
        try:
            return "short" if float(reaction) < 0 else "long"
        except (TypeError, ValueError):
            pass
    return "long"


def _matches_strategy_engine(
    row: dict[str, Any],
    strategy_engine: StrategyEngineConfig,
    event_type: str,
    timing_class: str,
    trade_direction: str,
) -> bool:
    if strategy_engine.entry_conventions:
        entry_convention = str(row.get("entry_convention", "")).lower()
        allowed_entry_conventions = {
            str(value).lower() for value in strategy_engine.entry_conventions if value
        }
        if entry_convention not in allowed_entry_conventions:
            return False

    if strategy_engine.event_types and event_type not in strategy_engine.event_types:
        return False

    event_direction = str(row.get("event_direction", "")).lower()
    if strategy_engine.event_directions:
        allowed_directions = {value.lower() for value in strategy_engine.event_directions}
        if event_direction not in allowed_directions:
            return False

    guidance_status = str(row.get("guidance_status", "")).lower()
    if strategy_engine.guidance_statuses:
        allowed_statuses = {value.lower() for value in strategy_engine.guidance_statuses}
        if guidance_status not in allowed_statuses:
            return False
    if strategy_engine.filing_time_buckets:
        allowed_buckets = {value.lower() for value in strategy_engine.filing_time_buckets}
        filing_time_bucket = str(row.get("filing_time_bucket", "")).lower()
        if filing_time_bucket not in allowed_buckets:
            return False
    if strategy_engine.allowed_exchanges:
        allowed_exchanges = {value.upper() for value in strategy_engine.allowed_exchanges}
        exchange = str(row.get("exchange_proxy") or row.get("exchange") or "").upper()
        if exchange not in allowed_exchanges:
            return False

    if strategy_engine.timing_class != "any" and timing_class != strategy_engine.timing_class:
        return False

    if strategy_engine.direction == "long_only" and trade_direction != "long":
        return False
    if strategy_engine.direction == "short_only" and trade_direction != "short":
        return False

    reaction_day_return = _safe_float(row.get("reaction_day_return"))
    if (
        strategy_engine.reaction_day_return_min is not None
        and reaction_day_return is not None
        and reaction_day_return < strategy_engine.reaction_day_return_min
    ):
        return False
    if (
        strategy_engine.reaction_day_return_max is not None
        and reaction_day_return is not None
        and reaction_day_return > strategy_engine.reaction_day_return_max
    ):
        return False

    close_location = _safe_float(row.get("close_location"))
    if (
        strategy_engine.close_location_min is not None
        and close_location is not None
        and close_location < strategy_engine.close_location_min
    ):
        return False
    if (
        strategy_engine.close_location_max is not None
        and close_location is not None
        and close_location > strategy_engine.close_location_max
    ):
        return False

    volume_ratio = _safe_float(row.get("volume_ratio_20d"))
    if (
        strategy_engine.volume_ratio_min is not None
        and volume_ratio is not None
        and volume_ratio < strategy_engine.volume_ratio_min
    ):
        return False
    if (
        strategy_engine.volume_ratio_max is not None
        and volume_ratio is not None
        and volume_ratio > strategy_engine.volume_ratio_max
    ):
        return False

    avg_dollar_volume = _safe_float(
        row.get("avg_dollar_volume_20d", row.get("avg_dollar_volume"))
    )
    if (
        strategy_engine.avg_dollar_volume_min is not None
        and avg_dollar_volume is not None
        and avg_dollar_volume < strategy_engine.avg_dollar_volume_min
    ):
        return False
    if (
        strategy_engine.avg_dollar_volume_max is not None
        and avg_dollar_volume is not None
        and avg_dollar_volume > strategy_engine.avg_dollar_volume_max
    ):
        return False

    gap_size = _safe_float(row.get("gap_size"))
    if (
        strategy_engine.gap_size_min is not None
        and gap_size is not None
        and gap_size < strategy_engine.gap_size_min
    ):
        return False
    if (
        strategy_engine.gap_size_max is not None
        and gap_size is not None
        and gap_size > strategy_engine.gap_size_max
    ):
        return False
    event_close = _safe_float(row.get("event_close"))
    reaction_day_low = _safe_float(row.get("reaction_day_low"))
    reaction_day_high = _safe_float(row.get("reaction_day_high"))
    reaction_day_range_pct = None
    upper_wick_pct = None
    if (
        event_close is not None
        and event_close > 0
        and reaction_day_low is not None
        and reaction_day_high is not None
        and reaction_day_high >= reaction_day_low
    ):
        reaction_day_range_pct = (reaction_day_high - reaction_day_low) / event_close
        upper_wick_pct = (reaction_day_high - event_close) / event_close
    if strategy_engine.reaction_day_range_pct_min is not None:
        if (
            reaction_day_range_pct is None
            or reaction_day_range_pct < strategy_engine.reaction_day_range_pct_min
        ):
            return False
    if strategy_engine.reaction_day_range_pct_max is not None:
        if (
            reaction_day_range_pct is None
            or reaction_day_range_pct > strategy_engine.reaction_day_range_pct_max
        ):
            return False
    if strategy_engine.upper_wick_pct_min is not None:
        if upper_wick_pct is None or upper_wick_pct < strategy_engine.upper_wick_pct_min:
            return False
    if strategy_engine.upper_wick_pct_max is not None:
        if upper_wick_pct is None or upper_wick_pct > strategy_engine.upper_wick_pct_max:
            return False

    market_cap_proxy = _safe_float(row.get("market_cap_proxy"))
    if strategy_engine.min_market_cap_proxy is not None:
        if market_cap_proxy is None or market_cap_proxy < strategy_engine.min_market_cap_proxy:
            return False
    if strategy_engine.max_market_cap_proxy is not None:
        if market_cap_proxy is None or market_cap_proxy > strategy_engine.max_market_cap_proxy:
            return False

    document_quality_score = _safe_float(row.get("document_quality_score"))
    if strategy_engine.document_quality_score_min is not None:
        if (
            document_quality_score is None
            or document_quality_score < strategy_engine.document_quality_score_min
        ):
            return False
    if strategy_engine.document_quality_score_max is not None:
        if (
            document_quality_score is None
            or document_quality_score > strategy_engine.document_quality_score_max
        ):
            return False

    signal_strength_score = _safe_float(row.get("signal_strength_score"))
    if strategy_engine.signal_strength_score_min is not None:
        if (
            signal_strength_score is None
            or signal_strength_score < strategy_engine.signal_strength_score_min
        ):
            return False
    if strategy_engine.signal_strength_score_max is not None:
        if (
            signal_strength_score is None
            or signal_strength_score > strategy_engine.signal_strength_score_max
        ):
            return False

    parse_confidence_overall = _safe_float(row.get("parse_confidence_overall"))
    if strategy_engine.parse_confidence_overall_min is not None:
        if (
            parse_confidence_overall is None
            or parse_confidence_overall < strategy_engine.parse_confidence_overall_min
        ):
            return False
    if strategy_engine.parse_confidence_overall_max is not None:
        if (
            parse_confidence_overall is None
            or parse_confidence_overall > strategy_engine.parse_confidence_overall_max
        ):
            return False

    macro_vix = _safe_float(row.get("macro_vix"))
    if strategy_engine.macro_vix_min is not None:
        if macro_vix is None or macro_vix < strategy_engine.macro_vix_min:
            return False
    if strategy_engine.macro_vix_max is not None:
        if macro_vix is None or macro_vix > strategy_engine.macro_vix_max:
            return False

    macro_hy_spread = _safe_float(row.get("macro_hy_spread"))
    if strategy_engine.macro_hy_spread_min is not None:
        if macro_hy_spread is None or macro_hy_spread < strategy_engine.macro_hy_spread_min:
            return False
    if strategy_engine.macro_hy_spread_max is not None:
        if macro_hy_spread is None or macro_hy_spread > strategy_engine.macro_hy_spread_max:
            return False

    if (
        strategy_engine.weak_reaction_threshold is not None
        and strategy_engine.weak_reaction_gap_max is not None
        and reaction_day_return is not None
        and gap_size is not None
        and reaction_day_return < strategy_engine.weak_reaction_threshold
        and gap_size > strategy_engine.weak_reaction_gap_max
    ):
        return False

    if (
        strategy_engine.unknown_direction_reaction_min is not None
        and event_direction == "unknown"
        and reaction_day_return is not None
        and reaction_day_return < strategy_engine.unknown_direction_reaction_min
    ):
        return False
    if (
        strategy_engine.unknown_direction_close_location_min is not None
        and event_direction == "unknown"
        and close_location is not None
        and close_location < strategy_engine.unknown_direction_close_location_min
    ):
        return False
    if (
        strategy_engine.unknown_direction_close_location_max is not None
        and event_direction == "unknown"
        and close_location is not None
        and close_location > strategy_engine.unknown_direction_close_location_max
    ):
        return False
    if (
        strategy_engine.unknown_direction_gap_size_min is not None
        and event_direction == "unknown"
        and gap_size is not None
        and gap_size < strategy_engine.unknown_direction_gap_size_min
    ):
        return False

    is_mixed_inline = (
        event_direction == "mixed"
        and guidance_status == "inline_or_maintained"
    )
    if (
        strategy_engine.mixed_inline_close_location_min is not None
        and is_mixed_inline
        and close_location is not None
        and close_location < strategy_engine.mixed_inline_close_location_min
    ):
        return False
    if (
        strategy_engine.mixed_inline_close_location_max is not None
        and is_mixed_inline
        and close_location is not None
        and close_location > strategy_engine.mixed_inline_close_location_max
    ):
        return False
    if (
        strategy_engine.mixed_inline_gap_size_max is not None
        and is_mixed_inline
        and gap_size is not None
        and gap_size > strategy_engine.mixed_inline_gap_size_max
    ):
        return False

    if (
        strategy_engine.entry_timing_policy == "reaction_close"
        and row.get("event_close") in (None, 0, 0.0, "")
    ):
        return False

    return True


def _classify_score_bucket(score: float) -> str:
    if score >= 0.8:
        return "high"
    if score >= 0.6:
        return "medium_high"
    if score >= 0.4:
        return "medium"
    if score >= 0.2:
        return "medium_low"
    return "low"


def _safe_float(raw: Any) -> float | None:
    try:
        return float(raw)
    except (TypeError, ValueError):
        return None


def rank_candidates(
    candidates: list[Candidate],
    ranking_fields: list[str] | None = None,
) -> list[Candidate]:
    """Sort candidates using configurable ranking fields.

    Field syntax:
    - `score` / `avg_dollar_volume` etc.: ascending
    - `-score` / `-close_location`: descending

    When no custom fields are provided, preserve the historical default:
    `score DESC, avg_dollar_volume DESC, symbol ASC`.
    """
    if not ranking_fields:
        return sorted(candidates, key=lambda c: (-c.score, -c.avg_dollar_volume, c.symbol))

    normalized_fields = list(ranking_fields) + ["symbol"]
    return sorted(candidates, key=lambda c: _candidate_sort_key(c, normalized_fields))


def _candidate_sort_key(candidate: Candidate, ranking_fields: list[str]) -> tuple[Any, ...]:
    keys: list[Any] = []
    for field in ranking_fields:
        descending = field.startswith("-")
        raw_field = field[1:] if descending else field
        value = _resolve_candidate_field(candidate, raw_field)
        if raw_field == "symbol":
            keys.append(str(value or ""))
            continue

        if isinstance(value, (int, float)):
            numeric = float(value)
            keys.append(-numeric if descending else numeric)
            continue

        text = str(value or "")
        keys.append(_invert_text_key(text) if descending else text)
    return tuple(keys)


def _resolve_candidate_field(candidate: Candidate, field: str) -> Any:
    if hasattr(candidate, field):
        return getattr(candidate, field)

    if field == "score_band_2dp":
        return math.floor(candidate.score * 100.0 + 1e-9)
    if field == "guidance_raised_flag":
        return 1.0 if str(candidate.features.get("guidance_status", "")).lower() == "raised" else 0.0
    if field == "bullish_direction_flag":
        return 1.0 if str(candidate.features.get("event_direction", "")).lower() == "bullish" else 0.0
    if field == "mixed_direction_flag":
        return 1.0 if str(candidate.features.get("event_direction", "")).lower() == "mixed" else 0.0
    if field == "unknown_direction_flag":
        return 1.0 if str(candidate.features.get("event_direction", "")).lower() == "unknown" else 0.0
    if field == "prior_positive_flag":
        prior = _safe_float(candidate.features.get("prior_event_fwd5d"))
        return 1.0 if prior is not None and prior > 0.02 else 0.0
    if field == "macro_favorable_flag":
        vix = _safe_float(candidate.features.get("macro_vix"))
        hy = _safe_float(candidate.features.get("macro_hy_spread"))
        return 1.0 if vix is not None and hy is not None and vix > 18.0 and hy > 3.25 else 0.0

    return candidate.features.get(field)


def _invert_text_key(value: str) -> tuple[int, ...]:
    return tuple(-ord(char) for char in value)


def filter_by_universe(
    candidates: list[Candidate],
    config: UniverseConfig,
) -> list[Candidate]:
    """Apply universe filters: min_price, min_avg_dollar_volume, exchange."""
    filtered = []
    for c in candidates:
        min_price = (
            c.engine_min_entry_price
            if c.engine_min_entry_price is not None
            else config.min_price
        )
        if c.entry_price_est < min_price:
            continue
        if (
            c.engine_max_entry_price is not None
            and c.entry_price_est > c.engine_max_entry_price
        ):
            continue
        if c.avg_dollar_volume < config.min_avg_dollar_volume:
            continue
        if config.min_market_cap_proxy is not None:
            market_cap_proxy = _safe_float(c.features.get("market_cap_proxy"))
            if market_cap_proxy is None or market_cap_proxy < config.min_market_cap_proxy:
                continue
        if config.allowed_exchanges:
            exchange = str(c.features.get("exchange_proxy") or c.features.get("exchange") or "").upper()
            if exchange and exchange not in {e.upper() for e in config.allowed_exchanges}:
                continue
        if config.exclude_asset_types:
            asset_type = str(c.features.get("asset_type_proxy") or c.features.get("asset_type") or "").upper()
            if asset_type and asset_type in {a.upper() for a in config.exclude_asset_types}:
                continue
        filtered.append(c)
    return filtered


def filter_by_score(
    candidates: list[Candidate],
    score_threshold: float,
) -> list[Candidate]:
    return [c for c in candidates if c.score >= score_threshold]


def filter_by_momentum(
    candidates: list[Candidate],
    max_momentum_20d: float | None,
) -> list[Candidate]:
    """Reject candidates where pre-event 20d return exceeds the cap."""
    if max_momentum_20d is None:
        return candidates
    filtered = []
    for c in candidates:
        mom = c.features.get("pre_event_momentum_20d")
        if mom is not None and float(mom) > max_momentum_20d:
            logger.debug("momentum_gate_reject", symbol=c.symbol, momentum=mom, cap=max_momentum_20d)
            continue
        filtered.append(c)
    return filtered


def truncate_candidates(
    candidates: list[Candidate],
    max_per_day: int,
) -> list[Candidate]:
    return candidates[:max_per_day]


def filter_by_event_type(
    candidates: list[Candidate],
    profiles: dict[str, EventTypeProfile],
) -> list[Candidate]:
    """Filter out candidates whose event_type is unknown, disabled, or below per-type threshold.

    Default-deny: if profiles dict is non-empty and event_type is not in profiles,
    the candidate is skipped (unknown event types are blocked).
    """
    if not profiles:
        return candidates
    filtered = []
    for c in candidates:
        profile = profiles.get(c.event_type)
        if profile is None:
            logger.debug("skip_unknown_event_type", symbol=c.symbol, event_type=c.event_type)
            continue
        if not profile.enabled:
            logger.debug("skip_disabled_event_type", symbol=c.symbol, event_type=c.event_type)
            continue
        if profile.score_threshold_override is not None:
            if c.score < profile.score_threshold_override:
                logger.debug(
                    "skip_event_type_score",
                    symbol=c.symbol,
                    event_type=c.event_type,
                    score=c.score,
                    threshold=profile.score_threshold_override,
                )
                continue
        filtered.append(c)
    return filtered


def select_candidates(
    raw_rows: list[dict[str, Any]],
    universe_config: UniverseConfig,
    signal_config: SignalConfig,
    event_type_profiles: dict[str, EventTypeProfile] | None = None,
    strategy_engine: StrategyEngineConfig | None = None,
    truncate_to: int | None = None,
    excluded_event_ids: set[str] | None = None,
    excluded_symbols: set[str] | None = None,
) -> list[Candidate]:
    """Full selection pipeline: build → filter → rank → truncate."""
    candidates = []
    excluded_event_ids = excluded_event_ids or set()
    excluded_symbols = {symbol.upper() for symbol in (excluded_symbols or set())}
    for row in raw_rows:
        event_id = str(row.get("event_id", ""))
        symbol = str(row.get("symbol", row.get("ticker", ""))).upper()
        if event_id and event_id in excluded_event_ids:
            continue
        if symbol and symbol in excluded_symbols:
            continue
        prepared_row = _prepare_row_for_strategy_engine(
            row,
            signal_config=signal_config,
            strategy_engine=strategy_engine,
        )
        c = build_candidate(prepared_row, strategy_engine=strategy_engine)
        if c is not None:
            candidates.append(c)

    candidates = filter_by_universe(candidates, universe_config)
    candidates = filter_by_score(
        candidates,
        _resolve_score_threshold(signal_config, strategy_engine),
    )
    candidates = filter_by_momentum(
        candidates,
        signal_config.pre_event_momentum_20d_max,
    )
    if event_type_profiles:
        candidates = filter_by_event_type(candidates, event_type_profiles)
    candidates = rank_candidates(candidates, signal_config.ranking_fields)
    candidates = truncate_candidates(
        candidates,
        truncate_to if truncate_to is not None else signal_config.max_candidates_per_day,
    )
    return candidates


def _resolve_score_threshold(
    signal_config: SignalConfig,
    strategy_engine: StrategyEngineConfig | None,
) -> float:
    if strategy_engine and strategy_engine.score_threshold_override is not None:
        return strategy_engine.score_threshold_override
    return signal_config.score_threshold


def _prepare_row_for_strategy_engine(
    row: dict[str, Any],
    signal_config: SignalConfig,
    strategy_engine: StrategyEngineConfig | None,
) -> dict[str, Any]:
    prepared = row
    if signal_config.ranking_model_path:
        from libs.backtest.ranking_models import compute_ranking_model_score

        ranking_score = compute_ranking_model_score(row, signal_config.ranking_model_path)
        if ranking_score is not None:
            prepared = dict(prepared)
            prepared["ranking_model_score"] = ranking_score
            prepared["learned_rank_score"] = ranking_score

    if strategy_engine is None or signal_config.scoring_model != "pead":
        return prepared

    reaction_threshold = (
        strategy_engine.pead_reaction_threshold_override
        if strategy_engine.pead_reaction_threshold_override is not None
        else signal_config.pead_reaction_threshold
    )
    volume_threshold = (
        strategy_engine.pead_volume_threshold_override
        if strategy_engine.pead_volume_threshold_override is not None
        else signal_config.pead_volume_threshold
    )

    if (
        reaction_threshold == signal_config.pead_reaction_threshold
        and volume_threshold == signal_config.pead_volume_threshold
    ):
        return prepared

    from libs.backtest.scoring import compute_pead_score

    if prepared is row:
        prepared = dict(row)
    prepared["score"] = compute_pead_score(
        prepared,
        reaction_threshold=reaction_threshold,
        volume_threshold=volume_threshold,
    )
    return prepared