#!/bin/bash
# Full midwide enrichment pipeline:
# midwide-liquid-long-v1 → tier2 → tier3 → macro → full (with prior_drift)
set -e

BASE="data/parquet/midwide-liquid-long-v1"
TIER2="data/parquet/midwide-liquid-long-v1_tier2"
TIER3="data/parquet/midwide-liquid-long-v1_tier3"
MACRO="data/parquet/midwide-liquid-long-v1_tier3_macro"
FULL="data/parquet/midwide-liquid-long-v1_full"

echo "=== STEP 1: Tier2 enrichment ==="
if [ -f "$TIER2/test.parquet" ]; then
    echo "Tier2 already done, skipping"
else
    PYTHONUNBUFFERED=1 uv run python3 scripts/enrich_tier2_features.py \
        --input "$BASE" --output "$TIER2"
fi

echo "=== STEP 2: Tier3 enrichment ==="
if [ -f "$TIER3/test.parquet" ]; then
    echo "Tier3 already done, skipping"
else
    PYTHONUNBUFFERED=1 uv run python3 scripts/enrich_tier3_features.py \
        --input "$TIER2" --output "$TIER3"
fi

echo "=== STEP 3: Macro enrichment ==="
if [ -f "$MACRO/test.parquet" ]; then
    echo "Macro already done, skipping"
else
    PYTHONUNBUFFERED=1 uv run python3 scripts/enrich_macro_features.py \
        --input "$TIER3" --output "$MACRO"
fi

echo "=== STEP 4: Prior drift enrichment ==="
if [ -f "$FULL/test.parquet" ]; then
    echo "Full already done, skipping"
else
    PYTHONUNBUFFERED=1 uv run python3 scripts/enrich_prior_drift.py \
        --input "$MACRO" --output "$FULL"
fi

echo "=== Pipeline complete: $FULL ==="
uv run python3 -c "
import pandas as pd, glob
total = 0
for split in ['train','valid','test']:
    df = pd.read_parquet('$FULL/' + split + '.parquet')
    print(f'{split}: {len(df)} rows, cols={len(df.columns)}')
    total += len(df)
print(f'Total: {total} rows')
key_cols = ['pre_event_entropy_60d','macro_vix','macro_hy_spread','prior_event_fwd5d','pre_event_hurst_60d']
df = pd.read_parquet('$FULL/train.parquet')
for c in key_cols:
    if c in df.columns:
        null_pct = df[c].isna().mean() * 100
        print(f'  {c}: {null_pct:.1f}% null')
    else:
        print(f'  {c}: MISSING')
"