You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
62 lines
1.9 KiB
Bash
62 lines
1.9 KiB
Bash
#!/bin/bash
|
|
# Full midwide enrichment pipeline:
|
|
# midwide-liquid-long-v1 → tier2 → tier3 → macro → full (with prior_drift)
|
|
set -e
|
|
|
|
BASE="data/parquet/midwide-liquid-long-v1"
|
|
TIER2="data/parquet/midwide-liquid-long-v1_tier2"
|
|
TIER3="data/parquet/midwide-liquid-long-v1_tier3"
|
|
MACRO="data/parquet/midwide-liquid-long-v1_tier3_macro"
|
|
FULL="data/parquet/midwide-liquid-long-v1_full"
|
|
|
|
echo "=== STEP 1: Tier2 enrichment ==="
|
|
if [ -f "$TIER2/test.parquet" ]; then
|
|
echo "Tier2 already done, skipping"
|
|
else
|
|
PYTHONUNBUFFERED=1 uv run python3 scripts/enrich_tier2_features.py \
|
|
--input "$BASE" --output "$TIER2"
|
|
fi
|
|
|
|
echo "=== STEP 2: Tier3 enrichment ==="
|
|
if [ -f "$TIER3/test.parquet" ]; then
|
|
echo "Tier3 already done, skipping"
|
|
else
|
|
PYTHONUNBUFFERED=1 uv run python3 scripts/enrich_tier3_features.py \
|
|
--input "$TIER2" --output "$TIER3"
|
|
fi
|
|
|
|
echo "=== STEP 3: Macro enrichment ==="
|
|
if [ -f "$MACRO/test.parquet" ]; then
|
|
echo "Macro already done, skipping"
|
|
else
|
|
PYTHONUNBUFFERED=1 uv run python3 scripts/enrich_macro_features.py \
|
|
--input "$TIER3" --output "$MACRO"
|
|
fi
|
|
|
|
echo "=== STEP 4: Prior drift enrichment ==="
|
|
if [ -f "$FULL/test.parquet" ]; then
|
|
echo "Full already done, skipping"
|
|
else
|
|
PYTHONUNBUFFERED=1 uv run python3 scripts/enrich_prior_drift.py \
|
|
--input "$MACRO" --output "$FULL"
|
|
fi
|
|
|
|
echo "=== Pipeline complete: $FULL ==="
|
|
uv run python3 -c "
|
|
import pandas as pd, glob
|
|
total = 0
|
|
for split in ['train','valid','test']:
|
|
df = pd.read_parquet('$FULL/' + split + '.parquet')
|
|
print(f'{split}: {len(df)} rows, cols={len(df.columns)}')
|
|
total += len(df)
|
|
print(f'Total: {total} rows')
|
|
key_cols = ['pre_event_entropy_60d','macro_vix','macro_hy_spread','prior_event_fwd5d','pre_event_hurst_60d']
|
|
df = pd.read_parquet('$FULL/train.parquet')
|
|
for c in key_cols:
|
|
if c in df.columns:
|
|
null_pct = df[c].isna().mean() * 100
|
|
print(f' {c}: {null_pct:.1f}% null')
|
|
else:
|
|
print(f' {c}: MISSING')
|
|
"
|