From 27de44c8d80900afc28efe8c77519d71f9bb574d Mon Sep 17 00:00:00 2001 From: I Luk Kim Date: Fri, 8 May 2026 22:08:52 -0700 Subject: [PATCH] Fix label_price_unavailable: preserve future entry_dates as pending MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-market label_generator runs request future-dated price windows from Stock Oracle, which correctly returns 404 because the data does not yet exist. The labeler was swallowing this as label_status='unavailable' with entry_date=None. Snapshot export then filtered these rows out, so live PEAD trading silently lost candidates whose entry_dates fell on later trading days (e.g., post-market 8-K filings late Friday → Monday open entry). This explains today's missed RKLB/SNDK/AKAM/MNST/AMD/MRNA even though their 8-Ks parsed correctly. Changes: - libs/labeler/label_generator.py: in 404/empty-bars path, when entry_date >= today, preserve entry_date and mark label_status='pending'. New log event label_price_pending_future_window distinguishes from real data-unavailable failures (past dates still log label_price_unavailable). - libs/export/snapshot_export.py: include 'pending' in the label_status filter so today's not-yet-labeled events flow into the live snapshot. - apps/pipeline/label_generator/main.py: regeneration logic also retries existing 'unavailable' rows whose entry_date is null or future to recover events already mis-labeled in the DB. - tests/unit/test_labeler.py: regression test reproducing the RKLB/SNDK/AKAM failure mode and asserting label_status='pending' with entry_date preserved. Co-Authored-By: Claude Opus 4.7 --- apps/pipeline/label_generator/main.py | 17 ++++++-- libs/export/snapshot_export.py | 5 ++- libs/labeler/label_generator.py | 29 ++++++++++++-- tests/unit/test_labeler.py | 58 +++++++++++++++++++++++++-- 4 files changed, 98 insertions(+), 11 deletions(-) diff --git a/apps/pipeline/label_generator/main.py b/apps/pipeline/label_generator/main.py index 822888a..976f36c 100644 --- a/apps/pipeline/label_generator/main.py +++ b/apps/pipeline/label_generator/main.py @@ -63,7 +63,11 @@ async def run_label_generator( stats["skipped"] += 1 continue - # Skip if label already exists (unless it's pending — regenerate those) + # Skip if label already exists. Regenerate when: + # - status == "pending" (market has since closed) + # - status == "unavailable" AND entry_date is null/future (recover + # from prior runs that 404'd on a future window — see + # label_price_pending_future_window flow in label_generator). existing = await session.execute( select(EventLabel).where( EventLabel.event_id == event.event_id, @@ -73,10 +77,17 @@ async def run_label_generator( ) existing_label = existing.scalar_one_or_none() if existing_label is not None: - if existing_label.label_status != "pending": + should_regenerate = existing_label.label_status == "pending" or ( + existing_label.label_status == "unavailable" + and ( + existing_label.entry_date is None + or existing_label.entry_date >= dt.date.today() + ) + ) + if not should_regenerate: stats["skipped"] += 1 continue - # Pending label — market has since closed; delete and regenerate + # Stale pending or recoverable unavailable — delete and regenerate await session.delete(existing_label) await session.flush() diff --git a/libs/export/snapshot_export.py b/libs/export/snapshot_export.py index f5962bb..90410e6 100644 --- a/libs/export/snapshot_export.py +++ b/libs/export/snapshot_export.py @@ -580,7 +580,10 @@ async def export_dataset_snapshot( .outerjoin(SymbolMaster, Event.symbol_id == SymbolMaster.symbol_id) .where(FeatureSnapshot.snapshot_name.in_(versions)) .where(EventLabel.label_version == label_version) - .where(EventLabel.label_status.in_(["ok", "truncated"])) + # "pending" included so today's not-yet-labeled events flow into the live + # snapshot for entry-signal use. Forward returns are null on pending rows; + # the post-close label_generator regenerates them with real values. + .where(EventLabel.label_status.in_(["ok", "truncated", "pending"])) .where(EventLabel.invalid_event_for_labeling.is_(False)) ) diff --git a/libs/labeler/label_generator.py b/libs/labeler/label_generator.py index c3d5d37..52f80ac 100644 --- a/libs/labeler/label_generator.py +++ b/libs/labeler/label_generator.py @@ -157,6 +157,27 @@ async def generate_labels( ) raw_bars = [b.model_dump() for b in price_resp.bars] except Exception as exc: + # If the requested window is entirely in the future (or starts today and the + # market hasn't closed yet), Oracle correctly returns 404 because the data + # doesn't exist yet. Mark as "pending" so the post-close label_generator + # run regenerates it instead of dropping the event from candidates entirely. + # This is NOT a transient error — retrying earlier won't help. + if entry_date >= dt.date.today(): + logger.info( + "label_price_pending_future_window", + ticker=ticker, + entry_date=entry_date.isoformat(), + error=str(exc), + ) + return EventLabel( + event_id=event.event_id, + entry_convention=entry_convention, + reaction_date=reaction_date, + entry_date=entry_date, + label_status="pending", + invalid_event_for_labeling=False, + label_version=LABEL_VERSION, + ) logger.warning("label_price_unavailable", ticker=ticker, error=str(exc)) return EventLabel( event_id=event.event_id, @@ -175,9 +196,11 @@ async def generate_labels( ) if not bars: - # For reaction_close, the bar may not exist yet if market hasn't closed. - # Return "pending" so the pipeline can regenerate after market close. - if entry_convention == "reaction_close" and entry_date >= dt.date.today(): + # If entry_date is today/future, the bar may not exist yet (market hasn't + # closed for reaction_close, or it's a next-trading-day entry that hasn't + # arrived). Return "pending" for both conventions so the post-close + # pipeline regenerates it. Use "unavailable" only for past-date misses. + if entry_date >= dt.date.today(): label_status_no_bar = "pending" else: label_status_no_bar = "unavailable" diff --git a/tests/unit/test_labeler.py b/tests/unit/test_labeler.py index 718e46f..35d2ae9 100644 --- a/tests/unit/test_labeler.py +++ b/tests/unit/test_labeler.py @@ -174,16 +174,21 @@ class TestGenerateLabels: @pytest.mark.asyncio async def test_generate_labels_unavailable_when_no_price_data(self) -> None: - """generate_labels returns 'unavailable' status on price fetch error.""" + """generate_labels returns 'unavailable' status on price fetch error + for an event with a past entry_date (Oracle has the data; 404 is real). + """ from libs.labeler.label_generator import generate_labels mock_event = MagicMock() mock_event.event_id = "EVT::test::002" - mock_event.event_date = dt.date(2026, 1, 5) - mock_event.filed_at_utc = dt.datetime(2026, 1, 5, 22, 0, tzinfo=dt.UTC) + # Far in the past — entry_date will also be in the past + mock_event.event_date = dt.date(2020, 1, 5) + mock_event.filed_at_utc = dt.datetime(2020, 1, 5, 22, 0, tzinfo=dt.UTC) mock_price_svc = AsyncMock() - mock_price_svc.get_daily_bars = AsyncMock(side_effect=Exception("Oracle unavailable")) + mock_price_svc.get_daily_bars = AsyncMock( + side_effect=Exception("Not found: /api/v1/price/data/AAPL") + ) mock_session = AsyncMock() @@ -195,6 +200,51 @@ class TestGenerateLabels: ) assert label.label_status == "unavailable" + assert label.entry_date is None + + @pytest.mark.asyncio + async def test_generate_labels_pending_on_future_entry_date_404(self) -> None: + """Reproduces the 2026-05-08 RKLB/SNDK/AKAM bug: pre-market label_generator + run requests prices for entry_date in today/future window; Oracle 404s + because those bars don't exist yet. Must return 'pending' (not 'unavailable') + with entry_date preserved so the post-close run can regenerate it AND the + live snapshot includes the candidate. + """ + from libs.labeler.label_generator import generate_labels + + # Post-market filing dated TODAY → reaction_date = next trading day + # entry_date for next_open_after_reaction_close = day after that → future + today = dt.date.today() + mock_event = MagicMock() + mock_event.event_id = "EVT::test::future" + mock_event.event_date = today + # 22:00 UTC = post_market in ET + mock_event.filed_at_utc = dt.datetime.combine( + today, dt.time(22, 0), tzinfo=dt.UTC + ) + + mock_price_svc = AsyncMock() + mock_price_svc.get_daily_bars = AsyncMock( + side_effect=Exception("Not found: /api/v1/price/data/RKLB") + ) + + mock_session = AsyncMock() + + label = await generate_labels( + session=mock_session, + event=mock_event, + price_svc=mock_price_svc, + ticker="RKLB", + ) + + assert label.label_status == "pending", ( + "Future-window 404 must yield 'pending' so the post-close pipeline " + "regenerates it and the live snapshot includes today's candidates" + ) + # entry_date must be preserved (downstream filters need it; old code set it None) + assert label.entry_date is not None + assert label.entry_date >= today + assert label.reaction_date is not None @pytest.mark.asyncio async def test_generate_labels_pre_market_same_day_reaction(self) -> None: