You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
415 lines
15 KiB
Python
415 lines
15 KiB
Python
"""Initial schema: all ACE-F v1 tables.
|
|
|
|
Revision ID: 0001
|
|
Revises:
|
|
Create Date: 2026-03-12
|
|
|
|
Tables created:
|
|
- issuer_master, symbol_master: company reference data
|
|
- job_runs: pipeline execution tracking
|
|
- documents, document_exhibits: SEC filing metadata
|
|
- exhibit_cache: local exhibit text cache metadata
|
|
- events, event_parses: parsed event data
|
|
- macro_series, macro_observations: FRED macro data
|
|
- short_sale_daily: FINRA short volume data
|
|
- feature_snapshots: computed ML features
|
|
- order_plans: future trade plans (schema only in Phase 1)
|
|
- sync_checkpoints: FRED/FINRA sync state
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Sequence
|
|
|
|
import sqlalchemy as sa
|
|
from alembic import op
|
|
from sqlalchemy.dialects.postgresql import JSONB, UUID
|
|
|
|
revision: str = "0001"
|
|
down_revision: str | None = None
|
|
branch_labels: str | Sequence[str] | None = None
|
|
depends_on: str | Sequence[str] | None = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.create_table(
|
|
"issuer_master",
|
|
sa.Column("issuer_id", sa.Text, primary_key=True),
|
|
sa.Column("cik", sa.Text, nullable=True, unique=True),
|
|
sa.Column("ticker", sa.Text, nullable=True),
|
|
sa.Column("issuer_name", sa.Text, nullable=False),
|
|
sa.Column("exchange", sa.Text, nullable=True),
|
|
sa.Column("country_code", sa.Text, nullable=True),
|
|
sa.Column("is_active", sa.Boolean, nullable=False, server_default=sa.text("true")),
|
|
sa.Column(
|
|
"created_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
sa.Column(
|
|
"updated_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
)
|
|
|
|
op.create_table(
|
|
"symbol_master",
|
|
sa.Column("symbol_id", sa.Text, primary_key=True),
|
|
sa.Column(
|
|
"issuer_id",
|
|
sa.Text,
|
|
sa.ForeignKey("issuer_master.issuer_id"),
|
|
nullable=True,
|
|
),
|
|
sa.Column("ticker", sa.Text, nullable=False),
|
|
sa.Column("venue", sa.Text, nullable=True),
|
|
sa.Column("asset_type", sa.Text, nullable=True),
|
|
sa.Column("currency", sa.Text, nullable=True),
|
|
sa.Column("start_date", sa.Date, nullable=True),
|
|
sa.Column("end_date", sa.Date, nullable=True),
|
|
sa.Column("is_primary", sa.Boolean, nullable=False, server_default=sa.text("true")),
|
|
sa.Column(
|
|
"created_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
sa.Column(
|
|
"updated_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
)
|
|
|
|
op.create_table(
|
|
"job_runs",
|
|
sa.Column("job_run_id", UUID(as_uuid=True), primary_key=True),
|
|
sa.Column("job_name", sa.Text, nullable=False),
|
|
sa.Column("source_name", sa.Text, nullable=True),
|
|
sa.Column("run_date", sa.Date, nullable=True),
|
|
sa.Column("status", sa.Text, nullable=False),
|
|
sa.Column(
|
|
"started_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
sa.Column("finished_at_utc", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("records_seen", sa.Integer, nullable=False, server_default=sa.text("0")),
|
|
sa.Column("records_written", sa.Integer, nullable=False, server_default=sa.text("0")),
|
|
sa.Column("records_skipped", sa.Integer, nullable=False, server_default=sa.text("0")),
|
|
sa.Column("error_count", sa.Integer, nullable=False, server_default=sa.text("0")),
|
|
sa.Column("error_summary", sa.Text, nullable=True),
|
|
sa.Column("metadata_json", JSONB, nullable=False, server_default=sa.text("'{}'")),
|
|
)
|
|
op.create_index("ix_job_runs_job_name_run_date", "job_runs", ["job_name", "run_date"])
|
|
|
|
op.create_table(
|
|
"documents",
|
|
sa.Column("document_id", sa.Text, primary_key=True),
|
|
sa.Column("source_name", sa.Text, nullable=False),
|
|
sa.Column(
|
|
"issuer_id",
|
|
sa.Text,
|
|
sa.ForeignKey("issuer_master.issuer_id"),
|
|
nullable=True,
|
|
),
|
|
sa.Column(
|
|
"symbol_id",
|
|
sa.Text,
|
|
sa.ForeignKey("symbol_master.symbol_id"),
|
|
nullable=True,
|
|
),
|
|
sa.Column("accession_no", sa.Text, nullable=True),
|
|
sa.Column("form_type", sa.Text, nullable=False),
|
|
sa.Column("filing_date", sa.Date, nullable=False),
|
|
sa.Column("accepted_at_utc", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("primary_document_name", sa.Text, nullable=True),
|
|
sa.Column("parsed_status", sa.Text, nullable=False, server_default=sa.text("'pending'")),
|
|
sa.Column(
|
|
"created_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
sa.Column(
|
|
"updated_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
sa.UniqueConstraint("accession_no", "form_type", name="uq_documents_accession_form"),
|
|
)
|
|
op.create_index(
|
|
"ix_documents_issuer_filing_date", "documents", ["issuer_id", "filing_date"]
|
|
)
|
|
op.create_index(
|
|
"ix_documents_form_type_filing_date", "documents", ["form_type", "filing_date"]
|
|
)
|
|
|
|
op.create_table(
|
|
"document_exhibits",
|
|
sa.Column("exhibit_id", sa.Text, primary_key=True),
|
|
sa.Column(
|
|
"document_id",
|
|
sa.Text,
|
|
sa.ForeignKey("documents.document_id"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("exhibit_code", sa.Text, nullable=False),
|
|
sa.Column("exhibit_name", sa.Text, nullable=True),
|
|
sa.Column(
|
|
"created_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
)
|
|
|
|
op.create_table(
|
|
"exhibit_cache",
|
|
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
|
|
sa.Column("accession_no", sa.Text, nullable=False),
|
|
sa.Column("exhibit_type", sa.Text, nullable=False),
|
|
sa.Column("content_hash", sa.Text, nullable=False),
|
|
sa.Column("cache_path", sa.Text, nullable=False),
|
|
sa.Column(
|
|
"fetched_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
sa.UniqueConstraint(
|
|
"accession_no", "exhibit_type", name="uq_exhibit_cache_accession_type"
|
|
),
|
|
)
|
|
|
|
op.create_table(
|
|
"events",
|
|
sa.Column("event_id", sa.Text, primary_key=True),
|
|
sa.Column(
|
|
"issuer_id",
|
|
sa.Text,
|
|
sa.ForeignKey("issuer_master.issuer_id"),
|
|
nullable=True,
|
|
),
|
|
sa.Column(
|
|
"symbol_id",
|
|
sa.Text,
|
|
sa.ForeignKey("symbol_master.symbol_id"),
|
|
nullable=True,
|
|
),
|
|
sa.Column(
|
|
"primary_document_id",
|
|
sa.Text,
|
|
sa.ForeignKey("documents.document_id"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("event_type", sa.Text, nullable=False),
|
|
sa.Column("event_direction", sa.Text, nullable=False),
|
|
sa.Column("event_date", sa.Date, nullable=False),
|
|
sa.Column("filed_at_utc", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("parser_version", sa.Text, nullable=False),
|
|
sa.Column("parse_confidence", sa.Numeric, nullable=True),
|
|
sa.Column("status", sa.Text, nullable=False, server_default=sa.text("'pending'")),
|
|
sa.Column(
|
|
"created_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
sa.Column(
|
|
"updated_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
)
|
|
op.create_index("ix_events_event_type_date", "events", ["event_type", "event_date"])
|
|
op.create_index(
|
|
"ix_events_primary_document_id", "events", ["primary_document_id"]
|
|
)
|
|
|
|
op.create_table(
|
|
"event_parses",
|
|
sa.Column("event_parse_id", sa.BigInteger, primary_key=True, autoincrement=True),
|
|
sa.Column(
|
|
"event_id", sa.Text, sa.ForeignKey("events.event_id"), nullable=False
|
|
),
|
|
sa.Column("parser_kind", sa.Text, nullable=False),
|
|
sa.Column("parser_version", sa.Text, nullable=False),
|
|
sa.Column("schema_version", sa.Text, nullable=False),
|
|
sa.Column("output_json", JSONB, nullable=False),
|
|
sa.Column("validation_status", sa.Text, nullable=False),
|
|
sa.Column("validation_errors", JSONB, nullable=True),
|
|
sa.Column(
|
|
"created_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
)
|
|
|
|
op.create_table(
|
|
"macro_series",
|
|
sa.Column("series_id", sa.Text, primary_key=True),
|
|
sa.Column("title", sa.Text, nullable=True),
|
|
sa.Column("frequency", sa.Text, nullable=True),
|
|
sa.Column("units", sa.Text, nullable=True),
|
|
sa.Column("source_name", sa.Text, nullable=False),
|
|
sa.Column("metadata_json", JSONB, nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column(
|
|
"created_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
)
|
|
|
|
op.create_table(
|
|
"macro_observations",
|
|
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
|
|
sa.Column(
|
|
"series_id",
|
|
sa.Text,
|
|
sa.ForeignKey("macro_series.series_id"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("observation_date", sa.Date, nullable=False),
|
|
sa.Column("value", sa.Numeric, nullable=True),
|
|
sa.Column(
|
|
"created_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
sa.UniqueConstraint(
|
|
"series_id", "observation_date", name="uq_macro_obs_series_date"
|
|
),
|
|
)
|
|
|
|
op.create_table(
|
|
"short_sale_daily",
|
|
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
|
|
sa.Column(
|
|
"symbol_id",
|
|
sa.Text,
|
|
sa.ForeignKey("symbol_master.symbol_id"),
|
|
nullable=True,
|
|
),
|
|
sa.Column("ticker_raw", sa.Text, nullable=False),
|
|
sa.Column("trade_date", sa.Date, nullable=False),
|
|
sa.Column("short_volume", sa.BigInteger, nullable=False),
|
|
sa.Column("short_exempt_volume", sa.BigInteger, nullable=True),
|
|
sa.Column("total_volume", sa.BigInteger, nullable=True),
|
|
sa.Column("source_name", sa.Text, nullable=False),
|
|
sa.Column(
|
|
"created_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
sa.UniqueConstraint(
|
|
"ticker_raw",
|
|
"trade_date",
|
|
"source_name",
|
|
name="uq_short_sale_ticker_date_source",
|
|
),
|
|
)
|
|
op.create_index(
|
|
"ix_short_sale_ticker_date", "short_sale_daily", ["ticker_raw", "trade_date"]
|
|
)
|
|
|
|
op.create_table(
|
|
"feature_snapshots",
|
|
sa.Column("feature_snapshot_id", sa.BigInteger, primary_key=True, autoincrement=True),
|
|
sa.Column(
|
|
"event_id", sa.Text, sa.ForeignKey("events.event_id"), nullable=False
|
|
),
|
|
sa.Column("snapshot_name", sa.Text, nullable=False),
|
|
sa.Column("snapshot_version", sa.Text, nullable=False),
|
|
sa.Column("feature_json", JSONB, nullable=False),
|
|
sa.Column(
|
|
"created_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
)
|
|
|
|
op.create_table(
|
|
"order_plans",
|
|
sa.Column("order_plan_id", sa.BigInteger, primary_key=True, autoincrement=True),
|
|
sa.Column(
|
|
"event_id", sa.Text, sa.ForeignKey("events.event_id"), nullable=False
|
|
),
|
|
sa.Column(
|
|
"symbol_id",
|
|
sa.Text,
|
|
sa.ForeignKey("symbol_master.symbol_id"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("side", sa.Text, nullable=False),
|
|
sa.Column("planned_entry_date", sa.Date, nullable=False),
|
|
sa.Column("planned_order_type", sa.Text, nullable=False),
|
|
sa.Column("planned_price", sa.Numeric, nullable=True),
|
|
sa.Column("stop_price", sa.Numeric, nullable=True),
|
|
sa.Column("take_profit_price", sa.Numeric, nullable=True),
|
|
sa.Column("quantity_plan", sa.Numeric, nullable=True),
|
|
sa.Column("status", sa.Text, nullable=False, server_default=sa.text("'draft'")),
|
|
sa.Column(
|
|
"created_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
)
|
|
|
|
op.create_table(
|
|
"sync_checkpoints",
|
|
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
|
|
sa.Column("domain", sa.Text, nullable=False),
|
|
sa.Column("last_sync_at_utc", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("last_sync_params", JSONB, nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column("status", sa.Text, nullable=False),
|
|
sa.Column(
|
|
"created_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
sa.Column(
|
|
"updated_at_utc",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("now()"),
|
|
),
|
|
sa.UniqueConstraint("domain", name="uq_sync_checkpoints_domain"),
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.drop_table("sync_checkpoints")
|
|
op.drop_table("order_plans")
|
|
op.drop_table("feature_snapshots")
|
|
op.drop_index("ix_short_sale_ticker_date", table_name="short_sale_daily")
|
|
op.drop_table("short_sale_daily")
|
|
op.drop_table("macro_observations")
|
|
op.drop_table("macro_series")
|
|
op.drop_table("event_parses")
|
|
op.drop_index("ix_events_primary_document_id", table_name="events")
|
|
op.drop_index("ix_events_event_type_date", table_name="events")
|
|
op.drop_table("events")
|
|
op.drop_table("exhibit_cache")
|
|
op.drop_table("document_exhibits")
|
|
op.drop_index("ix_documents_form_type_filing_date", table_name="documents")
|
|
op.drop_index("ix_documents_issuer_filing_date", table_name="documents")
|
|
op.drop_table("documents")
|
|
op.drop_index("ix_job_runs_job_name_run_date", table_name="job_runs")
|
|
op.drop_table("job_runs")
|
|
op.drop_table("symbol_master")
|
|
op.drop_table("issuer_master")
|