@ -11,10 +11,11 @@ Resolution pipeline:
import logging
import re
from datetime import datetime , timezone
from typing import Optional
import httpx
from sqlalchemy import , update
from sqlalchemy import or_, select, update
from sqlalchemy . dialects . postgresql import insert as pg_insert
from sqlalchemy . ext . asyncio import AsyncSession
@ -26,13 +27,28 @@ logger = logging.getLogger(__name__)
# Legal suffixes to strip for canonical name derivation
_SUFFIX_PATTERN = re . compile (
r " ,? \ s+ \ b(Inc \ .?|Corp \ .?|Corporation|Holdings?|Ltd \ .?|Limited|LLC|L \ .L \ .C \ .| "
r " Group|Co \ .?|Company|Technologies|Technology|International|Industries| "
r " L \ .P \ .?|LP| " # Limited Partnership (e.g. "Enterprise Products Partners L.P.")
r " Group|Co \ .?|Compan(?:y|ies)|Technologies|Technology|International|Industries| "
r " Pharmaceuticals?|Therapeutics?|Sciences?|Bancorp|Financial|Holding| "
r " Acquisition|Acquisitions|Capital|Partners|Trust| "
r " Nv|N \ .V \ .?|Plc \ .?|p \ .l \ .c \ .?| " # Dutch (N.V.) / British (Plc / p.l.c.)
r " S \ .A \ .?B?(?: \ s+de \ s+C \ .V \ .)?| " # Spanish (S.A., S.A.B., S.A.B. de C.V.)
r " Aktiengesellschaft|GmbH|AG|SE| " # German/Swiss/European
r " Com) \ s*$ " , # "Com" catches SEC-style ".com" artifacts (e.g. "AMAZON COM")
re . IGNORECASE ,
)
# SEC filing artifact: state-of-incorporation suffix after slash
# e.g. "Costco Wholesale Corp /New", "Wells Fargo & Company/Mn", "Applied Materials Inc /DE",
# "Canadian Imperial Bank Of Commerce /Can/"
_SEC_NEW_PATTERN = re . compile ( r " \ s*/ \ s*(?:New|[A-Za-z] { 2,4}) \ s*$ " , re . IGNORECASE )
# Danish/Norwegian corporate designation: "NOVO NORDISK A/S" → "NOVO NORDISK"
_AS_PATTERN = re . compile ( r " \ bA/S \ s*$ " , re . IGNORECASE )
# Belgian SA/NV: "Anheuser-Busch InBev SA/NV" → strip "/NV" first, then "SA" via suffix
_SA_NV_PATTERN = re . compile ( r " \ s*SA/NV \ s*$ " , re . IGNORECASE )
_COMPANY_KEYWORDS = {
" company " , " corporation " , " inc " , " corp " , " ltd " , " llc " , " holdings " ,
" stock " , " shares " , " nasdaq " , " nyse " , " ticker " , " finance " , " financial " ,
@ -47,7 +63,30 @@ def _normalize_name(raw_name: str) -> tuple[str, list[str]]:
Preserves the original and intermediate forms as aliases .
"""
aliases = [ ]
current = raw_name . strip ( )
# Normalize path separators: some DB records use backslash (e.g. "US BANCORP \DE\")
current = raw_name . strip ( ) . replace ( " \\ " , " / " ) . rstrip ( " / " ) . strip ( )
# Belgian "SA/NV" corporate designation (e.g. "Anheuser-Busch InBev SA/NV")
stripped_sanv = _SA_NV_PATTERN . sub ( " " , current ) . strip ( )
if stripped_sanv and stripped_sanv != current :
aliases . append ( current )
current = stripped_sanv
# Strip Danish/Norwegian "A/S" corporate designation (e.g. "NOVO NORDISK A/S")
stripped_as = _AS_PATTERN . sub ( " " , current ) . strip ( )
if stripped_as and stripped_as != current :
aliases . append ( current )
current = stripped_as
# Strip SEC reincorporation artifact "/New" (and state/province codes like "/DE", "/Can")
stripped_new = _SEC_NEW_PATTERN . sub ( " " , current ) . strip ( )
if stripped_new and stripped_new != current :
aliases . append ( current )
current = stripped_new
# Normalize SEC dot-com artifact: "Amazon.Com" / "Amazon.com" → "Amazon Com"
# so the iterative loop can strip "Com" as a regular suffix.
current = re . sub ( r " \ .com \ b " , " Com " , current , flags = re . IGNORECASE )
for _ in range ( 5 ) : # max 5 iterations to avoid infinite loops
stripped = _SUFFIX_PATTERN . sub ( " " , current ) . strip ( ) . rstrip ( " , " ) . strip ( )
@ -56,7 +95,9 @@ def _normalize_name(raw_name: str) -> tuple[str, list[str]]:
aliases . append ( current )
current = stripped
canonical = current
# Strip trailing punctuation artifacts left by suffix removal (e.g. "&" from
# "JPMorgan Chase & Co" → strip "Co" → "JPMorgan Chase &").
canonical = current . rstrip ( " &/, " ) . strip ( )
# Also add the fully original name if not already captured
if raw_name . strip ( ) != canonical and raw_name . strip ( ) not in aliases :
aliases . insert ( 0 , raw_name . strip ( ) )
@ -154,7 +195,11 @@ def _score_wiki_result(result: dict, canonical_name: str, aliases: list[str]) ->
"""
raw_title = result . get ( " title " , " " )
title = raw_title . lower ( )
snippet = result . get ( " snippet " , " " ) . lower ( )
# Wikipedia snippets contain <span class="searchmatch"> HTML — strip before matching.
# Replace each tag with a space then collapse runs so "Costco</span> <span>Wholesale"
# becomes "Costco Wholesale" rather than "Costco Wholesale" (breaking string match).
raw_snippet = re . sub ( r " \ s+ " , " " , re . sub ( r " <[^>]+> " , " " , result . get ( " snippet " , " " ) ) ) . strip ( )
snippet = raw_snippet . lower ( )
combined = title + " " + snippet
# Penalize obvious non-company pages immediately
@ -165,31 +210,56 @@ def _score_wiki_result(result: dict, canonical_name: str, aliases: list[str]) ->
if any ( sig in combined for sig in non_company_signals ) :
return 0.1
# Require snippet to have at least one finance keyword for non-exact-match titles
finance_signals = [ " company " , " corporation " , " stock " , " nasdaq " , " nyse " , " shares " ,
" investor " , " business " , " enterprise " , " holdings " , " inc. " ]
if not any ( sig in combined for sig in finance_signals ) :
return 0.15
# Penalize legal case titles: "X v. Y" format (e.g. "FSF v. Cisco Systems, Inc.")
if re . search ( r ' \ w v \ . \ w ' , raw_title ) :
return 0.05
canonical_lower = canonical_name . lower ( )
all_names = [ canonical_name ] + aliases
all_names_lower = [ n . lower ( ) for n in all_names ]
# Check for exact title match (e.g. "Apple Inc." == alias "Apple Inc.")
# Exact title match — checked BEFORE the finance-signal filter so that
# valid company pages whose snippet focuses on technical/product details
# (e.g. TSMC → fabs, JPMorgan → banking operations) still score 0.95.
raw_title_stripped = raw_title . strip ( )
for name in all_names :
if raw_title_stripped . lower ( ) == name . lower ( ) :
return 0.95 # exact match
return 0.95
# Space-collapsed match: handles merged brand names like "ExxonMobil" vs "Exxon Mobil"
title_no_space = raw_title_stripped . lower ( ) . replace ( " " , " " )
for name in all_names :
name_no_space = name . lower ( ) . replace ( " " , " " )
if len ( name_no_space ) > 4 and title_no_space == name_no_space :
return 0.90
# Require snippet to have at least one finance keyword for non-exact-match titles
finance_signals = [ " company " , " corporation " , " stock " , " nasdaq " , " nyse " , " shares " ,
" investor " , " business " , " enterprise " , " holdings " , " inc. " ]
if not any ( sig in combined for sig in finance_signals ) :
return 0.15
# Hyphen-normalized forms: "COCA COLA" matches "The Coca-Cola Company"
# because "coca cola" is in "the coca cola company" after replacing hyphens with spaces.
title_norm = title . replace ( " - " , " " )
canonical_norm = canonical_lower . replace ( " - " , " " )
all_names_norm = [ n . replace ( " - " , " " ) for n in all_names_lower ]
# Check if title starts with canonical name
title_starts_with_canonical = title . startswith ( canonical_lower )
title_contains_canonical = canonical_lower in title
title_starts_with_canonical = title _norm . startswith ( canonical_ norm )
title_contains_canonical = canonical_ norm in title_norm
# Also check aliases
title_starts_with_alias = any ( title . startswith ( n ) for n in all_names_lower )
title_contains_alias = any ( n in title for n in all_names_lower )
title_starts_with_alias = any ( title _norm . startswith ( n ) for n in all_names_ norm )
title_contains_alias = any ( n in title _norm for n in all_names_ norm )
if not ( title_contains_canonical or title_contains_alias ) :
# Snippet-contains fallback: handles acronym titles (e.g. "TSMC" article whose
# snippet reads "Taiwan Semiconductor Manufacturing Company Limited (TSMC)...")
# and short-title articles (e.g. "Costco" snippet contains "Costco Wholesale").
snippet_norm = snippet . replace ( " - " , " " )
if canonical_norm in snippet_norm or any ( n in snippet_norm for n in all_names_norm ) :
return 0.6
return 0.0
# Check for pages that are ABOUT the company (vs. lists, histories, etc.)
@ -207,11 +277,13 @@ def _score_wiki_result(result: dict, canonical_name: str, aliases: list[str]) ->
else :
base = 0.0
# Reward company/finance keywords in title or snippet
# Reward company/finance keywords in title or snippet.
# Cap at 0.89 so title-starts-with matches never outrank exact-title (0.95)
# or space-collapsed exact matches (0.90), regardless of keyword density.
keyword_hits = sum ( 1 for kw in _COMPANY_KEYWORDS if kw in combined )
keyword_score = min ( keyword_hits / 3 , 1.0 )
return min ( base + keyword_score * 0.3 , 1.0 )
return min ( base + keyword_score * 0.3 , 0.89 )
async def _resolve_wiki (
@ -311,41 +383,60 @@ async def resolve_entity(
ticker , wiki_title , confidence ,
)
# Upsert into company_entity_map
stmt = (
pg_insert ( CompanyEntityMap )
. values (
ticker = ticker ,
# Upsert into company_entity_map.
#
# No-downgrade guard: a transient Wikipedia failure (429/network) is
# swallowed by _search_wikipedia → returns [] → (wiki_title=None,
# confidence=0.0). Without this guard, re-running resolution while
# rate-limited would overwrite a previously-good mapping with NULL —
# which is exactly how the 2026-03-17 bulk run left 1423/1697 rows
# broken. The upsert therefore only updates when the new result is
# itself good (wiki_title not NULL) OR the existing row was already
# unresolved (wiki_title NULL). Manual overrides are never touched.
insert_stmt = pg_insert ( CompanyEntityMap ) . values (
ticker = ticker ,
canonical_name = canonical_name ,
wiki_title = wiki_title ,
gdelt_query = gdelt_query ,
aliases_json = aliases ,
resolver_confidence = confidence ,
is_manual_override = False ,
)
stmt = insert_stmt . on_conflict_do_update (
index_elements = [ " ticker " ] ,
set_ = dict (
canonical_name = canonical_name ,
wiki_title = wiki_title ,
gdelt_query = gdelt_query ,
aliases_json = aliases ,
resolver_confidence = confidence ,
is_manual_override = False ,
)
. on_conflict_do_update (
index_elements = [ " ticker " ] ,
set_ = dict (
canonical_name = canonical_name ,
wiki_title = wiki_title ,
gdelt_query = gdelt_query ,
aliases_json = aliases ,
resolver_confidence = confidence ,
) ,
where = CompanyEntityMap . is_manual_override == False , # noqa: E712
)
. returning ( CompanyEntityMap )
)
# Explicit: ORM `onupdate` does NOT fire for INSERT...ON CONFLICT,
# so stamp it here. The on-demand re-resolve recency guard in the
# /event endpoint relies on this reflecting the last attempt.
updated_at = datetime . now ( timezone . utc ) ,
) ,
where = (
( CompanyEntityMap . is_manual_override == False ) # noqa: E712
& or_ (
insert_stmt . excluded . wiki_title . isnot ( None ) ,
CompanyEntityMap . wiki_title . is_ ( None ) ,
)
) ,
) . returning ( CompanyEntityMap )
result = await db . execute ( stmt )
await db . execute ( stmt )
await db . commit ( )
row = result . scalars ( ) . first ( )
if row is None :
# Manual override prevented update — return existing
existing_result2 = await db . execute (
select ( CompanyEntityMap ) . where ( CompanyEntityMap . ticker == ticker )
)
row = existing_result2 . scalars ( ) . first ( )
# Read back via fresh SELECT with populate_existing=True.
# After commit, SQLAlchemy expires identity-map entries but does NOT evict them.
# A plain SELECT in the same session can return the expired (stale) cached object
# instead of reading the committed DB state. populate_existing forces the ORM to
# overwrite the identity-map entry with the fresh DB row.
fresh_result = await db . execute (
select ( CompanyEntityMap )
. where ( CompanyEntityMap . ticker == ticker )
. execution_options ( populate_existing = True )
)
row = fresh_result . scalars ( ) . first ( )
return row
I Luk Kim
3 months ago