You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
264 lines
9.3 KiB
Python
264 lines
9.3 KiB
Python
"""Canonical merger: combine rule parser output with LLM output."""
|
|
from __future__ import annotations
|
|
|
|
from copy import deepcopy
|
|
from typing import Any
|
|
|
|
_HIGH_CONFIDENCE = 0.70
|
|
_UNKNOWN_VALUES = {"unknown", "unclear", "not_provided"}
|
|
|
|
|
|
def _is_confident(output: dict[str, Any], field: str | None = None) -> bool:
|
|
"""Return True if output has overall confidence >= threshold."""
|
|
conf = output.get("confidence", {})
|
|
if field and field in conf:
|
|
return float(conf[field]) >= _HIGH_CONFIDENCE
|
|
return float(conf.get("overall", 0.0)) >= _HIGH_CONFIDENCE
|
|
|
|
|
|
def _field_is_unknown(value: Any) -> bool:
|
|
return str(value).lower() in _UNKNOWN_VALUES if value is not None else True
|
|
|
|
|
|
def _merge_scalar(
|
|
rule_val: Any,
|
|
llm_val: Any | None,
|
|
rule_conf: float,
|
|
llm_conf: float,
|
|
field_name: str,
|
|
warnings: list[str],
|
|
provenance: dict[str, str],
|
|
) -> Any:
|
|
"""Merge a single scalar field with provenance tracking.
|
|
|
|
Strategy:
|
|
- Both confident + disagree → conflict flag, rule wins (conservative).
|
|
- Only rule confident → rule wins.
|
|
- Only LLM confident → LLM wins (fills in unknown/low-confidence rule output).
|
|
- Neither confident → prefer non-unknown value, rule takes priority.
|
|
"""
|
|
rule_unknown = _field_is_unknown(rule_val)
|
|
llm_unknown = llm_val is None or _field_is_unknown(llm_val)
|
|
rule_confident = not rule_unknown and rule_conf >= _HIGH_CONFIDENCE
|
|
llm_confident = not llm_unknown and llm_conf >= _HIGH_CONFIDENCE
|
|
|
|
# Both confident and disagree → record conflict; rule wins but conflict is flagged
|
|
if rule_confident and llm_confident and str(rule_val) != str(llm_val):
|
|
warnings.append(f"rule_llm_conflict:{field_name} rule={rule_val} llm={llm_val}")
|
|
provenance[field_name] = "rule"
|
|
return rule_val
|
|
|
|
# Rule wins when confident (and no conflict detected above)
|
|
if rule_confident:
|
|
provenance[field_name] = "rule"
|
|
return rule_val
|
|
|
|
# LLM wins when confident (rule is not confident)
|
|
if llm_confident:
|
|
provenance[field_name] = "llm"
|
|
return llm_val
|
|
|
|
# Neither confident — prefer non-unknown, rule first
|
|
if not rule_unknown:
|
|
provenance[field_name] = "rule"
|
|
return rule_val
|
|
|
|
if not llm_unknown:
|
|
provenance[field_name] = "llm"
|
|
return llm_val
|
|
|
|
provenance[field_name] = "rule"
|
|
return rule_val # both unknown → keep rule default
|
|
|
|
|
|
def merge(
|
|
rule_output: dict[str, Any],
|
|
llm_output: dict[str, Any] | None,
|
|
) -> dict[str, Any]:
|
|
"""Merge rule parser output with optional LLM output into a canonical record.
|
|
|
|
Strategy:
|
|
- Rule parser wins when confident (overall >= 0.70).
|
|
- LLM fills in unknown fields when it is confident.
|
|
- Conflicts → conservative merge ("unknown"/"mixed") + warning.
|
|
- Adds ``provenance`` dict and ``rule_llm_conflict`` flag.
|
|
"""
|
|
result = deepcopy(rule_output)
|
|
result["parser_kind"] = "merged"
|
|
provenance: dict[str, str] = {}
|
|
warnings: list[str] = list(result.get("warnings", []))
|
|
has_conflict = False
|
|
|
|
if llm_output is None:
|
|
# No LLM — everything from rule
|
|
result["provenance"] = dict.fromkeys(_CANONICAL_FIELDS, "rule")
|
|
result["rule_llm_conflict"] = False
|
|
return result
|
|
|
|
rule_conf = float(rule_output.get("confidence", {}).get("overall", 0.0))
|
|
llm_conf = float(llm_output.get("confidence", {}).get("overall", 0.0))
|
|
|
|
# Merge top-level scalar fields
|
|
for field in ("event_type", "event_direction"):
|
|
merged_val = _merge_scalar(
|
|
rule_val=rule_output.get(field),
|
|
llm_val=llm_output.get(field),
|
|
rule_conf=float(rule_output.get("confidence", {}).get(field, rule_conf)),
|
|
llm_conf=float(llm_output.get("confidence", {}).get(field, llm_conf)),
|
|
field_name=field,
|
|
warnings=warnings,
|
|
provenance=provenance,
|
|
)
|
|
result[field] = merged_val
|
|
if any(f"rule_llm_conflict:{field}" in w for w in warnings):
|
|
has_conflict = True
|
|
|
|
# Headline / summary: prefer LLM (usually more informative)
|
|
if llm_output.get("headline") and not _field_is_unknown(llm_output.get("headline")):
|
|
result["headline"] = llm_output["headline"]
|
|
provenance["headline"] = "llm"
|
|
else:
|
|
provenance["headline"] = "rule"
|
|
|
|
if llm_output.get("summary") and not _field_is_unknown(llm_output.get("summary")):
|
|
result["summary"] = llm_output["summary"]
|
|
provenance["summary"] = "llm"
|
|
else:
|
|
provenance["summary"] = "rule"
|
|
|
|
# Guidance
|
|
rule_guidance = rule_output.get("guidance", {})
|
|
llm_guidance = llm_output.get("guidance", {})
|
|
merged_guidance = deepcopy(rule_guidance)
|
|
guidance_rule_conf = float(rule_output.get("confidence", {}).get("guidance", rule_conf))
|
|
guidance_llm_conf = float(llm_output.get("confidence", {}).get("guidance", llm_conf))
|
|
|
|
g_status = _merge_scalar(
|
|
rule_val=rule_guidance.get("status"),
|
|
llm_val=llm_guidance.get("status"),
|
|
rule_conf=guidance_rule_conf,
|
|
llm_conf=guidance_llm_conf,
|
|
field_name="guidance.status",
|
|
warnings=warnings,
|
|
provenance=provenance,
|
|
)
|
|
merged_guidance["status"] = g_status
|
|
if any("rule_llm_conflict:guidance.status" in w for w in warnings):
|
|
has_conflict = True
|
|
|
|
if llm_guidance.get("scope") and not _field_is_unknown(llm_guidance.get("scope")):
|
|
if _field_is_unknown(rule_guidance.get("scope")):
|
|
merged_guidance["scope"] = llm_guidance["scope"]
|
|
provenance["guidance.scope"] = "llm"
|
|
else:
|
|
provenance["guidance.scope"] = "rule"
|
|
else:
|
|
provenance["guidance.scope"] = "rule"
|
|
|
|
result["guidance"] = merged_guidance
|
|
|
|
# Signals: LLM fills in unknown rule signals
|
|
rule_signals = rule_output.get("signals", {})
|
|
llm_signals = llm_output.get("signals", {})
|
|
merged_signals = deepcopy(rule_signals)
|
|
for sig_field in (
|
|
"demand_strength",
|
|
"pricing_power",
|
|
"backlog_or_bookings",
|
|
"customer_expansion",
|
|
"margin_quality",
|
|
):
|
|
if _field_is_unknown(rule_signals.get(sig_field)) and not _field_is_unknown(
|
|
llm_signals.get(sig_field)
|
|
):
|
|
merged_signals[sig_field] = llm_signals[sig_field]
|
|
provenance[f"signals.{sig_field}"] = "llm"
|
|
else:
|
|
provenance[f"signals.{sig_field}"] = "rule"
|
|
result["signals"] = merged_signals
|
|
|
|
# Risk flags: OR of rule + LLM (either flagging = flagged)
|
|
rule_risks = rule_output.get("risk_flags", {})
|
|
llm_risks = llm_output.get("risk_flags", {})
|
|
merged_risks: dict[str, bool] = {}
|
|
for risk_field in (
|
|
"oneoff_item",
|
|
"tax_benefit",
|
|
"valuation_gain",
|
|
"non_gaap_heavy",
|
|
"financing_related",
|
|
"legal_or_regulatory_overhang",
|
|
):
|
|
rule_flag = bool(rule_risks.get(risk_field, False))
|
|
llm_flag = bool(llm_risks.get(risk_field, False))
|
|
merged_risks[risk_field] = rule_flag or llm_flag
|
|
if rule_flag and llm_flag:
|
|
provenance[f"risk_flags.{risk_field}"] = "merged"
|
|
elif rule_flag:
|
|
provenance[f"risk_flags.{risk_field}"] = "rule"
|
|
elif llm_flag:
|
|
provenance[f"risk_flags.{risk_field}"] = "llm"
|
|
else:
|
|
provenance[f"risk_flags.{risk_field}"] = "rule"
|
|
result["risk_flags"] = merged_risks
|
|
|
|
# Confidence: use higher overall
|
|
if llm_conf > rule_conf:
|
|
result["confidence"] = llm_output.get("confidence", result.get("confidence", {}))
|
|
provenance["confidence"] = "llm"
|
|
else:
|
|
provenance["confidence"] = "rule"
|
|
|
|
result["provenance"] = provenance
|
|
result["rule_llm_conflict"] = has_conflict
|
|
result["warnings"] = warnings
|
|
|
|
return result
|
|
|
|
|
|
# Canonical fields used for provenance initialization
|
|
_CANONICAL_FIELDS = [
|
|
"event_type",
|
|
"event_direction",
|
|
"headline",
|
|
"summary",
|
|
"guidance.status",
|
|
"guidance.scope",
|
|
"confidence",
|
|
*[f"signals.{f}" for f in ("demand_strength", "pricing_power", "backlog_or_bookings",
|
|
"customer_expansion", "margin_quality")],
|
|
*[f"risk_flags.{f}" for f in ("oneoff_item", "tax_benefit", "valuation_gain",
|
|
"non_gaap_heavy", "financing_related",
|
|
"legal_or_regulatory_overhang")],
|
|
]
|
|
|
|
|
|
def should_queue_for_review(merged: dict[str, Any]) -> tuple[bool, list[str]]:
|
|
"""Determine if a merged record needs human review.
|
|
|
|
Returns:
|
|
(should_queue, reason_codes) where reason_codes is a list of strings.
|
|
"""
|
|
reason_codes: list[str] = []
|
|
|
|
conf = float(merged.get("confidence", {}).get("overall", 0.0))
|
|
if conf < _HIGH_CONFIDENCE:
|
|
reason_codes.append("low_confidence")
|
|
|
|
if merged.get("rule_llm_conflict"):
|
|
reason_codes.append("rule_llm_conflict")
|
|
|
|
guidance = merged.get("guidance", {})
|
|
if guidance.get("status") in ("unclear",):
|
|
reason_codes.append("guidance_ambiguous")
|
|
|
|
risk_flags = merged.get("risk_flags", {})
|
|
if risk_flags.get("oneoff_item") or risk_flags.get("non_gaap_heavy"):
|
|
reason_codes.append("oneoff_likely")
|
|
|
|
event_direction = merged.get("event_direction", "unknown")
|
|
if event_direction in ("unknown", "mixed"):
|
|
reason_codes.append("direction_ambiguous")
|
|
|
|
return bool(reason_codes), reason_codes
|