56 lines
2.1 KiB
Python
56 lines
2.1 KiB
Python
"""Deterministic detection of contradictory extracted values within a cluster.
|
|
|
|
Extraction is a vision pass: quantities and sizes can be misread ("(2) 2x6" vs
|
|
"(5) 2x6"). Cluster members are supposed to describe the same real-world
|
|
element, so two members asserting different values for the same attribute are
|
|
a probable misread. Flag these so downstream text-only stages treat the value
|
|
as unverified instead of reasoning from one reading.
|
|
"""
|
|
|
|
import re
|
|
from typing import Dict, List
|
|
|
|
|
|
def _norm(value) -> str:
|
|
return re.sub(r"\s+", " ", ("" if value is None else str(value)).strip().lower())
|
|
|
|
|
|
def find_disputes(assertions: List[Dict]) -> List[Dict]:
|
|
"""Same attribute with >= 2 distinct normalized values = disputed."""
|
|
groups: Dict[str, Dict[str, Dict]] = {}
|
|
for assertion in assertions:
|
|
attribute = _norm(assertion.get("attribute"))
|
|
value = _norm(assertion.get("value"))
|
|
if not attribute or not value:
|
|
continue
|
|
# Group on the normalized value, but keep the original (whitespace-
|
|
# collapsed) text so disputes read like the sheet, not a lowercase munge.
|
|
original = re.sub(r"\s+", " ", str(assertion.get("value")).strip())
|
|
bucket = groups.setdefault(attribute, {}).setdefault(
|
|
value, {"original": original, "ids": set()}
|
|
)
|
|
bucket["ids"].add(assertion.get("id"))
|
|
disputes = []
|
|
for attribute, values in sorted(groups.items()):
|
|
if len(values) < 2:
|
|
continue
|
|
disputes.append({
|
|
"attribute": attribute,
|
|
"values": sorted(v["original"] for v in values.values()),
|
|
"assertion_ids": sorted(
|
|
aid for v in values.values() for aid in v["ids"] if aid
|
|
),
|
|
})
|
|
return disputes
|
|
|
|
|
|
def annotate_clusters(clusters: List[Dict]) -> int:
|
|
"""Attach disputed_attributes to each cluster that has any. Returns count."""
|
|
annotated = 0
|
|
for cluster in clusters:
|
|
disputes = find_disputes(cluster.get("assertions") or [])
|
|
if disputes:
|
|
cluster["disputed_attributes"] = disputes
|
|
annotated += 1
|
|
return annotated
|