From 48fefa40076589fd67350338867f9c4d9b316d20 Mon Sep 17 00:00:00 2001 From: woogi Date: Tue, 18 Aug 2026 13:35:21 -0500 Subject: [PATCH] feat: coverage ladder + vision-unverified stamping (classic path) --- backend/pipeline/extractor.py | 127 +++++++++++++++++++++--- tests/test_extraction_ladder_classic.py | 98 ++++++++++++++++++ tests/test_extractor_text_grounding.py | 36 +++++++ 3 files changed, 246 insertions(+), 15 deletions(-) create mode 100644 tests/test_extraction_ladder_classic.py diff --git a/backend/pipeline/extractor.py b/backend/pipeline/extractor.py index 491942f..3bb37b2 100644 --- a/backend/pipeline/extractor.py +++ b/backend/pipeline/extractor.py @@ -21,9 +21,18 @@ from backend.llm import call_json from backend.prompts import ( EXTRACTOR_SYSTEM_PROMPT, EXTRACTOR_USER_INSTRUCTION, + TEXT_STRUCTURING_SYSTEM_PROMPT, + TEXT_STRUCTURING_USER_INSTRUCTION, DISCIPLINE_PREFIXES, ATTRIBUTE_VOCAB, ) +from backend.text_coverage import ( + _norm, + fallback_objects, + merge_objects, + recover_sheet_number, + text_coverage, +) # prefix (upper) -> discipline, longest-prefix-first for greedy matching _PREFIX_TO_DISCIPLINE = sorted( @@ -164,6 +173,8 @@ def _normalize_sheet(parsed: Dict, page_number: int, clean: List[Dict] = [] dropped = 0 rescued = 0 + unverified = 0 + page_norm = _norm(page_text) if page_text else "" for idx, obj in enumerate(raw_objects): if not isinstance(obj, dict): @@ -179,9 +190,19 @@ def _normalize_sheet(parsed: Dict, page_number: int, page_text=page_text): dropped += 1 continue - grounding = _grounding_stamp(primary_val, source_text, page_text) - if grounding: + # Pre-set stamps (fallback/merge rungs) win; otherwise compute the + # text-layer rescue stamp. + grounding = obj.get("grounding") or _grounding_stamp( + primary_val, source_text, page_text) + if grounding == "text_layer": rescued += 1 + if not grounding and page_text and source_text: + # Vision-unverified: survived the digit guard, but the quoted + # source_text is not present in the deterministic text layer. + # Kept and stamped - the wave-5b verifier prioritizes these. + if _norm(str(source_text)) not in page_norm: + grounding = "vision_unverified" + unverified += 1 # --- location_key: new schema is richer; map to legacy shape + extras --- lk = obj.get("location_key") @@ -236,10 +257,11 @@ def _normalize_sheet(parsed: Dict, page_number: int, **({"grounding": grounding} if grounding else {}), }) - if dropped or rescued: + if dropped or rescued or unverified: print(f"[Extract] Page {page_number} ({sheet_number}): " f"dropped {dropped} ungrounded object(s)" - + (f", rescued {rescued} via text layer" if rescued else "")) + + (f", rescued {rescued} via text layer" if rescued else "") + + (f", {unverified} vision-unverified" if unverified else "")) unresolved = parsed.get("unresolved_items") or [] @@ -270,7 +292,25 @@ def _text_layer_block(page: Dict) -> str: + text[:config.TEXT_LAYER_MAX_CHARS]) +def _text_structuring_extract(page: Dict, sheet_hint: str = ""): + """Rung 2 of the extraction ladder: text-only structuring call (no + image). The text layer is authoritative for alphanumeric content - the + model segments it instead of transcribing pixels, so vision misreads + are impossible on this rung.""" + instruction = (TEXT_STRUCTURING_USER_INSTRUCTION + .replace("{sheet_hint}", sheet_hint or "") + .replace("{text_layer}", + (page.get("text_layer") or "") + [:config.TEXT_LAYER_MAX_CHARS])) + return call_json( + system_prompt=TEXT_STRUCTURING_SYSTEM_PROMPT, + user_text=instruction, + max_tokens=config.EXTRACT_MAX_TOKENS, + ) + + def _extract_one(page: Dict, sheet_hint: str = "") -> Dict: + page_text = page.get("text_layer") user_text = (EXTRACTOR_USER_INSTRUCTION.replace("{sheet_hint}", sheet_hint) + _text_layer_block(page)) parsed = call_json( @@ -280,17 +320,74 @@ def _extract_one(page: Dict, sheet_hint: str = "") -> Dict: max_tokens=config.EXTRACT_MAX_TOKENS, ) if not isinstance(parsed, dict): - return { - "page_number": page["page_number"], - "sheet_number": None, - "discipline": "Unknown", - "sheet_title": f"Page {page['page_number']} (extraction failed)", - "level": None, - "scale": None, - "assertions": [], - } - return _normalize_sheet(parsed, page["page_number"], - page_text=page.get("text_layer")) + if not page_text: + # Scanned/raster page: vision-only, keep the legacy failure shape. + return { + "page_number": page["page_number"], + "sheet_number": None, + "discipline": "Unknown", + "sheet_title": f"Page {page['page_number']} (extraction failed)", + "level": None, + "scale": None, + "assertions": [], + } + # Text-bearing page: climb the ladder instead of going dark. + parsed = {"sheet": {}, "objects": []} + + sheet = _normalize_sheet(parsed, page["page_number"], page_text=page_text) + cov = text_coverage(page_text or "", sheet["assertions"]) + sheet["coverage"] = cov + + # Rung 2: text-only structuring when coverage is below floor. MERGE, + # never replace - vision objects (graphical_basis content exists only + # in the image) are kept; the text pass fills what vision missed. + if (page_text and config.EXTRACT_TEXT_RETRY_ENABLED + and cov["ratio"] < config.EXTRACT_COVERAGE_FLOOR): + print(f"[Extract] Page {page['page_number']}: coverage " + f"{cov['ratio']:.0%} < floor - text-only structuring pass") + parsed2 = _text_structuring_extract(page, sheet_hint) + if isinstance(parsed2, dict): + sheet2 = _normalize_sheet(parsed2, page["page_number"], + page_text=page_text) + before = len(sheet["assertions"]) + sheet["assertions"] = merge_objects(sheet["assertions"], + sheet2["assertions"]) + for key in ("sheet_number", "sheet_title", "discipline", + "level", "scale", "drawing_type"): + if not sheet.get(key) and sheet2.get(key): + sheet[key] = sheet2[key] + cov = text_coverage(page_text, sheet["assertions"]) + sheet["coverage"] = cov + print(f"[Extract] Page {page['page_number']}: merged " + f"{len(sheet['assertions']) - before} text-structured " + f"object(s), coverage now {cov['ratio']:.0%}") + + # Rung 3: deterministic fallback - a dark text-bearing sheet is + # impossible. Stubs are deduped against earlier rungs. + if (page_text and config.EXTRACT_FALLBACK_ENABLED + and cov["ratio"] < config.EXTRACT_COVERAGE_FLOOR): + stubs = fallback_objects(page_text, page["page_number"], + config.EXTRACT_FALLBACK_MAX_OBJECTS) + stubs = _normalize_sheet({"sheet": {}, "objects": stubs}, + page["page_number"], + page_text=page_text)["assertions"] + before = len(sheet["assertions"]) + sheet["assertions"] = merge_objects(sheet["assertions"], stubs) + print(f"[Extract] Page {page['page_number']}: fallback merged " + f"{len(sheet['assertions']) - before} text-layer stub(s)") + sheet["coverage"] = text_coverage(page_text, sheet["assertions"]) + + # Identity recovery: never leave a text-bearing page sheet-less. + if not sheet.get("sheet_number") and page_text: + recovered = recover_sheet_number(page_text) + if recovered: + sheet["sheet_number"] = recovered + sheet["discipline"] = (discipline_from_sheet_number(recovered) + or sheet.get("discipline") or "Unknown") + print(f"[Extract] Page {page['page_number']}: sheet number " + f"recovered from text layer -> {recovered}") + + return sheet def extract_assertions(pages: List[Dict], on_progress=None) -> List[Dict]: diff --git a/tests/test_extraction_ladder_classic.py b/tests/test_extraction_ladder_classic.py new file mode 100644 index 0000000..49aab14 --- /dev/null +++ b/tests/test_extraction_ladder_classic.py @@ -0,0 +1,98 @@ +"""Coverage-driven extraction retry ladder — classic path (pipeline/extractor.py).""" + +from backend import config +from backend.pipeline import extractor + +PAGE_TEXT = ("1. \nALL SAWN LUMBER IN CONTACT WITH SOIL TO BE SOUTHERN PINE, " + "PRESSURE TREATED.\n2. \nROOF SHEATHING: 5/8\" PLYWOOD, C-D GRADE, " + "STRUCTURAL I.") + + +def _page(n=8, text=PAGE_TEXT): + return {"page_number": n, "base64": "AAAA", "text_layer": text} + + +def test_classic_fallback_when_vision_returns_nothing(monkeypatch): + """Vision pass returns an unusable bare list; text retry disabled -> + deterministic fallback stubs make a dark sheet impossible.""" + monkeypatch.setattr(extractor, "call_json", + lambda **kw: [{"name": "general notes", "value": "notes"}]) + monkeypatch.setattr(config, "EXTRACT_TEXT_RETRY_ENABLED", False) + sheet = extractor._extract_one(_page()) + assert sheet["assertions"], "dark sheet must be impossible with fallback enabled" + assert all(a.get("grounding") == "text_layer_fallback" + for a in sheet["assertions"]) + assert sheet["coverage"]["ratio"] >= 0.6 + + +def test_classic_merge_preserves_graphical_objects(monkeypatch): + """Rung-2 merge must never drop vision-only graphical objects.""" + def fake(**kw): + if kw.get("images_b64"): + return {"sheet": {}, "objects": [ + {"object_id": "g1", "object_type": "lighting_fixture", + "name": "pendant at grid C-4", "source_text": None, + "graphical_basis": "16in pendant symbol at grid C-4"}]} + return {"sheet": {}, "objects": [ + {"object_id": "t1", "object_type": "general_note", + "source_text": "ALL SAWN LUMBER IN CONTACT WITH SOIL TO BE " + "SOUTHERN PINE, PRESSURE TREATED.", + "name": "lumber note"}]} + + monkeypatch.setattr(extractor, "call_json", fake) + sheet = extractor._extract_one(_page()) + assert any(a.get("graphical_basis") for a in sheet["assertions"]) + assert any("SAWN LUMBER" in (a.get("source_text") or "") + for a in sheet["assertions"]) + + +def test_classic_recovers_sheet_number(monkeypatch): + text = ("REFLECTED CEILING PLAN\n" + "GYP. BD. CEILING 8'-11 3/8\" A.F.F. TYP. FOR ALL STOREFRONT\n" + "LED TAPE LIGHT. SEE ELEC. SCONCE 8'-0\" A.F.F., SEE ELEC.\n" + "A102") + + def fake(**kw): + if kw.get("images_b64"): + return {"sheet": {}, "objects": [ + {"object_id": "o1", "name": "RCP ceiling note", + "source_text": "GYP. BD. CEILING 8'-11 3/8\" A.F.F. TYP. " + "FOR ALL STOREFRONT", + "attributes": {"height": "8'-11 3/8\""}}]} + return {"sheet": {}, "objects": []} + + monkeypatch.setattr(extractor, "call_json", fake) + sheet = extractor._extract_one(_page(18, text)) + assert sheet["sheet_number"] == "A102" + + +def test_classic_skips_retry_when_coverage_healthy(monkeypatch): + calls = [] + + def fake(**kw): + calls.append(kw) + return {"sheet": {"sheet_number": "S202"}, "objects": [ + {"object_id": "o1", "name": "lumber note", + "source_text": "ALL SAWN LUMBER IN CONTACT WITH SOIL TO BE " + "SOUTHERN PINE, PRESSURE TREATED.", + "attributes": {"species": "southern pine"}}, + {"object_id": "o2", "name": "sheathing note", + "source_text": "ROOF SHEATHING: 5/8\" PLYWOOD, C-D GRADE, " + "STRUCTURAL I.", + "attributes": {"sheathing": "5/8 plywood"}}]} + + monkeypatch.setattr(extractor, "call_json", fake) + sheet = extractor._extract_one(_page()) + assert len(calls) == 1, "healthy coverage must not trigger the text-only rung" + assert sheet["coverage"]["ratio"] == 1.0 + assert sheet["sheet_number"] == "S202" + + +def test_classic_scanned_page_keeps_failed_sheet_shape(monkeypatch): + """No text layer (scanned page): total parse failure keeps the existing + 'extraction failed' empty-sheet return — ladder is text-layer-only.""" + monkeypatch.setattr(extractor, "call_json", lambda **kw: None) + sheet = extractor._extract_one({"page_number": 4, "base64": "AAAA", + "text_layer": None}) + assert sheet["assertions"] == [] + assert "extraction failed" in (sheet.get("sheet_title") or "") diff --git a/tests/test_extractor_text_grounding.py b/tests/test_extractor_text_grounding.py index e692cca..a33562a 100644 --- a/tests/test_extractor_text_grounding.py +++ b/tests/test_extractor_text_grounding.py @@ -84,3 +84,39 @@ def test_extractor_instruction_fully_substituted(): out = (EXTRACTOR_USER_INSTRUCTION.replace("{sheet_hint}", "") + _text_layer_block(page)) assert "{sheet_hint}" not in out + + +def test_vision_unverified_stamp_when_source_text_not_in_text_layer(): + """Digits ground the object against the page text, but its quoted + source_text is not actually present in the text layer: kept, stamped + vision_unverified (the wave-5b verifier consumes grounding stamps).""" + page_text = "WALL: 2X6 WD STUD @ 16\" O.C. WITH R-13 BATT INSULATION" + parsed = {"sheet": {}, "objects": [ + {"object_id": "x1", "name": "stud pack", + "source_text": "(5) 2X6 STUD PACK AT JAMB", # NOT in page text + "attributes": {"count": "5"}}]} + sheet = _normalize_sheet(parsed, 1, page_text=page_text) + assert len(sheet["assertions"]) == 1 + assert sheet["assertions"][0]["grounding"] == "vision_unverified" + + +def test_no_unverified_stamp_when_source_text_in_text_layer(): + page_text = "WALL: 2X6 WD STUD @ 16\" O.C. WITH R-13 BATT INSULATION" + parsed = {"sheet": {}, "objects": [ + {"object_id": "x1", "name": "stud note", + "source_text": "2X6 WD STUD @ 16\" O.C.", + "attributes": {"size": "2x6"}}]} + sheet = _normalize_sheet(parsed, 1, page_text=page_text) + assert len(sheet["assertions"]) == 1 + assert "grounding" not in sheet["assertions"][0] + + +def test_preset_grounding_stamp_survives_normalization(): + """Fallback/merge rungs stamp grounding upstream; normalization must + preserve a pre-set stamp instead of recomputing it away.""" + parsed = {"sheet": {}, "objects": [ + {"object_id": "f1", "name": "lumber note", + "source_text": "ALL LUMBER SOUTHERN PINE", + "grounding": "text_layer_fallback"}]} + sheet = _normalize_sheet(parsed, 1, page_text="ALL LUMBER SOUTHERN PINE") + assert sheet["assertions"][0]["grounding"] == "text_layer_fallback"