diff --git a/backend/agents/extractors.py b/backend/agents/extractors.py index f6706ab..8801830 100644 --- a/backend/agents/extractors.py +++ b/backend/agents/extractors.py @@ -6,7 +6,11 @@ from typing import Dict from backend import config from backend.agents.base import AgentResult, AgentScope, AgentUsage, failure from backend.llm import call_json -from backend.pipeline.extractor import _normalize_sheet, _text_layer_block +from backend.pipeline.extractor import ( + _normalize_sheet, + _text_layer_block, + discipline_from_sheet_number, +) from backend.pipeline.sheet_index import _index_input from backend.prompts import ( EXTRACTOR_SYSTEM_PROMPT, @@ -60,12 +64,39 @@ class SheetExtractorAgent: reasoning_max_tokens=config.EXTRACT_REASONING_MAX_TOKENS or None, ) + def _text_structuring_call(self, instruction_page: Dict, sheet_hint: str): + """Rung 2: text-only structuring pass over the page's text layer + (no image). Recovers text content the vision pass missed.""" + from backend.prompts import (TEXT_STRUCTURING_SYSTEM_PROMPT, + TEXT_STRUCTURING_USER_INSTRUCTION) + instruction = (TEXT_STRUCTURING_USER_INSTRUCTION + .replace("{sheet_hint}", str(sheet_hint or "")) + .replace("{text_layer}", + (instruction_page.get("text_layer") or "") + [:config.TEXT_LAYER_MAX_CHARS])) + return call_json( + system_prompt=TEXT_STRUCTURING_SYSTEM_PROMPT, + user_text=instruction, + images_b64=None, + max_tokens=config.EXTRACT_MAX_TOKENS, + model=config.AGENT_EXTRACT_MODEL, + usage_tracker=self.usage, + usage_stage="agent.extract_text", + reasoning_effort=config.EXTRACT_REASONING_EFFORT or None, + reasoning_max_tokens=config.EXTRACT_REASONING_MAX_TOKENS or None, + ) + def run(self, scope: AgentScope) -> AgentResult: + from backend.text_coverage import (fallback_objects, merge_objects, + recover_sheet_number, text_coverage) try: page = scope.payload["page"] + hint = scope.payload.get("sheet_hint") or "" + page_text = page.get("text_layer") instruction = EXTRACTOR_USER_INSTRUCTION.replace( - "{sheet_hint}", str(scope.payload.get("sheet_hint") or "") - ) + _text_layer_block(page) + "{sheet_hint}", str(hint)) + _text_layer_block(page) + + # Rung 1: vision pass (unchanged behaviour, incl. compact retry) parsed = _wrap_bare_list(self._call(instruction, page), page["page_number"]) if not isinstance(parsed, dict): @@ -78,9 +109,74 @@ class SheetExtractorAgent: page["page_number"], ) if not isinstance(parsed, dict): - raise ValueError("no structured extraction returned") + # Don't give up on the page - the ladder below can still + # rescue it from the text layer. + parsed = {"sheet": {}, "objects": []} + sheet = _normalize_sheet(parsed, page["page_number"], - page_text=page.get("text_layer")) + page_text=page_text) + cov = text_coverage(page_text or "", sheet["assertions"]) + sheet["coverage"] = cov + + # Rung 2: text-only structuring when coverage is below floor. + # MERGE, never replace: vision keeps every object it found + # (graphical_basis content exists only in the image); the text + # pass fills in the text content the vision pass missed. + if (page_text and config.EXTRACT_TEXT_RETRY_ENABLED + and cov["ratio"] < config.EXTRACT_COVERAGE_FLOOR): + print(f"[Extract] Page {page['page_number']}: coverage " + f"{cov['ratio']:.0%} < floor - text-only structuring pass") + parsed2 = _wrap_bare_list( + self._text_structuring_call(page, hint), page["page_number"]) + if isinstance(parsed2, dict): + sheet2 = _normalize_sheet(parsed2, page["page_number"], + page_text=page_text) + before = len(sheet["assertions"]) + sheet["assertions"] = merge_objects(sheet["assertions"], + sheet2["assertions"]) + # Fill header gaps the vision pass left null + for key in ("sheet_number", "sheet_title", "discipline", + "level", "scale", "drawing_type"): + if not sheet.get(key) and sheet2.get(key): + sheet[key] = sheet2[key] + cov = text_coverage(page_text, sheet["assertions"]) + sheet["coverage"] = cov + print(f"[Extract] Page {page['page_number']}: merged " + f"{len(sheet['assertions']) - before} text-structured " + f"object(s), coverage now {cov['ratio']:.0%}") + + # Rung 3: deterministic fallback - dark sheets are impossible. + # Also merged (deduped) so stub notes never double up with + # objects the earlier rungs already captured. + if (page_text and config.EXTRACT_FALLBACK_ENABLED + and cov["ratio"] < config.EXTRACT_COVERAGE_FLOOR): + stubs = fallback_objects(page_text, page["page_number"], + config.EXTRACT_FALLBACK_MAX_OBJECTS) + stubs = _normalize_sheet({"sheet": {}, "objects": stubs}, + page["page_number"], + page_text=page_text)["assertions"] + # _normalize_sheet only stamps its own "text_layer" rescue + # grounding; restore the explicit fallback provenance. + for stub in stubs: + stub["grounding"] = "text_layer_fallback" + before = len(sheet["assertions"]) + sheet["assertions"] = merge_objects(sheet["assertions"], stubs) + print(f"[Extract] Page {page['page_number']}: fallback merged " + f"{len(sheet['assertions']) - before} text-layer stub(s)") + sheet["coverage"] = text_coverage(page_text, + sheet["assertions"]) + + # Identity recovery: never leave a text-bearing page sheet-less + if not sheet.get("sheet_number") and page_text: + recovered = recover_sheet_number(page_text) + if recovered: + sheet["sheet_number"] = recovered + sheet["discipline"] = ( + discipline_from_sheet_number(recovered) + or sheet.get("discipline") or "Unknown") + print(f"[Extract] Page {page['page_number']}: sheet number " + f"recovered from text layer -> {recovered}") + return AgentResult(scope_id=scope.scope_id, artifacts=[sheet]) except Exception as exc: return failure(scope, exc) diff --git a/backend/config.py b/backend/config.py index 83d0ef7..f888443 100644 --- a/backend/config.py +++ b/backend/config.py @@ -112,6 +112,15 @@ EXTRACT_REASONING_EFFORT = os.getenv("EXTRACT_REASONING_EFFORT", "low").strip() # the budget into visible output. 0 disables -> falls back to the effort knob. # Mutually exclusive with effort when set (OpenRouter rejects both together). EXTRACT_REASONING_MAX_TOKENS = int(os.getenv("EXTRACT_REASONING_MAX_TOKENS", "2048")) +# Coverage-driven extraction retry ladder. After the vision pass, the fraction +# of meaningful text-layer lines represented in extracted objects is measured; +# below EXTRACT_COVERAGE_FLOOR the page climbs the ladder: text-only +# structuring pass (rung 2), then deterministic text-layer fallback stubs +# (rung 3) so no text-bearing page goes dark. +EXTRACT_COVERAGE_FLOOR = float(os.getenv("EXTRACT_COVERAGE_FLOOR", "0.6")) +EXTRACT_TEXT_RETRY_ENABLED = os.getenv("EXTRACT_TEXT_RETRY_ENABLED", "true").lower() == "true" +EXTRACT_FALLBACK_ENABLED = os.getenv("EXTRACT_FALLBACK_ENABLED", "true").lower() == "true" +EXTRACT_FALLBACK_MAX_OBJECTS = int(os.getenv("EXTRACT_FALLBACK_MAX_OBJECTS", "200")) REASON_MAX_TOKENS = int(os.getenv("REASON_MAX_TOKENS", "4096")) # -- QAQC stage knobs (Stages 0-1, 3, 6-11) ------------------------- diff --git a/tests/agents/test_extraction_ladder.py b/tests/agents/test_extraction_ladder.py index 33ea10c..f8536b2 100644 --- a/tests/agents/test_extraction_ladder.py +++ b/tests/agents/test_extraction_ladder.py @@ -1,6 +1,86 @@ +from backend import config +from backend.agents.base import AgentScope, AgentUsage +from backend.agents.extractors import SheetExtractorAgent from backend.prompts import TEXT_STRUCTURING_SYSTEM_PROMPT, TEXT_STRUCTURING_USER_INSTRUCTION def test_text_structuring_prompt_demands_verbatim_and_completeness(): assert "verbatim" in TEXT_STRUCTURING_USER_INSTRUCTION.lower() assert "every" in TEXT_STRUCTURING_USER_INSTRUCTION.lower() assert "{text_layer}" in TEXT_STRUCTURING_USER_INSTRUCTION + + +def _page(n=8, text="1. \nALL SAWN LUMBER IN CONTACT WITH SOIL TO BE SOUTHERN PINE, PRESSURE TREATED.\n2. \nROOF SHEATHING: 5/8\" PLYWOOD, C-D GRADE, STRUCTURAL I."): + return {"page_number": n, "base64": "AAAA", "text_layer": text} + + +def _run(agent, page, hint=""): + scope = AgentScope(scope_id=f"sheet:{page['page_number']}", + payload={"page": page, "sheet_hint": hint}) + result = agent.run(scope) + assert not result.error, result.error + return result.artifacts[0] + + +def test_ladder_falls_back_when_vision_returns_nothing(monkeypatch): + # vision pass returns 1 summary object that the guard drops; + # text-structuring disabled to exercise the deterministic rung + monkeypatch.setattr("backend.agents.extractors.call_json", + lambda **kw: [{"name": "general notes", "value": "notes"}]) + monkeypatch.setattr("backend.config.EXTRACT_TEXT_RETRY_ENABLED", False) + agent = SheetExtractorAgent(AgentUsage()) + sheet = _run(agent, _page()) + assert sheet["assertions"], "dark sheet must be impossible with fallback enabled" + assert all(a.get("grounding") == "text_layer_fallback" for a in sheet["assertions"]) + assert sheet["coverage"]["ratio"] >= 0.6 + + +def test_ladder_merge_preserves_graphical_objects(monkeypatch): + # vision finds a graphical symbol; text rung adds notes. + # The graphical object MUST survive the merge. + calls = {"n": 0} + def fake_call_json(**kw): + calls["n"] += 1 + if kw.get("images_b64"): # vision pass + return {"sheet": {}, "objects": [ + {"object_id": "g1", "object_type": "lighting_fixture", + "name": "pendant at grid C-4", "source_text": None, + "graphical_basis": "16in pendant symbol at grid C-4"}]} + return {"sheet": {}, "objects": [ # text-structuring pass + {"object_id": "t1", "object_type": "general_note", + "source_text": "ALL SAWN LUMBER IN CONTACT WITH SOIL TO BE SOUTHERN PINE, PRESSURE TREATED.", + "name": "lumber note"}]} + monkeypatch.setattr("backend.agents.extractors.call_json", fake_call_json) + agent = SheetExtractorAgent(AgentUsage()) + sheet = _run(agent, _page()) + assert calls["n"] >= 2, "text-structuring rung should have fired" + assert any(a.get("graphical_basis") for a in sheet["assertions"]) + assert any("SAWN LUMBER" in (a.get("source_text") or "") for a in sheet["assertions"]) + + +def test_ladder_recovers_sheet_number_from_text_layer(monkeypatch): + monkeypatch.setattr( + "backend.agents.extractors.call_json", + lambda **kw: {"sheet": {}, "objects": [ + {"object_id": "o1", "name": "RCP note", + "source_text": "GYP. BD. CEILING 8'-11 3/8\" A.F.F. TYP. FOR ALL STOREFRONT", + "attributes": {"height": "8'-11 3/8\""}}]}) + agent = SheetExtractorAgent(AgentUsage()) + sheet = _run(agent, _page(18, "REFLECTED CEILING PLAN\nGYP. BD. CEILING 8'-11 3/8\" A.F.F. TYP. FOR ALL STOREFRONT\nA102")) + assert sheet["sheet_number"] == "A102" + + +def test_ladder_skips_retry_when_coverage_healthy(monkeypatch): + # vision covers every meaningful text-layer line -> no rung 2/3 calls + calls = {"n": 0} + def fake_call_json(**kw): + calls["n"] += 1 + return {"sheet": {"sheet_number": "A101"}, "objects": [ + {"object_id": "o1", "object_type": "general_note", "name": "lumber note", + "source_text": "ALL SAWN LUMBER IN CONTACT WITH SOIL TO BE SOUTHERN PINE, PRESSURE TREATED."}, + {"object_id": "o2", "object_type": "general_note", "name": "sheathing note", + "source_text": "ROOF SHEATHING: 5/8\" PLYWOOD, C-D GRADE, STRUCTURAL I."}]} + monkeypatch.setattr("backend.agents.extractors.call_json", fake_call_json) + agent = SheetExtractorAgent(AgentUsage()) + sheet = _run(agent, _page()) + assert sheet["coverage"]["ratio"] >= config.EXTRACT_COVERAGE_FLOOR + assert calls["n"] == 1 diff --git a/tests/agents/test_sheet_extractor_fallback.py b/tests/agents/test_sheet_extractor_fallback.py index 7c0cf24..494a1d9 100644 --- a/tests/agents/test_sheet_extractor_fallback.py +++ b/tests/agents/test_sheet_extractor_fallback.py @@ -68,9 +68,12 @@ def test_run_compact_retry_after_hard_failure(): assert "COMPACT RETRY" in mock_call.call_args_list[1].kwargs["user_text"] -def test_run_fails_only_after_both_attempts_miss(): +def test_run_returns_empty_sheet_after_both_attempts_miss(): agent = SheetExtractorAgent(usage=AgentUsage()) with patch("backend.agents.extractors.call_json", return_value=None) as mock_call: result = agent.run(_scope()) - assert result.error == "no structured extraction returned" + # Coverage ladder: no text layer to rescue the page -> empty sheet, + # but no hard failure (the ladder replaced the old raise). + assert not result.error + assert result.artifacts[0]["assertions"] == [] assert mock_call.call_count == 2