woogi/Conflict_Checker
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: coverage-driven extraction retry ladder (agent path)
Docker Release / build-and-push (push) Successful in 1m14s
Details
Docker Release / release (push) Skipped
Details

Browse Source
This commit is contained in:
woogi
2026-08-18 13:38:30 -05:00
parent 48fefa4007
commit fe09e4a66b
4 changed files with 195 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/agents/extractors.py
+101 -5
View File
@@ -6,7 +6,11 @@ from typing import Dict
from backend import config
from backend.agents.base import AgentResult, AgentScope, AgentUsage, failure
from backend.llm import call_json
from backend.pipeline.extractor import _normalize_sheet, _text_layer_block
from backend.pipeline.extractor import (
_normalize_sheet,
_text_layer_block,
discipline_from_sheet_number,
)
from backend.pipeline.sheet_index import _index_input
from backend.prompts import (
EXTRACTOR_SYSTEM_PROMPT,
@@ -60,12 +64,39 @@ class SheetExtractorAgent:
reasoning_max_tokens=config.EXTRACT_REASONING_MAX_TOKENS or None,
)
def _text_structuring_call(self, instruction_page: Dict, sheet_hint: str):
"""Rung 2: text-only structuring pass over the page's text layer
(no image). Recovers text content the vision pass missed."""
from backend.prompts import (TEXT_STRUCTURING_SYSTEM_PROMPT,
TEXT_STRUCTURING_USER_INSTRUCTION)
instruction = (TEXT_STRUCTURING_USER_INSTRUCTION
.replace("{sheet_hint}", str(sheet_hint or ""))
.replace("{text_layer}",
(instruction_page.get("text_layer") or "")
[:config.TEXT_LAYER_MAX_CHARS]))
return call_json(
system_prompt=TEXT_STRUCTURING_SYSTEM_PROMPT,
user_text=instruction,
images_b64=None,
max_tokens=config.EXTRACT_MAX_TOKENS,
model=config.AGENT_EXTRACT_MODEL,
usage_tracker=self.usage,
usage_stage="agent.extract_text",
reasoning_effort=config.EXTRACT_REASONING_EFFORT or None,
reasoning_max_tokens=config.EXTRACT_REASONING_MAX_TOKENS or None,
)
def run(self, scope: AgentScope) -> AgentResult:
from backend.text_coverage import (fallback_objects, merge_objects,
recover_sheet_number, text_coverage)
try:
page = scope.payload["page"]
hint = scope.payload.get("sheet_hint") or ""
page_text = page.get("text_layer")
instruction = EXTRACTOR_USER_INSTRUCTION.replace(
"{sheet_hint}", str(scope.payload.get("sheet_hint") or "")
) + _text_layer_block(page)
"{sheet_hint}", str(hint)) + _text_layer_block(page)
# Rung 1: vision pass (unchanged behaviour, incl. compact retry)
parsed = _wrap_bare_list(self._call(instruction, page),
page["page_number"])
if not isinstance(parsed, dict):
@@ -78,9 +109,74 @@ class SheetExtractorAgent:
page["page_number"],
)
if not isinstance(parsed, dict):
raise ValueError("no structured extraction returned")
# Don't give up on the page - the ladder below can still
# rescue it from the text layer.
parsed = {"sheet": {}, "objects": []}
sheet = _normalize_sheet(parsed, page["page_number"],
page_text=page.get("text_layer"))
page_text=page_text)
cov = text_coverage(page_text or "", sheet["assertions"])
sheet["coverage"] = cov
# Rung 2: text-only structuring when coverage is below floor.
# MERGE, never replace: vision keeps every object it found
# (graphical_basis content exists only in the image); the text
# pass fills in the text content the vision pass missed.
if (page_text and config.EXTRACT_TEXT_RETRY_ENABLED
and cov["ratio"] < config.EXTRACT_COVERAGE_FLOOR):
print(f"[Extract] Page {page['page_number']}: coverage "
f"{cov['ratio']:.0%} < floor - text-only structuring pass")
parsed2 = _wrap_bare_list(
self._text_structuring_call(page, hint), page["page_number"])
if isinstance(parsed2, dict):
sheet2 = _normalize_sheet(parsed2, page["page_number"],
page_text=page_text)
before = len(sheet["assertions"])
sheet["assertions"] = merge_objects(sheet["assertions"],
sheet2["assertions"])
# Fill header gaps the vision pass left null
for key in ("sheet_number", "sheet_title", "discipline",
"level", "scale", "drawing_type"):
if not sheet.get(key) and sheet2.get(key):
sheet[key] = sheet2[key]
cov = text_coverage(page_text, sheet["assertions"])
sheet["coverage"] = cov
print(f"[Extract] Page {page['page_number']}: merged "
f"{len(sheet['assertions']) - before} text-structured "
f"object(s), coverage now {cov['ratio']:.0%}")
# Rung 3: deterministic fallback - dark sheets are impossible.
# Also merged (deduped) so stub notes never double up with
# objects the earlier rungs already captured.
if (page_text and config.EXTRACT_FALLBACK_ENABLED
and cov["ratio"] < config.EXTRACT_COVERAGE_FLOOR):
stubs = fallback_objects(page_text, page["page_number"],
config.EXTRACT_FALLBACK_MAX_OBJECTS)
stubs = _normalize_sheet({"sheet": {}, "objects": stubs},
page["page_number"],
page_text=page_text)["assertions"]
# _normalize_sheet only stamps its own "text_layer" rescue
# grounding; restore the explicit fallback provenance.
for stub in stubs:
stub["grounding"] = "text_layer_fallback"
before = len(sheet["assertions"])
sheet["assertions"] = merge_objects(sheet["assertions"], stubs)
print(f"[Extract] Page {page['page_number']}: fallback merged "
f"{len(sheet['assertions']) - before} text-layer stub(s)")
sheet["coverage"] = text_coverage(page_text,
sheet["assertions"])
# Identity recovery: never leave a text-bearing page sheet-less
if not sheet.get("sheet_number") and page_text:
recovered = recover_sheet_number(page_text)
if recovered:
sheet["sheet_number"] = recovered
sheet["discipline"] = (
discipline_from_sheet_number(recovered)
or sheet.get("discipline") or "Unknown")
print(f"[Extract] Page {page['page_number']}: sheet number "
f"recovered from text layer -> {recovered}")
return AgentResult(scope_id=scope.scope_id, artifacts=[sheet])
except Exception as exc:
return failure(scope, exc)
backend/config.py
+9
View File
@@ -112,6 +112,15 @@ EXTRACT_REASONING_EFFORT = os.getenv("EXTRACT_REASONING_EFFORT", "low").strip()
# the budget into visible output. 0 disables -> falls back to the effort knob.
# Mutually exclusive with effort when set (OpenRouter rejects both together).
EXTRACT_REASONING_MAX_TOKENS = int(os.getenv("EXTRACT_REASONING_MAX_TOKENS", "2048"))
# Coverage-driven extraction retry ladder. After the vision pass, the fraction
# of meaningful text-layer lines represented in extracted objects is measured;
# below EXTRACT_COVERAGE_FLOOR the page climbs the ladder: text-only
# structuring pass (rung 2), then deterministic text-layer fallback stubs
# (rung 3) so no text-bearing page goes dark.
EXTRACT_COVERAGE_FLOOR = float(os.getenv("EXTRACT_COVERAGE_FLOOR", "0.6"))
EXTRACT_TEXT_RETRY_ENABLED = os.getenv("EXTRACT_TEXT_RETRY_ENABLED", "true").lower() == "true"
EXTRACT_FALLBACK_ENABLED = os.getenv("EXTRACT_FALLBACK_ENABLED", "true").lower() == "true"
EXTRACT_FALLBACK_MAX_OBJECTS = int(os.getenv("EXTRACT_FALLBACK_MAX_OBJECTS", "200"))
REASON_MAX_TOKENS = int(os.getenv("REASON_MAX_TOKENS", "4096"))
# -- QAQC stage knobs (Stages 0-1, 3, 6-11) -------------------------
tests/agents/test_extraction_ladder.py
+80
View File
@@ -1,6 +1,86 @@
from backend import config
from backend.agents.base import AgentScope, AgentUsage
from backend.agents.extractors import SheetExtractorAgent
from backend.prompts import TEXT_STRUCTURING_SYSTEM_PROMPT, TEXT_STRUCTURING_USER_INSTRUCTION
def test_text_structuring_prompt_demands_verbatim_and_completeness():
assert "verbatim" in TEXT_STRUCTURING_USER_INSTRUCTION.lower()
assert "every" in TEXT_STRUCTURING_USER_INSTRUCTION.lower()
assert "{text_layer}" in TEXT_STRUCTURING_USER_INSTRUCTION
def _page(n=8, text="1. \nALL SAWN LUMBER IN CONTACT WITH SOIL TO BE SOUTHERN PINE, PRESSURE TREATED.\n2. \nROOF SHEATHING: 5/8\" PLYWOOD, C-D GRADE, STRUCTURAL I."):
return {"page_number": n, "base64": "AAAA", "text_layer": text}
def _run(agent, page, hint=""):
scope = AgentScope(scope_id=f"sheet:{page['page_number']}",
payload={"page": page, "sheet_hint": hint})
result = agent.run(scope)
assert not result.error, result.error
return result.artifacts[0]
def test_ladder_falls_back_when_vision_returns_nothing(monkeypatch):
# vision pass returns 1 summary object that the guard drops;
# text-structuring disabled to exercise the deterministic rung
monkeypatch.setattr("backend.agents.extractors.call_json",
lambda **kw: [{"name": "general notes", "value": "notes"}])
monkeypatch.setattr("backend.config.EXTRACT_TEXT_RETRY_ENABLED", False)
agent = SheetExtractorAgent(AgentUsage())
sheet = _run(agent, _page())
assert sheet["assertions"], "dark sheet must be impossible with fallback enabled"
assert all(a.get("grounding") == "text_layer_fallback" for a in sheet["assertions"])
assert sheet["coverage"]["ratio"] >= 0.6
def test_ladder_merge_preserves_graphical_objects(monkeypatch):
# vision finds a graphical symbol; text rung adds notes.
# The graphical object MUST survive the merge.
calls = {"n": 0}
def fake_call_json(**kw):
calls["n"] += 1
if kw.get("images_b64"): # vision pass
return {"sheet": {}, "objects": [
{"object_id": "g1", "object_type": "lighting_fixture",
"name": "pendant at grid C-4", "source_text": None,
"graphical_basis": "16in pendant symbol at grid C-4"}]}
return {"sheet": {}, "objects": [ # text-structuring pass
{"object_id": "t1", "object_type": "general_note",
"source_text": "ALL SAWN LUMBER IN CONTACT WITH SOIL TO BE SOUTHERN PINE, PRESSURE TREATED.",
"name": "lumber note"}]}
monkeypatch.setattr("backend.agents.extractors.call_json", fake_call_json)
agent = SheetExtractorAgent(AgentUsage())
sheet = _run(agent, _page())
assert calls["n"] >= 2, "text-structuring rung should have fired"
assert any(a.get("graphical_basis") for a in sheet["assertions"])
assert any("SAWN LUMBER" in (a.get("source_text") or "") for a in sheet["assertions"])
def test_ladder_recovers_sheet_number_from_text_layer(monkeypatch):
monkeypatch.setattr(
"backend.agents.extractors.call_json",
lambda **kw: {"sheet": {}, "objects": [
{"object_id": "o1", "name": "RCP note",
"source_text": "GYP. BD. CEILING 8'-11 3/8\" A.F.F. TYP. FOR ALL STOREFRONT",
"attributes": {"height": "8'-11 3/8\""}}]})
agent = SheetExtractorAgent(AgentUsage())
sheet = _run(agent, _page(18, "REFLECTED CEILING PLAN\nGYP. BD. CEILING 8'-11 3/8\" A.F.F. TYP. FOR ALL STOREFRONT\nA102"))
assert sheet["sheet_number"] == "A102"
def test_ladder_skips_retry_when_coverage_healthy(monkeypatch):
# vision covers every meaningful text-layer line -> no rung 2/3 calls
calls = {"n": 0}
def fake_call_json(**kw):
calls["n"] += 1
return {"sheet": {"sheet_number": "A101"}, "objects": [
{"object_id": "o1", "object_type": "general_note", "name": "lumber note",
"source_text": "ALL SAWN LUMBER IN CONTACT WITH SOIL TO BE SOUTHERN PINE, PRESSURE TREATED."},
{"object_id": "o2", "object_type": "general_note", "name": "sheathing note",
"source_text": "ROOF SHEATHING: 5/8\" PLYWOOD, C-D GRADE, STRUCTURAL I."}]}
monkeypatch.setattr("backend.agents.extractors.call_json", fake_call_json)
agent = SheetExtractorAgent(AgentUsage())
sheet = _run(agent, _page())
assert sheet["coverage"]["ratio"] >= config.EXTRACT_COVERAGE_FLOOR
assert calls["n"] == 1
tests/agents/test_sheet_extractor_fallback.py
+5 -2
View File
@@ -68,9 +68,12 @@ def test_run_compact_retry_after_hard_failure():
assert "COMPACT RETRY" in mock_call.call_args_list[1].kwargs["user_text"]
def test_run_fails_only_after_both_attempts_miss():
def test_run_returns_empty_sheet_after_both_attempts_miss():
agent = SheetExtractorAgent(usage=AgentUsage())
with patch("backend.agents.extractors.call_json", return_value=None) as mock_call:
result = agent.run(_scope())
assert result.error == "no structured extraction returned"
# Coverage ladder: no text layer to rescue the page -> empty sheet,
# but no hard failure (the ladder replaced the old raise).
assert not result.error
assert result.artifacts[0]["assertions"] == []
assert mock_call.call_count == 2