feat: text-layer grounding (extractor authority, guard rescue tier, verifier oracle + hi-DPI crops)

- backend/text_layer.py: PyMuPDF text-layer extraction, fuzzy evidence
  bbox matching, 300-DPI crop rendering, coverage-gap signal
- extractor (classic + agent): TEXT LAYER block appended at call sites;
  grounding guard gains text-layer rescue tier (grounding=text_layer stamp)
- verifier: {text_layer} oracle excerpt + evidence-located hi-DPI crops
  replacing full-page images (fallback preserved, I2 guard intact)
- coverage gaps: text-bearing pages with zero extraction -> failed-scope
  gap findings (agent) / log-only (classic)
- config knobs: TEXT_LAYER_ENABLED/MIN_CHARS/MAX_CHARS, VERIFY_TEXT_MAX_CHARS,
  VERIFY_HI_DPI_CROPS, VERIFY_CROP_DPI, VERIFY_CROP_MARGIN_PTS
- tests: 22 new (text_layer unit, grounding/render, runner-level flow)
Spec: docs/superpowers/specs/2026-08-12-text-layer-grounding-design.md

tests/agents/test_text_layer_flow.py

@@ -0,0 +1,128 @@
+"""Runner-level text-layer flow: excerpt into verify scopes, hi-DPI crop
+replacement with full-page fallback, and coverage-gap findings."""
+
+import pytest
+
+fitz = pytest.importorskip("pymupdf")
+
+import backend.agents.runner as runner_mod
+from backend.agents.base import AgentResult
+from backend.agents.runner import run_agent_pipeline
+
+PAGE_TEXT = "(5) 2X6 STUD PACK AT BEARING"
+
+
+def _make_pdf(path):
+    doc = fitz.open()
+    page = doc.new_page(width=612, height=792)
+    page.insert_text((72, 72), PAGE_TEXT, fontsize=11)
+    doc.save(str(path))
+    doc.close()
+    return str(path)
+
+
+def _finding(sheets, evidence_text):
+    return {
+        "issue_id": "C1", "severity": "critical", "confidence": "high",
+        "source_stage": "constructability", "sheets": sheets,
+        "description": "stud pack conflict",
+        "evidence": [{"sheet": sheets[0], "source_text": evidence_text}],
+    }
+
+
+def _stub_agent(artifacts):
+    return lambda usage: type("S", (), {
+        "name": "stub",
+        "run": lambda self, scope: AgentResult(
+            scope_id=scope.scope_id, artifacts=list(artifacts)),
+    })()
+
+
+def _patch_pipeline(monkeypatch, finding, verify_sink):
+    monkeypatch.setattr(
+        runner_mod, "convert_pdf_to_images",
+        lambda path: [{"page_number": 1, "base64": "QUJD"}])
+    monkeypatch.setattr(runner_mod, "SheetExtractorAgent", _stub_agent([
+        {"sheet_number": "S401", "page_number": 1, "level": "roof",
+         "discipline": "S", "assertions": [
+             {"text": "(5) 2X6 STUD PACK", "object_type": "framing"},
+             {"text": "HSS16X4 beam", "object_type": "framing"},
+         ]},
+    ]))
+    monkeypatch.setattr(runner_mod, "SheetIndexAgent", _stub_agent([{}]))
+    monkeypatch.setattr(runner_mod, "JurisdictionAgent", _stub_agent([{}]))
+    monkeypatch.setattr(runner_mod, "LinkerAgent", _stub_agent([
+        {"key": "c1", "location": "roof beam pocket", "assertions": []},
+    ]))
+    monkeypatch.setattr(runner_mod, "ConflictCriticAgent", _stub_agent([]))
+    monkeypatch.setattr(runner_mod, "CodeAgent", _stub_agent([]))
+    monkeypatch.setattr(runner_mod, "ConstructabilityAgent",
+                        _stub_agent([finding]))
+    monkeypatch.setattr(runner_mod, "CompletenessAgent", _stub_agent([]))
+    monkeypatch.setattr(
+        runner_mod, "BrainAgent",
+        lambda usage: type("B", (), {
+            "run": lambda self, findings, sheet_index, jurisdiction:
+                (list(findings), [])})())
+
+    class _RecordingVerifier:
+        name = "verify"
+
+        def __init__(self, usage):
+            pass
+
+        def run(self, scope):
+            verify_sink.append(scope.payload)
+            return AgentResult(scope_id=scope.scope_id, artifacts=[{
+                "finding_index": scope.payload["finding_index"],
+                "status": "confirmed",
+                "verdicts": [],
+            }])
+
+    monkeypatch.setattr(runner_mod, "EvidenceVerifierAgent",
+                        lambda usage: _RecordingVerifier(usage))
+
+
+def test_verify_scope_carries_text_excerpt_and_crop(monkeypatch, tmp_path):
+    """Evidence text matches the page text layer -> excerpt present and the
+    full-page image is replaced by a hi-DPI crop."""
+    sink = []
+    _patch_pipeline(monkeypatch,
+                    _finding(["S401"], "(5) 2X6 STUD PACK AT BEARING"), sink)
+    pdf = _make_pdf(tmp_path / "set.pdf")
+    run_agent_pipeline(pdf, out_dir=str(tmp_path), require_review=False)
+    assert len(sink) == 1
+    payload = sink[0]
+    assert "2X6 STUD PACK" in payload["text_layer_excerpt"]
+    assert payload["images_b64"], "crop must never drop all images"
+    assert payload["images_b64"][0] != "QUJD", "expected crop, not full page"
+
+
+def test_verify_scope_falls_back_to_full_page(monkeypatch, tmp_path):
+    """Evidence text not in the text layer -> keep the full-page image."""
+    sink = []
+    _patch_pipeline(monkeypatch,
+                    _finding(["S401"], "PENTHOUSE EXHAUST FAN EF-9"), sink)
+    pdf = _make_pdf(tmp_path / "set.pdf")
+    run_agent_pipeline(pdf, out_dir=str(tmp_path), require_review=False)
+    assert len(sink) == 1
+    assert sink[0]["images_b64"] == ["QUJD"]
+
+
+def test_coverage_gap_becomes_gap_finding(monkeypatch, tmp_path):
+    """Text layer present but zero objects extracted -> failed-scope gap
+    finding survives into the report."""
+    sink = []
+    _patch_pipeline(monkeypatch, _finding(["S401"], PAGE_TEXT), sink)
+    # Extractor returns a sheet with NO objects despite a real text layer.
+    monkeypatch.setattr(runner_mod, "SheetExtractorAgent", _stub_agent([
+        {"sheet_number": "S401", "page_number": 1, "level": "roof",
+         "discipline": "S", "assertions": []},
+    ]))
+    pdf = _make_pdf(tmp_path / "set.pdf")
+    report = run_agent_pipeline(pdf, out_dir=str(tmp_path),
+                                require_review=False)
+    gaps = [f for f in (report.get("validated_issues") or [])
+            if f.get("category") == "analysis_gap"]
+    assert any("extraction gap" in (g.get("description") or "")
+               for g in gaps)

tests/test_extractor_text_grounding.py

@@ -0,0 +1,86 @@
+"""Grounding-guard rescue tier, text-layer prompt block, and render hygiene."""
+
+from backend import config
+from backend.pipeline._stage import render
+from backend.pipeline.extractor import (
+    _is_grounded,
+    _normalize_sheet,
+    _text_layer_block,
+)
+from backend.prompts import EXTRACTOR_USER_INSTRUCTION, VERIFY_USER_INSTRUCTION
+
+PAGE_TEXT = "NOTES: (5) 2X6 STUD PACK AT BEARING. HSS16X4 BEAM. 7'-0\" AFF."
+
+
+def _parsed(value, source_text):
+    return {
+        "sheet": {"sheet_number": "S401"},
+        "objects": [{
+            "object_id": "o1",
+            "object_type": "framing",
+            "name": "stud pack",
+            "attributes": {"count": value},
+            "source_text": source_text,
+        }],
+    }
+
+
+def test_rescue_tier_keeps_and_stamps():
+    """Digits absent from source_text but present in the page text layer:
+    kept, stamped grounding=text_layer (vision quoted imperfectly)."""
+    sheet = _normalize_sheet(_parsed("(2)", "(2) 2x6 STUD PACK"), 1,
+                             page_text=PAGE_TEXT)
+    # "(2)" is not grounded by its own source_text alone? it is - use a value
+    # whose digits differ from the quote to exercise the rescue path.
+    sheet = _normalize_sheet(_parsed("5", "(2) 2x6 STUD PACK"), 1,
+                             page_text=PAGE_TEXT)
+    assert len(sheet["assertions"]) == 1
+    assert sheet["assertions"][0]["grounding"] == "text_layer"
+
+
+def test_no_rescue_without_page_text():
+    sheet = _normalize_sheet(_parsed("5", "(2) 2x6 STUD PACK"), 1)
+    assert sheet["assertions"] == []
+
+
+def test_still_dropped_when_digits_nowhere():
+    sheet = _normalize_sheet(_parsed("99", "(2) 2x6 STUD PACK"), 1,
+                             page_text=PAGE_TEXT)
+    assert sheet["assertions"] == []
+
+
+def test_is_grounded_backward_compatible():
+    assert _is_grounded("(5)", "(5) 2x6 STUD PACK") is True
+    # Digit-run guard is a set check: "(3)" has no support anywhere.
+    assert _is_grounded("(3)", "(5) 2x6 STUD PACK") is False
+    assert _is_grounded("(3)", "(5) 2x6 STUD PACK",
+                        page_text="(3) 2x6 STUD PACK") is True
+
+
+def test_text_layer_block_empty_without_layer():
+    assert _text_layer_block({"page_number": 1}) == ""
+    assert _text_layer_block({"page_number": 1, "text_layer": None}) == ""
+
+
+def test_text_layer_block_appends_and_caps(monkeypatch):
+    block = _text_layer_block({"page_number": 1, "text_layer": PAGE_TEXT})
+    assert "TEXT LAYER" in block and "STUD PACK" in block
+    monkeypatch.setattr(config, "TEXT_LAYER_MAX_CHARS", 50)
+    block = _text_layer_block({"page_number": 1, "text_layer": "x" * 500})
+    assert len(block.split(":\n", 1)[1]) == 50
+
+
+def test_verify_instruction_fully_rendered():
+    """render() silently leaves missing keys as literals - both placeholders
+    must be substituted at the (single) verify render site."""
+    out = render(VERIFY_USER_INSTRUCTION,
+                 {"finding": "FINDING_JSON", "text_layer": "PAGE_TEXT"})
+    assert "{finding}" not in out and "{text_layer}" not in out
+    assert "FINDING_JSON" in out and "PAGE_TEXT" in out
+
+
+def test_extractor_instruction_fully_substituted():
+    page = {"page_number": 1, "text_layer": PAGE_TEXT}
+    out = (EXTRACTOR_USER_INSTRUCTION.replace("{sheet_hint}", "")
+           + _text_layer_block(page))
+    assert "{sheet_hint}" not in out

tests/test_text_layer.py

@@ -0,0 +1,111 @@
+"""Text-layer extraction, evidence bbox matching, and crop rendering."""
+
+import os
+
+import pytest
+
+fitz = pytest.importorskip("pymupdf")
+
+from backend import config
+from backend.text_layer import (
+    attach_text_layers,
+    coverage_gaps,
+    extract_text_layers,
+    find_evidence_bbox,
+    render_crop,
+)
+
+EVIDENCE = "(5) 2X6 STUD PACK @ 16 IN O.C."
+
+
+def _make_pdf(path, pages):
+    """pages: list of str ('' = effectively blank page)."""
+    doc = fitz.open()
+    for text in pages:
+        page = doc.new_page(width=612, height=792)
+        if text:
+            page.insert_text((72, 72), text, fontsize=11)
+    doc.save(str(path))
+    doc.close()
+    return str(path)
+
+
+@pytest.fixture
+def text_pdf(tmp_path):
+    return _make_pdf(tmp_path / "set.pdf", [EVIDENCE, ""])
+
+
+def test_extract_text_layers(text_pdf):
+    layers = extract_text_layers(text_pdf)
+    assert set(layers) == {1, 2}
+    assert layers[1]["has_text_layer"] is True
+    assert "2X6 STUD PACK" in layers[1]["text"]
+    assert layers[1]["words"], "expected word-level bboxes"
+    assert all("bbox" in w and len(w["bbox"]) == 4 for w in layers[1]["words"])
+
+
+def test_blank_page_below_min_chars(text_pdf):
+    layers = extract_text_layers(text_pdf)
+    assert layers[2]["has_text_layer"] is False
+
+
+def test_disabled_returns_empty(text_pdf, monkeypatch):
+    monkeypatch.setattr(config, "TEXT_LAYER_ENABLED", False)
+    assert extract_text_layers(text_pdf) == {}
+
+
+def test_attach_text_layers(text_pdf, tmp_path):
+    pages = [{"page_number": 1}, {"page_number": 2}]
+    words = attach_text_layers(text_pdf, pages,
+                               text_dir=str(tmp_path / "text"))
+    assert pages[0]["text_layer"] and "STUD PACK" in pages[0]["text_layer"]
+    assert pages[1]["text_layer"] is None
+    assert words[1] and not words[2]
+    assert os.path.isfile(tmp_path / "text" / "page-001.txt")
+    assert not os.path.exists(tmp_path / "text" / "page-002.txt")
+
+
+def test_find_evidence_bbox_exact(text_pdf):
+    words = extract_text_layers(text_pdf)[1]["words"]
+    bbox = find_evidence_bbox(words, EVIDENCE)
+    assert bbox is not None
+    assert bbox[2] > bbox[0] and bbox[3] > bbox[1]
+
+
+def test_find_evidence_bbox_fuzzy(text_pdf):
+    # Vision quotes imperfectly: wrong count token, rest exact.
+    words = extract_text_layers(text_pdf)[1]["words"]
+    bbox = find_evidence_bbox(words, "(2) 2X6 STUD PACK @ 16 IN O.C.")
+    assert bbox is not None
+
+
+def test_find_evidence_bbox_miss(text_pdf):
+    words = extract_text_layers(text_pdf)[1]["words"]
+    assert find_evidence_bbox(words, "PENTHOUSE EXHAUST FAN EF-9") is None
+    assert find_evidence_bbox([], EVIDENCE) is None
+    assert find_evidence_bbox(words, "") is None
+
+
+def test_render_crop(text_pdf):
+    words = extract_text_layers(text_pdf)[1]["words"]
+    bbox = find_evidence_bbox(words, EVIDENCE)
+    crop = render_crop(text_pdf, 1, bbox)
+    assert crop is not None
+    # Decodes as an image of plausible size (margin around the text line).
+    doc = fitz.open(stream=crop, filetype="jpeg")
+    pix = doc[0].get_pixmap()
+    assert pix.width > 100 and pix.height > 20
+    doc.close()
+
+
+def test_render_crop_bad_page(text_pdf):
+    assert render_crop(text_pdf, 99, (0, 0, 10, 10)) is None
+
+
+def test_coverage_gaps():
+    pages = [{"page_number": 1, "text_layer": "some real text"},
+             {"page_number": 2, "text_layer": "more text"},
+             {"page_number": 3, "text_layer": None}]
+    sheets = [{"page_number": 1, "assertions": [{"id": "a"}]},
+              {"page_number": 2, "assertions": []}]
+    assert coverage_gaps(pages, sheets) == [2]