feat: text-layer grounding (extractor authority, guard rescue tier, verifier oracle + hi-DPI crops)
Docker Release / build-and-push (push) Successful in 1m25s
Docker Release / release (push) Skipped

- backend/text_layer.py: PyMuPDF text-layer extraction, fuzzy evidence
  bbox matching, 300-DPI crop rendering, coverage-gap signal
- extractor (classic + agent): TEXT LAYER block appended at call sites;
  grounding guard gains text-layer rescue tier (grounding=text_layer stamp)
- verifier: {text_layer} oracle excerpt + evidence-located hi-DPI crops
  replacing full-page images (fallback preserved, I2 guard intact)
- coverage gaps: text-bearing pages with zero extraction -> failed-scope
  gap findings (agent) / log-only (classic)
- config knobs: TEXT_LAYER_ENABLED/MIN_CHARS/MAX_CHARS, VERIFY_TEXT_MAX_CHARS,
  VERIFY_HI_DPI_CROPS, VERIFY_CROP_DPI, VERIFY_CROP_MARGIN_PTS
- tests: 22 new (text_layer unit, grounding/render, runner-level flow)
Spec: docs/superpowers/specs/2026-08-12-text-layer-grounding-design.md
This commit is contained in:
2026-08-12 14:27:00 -05:00
parent 349b357e5c
commit 570300324f
14 changed files with 1665 additions and 16 deletions
+15
View File
@@ -78,3 +78,18 @@ AGENT_VERIFY_MAX_CHECKS=20
AGENT_VERIFY_SEVERITIES=critical,high
AGENT_VERIFY_REASONING_EFFORT=low
VERIFY_MAX_TOKENS=8192
# Text-layer grounding (deterministic PDF text layer via PyMuPDF)
# TEXT_LAYER_ENABLED: master switch for text-layer extraction/grounding
# TEXT_LAYER_MIN_CHARS: below this per page the sheet stays vision-only
# TEXT_LAYER_MAX_CHARS: cap of text layer injected into the extractor prompt
# VERIFY_TEXT_MAX_CHARS: cap of the text-layer excerpt in verify scopes
# VERIFY_HI_DPI_CROPS: evidence-located high-DPI crops in the verifier
# VERIFY_CROP_DPI / VERIFY_CROP_MARGIN_PTS: crop render DPI / padding (PDF points)
TEXT_LAYER_ENABLED=true
TEXT_LAYER_MIN_CHARS=20
TEXT_LAYER_MAX_CHARS=12000
VERIFY_TEXT_MAX_CHARS=8000
VERIFY_HI_DPI_CROPS=true
VERIFY_CROP_DPI=300
VERIFY_CROP_MARGIN_PTS=36
+4 -3
View File
@@ -6,7 +6,7 @@ from typing import Dict
from backend import config
from backend.agents.base import AgentResult, AgentScope, AgentUsage, failure
from backend.llm import call_json
from backend.pipeline.extractor import _normalize_sheet
from backend.pipeline.extractor import _normalize_sheet, _text_layer_block
from backend.pipeline.sheet_index import _index_input
from backend.prompts import (
EXTRACTOR_SYSTEM_PROMPT,
@@ -65,7 +65,7 @@ class SheetExtractorAgent:
page = scope.payload["page"]
instruction = EXTRACTOR_USER_INSTRUCTION.replace(
"{sheet_hint}", str(scope.payload.get("sheet_hint") or "")
)
) + _text_layer_block(page)
parsed = _wrap_bare_list(self._call(instruction, page),
page["page_number"])
if not isinstance(parsed, dict):
@@ -79,7 +79,8 @@ class SheetExtractorAgent:
)
if not isinstance(parsed, dict):
raise ValueError("no structured extraction returned")
sheet = _normalize_sheet(parsed, page["page_number"])
sheet = _normalize_sheet(parsed, page["page_number"],
page_text=page.get("text_layer"))
return AgentResult(scope_id=scope.scope_id, artifacts=[sheet])
except Exception as exc:
return failure(scope, exc)
+72 -3
View File
@@ -1,5 +1,6 @@
"""Public entry point for the scoped Agent-mode pipeline."""
import base64
import json
import os
from typing import Callable, Dict, Optional
@@ -30,6 +31,9 @@ from backend.pipeline.report import build_report, to_markdown
from backend.pipeline.sheet_index import derive_project_meta_from_cover
from backend.review.gate import build_review_queue
from backend.review.store import ReviewStore
from backend.text_layer import (
attach_text_layers, coverage_gaps, find_evidence_bbox, render_crop,
)
def run_agent_pipeline(
@@ -57,6 +61,9 @@ def run_agent_pipeline(
orchestrator.stage("Agent ingest: PDF -> images")
pages = convert_pdf_to_images(pdf_path)
page_to_b64 = {page["page_number"]: page["base64"] for page in pages}
text_dir = os.path.join(agent_dir, "text") if agent_dir else None
page_words = attach_text_layers(pdf_path, pages, text_dir=text_dir)
page_to_text = {page["page_number"]: page.get("text_layer") for page in pages}
orchestrator.stage("Agent wave 1: extract sheets")
extract_scopes = [
@@ -77,6 +84,13 @@ def run_agent_pipeline(
sheets.sort(key=lambda sheet: sheet.get("page_number") or 0)
memory.replace("sheets", sheets)
memory.dump("01-extract.json")
# Coverage signal: text layer present but extraction failed/empty reuses
# the failed-scopes gap-finding path (finding built below wave 6).
for gap_page in coverage_gaps(pages, sheets):
orchestrator.stats.failed_scopes.append(
f"sheet_extractor:sheet:{gap_page}: extraction gap "
f"(text layer present, no objects extracted)"
)
cover_meta = derive_project_meta_from_cover(
sheets, source_name or os.path.basename(pdf_path)
@@ -183,19 +197,34 @@ def run_agent_pipeline(
target_indexes = {id(f): i for i, f in enumerate(specialist_findings)}
verify_scopes = []
for finding in verify_targets:
images = [
page_to_b64[sheet_to_page[str(name)]]
for name in (finding.get("sheets") or [])[:config.AGENT_CONFLICT_MAX_IMAGES]
cited_pages = [
sheet_to_page[str(name)]
for name in (finding.get("sheets") or [])
if sheet_to_page.get(str(name)) in page_to_b64
]
images = [
page_to_b64[p]
for p in cited_pages[:config.AGENT_CONFLICT_MAX_IMAGES]
]
if not images:
continue # never judge evidence against images we could not load
# Text oracle: concatenated text layer of the cited sheets, capped.
excerpt = "\n\n".join(
f"--- Page {p} ---\n{page_to_text[p]}"
for p in cited_pages
if page_to_text.get(p)
)[:config.VERIFY_TEXT_MAX_CHARS]
if config.VERIFY_HI_DPI_CROPS:
images = _evidence_crops(finding, cited_pages, sheet_to_page,
page_words, page_to_b64, pdf_path,
fallback=images)
verify_scopes.append(AgentScope(
scope_id=f"verify:{target_indexes[id(finding)]}",
payload={
"finding_index": target_indexes[id(finding)],
"finding": finding,
"images_b64": images,
"text_layer_excerpt": excerpt,
},
))
verify_results = orchestrator.run_scopes(
@@ -394,6 +423,46 @@ def _dump(out_dir: str, name: str, value) -> None:
json.dump(value, f, indent=2)
def _evidence_crops(
finding: Dict,
cited_pages: list,
sheet_to_page: Dict,
page_words: Dict,
page_to_b64: Dict,
pdf_path: str,
fallback: list,
) -> list:
"""High-DPI crops around each evidence item's source_text, located via the
page text layer. Crops REPLACE full-page images when at least one evidence
location resolves confidently; otherwise the full-page fallback is kept.
Never returns an empty list when fallback is non-empty (I2 guard)."""
crops: list = []
for item in finding.get("evidence") or []:
if len(crops) >= config.AGENT_CONFLICT_MAX_IMAGES:
break
if not isinstance(item, dict):
continue
source_text = item.get("source_text") or ""
if not source_text:
continue
# Prefer the page named on the evidence item, then any cited page.
candidates = []
named_page = sheet_to_page.get(str(item.get("sheet") or ""))
if named_page in cited_pages:
candidates.append(named_page)
candidates.extend(p for p in cited_pages if p not in candidates)
for page in candidates:
bbox = find_evidence_bbox(page_words.get(page) or [], source_text)
if bbox is None:
continue
crop = render_crop(pdf_path, page, bbox)
if not crop:
continue
crops.append(base64.b64encode(crop).decode("utf-8"))
break
return crops or fallback
def _counts(items, key: str) -> Dict[str, int]:
counts: Dict[str, int] = {}
for item in items:
+5 -1
View File
@@ -55,7 +55,11 @@ class EvidenceVerifierAgent:
def run(self, scope: AgentScope) -> AgentResult:
try:
finding = scope.payload["finding"]
instruction = render(VERIFY_USER_INSTRUCTION, {"finding": dumps(finding)})
instruction = render(VERIFY_USER_INSTRUCTION, {
"finding": dumps(finding),
"text_layer": scope.payload.get("text_layer_excerpt")
or "(no text layer available for the cited sheets)",
})
parsed = call_json(
system_prompt=VERIFY_SYSTEM_PROMPT,
user_text=instruction,
+12
View File
@@ -59,6 +59,18 @@ AGENT_VERIFY_SEVERITIES = {
AGENT_VERIFY_REASONING_EFFORT = os.getenv("AGENT_VERIFY_REASONING_EFFORT", "low").strip()
VERIFY_MAX_TOKENS = int(os.getenv("VERIFY_MAX_TOKENS", "8192"))
# -- Text-layer grounding (deterministic PDF text layer via PyMuPDF) ----
# The vector text layer is extracted once per job and grounds the extractor,
# rescues misquoted-but-real values in the grounding guard, and serves the
# wave-5b verifier as a text oracle plus high-DPI evidence crops.
TEXT_LAYER_ENABLED = os.getenv("TEXT_LAYER_ENABLED", "true").strip().lower() in ("1", "true", "yes")
TEXT_LAYER_MIN_CHARS = int(os.getenv("TEXT_LAYER_MIN_CHARS", "20")) # below this per page -> no text layer
TEXT_LAYER_MAX_CHARS = int(os.getenv("TEXT_LAYER_MAX_CHARS", "12000")) # cap per sheet in extractor prompt
VERIFY_TEXT_MAX_CHARS = int(os.getenv("VERIFY_TEXT_MAX_CHARS", "8000"))# cap of excerpt in verify scope
VERIFY_HI_DPI_CROPS = os.getenv("VERIFY_HI_DPI_CROPS", "true").strip().lower() in ("1", "true", "yes")
VERIFY_CROP_DPI = int(os.getenv("VERIFY_CROP_DPI", "300"))
VERIFY_CROP_MARGIN_PTS = int(os.getenv("VERIFY_CROP_MARGIN_PTS", "36"))# padding around evidence bbox (PDF points)
# Agent-mode human-review gate. When on (default), Agent runs stop after the
# Brain merge and wait for human decisions before RFIs/final report/email go
# out. AGENT_REVIEW_AUDIT_SAMPLE caps how many clean clusters get added to the
+57 -8
View File
@@ -64,7 +64,8 @@ def discipline_from_sheet_number(sheet_number: Optional[str]) -> Optional[str]:
return None
def _is_grounded(value: str, source_text: str, graphical_basis: str = "") -> bool:
def _is_grounded(value: str, source_text: str, graphical_basis: str = "",
page_text: Optional[str] = None) -> bool:
"""
Keep an object only if its primary value is supported by its source_text,
OR it is a graphical object (has graphical_basis with no text to quote).
@@ -72,6 +73,9 @@ def _is_grounded(value: str, source_text: str, graphical_basis: str = "") -> boo
- If graphical_basis is set and source_text is absent, the object is valid.
- If the value contains digits, every distinct digit-run must appear in
source_text (catches invented dimensions/counts/elevations).
- Rescue tier: when page_text (the deterministic text layer) is given,
digit-runs absent from source_text but present in the page text are
still grounded - vision quoted imperfectly but the value is real.
- If the value has no digits, require some alphabetic-token overlap.
"""
# Graphical objects (no readable text on sheet) are always allowed through.
@@ -85,7 +89,11 @@ def _is_grounded(value: str, source_text: str, graphical_basis: str = "") -> boo
val_digits = set(_DIGITS_RE.findall(value))
if val_digits:
src_digits = set(_DIGITS_RE.findall(source_text))
return val_digits.issubset(src_digits)
if val_digits.issubset(src_digits):
return True
if page_text:
return val_digits.issubset(set(_DIGITS_RE.findall(page_text)))
return False
# No digits: text-based grounding.
val_norm = re.sub(r"[^a-z0-9]+", " ", value.lower()).strip()
@@ -109,7 +117,24 @@ def _primary_value(obj: Dict) -> str:
or obj.get("name") or obj.get("tag") or "")
def _normalize_sheet(parsed: Dict, page_number: int) -> Dict:
def _grounding_stamp(value: str, source_text: str,
page_text: Optional[str]) -> Optional[str]:
"""\"text_layer\" when the object survived only via the text-layer rescue
tier (digits absent from source_text but present in the page text)."""
if not page_text:
return None
val_digits = set(_DIGITS_RE.findall(str(value)))
if not val_digits:
return None
if val_digits.issubset(set(_DIGITS_RE.findall(source_text))):
return None
if val_digits.issubset(set(_DIGITS_RE.findall(page_text))):
return "text_layer"
return None
def _normalize_sheet(parsed: Dict, page_number: int,
page_text: Optional[str] = None) -> Dict:
"""
Validate + clean one parsed sheet result, attaching page_number and ids.
@@ -138,6 +163,7 @@ def _normalize_sheet(parsed: Dict, page_number: int) -> Dict:
raw_objects = parsed.get("objects") or parsed.get("assertions") or []
clean: List[Dict] = []
dropped = 0
rescued = 0
for idx, obj in enumerate(raw_objects):
if not isinstance(obj, dict):
@@ -149,9 +175,13 @@ def _normalize_sheet(parsed: Dict, page_number: int) -> Dict:
# Derive a primary value for the grounding check
primary_val = _primary_value(obj)
if not _is_grounded(primary_val, source_text, graphical_basis):
if not _is_grounded(primary_val, source_text, graphical_basis,
page_text=page_text):
dropped += 1
continue
grounding = _grounding_stamp(primary_val, source_text, page_text)
if grounding:
rescued += 1
# --- location_key: new schema is richer; map to legacy shape + extras ---
lk = obj.get("location_key")
@@ -203,10 +233,13 @@ def _normalize_sheet(parsed: Dict, page_number: int) -> Dict:
"object_attributes": attrs,
"graphical_basis": graphical_basis or None,
"review_uses": obj.get("review_uses") or [],
**({"grounding": grounding} if grounding else {}),
})
if dropped:
print(f"[Extract] Page {page_number} ({sheet_number}): dropped {dropped} ungrounded object(s)")
if dropped or rescued:
print(f"[Extract] Page {page_number} ({sheet_number}): "
f"dropped {dropped} ungrounded object(s)"
+ (f", rescued {rescued} via text layer" if rescued else ""))
unresolved = parsed.get("unresolved_items") or []
@@ -223,8 +256,23 @@ def _normalize_sheet(parsed: Dict, page_number: int) -> Dict:
}
def _text_layer_block(page: Dict) -> str:
"""
The TEXT LAYER block appended to the extractor instruction at call sites
(NOT a template placeholder - render() silently leaves missing keys as
literals). Empty string when the page has no usable text layer.
"""
text = (page.get("text_layer") or "").strip()
if not text:
return ""
return ("\n\nTEXT LAYER (authoritative for alphanumeric content — trust it "
"over the image for numbers, tags, and note text):\n"
+ text[:config.TEXT_LAYER_MAX_CHARS])
def _extract_one(page: Dict, sheet_hint: str = "") -> Dict:
user_text = EXTRACTOR_USER_INSTRUCTION.replace("{sheet_hint}", sheet_hint)
user_text = (EXTRACTOR_USER_INSTRUCTION.replace("{sheet_hint}", sheet_hint)
+ _text_layer_block(page))
parsed = call_json(
system_prompt=EXTRACTOR_SYSTEM_PROMPT,
user_text=user_text,
@@ -241,7 +289,8 @@ def _extract_one(page: Dict, sheet_hint: str = "") -> Dict:
"scale": None,
"assertions": [],
}
return _normalize_sheet(parsed, page["page_number"])
return _normalize_sheet(parsed, page["page_number"],
page_text=page.get("text_layer"))
def extract_assertions(pages: List[Dict], on_progress=None) -> List[Dict]:
+4
View File
@@ -27,6 +27,7 @@ from typing import Dict, Optional, Callable
from backend.pipeline.pdf_processor import convert_pdf_to_images
from backend.pipeline.extractor import extract_assertions
from backend.text_layer import attach_text_layers, coverage_gaps
from backend.pipeline.sheet_index import classify_sheets, derive_project_meta_from_cover
from backend.pipeline.jurisdiction import run_jurisdiction
from backend.pipeline.normalizer import normalize_assertions, build_project_intelligence
@@ -103,9 +104,12 @@ def _run_stages(
) -> Dict:
stage("PDF -> images")
pages = convert_pdf_to_images(pdf_path)
text_dir = os.path.join(out_dir, "text") if out_dir else None
attach_text_layers(pdf_path, pages, text_dir=text_dir)
stage("Extract assertions")
sheets = extract_assertions(pages)
coverage_gaps(pages, sheets) # classic: log-only recall signal
stage("Classify sheet index")
sheet_index = classify_sheets(sheets)
+4 -1
View File
@@ -230,6 +230,7 @@ Rules you must never break:
- Every object must include source_text copied verbatim from the sheet whenever text is available.
- If the object is graphical and has no text, describe it visually and mark confidence low or medium.
- Preserve tags, marks, room numbers, sheet numbers, detail references, and abbreviations exactly as shown.
TEXT LAYER GROUNDING: when a TEXT LAYER block is present in the user message, it is the sheet's deterministic PDF text layer and is authoritative for alphanumeric content (counts, dimensions, member tags, note text). Trust it over your reading of the image for numbers, tags, and note text; quote source_text from it verbatim. Use the image for geometry, symbols, linework, and anything absent from the text layer.
- Use null when information is not determinable.
- Keep objects atomic.
- Use plain ASCII only.
@@ -467,7 +468,9 @@ Respond only with valid JSON."""
VERIFY_USER_INSTRUCTION = """Verify this finding's evidence against the attached sheet images.
Respond ONLY with a valid JSON object - no markdown fences, no explanation:
{ "verdicts": [ { "sheet": "string", "source_text": "the evidence text judged", "verdict": "confirmed | corrected | not_found", "actual_text": "verbatim sheet text when corrected, else null", "notes": "string or null" } ] }
Finding: {finding}"""
Finding: {finding}
TEXT LAYER (deterministic page text extracted from the PDF - an oracle for alphanumeric content such as counts, dimensions, and member tags; when it disagrees with the extracted evidence, trust it and cite it as actual_text):
{text_layer}"""
# ---------------------------------------------------------------------------
+230
View File
@@ -0,0 +1,230 @@
"""
text_layer.py - deterministic PDF text-layer extraction (PyMuPDF, no LLM).
Most CAD-produced drawing sets carry a real vector text layer. We extract it
once per job and feed it to the extractor (grounding), the grounding guard
(rescue tier), and the wave-5b verifier (text oracle + high-DPI evidence
crops). Pages below TEXT_LAYER_MIN_CHARS of text are treated as having no
text layer (scanned/raster sheets stay vision-only).
If PyMuPDF is unavailable the module degrades gracefully: every public
function returns empty/None, equivalent to TEXT_LAYER_ENABLED=false.
"""
import re
from typing import Dict, List, Optional, Tuple
from backend import config
try: # PyMuPDF >= 1.24 prefers the pymupdf name; fitz works everywhere.
import pymupdf as fitz
except ImportError: # pragma: no cover - older PyMuPDF
try:
import fitz
except ImportError: # pragma: no cover - PyMuPDF not installed
fitz = None
_warned_unavailable = False
# Word token normalization for evidence matching: lowercase alphanumeric only.
_TOKEN_RE = re.compile(r"[^a-z0-9]+")
# Fuzzy match floor: fraction of needle tokens that must align with the page's
# word sequence for a bbox to count as a confident evidence location.
_FUZZY_MIN_RATIO = 0.6
def _fitz_or_none():
"""Return the fitz module, logging once if PyMuPDF is missing."""
global _warned_unavailable
if fitz is None and not _warned_unavailable:
print("[TextLayer] PyMuPDF not available - text-layer grounding disabled")
_warned_unavailable = True
return fitz
def extract_text_layers(pdf_path: str) -> Dict[int, Dict]:
"""
Extract the text layer of every page. Returns {1-based page_number:
{"text": str, "words": [{"text", "bbox": (x0,y0,x1,y1)}, ...],
"has_text_layer": bool}}. Returns {} when disabled or unavailable.
"""
if not config.TEXT_LAYER_ENABLED:
return {}
f = _fitz_or_none()
if f is None:
return {}
try:
doc = f.open(pdf_path)
except Exception as exc:
print(f"[TextLayer] could not open {pdf_path}: {exc}")
return {}
layers: Dict[int, Dict] = {}
try:
for index in range(doc.page_count):
page = doc[index]
text = page.get_text("text") or ""
words = [
{"text": w[4], "bbox": (w[0], w[1], w[2], w[3])}
for w in (page.get_text("words") or [])
]
has_text_layer = len(text.strip()) >= config.TEXT_LAYER_MIN_CHARS
if not has_text_layer:
print(f"[TextLayer] Page {index + 1}: {len(text.strip())} chars "
f"(< TEXT_LAYER_MIN_CHARS={config.TEXT_LAYER_MIN_CHARS}) - "
f"vision-only")
layers[index + 1] = {
"text": text,
"words": words,
"has_text_layer": has_text_layer,
}
finally:
doc.close()
return layers
def attach_text_layers(
pdf_path: str,
pages: List[Dict],
text_dir: Optional[str] = None,
) -> Dict[int, List[Dict]]:
"""
Attach page["text_layer"] (text or None) to each converted page dict and
return the runner-local {page_number: words} map (kept off page dicts -
those get serialized). When text_dir is set, dump one .txt per page there
(plain file writes; ProjectMemory is a closed registry).
"""
layers = extract_text_layers(pdf_path)
page_words: Dict[int, List[Dict]] = {}
for page in pages:
layer = layers.get(page["page_number"]) or {}
page["text_layer"] = layer.get("text") if layer.get("has_text_layer") else None
page_words[page["page_number"]] = layer.get("words") or []
if text_dir and layers:
import os
os.makedirs(text_dir, exist_ok=True)
for page_number, layer in layers.items():
if not layer.get("has_text_layer"):
continue
with open(os.path.join(text_dir, f"page-{page_number:03d}.txt"),
"w", encoding="utf-8") as fh:
fh.write(layer.get("text") or "")
return page_words
def _tokens(text: str) -> List[str]:
return [t for t in _TOKEN_RE.split(text.lower()) if t]
def _union_bbox(boxes: List[Tuple[float, float, float, float]]):
return (
min(b[0] for b in boxes),
min(b[1] for b in boxes),
max(b[2] for b in boxes),
max(b[3] for b in boxes),
)
def find_evidence_bbox(
words: List[Dict],
needle: str,
) -> Optional[Tuple[float, float, float, float]]:
"""
Best-effort fuzzy substring match of an evidence source_text against the
page's word sequence. Returns the union bbox of the matched words, or
None when nothing aligns confidently.
Exact contiguous token runs win; otherwise the best-scoring window with
>= _FUZZY_MIN_RATIO token alignment is accepted (vision quotes imperfectly
but the value is real page text).
"""
if not words or not needle:
return None
needle_tokens = _tokens(str(needle))
if not needle_tokens:
return None
page_tokens = [_tokens(w.get("text") or "") for w in words]
# Flatten multi-token words, remembering which word each token came from.
flat: List[Tuple[str, int]] = []
for word_index, parts in enumerate(page_tokens):
for part in parts:
flat.append((part, word_index))
if not flat:
return None
n = len(needle_tokens)
best_span = None
best_score = 0.0
for start in range(0, len(flat)):
window = flat[start:start + n]
if not window:
break
score = sum(1 for i, tok in enumerate(needle_tokens)
if i < len(window) and window[i][0] == tok) / n
if score > best_score:
best_score = score
best_span = window
if best_score == 1.0:
break
if best_span is None or best_score < _FUZZY_MIN_RATIO:
return None
word_indexes = {word_index for _, word_index in best_span}
return _union_bbox([words[i]["bbox"] for i in sorted(word_indexes)])
def render_crop(
pdf_path: str,
page_number: int,
bbox: Tuple[float, float, float, float],
dpi: Optional[int] = None,
margin_pts: Optional[float] = None,
) -> Optional[bytes]:
"""
Render a clip of one page around bbox (+ margin, clamped to the page) at
the given DPI and return JPEG bytes, or None on any failure.
"""
f = _fitz_or_none()
if f is None:
return None
dpi = dpi or config.VERIFY_CROP_DPI
margin_pts = config.VERIFY_CROP_MARGIN_PTS if margin_pts is None else margin_pts
try:
doc = f.open(pdf_path)
try:
page = doc[page_number - 1]
rect = f.Rect(
bbox[0] - margin_pts,
bbox[1] - margin_pts,
bbox[2] + margin_pts,
bbox[3] + margin_pts,
) & page.rect
if rect.is_empty:
return None
pix = page.get_pixmap(clip=rect, dpi=dpi)
return pix.tobytes("jpeg")
finally:
doc.close()
except Exception as exc:
print(f"[TextLayer] render_crop failed on page {page_number}: {exc}")
return None
def coverage_gaps(pages: List[Dict], sheets: List[Dict]) -> List[int]:
"""
Page numbers that have a text layer but whose extraction failed or
returned 0 objects - the silent extraction-loss signal. Logs one
[TextLayer] line per gap.
"""
by_page = {s.get("page_number"): s for s in sheets or []}
gaps: List[int] = []
for page in pages:
text = page.get("text_layer")
if not text:
continue
sheet = by_page.get(page["page_number"])
extracted = len(sheet.get("assertions") or []) if sheet else 0
if extracted == 0:
gaps.append(page["page_number"])
print(f"[TextLayer] Page {page['page_number']}: text layer present "
f"({len(text)} chars) but no objects extracted — possible "
f"extraction gap")
return gaps