feat: text-layer grounding (extractor authority, guard rescue tier, verifier oracle + hi-DPI crops)
- backend/text_layer.py: PyMuPDF text-layer extraction, fuzzy evidence
bbox matching, 300-DPI crop rendering, coverage-gap signal
- extractor (classic + agent): TEXT LAYER block appended at call sites;
grounding guard gains text-layer rescue tier (grounding=text_layer stamp)
- verifier: {text_layer} oracle excerpt + evidence-located hi-DPI crops
replacing full-page images (fallback preserved, I2 guard intact)
- coverage gaps: text-bearing pages with zero extraction -> failed-scope
gap findings (agent) / log-only (classic)
- config knobs: TEXT_LAYER_ENABLED/MIN_CHARS/MAX_CHARS, VERIFY_TEXT_MAX_CHARS,
VERIFY_HI_DPI_CROPS, VERIFY_CROP_DPI, VERIFY_CROP_MARGIN_PTS
- tests: 22 new (text_layer unit, grounding/render, runner-level flow)
Spec: docs/superpowers/specs/2026-08-12-text-layer-grounding-design.md
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
"""Public entry point for the scoped Agent-mode pipeline."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from typing import Callable, Dict, Optional
|
||||
@@ -30,6 +31,9 @@ from backend.pipeline.report import build_report, to_markdown
|
||||
from backend.pipeline.sheet_index import derive_project_meta_from_cover
|
||||
from backend.review.gate import build_review_queue
|
||||
from backend.review.store import ReviewStore
|
||||
from backend.text_layer import (
|
||||
attach_text_layers, coverage_gaps, find_evidence_bbox, render_crop,
|
||||
)
|
||||
|
||||
|
||||
def run_agent_pipeline(
|
||||
@@ -57,6 +61,9 @@ def run_agent_pipeline(
|
||||
orchestrator.stage("Agent ingest: PDF -> images")
|
||||
pages = convert_pdf_to_images(pdf_path)
|
||||
page_to_b64 = {page["page_number"]: page["base64"] for page in pages}
|
||||
text_dir = os.path.join(agent_dir, "text") if agent_dir else None
|
||||
page_words = attach_text_layers(pdf_path, pages, text_dir=text_dir)
|
||||
page_to_text = {page["page_number"]: page.get("text_layer") for page in pages}
|
||||
|
||||
orchestrator.stage("Agent wave 1: extract sheets")
|
||||
extract_scopes = [
|
||||
@@ -77,6 +84,13 @@ def run_agent_pipeline(
|
||||
sheets.sort(key=lambda sheet: sheet.get("page_number") or 0)
|
||||
memory.replace("sheets", sheets)
|
||||
memory.dump("01-extract.json")
|
||||
# Coverage signal: text layer present but extraction failed/empty reuses
|
||||
# the failed-scopes gap-finding path (finding built below wave 6).
|
||||
for gap_page in coverage_gaps(pages, sheets):
|
||||
orchestrator.stats.failed_scopes.append(
|
||||
f"sheet_extractor:sheet:{gap_page}: extraction gap "
|
||||
f"(text layer present, no objects extracted)"
|
||||
)
|
||||
|
||||
cover_meta = derive_project_meta_from_cover(
|
||||
sheets, source_name or os.path.basename(pdf_path)
|
||||
@@ -183,19 +197,34 @@ def run_agent_pipeline(
|
||||
target_indexes = {id(f): i for i, f in enumerate(specialist_findings)}
|
||||
verify_scopes = []
|
||||
for finding in verify_targets:
|
||||
images = [
|
||||
page_to_b64[sheet_to_page[str(name)]]
|
||||
for name in (finding.get("sheets") or [])[:config.AGENT_CONFLICT_MAX_IMAGES]
|
||||
cited_pages = [
|
||||
sheet_to_page[str(name)]
|
||||
for name in (finding.get("sheets") or [])
|
||||
if sheet_to_page.get(str(name)) in page_to_b64
|
||||
]
|
||||
images = [
|
||||
page_to_b64[p]
|
||||
for p in cited_pages[:config.AGENT_CONFLICT_MAX_IMAGES]
|
||||
]
|
||||
if not images:
|
||||
continue # never judge evidence against images we could not load
|
||||
# Text oracle: concatenated text layer of the cited sheets, capped.
|
||||
excerpt = "\n\n".join(
|
||||
f"--- Page {p} ---\n{page_to_text[p]}"
|
||||
for p in cited_pages
|
||||
if page_to_text.get(p)
|
||||
)[:config.VERIFY_TEXT_MAX_CHARS]
|
||||
if config.VERIFY_HI_DPI_CROPS:
|
||||
images = _evidence_crops(finding, cited_pages, sheet_to_page,
|
||||
page_words, page_to_b64, pdf_path,
|
||||
fallback=images)
|
||||
verify_scopes.append(AgentScope(
|
||||
scope_id=f"verify:{target_indexes[id(finding)]}",
|
||||
payload={
|
||||
"finding_index": target_indexes[id(finding)],
|
||||
"finding": finding,
|
||||
"images_b64": images,
|
||||
"text_layer_excerpt": excerpt,
|
||||
},
|
||||
))
|
||||
verify_results = orchestrator.run_scopes(
|
||||
@@ -394,6 +423,46 @@ def _dump(out_dir: str, name: str, value) -> None:
|
||||
json.dump(value, f, indent=2)
|
||||
|
||||
|
||||
def _evidence_crops(
|
||||
finding: Dict,
|
||||
cited_pages: list,
|
||||
sheet_to_page: Dict,
|
||||
page_words: Dict,
|
||||
page_to_b64: Dict,
|
||||
pdf_path: str,
|
||||
fallback: list,
|
||||
) -> list:
|
||||
"""High-DPI crops around each evidence item's source_text, located via the
|
||||
page text layer. Crops REPLACE full-page images when at least one evidence
|
||||
location resolves confidently; otherwise the full-page fallback is kept.
|
||||
Never returns an empty list when fallback is non-empty (I2 guard)."""
|
||||
crops: list = []
|
||||
for item in finding.get("evidence") or []:
|
||||
if len(crops) >= config.AGENT_CONFLICT_MAX_IMAGES:
|
||||
break
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
source_text = item.get("source_text") or ""
|
||||
if not source_text:
|
||||
continue
|
||||
# Prefer the page named on the evidence item, then any cited page.
|
||||
candidates = []
|
||||
named_page = sheet_to_page.get(str(item.get("sheet") or ""))
|
||||
if named_page in cited_pages:
|
||||
candidates.append(named_page)
|
||||
candidates.extend(p for p in cited_pages if p not in candidates)
|
||||
for page in candidates:
|
||||
bbox = find_evidence_bbox(page_words.get(page) or [], source_text)
|
||||
if bbox is None:
|
||||
continue
|
||||
crop = render_crop(pdf_path, page, bbox)
|
||||
if not crop:
|
||||
continue
|
||||
crops.append(base64.b64encode(crop).decode("utf-8"))
|
||||
break
|
||||
return crops or fallback
|
||||
|
||||
|
||||
def _counts(items, key: str) -> Dict[str, int]:
|
||||
counts: Dict[str, int] = {}
|
||||
for item in items:
|
||||
|
||||
Reference in New Issue
Block a user