From 3df359500c55af35dede68959cde689e06be15ae Mon Sep 17 00:00:00 2001 From: woogi Date: Mon, 10 Aug 2026 11:30:28 -0500 Subject: [PATCH] feat: wave 5b evidence verification - vision fact-check of cited sheet text --- backend/agents/construct_agent.py | 3 +- backend/agents/runner.py | 38 ++++++++++++- backend/agents/verifier.py | 95 +++++++++++++++++++++++++++++++ tests/agents/test_verifier.py | 68 ++++++++++++++++++++++ 4 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 backend/agents/verifier.py create mode 100644 tests/agents/test_verifier.py diff --git a/backend/agents/construct_agent.py b/backend/agents/construct_agent.py index 2bd15cf..5d3487d 100644 --- a/backend/agents/construct_agent.py +++ b/backend/agents/construct_agent.py @@ -65,7 +65,8 @@ class ConstructabilityAgent: lambda item: validate_issue(item, "constructability"), ) for finding in findings: - finding.update(agent=self.name, scope_id=scope.scope_id) + finding.update(agent=self.name, scope_id=scope.scope_id, + cluster_key=cluster.get("key")) return AgentResult(scope_id=scope.scope_id, artifacts=findings) except Exception as exc: return failure(scope, exc) diff --git a/backend/agents/runner.py b/backend/agents/runner.py index 538a0ec..29238ac 100644 --- a/backend/agents/runner.py +++ b/backend/agents/runner.py @@ -21,6 +21,9 @@ from backend.agents.linker import LinkerAgent, build_link_scopes, build_object_g from backend.agents.memory import ProjectMemory from backend.agents.orchestrator import Orchestrator from backend.agents.rfi_writer import RFIWriterAgent +from backend.agents.verifier import ( + EvidenceVerifierAgent, apply_verdicts, select_findings, +) from backend.llm import reset_cost from backend.pipeline.pdf_processor import convert_pdf_to_images from backend.pipeline.report import build_report, to_markdown @@ -168,6 +171,38 @@ def run_agent_pipeline( for result in code_results + construct_results + completeness_results for artifact in result.artifacts ] + + orchestrator.stage("Agent wave 5b: evidence verification") + sheet_to_page = {s.get("sheet_number"): s.get("page_number") for s in sheets} + verify_targets = select_findings( + specialist_findings, clusters, + max_checks=config.AGENT_VERIFY_MAX_CHECKS, + severities=config.AGENT_VERIFY_SEVERITIES, + ) + target_indexes = {id(f): i for i, f in enumerate(specialist_findings)} + verify_scopes = [ + AgentScope( + scope_id=f"verify:{target_indexes[id(finding)]}", + payload={ + "finding_index": target_indexes[id(finding)], + "finding": finding, + "images_b64": [ + page_to_b64[sheet_to_page[name]] + for name in (finding.get("sheets") or [])[:config.AGENT_CONFLICT_MAX_IMAGES] + if sheet_to_page.get(name) in page_to_b64 + ], + }, + ) + for finding in verify_targets + ] + verify_results = orchestrator.run_scopes( + EvidenceVerifierAgent(usage), verify_scopes, config.AGENT_VERIFY_CONCURRENCY) + suppressed = apply_verdicts(specialist_findings, verify_results) + if suppressed: + suppressed_ids = {id(f) for f in suppressed} + specialist_findings = [f for f in specialist_findings if id(f) not in suppressed_ids] + memory.replace("suppressed", suppressed) + memory.extend("findings", specialist_findings) gap_findings = [ { @@ -225,7 +260,7 @@ def run_agent_pipeline( "project_intelligence": object_graph, "validated_issues": prioritized, "rfis": [], - "suppressed_issues": [], + "suppressed_issues": memory.snapshot().get("suppressed") or [], }) progress = store.progress(queue) # Same usage/stats summary block as the wave-7 path (rfis: 0 — they @@ -301,6 +336,7 @@ def run_agent_pipeline( "project_intelligence": object_graph, "validated_issues": prioritized, "rfis": rfis, + "suppressed_issues": memory.snapshot().get("suppressed") or [], }) cost = usage.snapshot() orchestrator.stats.calls = cost["calls"] diff --git a/backend/agents/verifier.py b/backend/agents/verifier.py new file mode 100644 index 0000000..365fea5 --- /dev/null +++ b/backend/agents/verifier.py @@ -0,0 +1,95 @@ +"""Wave 5b: vision fact-check of extracted evidence against cited sheet images.""" + +from backend import config +from backend.agents.base import AgentResult, AgentScope, AgentUsage, failure +from backend.llm import call_json +from backend.pipeline._serialize import dumps +from backend.pipeline._stage import collect_list, render +from backend.prompts import VERIFY_SYSTEM_PROMPT, VERIFY_USER_INSTRUCTION + +_SEVERITY_RANK = {"critical": 0, "high": 1, "medium": 2, "low": 3} +_VERDICTS = ("confirmed", "corrected", "not_found") + + +def select_findings(findings, clusters, max_checks, severities): + """Severity-gated selection plus any finding tied to a disputed cluster.""" + disputed_keys = {c.get("key") for c in clusters if c.get("disputed_attributes")} + selected = [f for f in findings + if str(f.get("severity") or "").lower() in severities + or f.get("cluster_key") in disputed_keys] + selected.sort(key=lambda f: _SEVERITY_RANK.get( + str(f.get("severity") or "").lower(), 9)) + return selected[:max_checks] + + +def _valid_verdict(item): + if not isinstance(item, dict): + return None + verdict = str(item.get("verdict") or "").lower() + if verdict not in _VERDICTS: + return None + return {"sheet": item.get("sheet") or "", + "source_text": item.get("source_text") or "", + "verdict": verdict, + "actual_text": item.get("actual_text"), + "notes": item.get("notes")} + + +def _status(verdicts): + if not verdicts: + return "unverified" + confirmed = sum(1 for v in verdicts if v["verdict"] == "confirmed") + if confirmed == len(verdicts): + return "confirmed" + if confirmed == 0: + return "refuted" + return "mixed" + + +class EvidenceVerifierAgent: + name = "verify" + + def __init__(self, usage: AgentUsage) -> None: + self.usage = usage + + def run(self, scope: AgentScope) -> AgentResult: + try: + finding = scope.payload["finding"] + instruction = render(VERIFY_USER_INSTRUCTION, {"finding": dumps(finding)}) + parsed = call_json( + system_prompt=VERIFY_SYSTEM_PROMPT, + user_text=instruction, + images_b64=scope.payload.get("images_b64") or [], + max_tokens=config.VERIFY_MAX_TOKENS, + model=config.AGENT_VERIFY_MODEL, + reasoning_effort=config.AGENT_VERIFY_REASONING_EFFORT or None, + usage_tracker=self.usage, + usage_stage="agent.verify", + ) + verdicts = collect_list(parsed, "verdicts", _valid_verdict) + return AgentResult(scope_id=scope.scope_id, artifacts=[{ + "finding_index": scope.payload["finding_index"], + "status": _status(verdicts), + "verdicts": verdicts, + }]) + except Exception as exc: + return failure(scope, exc) + + +def apply_verdicts(findings, verify_results): + """Annotate findings with verification; return refuted ones to suppress.""" + by_index = {} + for result in verify_results: + for artifact in result.artifacts: + by_index[artifact["finding_index"]] = artifact + suppressed = [] + for index, finding in enumerate(findings): + artifact = by_index.get(index) + if not artifact: + continue + finding["verification"] = {"status": artifact["status"], + "verdicts": artifact["verdicts"]} + if artifact["status"] == "refuted": + finding["confidence"] = "low" + suppressed.append(finding) + return suppressed diff --git a/tests/agents/test_verifier.py b/tests/agents/test_verifier.py new file mode 100644 index 0000000..a4388a5 --- /dev/null +++ b/tests/agents/test_verifier.py @@ -0,0 +1,68 @@ +from unittest.mock import patch + +from backend.agents.base import AgentScope, AgentUsage + +from backend.agents.verifier import ( + EvidenceVerifierAgent, apply_verdicts, select_findings, +) + + +def _finding(sev="critical", issue_id="i1", sheets=("S401",), cluster_key=None): + f = {"issue_id": issue_id, "severity": sev, "confidence": "high", + "source_stage": "constructability", "sheets": list(sheets), + "description": "HSS16x4 on (2) 2x6 STUD PACK is unbuildable", + "evidence": [{"sheet": "S401", "source_text": "(2) 2x6 STUD PACK", + "asserted_value": "3-inch width"}]} + if cluster_key: + f["cluster_key"] = cluster_key + return f + + +def test_select_findings_by_severity_and_dispute(): + findings = [_finding("critical"), _finding("low", "i2"), + _finding("medium", "i3", cluster_key="c9")] + clusters = [{"key": "c9", "disputed_attributes": [{"attribute": "a"}]}] + selected = select_findings(findings, clusters, max_checks=20, + severities={"critical", "high"}) + assert [f["issue_id"] for f in selected] == ["i1", "i3"] + + +def test_select_findings_respects_cap(): + findings = [_finding("critical", f"i{n}") for n in range(30)] + selected = select_findings(findings, [], max_checks=5, + severities={"critical"}) + assert len(selected) == 5 + + +def test_run_attaches_verdicts_and_marks_refuted(): + agent = EvidenceVerifierAgent(usage=AgentUsage()) + scope = AgentScope(scope_id="verify:0", payload={ + "finding_index": 0, + "finding": _finding(), + "images_b64": ["QUJD"], + }) + verdicts = {"verdicts": [ + {"sheet": "S401", "source_text": "(2) 2x6 STUD PACK", + "verdict": "corrected", "actual_text": "(5) 2x6 STUD PACK", + "notes": "callout reads (5)"}, + ]} + with patch("backend.agents.verifier.call_json", return_value=verdicts): + result = agent.run(scope) + assert not result.error + artifact = result.artifacts[0] + assert artifact["finding_index"] == 0 + assert artifact["status"] == "refuted" # no evidence confirmed + assert artifact["verdicts"][0]["actual_text"] == "(5) 2x6 STUD PACK" + + +def test_apply_verdicts_annotates_and_suppresses(): + from backend.agents.base import AgentResult + findings = [_finding("critical", "i1"), _finding("high", "i2")] + results = [AgentResult(scope_id="verify:0", artifacts=[ + {"finding_index": 0, "status": "refuted", "verdicts": []}, + {"finding_index": 1, "status": "confirmed", "verdicts": []}, + ])] + suppressed = apply_verdicts(findings, results) + assert suppressed == [findings[0]] + assert findings[0]["verification"]["status"] == "refuted" + assert findings[1]["verification"]["status"] == "confirmed"