feat: refocus on drawings — code/ADA gated off, drawing-integrity wave, Brain-directed clarification
- ENABLE_CODE_REVIEW flag (default off): skips code/ADA/jurisdiction review path in both pipelines; nothing deleted, one env flag to restore. - Per-sheet Drawing Integrity QA wave (agent + classic, default on): dangling refs, on-sheet contradictions, dimension sanity, missing sheet essentials, tag hygiene. New DrawingIntegrityAgent + classic stage. - Broadened conflict critic: intra-sheet + same-discipline contradictions, not just cross-discipline. - Wave 6.5 Brain-directed clarification (bounded hub-and-spoke): Brain names uncertain findings, verify_evidence requests route through the wave-5b verifier; refuted findings suppressed. One planning call + capped verifies, single iteration. Shared _build_verify_scopes across 5b and 6.5. - Config knobs, .env.example, frontend copy, tests (182 passing).
This commit is contained in:
@@ -0,0 +1,198 @@
|
||||
"""Wave 6.5 Brain-directed clarification: planning unit + runner integration."""
|
||||
|
||||
import backend.agents.brain as brain_mod
|
||||
import backend.agents.runner as runner_mod
|
||||
from backend import config
|
||||
from backend.agents.base import AgentResult
|
||||
from backend.agents.base import AgentUsage
|
||||
from backend.agents.brain import BrainAgent
|
||||
from backend.agents.runner import run_agent_pipeline
|
||||
|
||||
|
||||
# --- plan_clarifications unit tests ---------------------------------------
|
||||
|
||||
def _finding(issue_id, **kw):
|
||||
base = {"issue_id": issue_id, "severity": "medium", "confidence": "low",
|
||||
"source_stage": "conflict", "description": "d",
|
||||
"evidence": [{"sheet": "A1", "source_text": "x"}]}
|
||||
base.update(kw)
|
||||
return base
|
||||
|
||||
|
||||
def test_plan_caps_and_filters_unknown_ids(monkeypatch):
|
||||
monkeypatch.setattr(config, "BRAIN_CLARIFY_MAX_REQUESTS", 2)
|
||||
monkeypatch.setattr(brain_mod, "call_json", lambda **k: {"requests": [
|
||||
{"issue_id": "A", "request_type": "verify_evidence", "reason": "thin"},
|
||||
{"issue_id": "GHOST", "request_type": "verify_evidence", "reason": "x"},
|
||||
{"issue_id": "B", "request_type": "verify_evidence", "reason": "amb"},
|
||||
{"issue_id": "C", "request_type": "verify_evidence", "reason": "over cap"},
|
||||
]})
|
||||
prioritized = [_finding("A"), _finding("B"), _finding("C")]
|
||||
reqs = BrainAgent(AgentUsage()).plan_clarifications(prioritized)
|
||||
ids = [r["issue_id"] for r in reqs]
|
||||
assert ids == ["A", "B"] # GHOST filtered, capped at 2
|
||||
|
||||
|
||||
def test_plan_skips_already_verified(monkeypatch):
|
||||
monkeypatch.setattr(config, "BRAIN_CLARIFY_MAX_REQUESTS", 8)
|
||||
captured = {}
|
||||
|
||||
def fake(**kwargs):
|
||||
captured["user_text"] = kwargs["user_text"]
|
||||
return {"requests": [
|
||||
{"issue_id": "A", "request_type": "verify_evidence", "reason": "y"},
|
||||
]}
|
||||
|
||||
monkeypatch.setattr(brain_mod, "call_json", fake)
|
||||
prioritized = [
|
||||
_finding("A"),
|
||||
_finding("V", verification={"status": "confirmed", "verdicts": []}),
|
||||
]
|
||||
reqs = BrainAgent(AgentUsage()).plan_clarifications(prioritized)
|
||||
assert [r["issue_id"] for r in reqs] == ["A"]
|
||||
# The already-verified finding must not even be offered to the model.
|
||||
assert '"V"' not in captured["user_text"]
|
||||
|
||||
|
||||
def test_plan_empty_on_call_failure(monkeypatch):
|
||||
monkeypatch.setattr(config, "BRAIN_CLARIFY_MAX_REQUESTS", 8)
|
||||
|
||||
def boom(**k):
|
||||
raise RuntimeError("brain down")
|
||||
monkeypatch.setattr(brain_mod, "call_json", boom)
|
||||
assert BrainAgent(AgentUsage()).plan_clarifications([_finding("A")]) == []
|
||||
|
||||
|
||||
def test_plan_no_requests_returns_empty(monkeypatch):
|
||||
monkeypatch.setattr(config, "BRAIN_CLARIFY_MAX_REQUESTS", 8)
|
||||
monkeypatch.setattr(brain_mod, "call_json", lambda **k: {"requests": []})
|
||||
assert BrainAgent(AgentUsage()).plan_clarifications([_finding("A")]) == []
|
||||
|
||||
|
||||
# --- runner-level integration ---------------------------------------------
|
||||
|
||||
def _stub_agent(artifacts):
|
||||
return lambda usage: type("S", (), {
|
||||
"name": "stub",
|
||||
"run": lambda self, scope: AgentResult(
|
||||
scope_id=scope.scope_id, artifacts=list(artifacts)),
|
||||
})()
|
||||
|
||||
|
||||
def _patch_pipeline(monkeypatch, brain_finding, plan_requests):
|
||||
monkeypatch.setattr(
|
||||
runner_mod, "convert_pdf_to_images",
|
||||
lambda path: [{"page_number": 1, "base64": "QUJD"}])
|
||||
monkeypatch.setattr(runner_mod, "SheetExtractorAgent", _stub_agent([
|
||||
{"sheet_number": "S401", "page_number": 1, "level": "roof",
|
||||
"discipline": "S", "assertions": [
|
||||
{"text": "(2) 2x6 STUD PACK", "object_type": "framing"},
|
||||
{"text": "HSS16X4 beam", "object_type": "framing"},
|
||||
]},
|
||||
]))
|
||||
monkeypatch.setattr(runner_mod, "SheetIndexAgent", _stub_agent([{}]))
|
||||
monkeypatch.setattr(runner_mod, "JurisdictionAgent", _stub_agent([{}]))
|
||||
monkeypatch.setattr(runner_mod, "LinkerAgent", _stub_agent([
|
||||
{"key": "c1", "location": "roof beam pocket", "assertions": []},
|
||||
]))
|
||||
# A filler conflict finding so memory["findings"] is non-empty and wave 6
|
||||
# actually invokes Brain.run (which our stub replaces with brain_finding).
|
||||
monkeypatch.setattr(runner_mod, "ConflictCriticAgent", _stub_agent([
|
||||
{"issue_id": "FILLER", "severity": "low", "confidence": "low",
|
||||
"source_stage": "conflict", "sheets": [], "description": "filler",
|
||||
"evidence": []},
|
||||
]))
|
||||
monkeypatch.setattr(runner_mod, "CodeAgent", _stub_agent([]))
|
||||
monkeypatch.setattr(runner_mod, "ConstructabilityAgent", _stub_agent([]))
|
||||
monkeypatch.setattr(runner_mod, "CompletenessAgent", _stub_agent([]))
|
||||
monkeypatch.setattr(runner_mod, "DrawingIntegrityAgent", _stub_agent([]))
|
||||
# Brain.run returns our finding; plan_clarifications returns the requests.
|
||||
monkeypatch.setattr(
|
||||
runner_mod, "BrainAgent",
|
||||
lambda usage: type("B", (), {
|
||||
"run": lambda self, findings, si, ju: ([dict(brain_finding)], []),
|
||||
"plan_clarifications": lambda self, prioritized: list(plan_requests),
|
||||
})())
|
||||
|
||||
|
||||
def test_brain_clarify_refutes_and_suppresses(monkeypatch, tmp_path):
|
||||
"""Brain flags a MEDIUM finding wave-5b's severity gate skipped; the
|
||||
clarification verifier refutes it, so it moves to suppressed_issues."""
|
||||
monkeypatch.setattr(config, "ENABLE_BRAIN_CLARIFY", True)
|
||||
finding = {
|
||||
"issue_id": "M1", "severity": "medium", "confidence": "low",
|
||||
"source_stage": "conflict", "sheets": ["S401"],
|
||||
"description": "beam bears on (2) 2x6 stud pack",
|
||||
"evidence": [{"sheet": "S401", "source_text": "(2) 2x6 STUD PACK"}],
|
||||
}
|
||||
_patch_pipeline(monkeypatch, finding, [
|
||||
{"issue_id": "M1", "request_type": "verify_evidence", "reason": "misread?"},
|
||||
])
|
||||
# The clarification verifier returns a 'corrected' verdict -> refuted.
|
||||
monkeypatch.setattr(
|
||||
"backend.agents.verifier.call_json",
|
||||
lambda **kwargs: {"verdicts": [
|
||||
{"sheet": "S401", "source_text": "(2) 2x6 STUD PACK",
|
||||
"verdict": "corrected", "actual_text": "(5) 2x6 STUD PACK",
|
||||
"notes": "reads (5)"},
|
||||
]})
|
||||
pdf = tmp_path / "d.pdf"
|
||||
pdf.write_bytes(b"%PDF-1.4\n")
|
||||
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path),
|
||||
require_review=False)
|
||||
validated = report.get("validated_issues") or []
|
||||
assert all(f.get("issue_id") != "M1" for f in validated) # dropped
|
||||
assert [f["issue_id"] for f in report["suppressed_issues"]] == ["M1"]
|
||||
assert report["suppressed_issues"][0]["verification"]["status"] == "refuted"
|
||||
|
||||
|
||||
def test_brain_clarify_confirms_keeps_finding(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(config, "ENABLE_BRAIN_CLARIFY", True)
|
||||
finding = {
|
||||
"issue_id": "M2", "severity": "medium", "confidence": "low",
|
||||
"source_stage": "conflict", "sheets": ["S401"],
|
||||
"description": "beam bears on (5) 2x6 stud pack",
|
||||
"evidence": [{"sheet": "S401", "source_text": "(5) 2x6 STUD PACK"}],
|
||||
}
|
||||
_patch_pipeline(monkeypatch, finding, [
|
||||
{"issue_id": "M2", "request_type": "verify_evidence", "reason": "check"},
|
||||
])
|
||||
monkeypatch.setattr(
|
||||
"backend.agents.verifier.call_json",
|
||||
lambda **kwargs: {"verdicts": [
|
||||
{"sheet": "S401", "source_text": "(5) 2x6 STUD PACK",
|
||||
"verdict": "confirmed", "actual_text": None, "notes": None},
|
||||
]})
|
||||
pdf = tmp_path / "d.pdf"
|
||||
pdf.write_bytes(b"%PDF-1.4\n")
|
||||
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path),
|
||||
require_review=False)
|
||||
validated = report.get("validated_issues") or []
|
||||
kept = [f for f in validated if f.get("issue_id") == "M2"]
|
||||
assert len(kept) == 1
|
||||
assert kept[0]["verification"]["status"] == "confirmed"
|
||||
assert report["suppressed_issues"] == []
|
||||
|
||||
|
||||
def test_brain_clarify_disabled_is_noop(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(config, "ENABLE_BRAIN_CLARIFY", False)
|
||||
finding = {
|
||||
"issue_id": "M3", "severity": "medium", "confidence": "low",
|
||||
"source_stage": "conflict", "sheets": ["S401"],
|
||||
"description": "d", "evidence": [{"sheet": "S401", "source_text": "t"}],
|
||||
}
|
||||
# plan_clarifications should never be consulted; give it a bomb to prove it.
|
||||
def _bomb(self, prioritized):
|
||||
raise AssertionError("plan_clarifications must not run when disabled")
|
||||
_patch_pipeline(monkeypatch, finding, [])
|
||||
monkeypatch.setattr(
|
||||
runner_mod, "BrainAgent",
|
||||
lambda usage: type("B", (), {
|
||||
"run": lambda self, findings, si, ju: ([dict(finding)], []),
|
||||
"plan_clarifications": _bomb})())
|
||||
pdf = tmp_path / "d.pdf"
|
||||
pdf.write_bytes(b"%PDF-1.4\n")
|
||||
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path),
|
||||
require_review=False)
|
||||
validated = report.get("validated_issues") or []
|
||||
assert any(f.get("issue_id") == "M3" for f in validated)
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Unit tests for the per-sheet DrawingIntegrityAgent and scope builder."""
|
||||
|
||||
import backend.agents.integrity_agent as integ
|
||||
from backend.agents.base import AgentScope, AgentUsage
|
||||
from backend.agents.integrity_agent import (
|
||||
DrawingIntegrityAgent, build_integrity_scopes,
|
||||
)
|
||||
from backend import config
|
||||
|
||||
|
||||
def _sheet(page, sheet_number, n_assertions):
|
||||
return {
|
||||
"sheet_number": sheet_number,
|
||||
"page_number": page,
|
||||
"sheet_title": f"Sheet {sheet_number}",
|
||||
"discipline": "Architectural",
|
||||
"assertions": [
|
||||
{"object_type": "note", "source_text": f"note {i}"}
|
||||
for i in range(n_assertions)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_build_scopes_skips_sparse_sheets(monkeypatch):
|
||||
monkeypatch.setattr(config, "INTEGRITY_MIN_ASSERTIONS", 3)
|
||||
sheets = [
|
||||
_sheet(1, "A101", 5), # kept
|
||||
_sheet(2, "A102", 2), # skipped (too sparse)
|
||||
_sheet(3, "A103", 3), # kept (== floor)
|
||||
]
|
||||
page_to_b64 = {1: "IMG1", 2: "IMG2", 3: "IMG3"}
|
||||
page_to_text = {1: "text one", 2: "text two", 3: "text three"}
|
||||
scopes = build_integrity_scopes(sheets, page_to_b64, page_to_text)
|
||||
ids = sorted(s.scope_id for s in scopes)
|
||||
assert ids == ["integrity:1", "integrity:3"]
|
||||
# Each scope carries its own page image + text layer.
|
||||
by_id = {s.scope_id: s for s in scopes}
|
||||
assert by_id["integrity:1"].payload["image_b64"] == "IMG1"
|
||||
assert by_id["integrity:1"].payload["text_layer"] == "text one"
|
||||
|
||||
|
||||
def test_agent_parses_and_anchors_sheet(monkeypatch):
|
||||
"""Findings with blank sheets get anchored to the scope's sheet number."""
|
||||
captured = {}
|
||||
|
||||
def fake_call_json(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"issues": [
|
||||
{"issue_id": "DI-1", "severity": "high", "confidence": "high",
|
||||
"category": "dangling_reference", "sheets": [],
|
||||
"description": "Detail callout 5/A101 has no detail 5 on this sheet",
|
||||
"evidence": [{"sheet": "A101", "source_text": "5/A101"}]},
|
||||
]}
|
||||
|
||||
monkeypatch.setattr(integ, "call_json", fake_call_json)
|
||||
sheet = _sheet(1, "A101", 5)
|
||||
scope = AgentScope("integrity:1", {
|
||||
"sheet": sheet, "page_number": 1,
|
||||
"image_b64": "IMG1", "text_layer": "the deterministic text layer",
|
||||
})
|
||||
result = DrawingIntegrityAgent(AgentUsage()).run(scope)
|
||||
assert result.error == ""
|
||||
assert len(result.artifacts) == 1
|
||||
finding = result.artifacts[0]
|
||||
assert finding["source_stage"] == "drawing_integrity"
|
||||
assert finding["sheets"] == ["A101"] # anchored
|
||||
assert finding["agent"] == "drawing_integrity"
|
||||
assert finding["scope_id"] == "integrity:1"
|
||||
# The image + text layer reached the model.
|
||||
assert captured["images_b64"] == ["IMG1"]
|
||||
assert "the deterministic text layer" in captured["user_text"]
|
||||
|
||||
|
||||
def test_agent_empty_issues_is_clean(monkeypatch):
|
||||
monkeypatch.setattr(integ, "call_json", lambda **k: {"issues": []})
|
||||
scope = AgentScope("integrity:1", {
|
||||
"sheet": _sheet(1, "A101", 5), "page_number": 1,
|
||||
"image_b64": "IMG1", "text_layer": "t",
|
||||
})
|
||||
result = DrawingIntegrityAgent(AgentUsage()).run(scope)
|
||||
assert result.error == ""
|
||||
assert result.artifacts == []
|
||||
|
||||
|
||||
def test_agent_survives_call_failure(monkeypatch):
|
||||
def boom(**kwargs):
|
||||
raise RuntimeError("model exploded")
|
||||
monkeypatch.setattr(integ, "call_json", boom)
|
||||
scope = AgentScope("integrity:1", {
|
||||
"sheet": _sheet(1, "A101", 5), "page_number": 1,
|
||||
"image_b64": "IMG1", "text_layer": "t",
|
||||
})
|
||||
result = DrawingIntegrityAgent(AgentUsage()).run(scope)
|
||||
assert "model exploded" in result.error
|
||||
assert result.artifacts == []
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Flag-gating tests: ENABLE_CODE_REVIEW off skips code, drawing_integrity runs.
|
||||
|
||||
Runner-level smoke tests using stubbed agents (same pattern as
|
||||
test_wave5b_suppression). Verifies the code/ADA wave is skipped when
|
||||
ENABLE_CODE_REVIEW is false and the Drawing Integrity wave feeds findings
|
||||
into the report by_stage counters.
|
||||
"""
|
||||
|
||||
import backend.agents.runner as runner_mod
|
||||
from backend import config
|
||||
from backend.agents.base import AgentResult
|
||||
from backend.agents.runner import run_agent_pipeline
|
||||
|
||||
|
||||
def _stub_agent(artifacts):
|
||||
return lambda usage: type("S", (), {
|
||||
"name": "stub",
|
||||
"run": lambda self, scope: AgentResult(
|
||||
scope_id=scope.scope_id, artifacts=list(artifacts)),
|
||||
})()
|
||||
|
||||
|
||||
def _integrity_finding():
|
||||
return {
|
||||
"issue_id": "DI-1", "severity": "high", "confidence": "high",
|
||||
"source_stage": "drawing_integrity", "sheets": ["A101"],
|
||||
"category": "dangling_reference",
|
||||
"description": "Detail callout 5/A101 has no detail 5 on this sheet",
|
||||
"evidence": [{"sheet": "A101", "source_text": "5/A101"}],
|
||||
}
|
||||
|
||||
|
||||
def _patch(monkeypatch, code_should_raise):
|
||||
monkeypatch.setattr(
|
||||
runner_mod, "convert_pdf_to_images",
|
||||
lambda path: [{"page_number": 1, "base64": "QUJD"}])
|
||||
# Sheet has 3+ assertions so the integrity wave does NOT skip it.
|
||||
monkeypatch.setattr(runner_mod, "SheetExtractorAgent", _stub_agent([
|
||||
{"sheet_number": "A101", "page_number": 1, "level": "1",
|
||||
"discipline": "A", "assertions": [
|
||||
{"text": "5/A101", "object_type": "detail_marker"},
|
||||
{"text": "ROOM 101", "object_type": "room"},
|
||||
{"text": "DOOR 101A", "object_type": "door"},
|
||||
]},
|
||||
]))
|
||||
monkeypatch.setattr(runner_mod, "SheetIndexAgent", _stub_agent([{}]))
|
||||
monkeypatch.setattr(runner_mod, "JurisdictionAgent", _stub_agent([{}]))
|
||||
monkeypatch.setattr(runner_mod, "LinkerAgent", _stub_agent([]))
|
||||
monkeypatch.setattr(runner_mod, "ConflictCriticAgent", _stub_agent([]))
|
||||
|
||||
def code_boom(usage):
|
||||
if code_should_raise:
|
||||
raise AssertionError("CodeAgent must not run when gated off")
|
||||
return _stub_agent([])(usage)
|
||||
monkeypatch.setattr(runner_mod, "CodeAgent", code_boom)
|
||||
|
||||
monkeypatch.setattr(runner_mod, "DrawingIntegrityAgent",
|
||||
_stub_agent([_integrity_finding()]))
|
||||
monkeypatch.setattr(runner_mod, "ConstructabilityAgent", _stub_agent([]))
|
||||
monkeypatch.setattr(runner_mod, "CompletenessAgent", _stub_agent([]))
|
||||
monkeypatch.setattr(
|
||||
runner_mod, "BrainAgent",
|
||||
lambda usage: type("B", (), {
|
||||
"run": lambda self, findings, si, ju: (list(findings), []),
|
||||
"plan_clarifications": lambda self, prioritized: []})())
|
||||
# Stub the wave-5b verifier so the high-severity integrity finding is
|
||||
# confirmed (never a live network call).
|
||||
monkeypatch.setattr(
|
||||
"backend.agents.verifier.call_json",
|
||||
lambda **kwargs: {"verdicts": [
|
||||
{"sheet": "A101", "source_text": "5/A101",
|
||||
"verdict": "confirmed", "actual_text": None, "notes": None},
|
||||
]})
|
||||
|
||||
|
||||
def test_code_gated_off_integrity_on(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(config, "ENABLE_CODE_REVIEW", False)
|
||||
monkeypatch.setattr(config, "ENABLE_DRAWING_INTEGRITY", True)
|
||||
_patch(monkeypatch, code_should_raise=True)
|
||||
pdf = tmp_path / "d.pdf"
|
||||
pdf.write_bytes(b"%PDF-1.4\n")
|
||||
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path),
|
||||
require_review=False)
|
||||
by_stage = report["summary"]["by_stage"]
|
||||
assert by_stage["code"] == 0
|
||||
assert by_stage["drawing_integrity"] == 1
|
||||
# The integrity finding survived into the validated set.
|
||||
assert any(f.get("issue_id") == "DI-1"
|
||||
for f in report.get("validated_issues") or [])
|
||||
|
||||
|
||||
def test_code_enabled_runs(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(config, "ENABLE_CODE_REVIEW", True)
|
||||
monkeypatch.setattr(config, "ENABLE_DRAWING_INTEGRITY", True)
|
||||
_patch(monkeypatch, code_should_raise=False)
|
||||
# build_code_scopes runs on the real sheet; CodeAgent is stubbed to []
|
||||
pdf = tmp_path / "d.pdf"
|
||||
pdf.write_bytes(b"%PDF-1.4\n")
|
||||
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path),
|
||||
require_review=False)
|
||||
# No crash; integrity still reported.
|
||||
assert report["summary"]["by_stage"]["drawing_integrity"] == 1
|
||||
@@ -4,7 +4,7 @@ from backend.agents.runner import run_agent_pipeline
|
||||
|
||||
def _patch_brain(monkeypatch):
|
||||
monkeypatch.setattr("backend.agents.runner.convert_pdf_to_images", lambda path: [{"page_number": 1, "base64": "x"}])
|
||||
monkeypatch.setattr("backend.agents.runner.BrainAgent", lambda usage: type("B", (), {"run": lambda self, findings, sheet_index, jurisdiction: ([{"issue_id": "AGENT-0001", "severity": "high", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"}], [])})())
|
||||
monkeypatch.setattr("backend.agents.runner.BrainAgent", lambda usage: type("B", (), {"run": lambda self, findings, sheet_index, jurisdiction: ([{"issue_id": "AGENT-0001", "severity": "high", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"}], []), "plan_clarifications": lambda self, prioritized: []})())
|
||||
|
||||
|
||||
def test_agent_runner_can_enter_review_mode(monkeypatch, tmp_path):
|
||||
|
||||
@@ -63,7 +63,8 @@ def _patch_pipeline(monkeypatch, finding, verify_sink):
|
||||
runner_mod, "BrainAgent",
|
||||
lambda usage: type("B", (), {
|
||||
"run": lambda self, findings, sheet_index, jurisdiction:
|
||||
(list(findings), [])})())
|
||||
(list(findings), []),
|
||||
"plan_clarifications": lambda self, prioritized: []})())
|
||||
|
||||
class _RecordingVerifier:
|
||||
name = "verify"
|
||||
|
||||
@@ -46,7 +46,8 @@ def _patch_pipeline(monkeypatch, finding):
|
||||
runner_mod, "BrainAgent",
|
||||
lambda usage: type("B", (), {
|
||||
"run": lambda self, findings, sheet_index, jurisdiction:
|
||||
(list(findings), [])})())
|
||||
(list(findings), []),
|
||||
"plan_clarifications": lambda self, prioritized: []})())
|
||||
|
||||
|
||||
def test_refuted_finding_is_suppressed_not_crash(monkeypatch, tmp_path):
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Tests for the classic-path drawing_integrity_review stage + gating."""
|
||||
|
||||
import backend.pipeline.drawing_integrity as di
|
||||
from backend import config
|
||||
from backend.pipeline.drawing_integrity import drawing_integrity_review
|
||||
|
||||
|
||||
def _sheet(page, sheet_number, n):
|
||||
return {
|
||||
"sheet_number": sheet_number,
|
||||
"page_number": page,
|
||||
"discipline": "Architectural",
|
||||
"assertions": [{"source_text": f"n{i}"} for i in range(n)],
|
||||
}
|
||||
|
||||
|
||||
def _pages(pages):
|
||||
return [{"page_number": p, "base64": f"IMG{p}", "text_layer": f"txt{p}"}
|
||||
for p in pages]
|
||||
|
||||
|
||||
def test_disabled_returns_empty(monkeypatch):
|
||||
monkeypatch.setattr(config, "ENABLE_DRAWING_INTEGRITY", False)
|
||||
called = []
|
||||
monkeypatch.setattr(di, "call_stage", lambda *a, **k: called.append(1) or {})
|
||||
out = drawing_integrity_review([_sheet(1, "A101", 5)], _pages([1]))
|
||||
assert out == []
|
||||
assert called == [] # no LLM calls when disabled
|
||||
|
||||
|
||||
def test_reviews_only_dense_sheets(monkeypatch):
|
||||
monkeypatch.setattr(config, "ENABLE_DRAWING_INTEGRITY", True)
|
||||
monkeypatch.setattr(config, "INTEGRITY_MIN_ASSERTIONS", 3)
|
||||
seen_sheets = []
|
||||
|
||||
def fake_call_stage(system, user, subs=None, images_b64=None, **k):
|
||||
# Record which sheet_meta reached the model.
|
||||
seen_sheets.append(subs["sheet_meta"])
|
||||
return {"issues": [
|
||||
{"severity": "medium", "confidence": "high",
|
||||
"category": "on_sheet_contradiction", "sheets": [],
|
||||
"description": "plan disagrees with same-sheet schedule"},
|
||||
]}
|
||||
|
||||
monkeypatch.setattr(di, "call_stage", fake_call_stage)
|
||||
sheets = [_sheet(1, "A101", 5), _sheet(2, "A102", 1), _sheet(3, "A103", 4)]
|
||||
out = drawing_integrity_review(sheets, _pages([1, 2, 3]))
|
||||
# Two dense sheets reviewed, one skipped; each produced one anchored finding.
|
||||
assert len(out) == 2
|
||||
assert all(f["source_stage"] == "drawing_integrity" for f in out)
|
||||
assert {tuple(f["sheets"]) for f in out} == {("A101",), ("A103",)}
|
||||
assert len(seen_sheets) == 2
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Regression tests for defects found in the Aug 2026 agent-mode code review.
|
||||
|
||||
R2 - text_coverage._SHEET_ID_RE could not match hyphenated sheet ids (C-001),
|
||||
leaving civil/landscape pages sheet_number=None and producing false
|
||||
"declared but not in set" reconciliation warnings.
|
||||
R3 - config bool knobs mixed `== "true"` with the 1/true/yes set, so setting
|
||||
EXTRACT_TEXT_RETRY_ENABLED=1 silently DISABLED the retry ladder.
|
||||
|
||||
See also tests/agents/test_verifier.py for the verifier's "corrected"
|
||||
semantics, which are intentional and pinned there.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import os
|
||||
from unittest import mock
|
||||
|
||||
from backend.sheet_reconcile import declared_sheet_list, reconcile_sheets
|
||||
from backend.text_coverage import recover_sheet_number
|
||||
|
||||
|
||||
# --- R2: hyphenated sheet ids ----------------------------------------------
|
||||
|
||||
def test_recover_sheet_number_handles_hyphenated_civil_id():
|
||||
page_text = ("GENERAL NOTES\n" * 40) + "PROJECT NO 2024-118\nSHEET\nC-001\n"
|
||||
assert recover_sheet_number(page_text) == "C-001"
|
||||
|
||||
|
||||
def test_recover_sheet_number_still_handles_plain_ids():
|
||||
page_text = ("NOTES\n" * 40) + "SHEET\nS302\n"
|
||||
assert recover_sheet_number(page_text) == "S302"
|
||||
|
||||
|
||||
def test_recovered_hyphenated_id_reconciles_against_declared_index():
|
||||
"""The whole point: a recovered C-001 must not read as a missing sheet."""
|
||||
declared = declared_sheet_list({1: "SHEET LIST\nC-001 CIVIL\nA102 PLAN\n"})
|
||||
assert declared == ["C-001", "A102"]
|
||||
recovered = recover_sheet_number(("X\n" * 40) + "SHEET\nC-001\n")
|
||||
recon = reconcile_sheets(
|
||||
[{"sheet_number": recovered}, {"sheet_number": "A102"}], declared)
|
||||
assert recon["declared_not_in_set"] == []
|
||||
assert recon["in_set_not_declared"] == []
|
||||
|
||||
|
||||
# --- R3: boolean env knob parsing ------------------------------------------
|
||||
|
||||
def test_numeric_one_enables_ladder_knobs():
|
||||
with mock.patch.dict(os.environ, {
|
||||
"EXTRACT_TEXT_RETRY_ENABLED": "1",
|
||||
"EXTRACT_FALLBACK_ENABLED": "yes",
|
||||
}):
|
||||
cfg = importlib.reload(importlib.import_module("backend.config"))
|
||||
try:
|
||||
assert cfg.EXTRACT_TEXT_RETRY_ENABLED is True
|
||||
assert cfg.EXTRACT_FALLBACK_ENABLED is True
|
||||
finally:
|
||||
importlib.reload(cfg)
|
||||
|
||||
|
||||
def test_false_values_still_disable_ladder_knobs():
|
||||
with mock.patch.dict(os.environ, {
|
||||
"EXTRACT_TEXT_RETRY_ENABLED": "false",
|
||||
"EXTRACT_FALLBACK_ENABLED": "0",
|
||||
}):
|
||||
cfg = importlib.reload(importlib.import_module("backend.config"))
|
||||
try:
|
||||
assert cfg.EXTRACT_TEXT_RETRY_ENABLED is False
|
||||
assert cfg.EXTRACT_FALLBACK_ENABLED is False
|
||||
finally:
|
||||
importlib.reload(cfg)
|
||||
Reference in New Issue
Block a user