feat: refocus on drawings — code/ADA gated off, drawing-integrity wave, Brain-directed clarification
Docker Release / build-and-push (push) Successful in 1m8s
Docker Release / release (push) Skipped

- ENABLE_CODE_REVIEW flag (default off): skips code/ADA/jurisdiction review
  path in both pipelines; nothing deleted, one env flag to restore.
- Per-sheet Drawing Integrity QA wave (agent + classic, default on):
  dangling refs, on-sheet contradictions, dimension sanity, missing sheet
  essentials, tag hygiene. New DrawingIntegrityAgent + classic stage.
- Broadened conflict critic: intra-sheet + same-discipline contradictions,
  not just cross-discipline.
- Wave 6.5 Brain-directed clarification (bounded hub-and-spoke): Brain names
  uncertain findings, verify_evidence requests route through the wave-5b
  verifier; refuted findings suppressed. One planning call + capped verifies,
  single iteration. Shared _build_verify_scopes across 5b and 6.5.
- Config knobs, .env.example, frontend copy, tests (182 passing).
This commit is contained in:
2026-08-20 15:10:32 -05:00
parent bae608a505
commit d37ac8c1c7
22 changed files with 1917 additions and 63 deletions
+198
View File
@@ -0,0 +1,198 @@
"""Wave 6.5 Brain-directed clarification: planning unit + runner integration."""
import backend.agents.brain as brain_mod
import backend.agents.runner as runner_mod
from backend import config
from backend.agents.base import AgentResult
from backend.agents.base import AgentUsage
from backend.agents.brain import BrainAgent
from backend.agents.runner import run_agent_pipeline
# --- plan_clarifications unit tests ---------------------------------------
def _finding(issue_id, **kw):
base = {"issue_id": issue_id, "severity": "medium", "confidence": "low",
"source_stage": "conflict", "description": "d",
"evidence": [{"sheet": "A1", "source_text": "x"}]}
base.update(kw)
return base
def test_plan_caps_and_filters_unknown_ids(monkeypatch):
monkeypatch.setattr(config, "BRAIN_CLARIFY_MAX_REQUESTS", 2)
monkeypatch.setattr(brain_mod, "call_json", lambda **k: {"requests": [
{"issue_id": "A", "request_type": "verify_evidence", "reason": "thin"},
{"issue_id": "GHOST", "request_type": "verify_evidence", "reason": "x"},
{"issue_id": "B", "request_type": "verify_evidence", "reason": "amb"},
{"issue_id": "C", "request_type": "verify_evidence", "reason": "over cap"},
]})
prioritized = [_finding("A"), _finding("B"), _finding("C")]
reqs = BrainAgent(AgentUsage()).plan_clarifications(prioritized)
ids = [r["issue_id"] for r in reqs]
assert ids == ["A", "B"] # GHOST filtered, capped at 2
def test_plan_skips_already_verified(monkeypatch):
monkeypatch.setattr(config, "BRAIN_CLARIFY_MAX_REQUESTS", 8)
captured = {}
def fake(**kwargs):
captured["user_text"] = kwargs["user_text"]
return {"requests": [
{"issue_id": "A", "request_type": "verify_evidence", "reason": "y"},
]}
monkeypatch.setattr(brain_mod, "call_json", fake)
prioritized = [
_finding("A"),
_finding("V", verification={"status": "confirmed", "verdicts": []}),
]
reqs = BrainAgent(AgentUsage()).plan_clarifications(prioritized)
assert [r["issue_id"] for r in reqs] == ["A"]
# The already-verified finding must not even be offered to the model.
assert '"V"' not in captured["user_text"]
def test_plan_empty_on_call_failure(monkeypatch):
monkeypatch.setattr(config, "BRAIN_CLARIFY_MAX_REQUESTS", 8)
def boom(**k):
raise RuntimeError("brain down")
monkeypatch.setattr(brain_mod, "call_json", boom)
assert BrainAgent(AgentUsage()).plan_clarifications([_finding("A")]) == []
def test_plan_no_requests_returns_empty(monkeypatch):
monkeypatch.setattr(config, "BRAIN_CLARIFY_MAX_REQUESTS", 8)
monkeypatch.setattr(brain_mod, "call_json", lambda **k: {"requests": []})
assert BrainAgent(AgentUsage()).plan_clarifications([_finding("A")]) == []
# --- runner-level integration ---------------------------------------------
def _stub_agent(artifacts):
return lambda usage: type("S", (), {
"name": "stub",
"run": lambda self, scope: AgentResult(
scope_id=scope.scope_id, artifacts=list(artifacts)),
})()
def _patch_pipeline(monkeypatch, brain_finding, plan_requests):
monkeypatch.setattr(
runner_mod, "convert_pdf_to_images",
lambda path: [{"page_number": 1, "base64": "QUJD"}])
monkeypatch.setattr(runner_mod, "SheetExtractorAgent", _stub_agent([
{"sheet_number": "S401", "page_number": 1, "level": "roof",
"discipline": "S", "assertions": [
{"text": "(2) 2x6 STUD PACK", "object_type": "framing"},
{"text": "HSS16X4 beam", "object_type": "framing"},
]},
]))
monkeypatch.setattr(runner_mod, "SheetIndexAgent", _stub_agent([{}]))
monkeypatch.setattr(runner_mod, "JurisdictionAgent", _stub_agent([{}]))
monkeypatch.setattr(runner_mod, "LinkerAgent", _stub_agent([
{"key": "c1", "location": "roof beam pocket", "assertions": []},
]))
# A filler conflict finding so memory["findings"] is non-empty and wave 6
# actually invokes Brain.run (which our stub replaces with brain_finding).
monkeypatch.setattr(runner_mod, "ConflictCriticAgent", _stub_agent([
{"issue_id": "FILLER", "severity": "low", "confidence": "low",
"source_stage": "conflict", "sheets": [], "description": "filler",
"evidence": []},
]))
monkeypatch.setattr(runner_mod, "CodeAgent", _stub_agent([]))
monkeypatch.setattr(runner_mod, "ConstructabilityAgent", _stub_agent([]))
monkeypatch.setattr(runner_mod, "CompletenessAgent", _stub_agent([]))
monkeypatch.setattr(runner_mod, "DrawingIntegrityAgent", _stub_agent([]))
# Brain.run returns our finding; plan_clarifications returns the requests.
monkeypatch.setattr(
runner_mod, "BrainAgent",
lambda usage: type("B", (), {
"run": lambda self, findings, si, ju: ([dict(brain_finding)], []),
"plan_clarifications": lambda self, prioritized: list(plan_requests),
})())
def test_brain_clarify_refutes_and_suppresses(monkeypatch, tmp_path):
"""Brain flags a MEDIUM finding wave-5b's severity gate skipped; the
clarification verifier refutes it, so it moves to suppressed_issues."""
monkeypatch.setattr(config, "ENABLE_BRAIN_CLARIFY", True)
finding = {
"issue_id": "M1", "severity": "medium", "confidence": "low",
"source_stage": "conflict", "sheets": ["S401"],
"description": "beam bears on (2) 2x6 stud pack",
"evidence": [{"sheet": "S401", "source_text": "(2) 2x6 STUD PACK"}],
}
_patch_pipeline(monkeypatch, finding, [
{"issue_id": "M1", "request_type": "verify_evidence", "reason": "misread?"},
])
# The clarification verifier returns a 'corrected' verdict -> refuted.
monkeypatch.setattr(
"backend.agents.verifier.call_json",
lambda **kwargs: {"verdicts": [
{"sheet": "S401", "source_text": "(2) 2x6 STUD PACK",
"verdict": "corrected", "actual_text": "(5) 2x6 STUD PACK",
"notes": "reads (5)"},
]})
pdf = tmp_path / "d.pdf"
pdf.write_bytes(b"%PDF-1.4\n")
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path),
require_review=False)
validated = report.get("validated_issues") or []
assert all(f.get("issue_id") != "M1" for f in validated) # dropped
assert [f["issue_id"] for f in report["suppressed_issues"]] == ["M1"]
assert report["suppressed_issues"][0]["verification"]["status"] == "refuted"
def test_brain_clarify_confirms_keeps_finding(monkeypatch, tmp_path):
monkeypatch.setattr(config, "ENABLE_BRAIN_CLARIFY", True)
finding = {
"issue_id": "M2", "severity": "medium", "confidence": "low",
"source_stage": "conflict", "sheets": ["S401"],
"description": "beam bears on (5) 2x6 stud pack",
"evidence": [{"sheet": "S401", "source_text": "(5) 2x6 STUD PACK"}],
}
_patch_pipeline(monkeypatch, finding, [
{"issue_id": "M2", "request_type": "verify_evidence", "reason": "check"},
])
monkeypatch.setattr(
"backend.agents.verifier.call_json",
lambda **kwargs: {"verdicts": [
{"sheet": "S401", "source_text": "(5) 2x6 STUD PACK",
"verdict": "confirmed", "actual_text": None, "notes": None},
]})
pdf = tmp_path / "d.pdf"
pdf.write_bytes(b"%PDF-1.4\n")
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path),
require_review=False)
validated = report.get("validated_issues") or []
kept = [f for f in validated if f.get("issue_id") == "M2"]
assert len(kept) == 1
assert kept[0]["verification"]["status"] == "confirmed"
assert report["suppressed_issues"] == []
def test_brain_clarify_disabled_is_noop(monkeypatch, tmp_path):
monkeypatch.setattr(config, "ENABLE_BRAIN_CLARIFY", False)
finding = {
"issue_id": "M3", "severity": "medium", "confidence": "low",
"source_stage": "conflict", "sheets": ["S401"],
"description": "d", "evidence": [{"sheet": "S401", "source_text": "t"}],
}
# plan_clarifications should never be consulted; give it a bomb to prove it.
def _bomb(self, prioritized):
raise AssertionError("plan_clarifications must not run when disabled")
_patch_pipeline(monkeypatch, finding, [])
monkeypatch.setattr(
runner_mod, "BrainAgent",
lambda usage: type("B", (), {
"run": lambda self, findings, si, ju: ([dict(finding)], []),
"plan_clarifications": _bomb})())
pdf = tmp_path / "d.pdf"
pdf.write_bytes(b"%PDF-1.4\n")
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path),
require_review=False)
validated = report.get("validated_issues") or []
assert any(f.get("issue_id") == "M3" for f in validated)
+95
View File
@@ -0,0 +1,95 @@
"""Unit tests for the per-sheet DrawingIntegrityAgent and scope builder."""
import backend.agents.integrity_agent as integ
from backend.agents.base import AgentScope, AgentUsage
from backend.agents.integrity_agent import (
DrawingIntegrityAgent, build_integrity_scopes,
)
from backend import config
def _sheet(page, sheet_number, n_assertions):
return {
"sheet_number": sheet_number,
"page_number": page,
"sheet_title": f"Sheet {sheet_number}",
"discipline": "Architectural",
"assertions": [
{"object_type": "note", "source_text": f"note {i}"}
for i in range(n_assertions)
],
}
def test_build_scopes_skips_sparse_sheets(monkeypatch):
monkeypatch.setattr(config, "INTEGRITY_MIN_ASSERTIONS", 3)
sheets = [
_sheet(1, "A101", 5), # kept
_sheet(2, "A102", 2), # skipped (too sparse)
_sheet(3, "A103", 3), # kept (== floor)
]
page_to_b64 = {1: "IMG1", 2: "IMG2", 3: "IMG3"}
page_to_text = {1: "text one", 2: "text two", 3: "text three"}
scopes = build_integrity_scopes(sheets, page_to_b64, page_to_text)
ids = sorted(s.scope_id for s in scopes)
assert ids == ["integrity:1", "integrity:3"]
# Each scope carries its own page image + text layer.
by_id = {s.scope_id: s for s in scopes}
assert by_id["integrity:1"].payload["image_b64"] == "IMG1"
assert by_id["integrity:1"].payload["text_layer"] == "text one"
def test_agent_parses_and_anchors_sheet(monkeypatch):
"""Findings with blank sheets get anchored to the scope's sheet number."""
captured = {}
def fake_call_json(**kwargs):
captured.update(kwargs)
return {"issues": [
{"issue_id": "DI-1", "severity": "high", "confidence": "high",
"category": "dangling_reference", "sheets": [],
"description": "Detail callout 5/A101 has no detail 5 on this sheet",
"evidence": [{"sheet": "A101", "source_text": "5/A101"}]},
]}
monkeypatch.setattr(integ, "call_json", fake_call_json)
sheet = _sheet(1, "A101", 5)
scope = AgentScope("integrity:1", {
"sheet": sheet, "page_number": 1,
"image_b64": "IMG1", "text_layer": "the deterministic text layer",
})
result = DrawingIntegrityAgent(AgentUsage()).run(scope)
assert result.error == ""
assert len(result.artifacts) == 1
finding = result.artifacts[0]
assert finding["source_stage"] == "drawing_integrity"
assert finding["sheets"] == ["A101"] # anchored
assert finding["agent"] == "drawing_integrity"
assert finding["scope_id"] == "integrity:1"
# The image + text layer reached the model.
assert captured["images_b64"] == ["IMG1"]
assert "the deterministic text layer" in captured["user_text"]
def test_agent_empty_issues_is_clean(monkeypatch):
monkeypatch.setattr(integ, "call_json", lambda **k: {"issues": []})
scope = AgentScope("integrity:1", {
"sheet": _sheet(1, "A101", 5), "page_number": 1,
"image_b64": "IMG1", "text_layer": "t",
})
result = DrawingIntegrityAgent(AgentUsage()).run(scope)
assert result.error == ""
assert result.artifacts == []
def test_agent_survives_call_failure(monkeypatch):
def boom(**kwargs):
raise RuntimeError("model exploded")
monkeypatch.setattr(integ, "call_json", boom)
scope = AgentScope("integrity:1", {
"sheet": _sheet(1, "A101", 5), "page_number": 1,
"image_b64": "IMG1", "text_layer": "t",
})
result = DrawingIntegrityAgent(AgentUsage()).run(scope)
assert "model exploded" in result.error
assert result.artifacts == []
+102
View File
@@ -0,0 +1,102 @@
"""Flag-gating tests: ENABLE_CODE_REVIEW off skips code, drawing_integrity runs.
Runner-level smoke tests using stubbed agents (same pattern as
test_wave5b_suppression). Verifies the code/ADA wave is skipped when
ENABLE_CODE_REVIEW is false and the Drawing Integrity wave feeds findings
into the report by_stage counters.
"""
import backend.agents.runner as runner_mod
from backend import config
from backend.agents.base import AgentResult
from backend.agents.runner import run_agent_pipeline
def _stub_agent(artifacts):
return lambda usage: type("S", (), {
"name": "stub",
"run": lambda self, scope: AgentResult(
scope_id=scope.scope_id, artifacts=list(artifacts)),
})()
def _integrity_finding():
return {
"issue_id": "DI-1", "severity": "high", "confidence": "high",
"source_stage": "drawing_integrity", "sheets": ["A101"],
"category": "dangling_reference",
"description": "Detail callout 5/A101 has no detail 5 on this sheet",
"evidence": [{"sheet": "A101", "source_text": "5/A101"}],
}
def _patch(monkeypatch, code_should_raise):
monkeypatch.setattr(
runner_mod, "convert_pdf_to_images",
lambda path: [{"page_number": 1, "base64": "QUJD"}])
# Sheet has 3+ assertions so the integrity wave does NOT skip it.
monkeypatch.setattr(runner_mod, "SheetExtractorAgent", _stub_agent([
{"sheet_number": "A101", "page_number": 1, "level": "1",
"discipline": "A", "assertions": [
{"text": "5/A101", "object_type": "detail_marker"},
{"text": "ROOM 101", "object_type": "room"},
{"text": "DOOR 101A", "object_type": "door"},
]},
]))
monkeypatch.setattr(runner_mod, "SheetIndexAgent", _stub_agent([{}]))
monkeypatch.setattr(runner_mod, "JurisdictionAgent", _stub_agent([{}]))
monkeypatch.setattr(runner_mod, "LinkerAgent", _stub_agent([]))
monkeypatch.setattr(runner_mod, "ConflictCriticAgent", _stub_agent([]))
def code_boom(usage):
if code_should_raise:
raise AssertionError("CodeAgent must not run when gated off")
return _stub_agent([])(usage)
monkeypatch.setattr(runner_mod, "CodeAgent", code_boom)
monkeypatch.setattr(runner_mod, "DrawingIntegrityAgent",
_stub_agent([_integrity_finding()]))
monkeypatch.setattr(runner_mod, "ConstructabilityAgent", _stub_agent([]))
monkeypatch.setattr(runner_mod, "CompletenessAgent", _stub_agent([]))
monkeypatch.setattr(
runner_mod, "BrainAgent",
lambda usage: type("B", (), {
"run": lambda self, findings, si, ju: (list(findings), []),
"plan_clarifications": lambda self, prioritized: []})())
# Stub the wave-5b verifier so the high-severity integrity finding is
# confirmed (never a live network call).
monkeypatch.setattr(
"backend.agents.verifier.call_json",
lambda **kwargs: {"verdicts": [
{"sheet": "A101", "source_text": "5/A101",
"verdict": "confirmed", "actual_text": None, "notes": None},
]})
def test_code_gated_off_integrity_on(monkeypatch, tmp_path):
monkeypatch.setattr(config, "ENABLE_CODE_REVIEW", False)
monkeypatch.setattr(config, "ENABLE_DRAWING_INTEGRITY", True)
_patch(monkeypatch, code_should_raise=True)
pdf = tmp_path / "d.pdf"
pdf.write_bytes(b"%PDF-1.4\n")
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path),
require_review=False)
by_stage = report["summary"]["by_stage"]
assert by_stage["code"] == 0
assert by_stage["drawing_integrity"] == 1
# The integrity finding survived into the validated set.
assert any(f.get("issue_id") == "DI-1"
for f in report.get("validated_issues") or [])
def test_code_enabled_runs(monkeypatch, tmp_path):
monkeypatch.setattr(config, "ENABLE_CODE_REVIEW", True)
monkeypatch.setattr(config, "ENABLE_DRAWING_INTEGRITY", True)
_patch(monkeypatch, code_should_raise=False)
# build_code_scopes runs on the real sheet; CodeAgent is stubbed to []
pdf = tmp_path / "d.pdf"
pdf.write_bytes(b"%PDF-1.4\n")
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path),
require_review=False)
# No crash; integrity still reported.
assert report["summary"]["by_stage"]["drawing_integrity"] == 1
+1 -1
View File
@@ -4,7 +4,7 @@ from backend.agents.runner import run_agent_pipeline
def _patch_brain(monkeypatch):
monkeypatch.setattr("backend.agents.runner.convert_pdf_to_images", lambda path: [{"page_number": 1, "base64": "x"}])
monkeypatch.setattr("backend.agents.runner.BrainAgent", lambda usage: type("B", (), {"run": lambda self, findings, sheet_index, jurisdiction: ([{"issue_id": "AGENT-0001", "severity": "high", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"}], [])})())
monkeypatch.setattr("backend.agents.runner.BrainAgent", lambda usage: type("B", (), {"run": lambda self, findings, sheet_index, jurisdiction: ([{"issue_id": "AGENT-0001", "severity": "high", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"}], []), "plan_clarifications": lambda self, prioritized: []})())
def test_agent_runner_can_enter_review_mode(monkeypatch, tmp_path):
+2 -1
View File
@@ -63,7 +63,8 @@ def _patch_pipeline(monkeypatch, finding, verify_sink):
runner_mod, "BrainAgent",
lambda usage: type("B", (), {
"run": lambda self, findings, sheet_index, jurisdiction:
(list(findings), [])})())
(list(findings), []),
"plan_clarifications": lambda self, prioritized: []})())
class _RecordingVerifier:
name = "verify"
+2 -1
View File
@@ -46,7 +46,8 @@ def _patch_pipeline(monkeypatch, finding):
runner_mod, "BrainAgent",
lambda usage: type("B", (), {
"run": lambda self, findings, sheet_index, jurisdiction:
(list(findings), [])})())
(list(findings), []),
"plan_clarifications": lambda self, prioritized: []})())
def test_refuted_finding_is_suppressed_not_crash(monkeypatch, tmp_path):