Add required human review gate to the Agent pipeline.
Agent web jobs now stop after Brain consolidation and enter needs_review with a persisted review queue (blocking: high-severity, low-confidence, sensitive-category findings; audit sample of clean clusters). Humans decide confirm/reject/unsure/needs_clarification via new review API and frontend queue; a finalizer applies decisions (rejections suppressed with reason codes), performs bounded targeted reruns for clarifications, drafts RFIs only for kept issues, and only then marks the job done and sends the final email. Two-phase email (review-required, then final report), per-decision feedback labels with redacted aggregate metrics, restart recovery from job artifacts, and CLI --no-review bypass. Classic pipeline unchanged. 65 non-LLM tests.
This commit is contained in:
@@ -23,6 +23,8 @@ from backend.agents.rfi_writer import RFIWriterAgent
|
||||
from backend.pipeline.pdf_processor import convert_pdf_to_images
|
||||
from backend.pipeline.report import build_report, to_markdown
|
||||
from backend.pipeline.sheet_index import derive_project_meta_from_cover
|
||||
from backend.review.gate import build_review_queue
|
||||
from backend.review.store import ReviewStore
|
||||
|
||||
|
||||
def run_agent_pipeline(
|
||||
@@ -31,6 +33,7 @@ def run_agent_pipeline(
|
||||
on_stage: Optional[Callable[[str], None]] = None,
|
||||
project_input: Optional[Dict] = None,
|
||||
source_name: Optional[str] = None,
|
||||
require_review: bool = True,
|
||||
) -> Dict:
|
||||
"""Run all scoped specialist waves and return a Classic-compatible report."""
|
||||
if not os.path.isfile(pdf_path):
|
||||
@@ -192,6 +195,71 @@ def run_agent_pipeline(
|
||||
1 for decision in decisions if decision.get("action") == "merged"
|
||||
)
|
||||
|
||||
if require_review:
|
||||
orchestrator.stage("Agent review gate: build human-review queue")
|
||||
memory_snapshot = memory.snapshot()
|
||||
queue = build_review_queue(memory_snapshot, prioritized, decisions,
|
||||
limit=config.AGENT_REVIEW_AUDIT_SAMPLE)
|
||||
store = ReviewStore(out_dir)
|
||||
store.write_queue(queue)
|
||||
candidate_conflicts = [_finding_as_conflict(item) for item in conflict_findings]
|
||||
report = build_report(
|
||||
conflicts=candidate_conflicts,
|
||||
sheets=sheets,
|
||||
clusters=clusters,
|
||||
source=source_name or os.path.basename(pdf_path),
|
||||
)
|
||||
report.update({
|
||||
"project_input": merged_input,
|
||||
"jurisdiction": jurisdiction,
|
||||
"sheet_index": sheet_index,
|
||||
"project_intelligence": object_graph,
|
||||
"validated_issues": prioritized,
|
||||
"rfis": [],
|
||||
"suppressed_issues": [],
|
||||
})
|
||||
progress = store.progress(queue)
|
||||
# Same usage/stats summary block as the wave-7 path (rfis: 0 — they
|
||||
# are drafted only after human review finalizes the run).
|
||||
cost = usage.snapshot()
|
||||
orchestrator.stats.calls = cost["calls"]
|
||||
stats = orchestrator.stats.as_dict()
|
||||
report["summary"].update({
|
||||
"pipeline_mode": "agent",
|
||||
"agent_status": "needs_review",
|
||||
"review": progress,
|
||||
"agent_stats": stats,
|
||||
"by_stage": {
|
||||
"conflicts": len(conflict_findings),
|
||||
"qaqc": sum(
|
||||
1 for item in specialist_findings
|
||||
if item.get("source_stage") == "qaqc"
|
||||
),
|
||||
"code": sum(
|
||||
1 for item in specialist_findings
|
||||
if item.get("source_stage") == "code"
|
||||
),
|
||||
"constructability": sum(
|
||||
1 for item in specialist_findings
|
||||
if item.get("source_stage") == "constructability"
|
||||
),
|
||||
"validated": len(prioritized),
|
||||
"rfis": 0,
|
||||
},
|
||||
"cost_usd": round(cost["usd"], 4),
|
||||
"llm_calls": cost["calls"],
|
||||
"cached_calls": cost["cached"],
|
||||
"cost_by_stage": cost["by_stage"],
|
||||
"text_backend": "openrouter",
|
||||
"models_used": cost["models"],
|
||||
})
|
||||
if out_dir:
|
||||
_dump(out_dir, "conflicts.json", report)
|
||||
_dump(out_dir, "validated_issues.json", prioritized)
|
||||
# Snapshot for the review finalizer's targeted clarification reruns.
|
||||
memory.dump("memory.json")
|
||||
return report
|
||||
|
||||
orchestrator.stage("Agent wave 7: per-finding RFI writers")
|
||||
rfi_scopes = [
|
||||
AgentScope(
|
||||
|
||||
Reference in New Issue
Block a user