Add required human review gate to the Agent pipeline.
Docker Release / build-and-push (push) Successful in 1m10s
Docker Release / release (push) Skipped

Agent web jobs now stop after Brain consolidation and enter needs_review
with a persisted review queue (blocking: high-severity, low-confidence,
sensitive-category findings; audit sample of clean clusters). Humans
decide confirm/reject/unsure/needs_clarification via new review API and
frontend queue; a finalizer applies decisions (rejections suppressed with
reason codes), performs bounded targeted reruns for clarifications,
drafts RFIs only for kept issues, and only then marks the job done and
sends the final email. Two-phase email (review-required, then final
report), per-decision feedback labels with redacted aggregate metrics,
restart recovery from job artifacts, and CLI --no-review bypass.
Classic pipeline unchanged. 65 non-LLM tests.
This commit is contained in:
John Wilganowski
2026-07-28 19:23:57 +00:00
parent ac328d34fd
commit 1c1d2ff21b
27 changed files with 3344 additions and 22 deletions
+52 -11
View File
@@ -11,6 +11,7 @@ to outputs/<job_id>/ so results survive a restart even though live status does
not. No external queue/DB.
"""
import json
import os
import time
import uuid
@@ -21,7 +22,7 @@ from typing import Dict, Optional
from backend import config
from backend.agents.runner import run_agent_pipeline
from backend.pipeline.runner import run_pipeline
from backend.email_sender import send_conflict_report
from backend.email_sender import send_conflict_report, send_review_required
_jobs: Dict[str, Dict] = {}
_lock = threading.Lock()
@@ -46,7 +47,7 @@ def create_job(pdf_path: str, source_filename: str, email: Optional[str] = None,
with _lock:
_jobs[job_id] = {
"job_id": job_id,
"status": "queued", # queued -> running -> done | error
"status": "queued", # queued -> running -> done | needs_review | error
"source": source_filename,
"email": email or None,
"project_input": project_input or {},
@@ -72,6 +73,16 @@ def _run(job_id: str, pdf_path: str, project_input: Optional[Dict] = None,
_set(job_id, status="running")
# Keep a copy of the source PDF so its sheets can be viewed later.
os.makedirs(out_dir, exist_ok=True)
# Persist minimal job metadata so the disk fallback in get_job can
# recover the recipient email / pipeline mode after a server restart
# (plain json.dump, matching the _dump style used elsewhere).
with open(os.path.join(out_dir, "job.json"), "w", encoding="utf-8") as f:
json.dump({
"job_id": job_id,
"email": _jobs[job_id].get("email"),
"pipeline_mode": pipeline_mode,
"source": _jobs[job_id].get("source"),
}, f, indent=2)
shutil.copy2(pdf_path, os.path.join(out_dir, "source.pdf"))
runner = run_agent_pipeline if pipeline_mode == "agent" else run_pipeline
runner_kwargs = {
@@ -82,10 +93,21 @@ def _run(job_id: str, pdf_path: str, project_input: Optional[Dict] = None,
}
if pipeline_mode == "classic":
runner_kwargs["text_local"] = text_local
else:
runner_kwargs["require_review"] = config.AGENT_REQUIRE_REVIEW
report = runner(pdf_path, **runner_kwargs)
report.setdefault("summary", {})["pipeline_mode"] = pipeline_mode
_set(job_id, status="done", report=report, finished_at=time.time(), stage=None)
_notify(job_id, report, out_dir)
if report["summary"].get("agent_status") == "needs_review":
# Human-review gate: hold the job, don't email the unreviewed report.
_set(job_id, status="needs_review", report=report,
finished_at=time.time(), stage=None)
email = _jobs[job_id].get("email")
if email:
review_url = f"{config.APP_BASE_URL.rstrip('/')}/?job={job_id}"
send_review_required(email, report, review_url)
else:
_set(job_id, status="done", report=report, finished_at=time.time(), stage=None)
_notify(job_id, report, out_dir)
except Exception as e:
print(f"[Jobs] Job {job_id} failed: {e}")
_set(job_id, status="error", error=str(e), finished_at=time.time())
@@ -158,24 +180,43 @@ def get_job(job_id: str) -> Optional[Dict]:
if not os.path.isfile(report_path):
return None
try:
import json
with open(report_path, encoding="utf-8") as f:
report = json.load(f)
summary = report.get("summary", {})
# Recover the job's real state: a job that stopped at the review gate
# must come back as needs_review (not done) or it can never finalize.
status = "needs_review" if summary.get("agent_status") == "needs_review" else "done"
# job.json (written at job start) carries the recipient email and
# pipeline mode so the final notification still fires after a restart.
# Missing/corrupt job.json degrades to the previous derivations.
meta: Dict = {}
meta_path = os.path.join(config.OUTPUT_DIR, job_id, "job.json")
try:
with open(meta_path, encoding="utf-8") as f:
loaded = json.load(f)
if isinstance(loaded, dict):
meta = loaded
except (OSError, json.JSONDecodeError):
pass
source_pdf = os.path.join(config.OUTPUT_DIR, job_id, "source.pdf")
return {
job = {
"job_id": job_id,
"status": "done",
"source": report.get("source", os.path.basename(report_path)),
"email": None,
"status": status,
"source": meta.get("source") or report.get("source", os.path.basename(report_path)),
"email": meta.get("email"),
"project_input": report.get("project_input", {}),
"text_local": report.get("summary", {}).get("text_backend") == "local",
"pipeline_mode": report.get("summary", {}).get("pipeline_mode", "classic"),
"text_local": summary.get("text_backend") == "local",
"pipeline_mode": meta.get("pipeline_mode") or summary.get("pipeline_mode", "classic"),
"stage": None,
"created_at": os.path.getmtime(source_pdf) if os.path.isfile(source_pdf) else None,
"finished_at": os.path.getmtime(report_path),
"report": report,
"error": None,
}
# Hydrate the in-memory registry so _set(...) transitions (reviewing,
# finalizing, done) work for restart-recovered jobs.
with _lock:
return dict(_jobs.setdefault(job_id, job))
except Exception as e:
print(f"[Jobs] Failed to load job {job_id} from disk: {e}")
return None