Add required human review gate to the Agent pipeline.
Agent web jobs now stop after Brain consolidation and enter needs_review with a persisted review queue (blocking: high-severity, low-confidence, sensitive-category findings; audit sample of clean clusters). Humans decide confirm/reject/unsure/needs_clarification via new review API and frontend queue; a finalizer applies decisions (rejections suppressed with reason codes), performs bounded targeted reruns for clarifications, drafts RFIs only for kept issues, and only then marks the job done and sends the final email. Two-phase email (review-required, then final report), per-decision feedback labels with redacted aggregate metrics, restart recovery from job artifacts, and CLI --no-review bypass. Classic pipeline unchanged. 65 non-LLM tests.
This commit is contained in:
John Wilganowski
@@ -0,0 +1,439 @@
|
||||
"""API tests for the human-review endpoints and review-aware job states."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import backend.jobs
|
||||
from backend.main import app
|
||||
from backend.review.store import ReviewStore
|
||||
|
||||
|
||||
class _SyncThread:
|
||||
"""Drop-in threading.Thread replacement that runs the target inline."""
|
||||
|
||||
def __init__(self, target=None, args=(), **kwargs):
|
||||
self._target = target
|
||||
self._args = args
|
||||
|
||||
def start(self):
|
||||
self._target(*self._args)
|
||||
|
||||
|
||||
def _queue_item(item_id: str) -> dict:
|
||||
return {"review_item_id": item_id, "kind": "finding",
|
||||
"blocking": True, "reasons": ["high_severity"], "payload": {}}
|
||||
|
||||
|
||||
def test_review_queue_and_decision_save(monkeypatch, tmp_path):
|
||||
store = ReviewStore(str(tmp_path))
|
||||
store.write_queue([_queue_item("finding:AGENT-0001")])
|
||||
client = TestClient(app)
|
||||
monkeypatch.setattr("backend.main.get_job", lambda job_id: {"job_id": job_id, "status": "needs_review", "report": {"summary": {}}, "out_dir": str(tmp_path)})
|
||||
queue_response = client.get("/jobs/job1/review")
|
||||
assert queue_response.status_code == 200
|
||||
decision_response = client.post("/jobs/job1/review-decisions", json={"decisions": [{"review_item_id": "finding:AGENT-0001", "decision": "confirm"}]})
|
||||
assert decision_response.status_code == 200
|
||||
|
||||
|
||||
def test_review_decision_emits_feedback_label(monkeypatch, tmp_path):
|
||||
"""Every saved decision appends one feedback label under review/."""
|
||||
store = ReviewStore(str(tmp_path))
|
||||
store.write_queue([_queue_item("finding:AGENT-0001")])
|
||||
client = TestClient(app)
|
||||
monkeypatch.setattr("backend.main.get_job", lambda job_id: {"job_id": job_id, "status": "needs_review", "report": {"summary": {}}, "out_dir": str(tmp_path)})
|
||||
response = client.post("/jobs/job1/review-decisions", json={"decisions": [{"review_item_id": "finding:AGENT-0001", "decision": "confirm"}]})
|
||||
assert response.status_code == 200
|
||||
path = os.path.join(str(tmp_path), "review", "feedback_labels.jsonl")
|
||||
with open(path, encoding="utf-8") as f:
|
||||
labels = [json.loads(line) for line in f if line.strip()]
|
||||
assert len(labels) == 1
|
||||
assert labels[0]["review_item_id"] == "finding:AGENT-0001"
|
||||
assert labels[0]["decision"] == "confirm"
|
||||
|
||||
|
||||
def test_review_endpoints_404_for_unknown_job(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("backend.config.OUTPUT_DIR", str(tmp_path))
|
||||
client = TestClient(app)
|
||||
assert client.get("/jobs/nope/review").status_code == 404
|
||||
assert client.post("/jobs/nope/review-decisions", json={"decisions": []}).status_code == 404
|
||||
|
||||
|
||||
def test_review_decision_invalid_returns_422(monkeypatch, tmp_path):
|
||||
client = TestClient(app)
|
||||
monkeypatch.setattr("backend.main.get_job", lambda job_id: {"job_id": job_id, "status": "needs_review", "report": {"summary": {}}, "out_dir": str(tmp_path)})
|
||||
response = client.post("/jobs/job1/review-decisions", json={"decisions": [{"review_item_id": "finding:AGENT-0001", "decision": "bogus"}]})
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
def test_partial_review_moves_job_to_reviewing(monkeypatch, tmp_path):
|
||||
"""Saving some but not all required decisions flips needs_review -> reviewing."""
|
||||
store = ReviewStore(str(tmp_path))
|
||||
store.write_queue([_queue_item("finding:AGENT-0001"), _queue_item("finding:AGENT-0002")])
|
||||
job_id = "jobreviewing"
|
||||
backend.jobs._jobs[job_id] = {
|
||||
"job_id": job_id,
|
||||
"status": "needs_review",
|
||||
"out_dir": str(tmp_path),
|
||||
"report": {"summary": {}},
|
||||
}
|
||||
try:
|
||||
client = TestClient(app)
|
||||
response = client.post(f"/jobs/{job_id}/review-decisions", json={
|
||||
"decisions": [{"review_item_id": "finding:AGENT-0001", "decision": "confirm"}],
|
||||
})
|
||||
assert response.status_code == 200
|
||||
assert response.json()["progress"]["remaining"] == 1
|
||||
assert backend.jobs.get_job(job_id)["status"] == "reviewing"
|
||||
finally:
|
||||
backend.jobs._jobs.pop(job_id, None)
|
||||
|
||||
|
||||
def test_agent_job_needs_review_skips_notify(monkeypatch, tmp_path):
|
||||
"""Carried finding from Task 4: an agent report needing review must not be emailed."""
|
||||
sent = []
|
||||
monkeypatch.setattr("backend.config.OUTPUT_DIR", str(tmp_path))
|
||||
monkeypatch.setattr("backend.jobs.run_agent_pipeline", lambda pdf_path, **kw: {
|
||||
"summary": {"agent_status": "needs_review"}, "conflicts": [],
|
||||
})
|
||||
monkeypatch.setattr("backend.jobs.send_conflict_report", lambda *a, **kw: sent.append((a, kw)))
|
||||
monkeypatch.setattr(threading, "Thread", _SyncThread)
|
||||
|
||||
pdf = tmp_path / "upload.pdf"
|
||||
pdf.write_bytes(b"%PDF-1.4 dummy")
|
||||
job_id = backend.jobs.create_job(str(pdf), source_filename="set.pdf",
|
||||
email="arch@example.com", pipeline_mode="agent")
|
||||
try:
|
||||
job = backend.jobs.get_job(job_id)
|
||||
assert job["status"] == "needs_review"
|
||||
assert job["report"]["summary"]["agent_status"] == "needs_review"
|
||||
assert sent == []
|
||||
finally:
|
||||
backend.jobs._jobs.pop(job_id, None)
|
||||
|
||||
|
||||
def test_classic_job_still_completes_and_notifies(monkeypatch, tmp_path):
|
||||
"""Classic pipeline behavior is unchanged: done status + completion email."""
|
||||
sent = []
|
||||
monkeypatch.setattr("backend.config.OUTPUT_DIR", str(tmp_path))
|
||||
monkeypatch.setattr("backend.jobs.run_pipeline", lambda pdf_path, **kw: {
|
||||
"summary": {}, "conflicts": [],
|
||||
})
|
||||
monkeypatch.setattr("backend.jobs.send_conflict_report", lambda *a, **kw: sent.append((a, kw)))
|
||||
monkeypatch.setattr(threading, "Thread", _SyncThread)
|
||||
|
||||
pdf = tmp_path / "upload.pdf"
|
||||
pdf.write_bytes(b"%PDF-1.4 dummy")
|
||||
job_id = backend.jobs.create_job(str(pdf), source_filename="set.pdf",
|
||||
email="arch@example.com", pipeline_mode="classic")
|
||||
try:
|
||||
assert backend.jobs.get_job(job_id)["status"] == "done"
|
||||
assert len(sent) == 1
|
||||
finally:
|
||||
backend.jobs._jobs.pop(job_id, None)
|
||||
|
||||
|
||||
def test_job_status_includes_review_progress(monkeypatch, tmp_path):
|
||||
"""GET /jobs/{id} surfaces report.summary.review for a needs_review job."""
|
||||
review = {"required": 1, "completed": 0, "remaining": 1, "total": 1}
|
||||
client = TestClient(app)
|
||||
monkeypatch.setattr("backend.main.get_job", lambda job_id: {
|
||||
"job_id": job_id, "status": "needs_review",
|
||||
"report": {"summary": {"agent_status": "needs_review", "review": review}},
|
||||
"out_dir": str(tmp_path),
|
||||
})
|
||||
response = client.get("/jobs/job1")
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["status"] == "needs_review"
|
||||
assert body["report"]["summary"]["review"] == review
|
||||
|
||||
|
||||
def test_review_response_includes_saved_decisions(monkeypatch, tmp_path):
|
||||
"""GET /jobs/{id}/review also returns the decisions map for UI pre-population."""
|
||||
store = ReviewStore(str(tmp_path))
|
||||
store.write_queue([_queue_item("finding:AGENT-0001")])
|
||||
client = TestClient(app)
|
||||
monkeypatch.setattr("backend.main.get_job", lambda job_id: {"job_id": job_id, "status": "needs_review", "report": {"summary": {}}, "out_dir": str(tmp_path)})
|
||||
post = client.post("/jobs/job1/review-decisions", json={"decisions": [
|
||||
{"review_item_id": "finding:AGENT-0001", "decision": "confirm", "comment": "looks right"},
|
||||
]})
|
||||
assert post.status_code == 200
|
||||
get = client.get("/jobs/job1/review")
|
||||
assert get.status_code == 200
|
||||
decisions = get.json()["decisions"]
|
||||
assert decisions["finding:AGENT-0001"]["decision"] == "confirm"
|
||||
assert decisions["finding:AGENT-0001"]["comment"] == "looks right"
|
||||
|
||||
|
||||
def test_review_flow_via_disk_fallback(monkeypatch, tmp_path):
|
||||
"""Smoke: a synthetic on-disk needs_review job served by the REAL get_job
|
||||
(disk fallback), with decisions persisting across review GETs."""
|
||||
job_id = "jobdisk"
|
||||
out_dir = os.path.join(str(tmp_path), job_id)
|
||||
os.makedirs(out_dir)
|
||||
report = {
|
||||
"source": "set.pdf",
|
||||
"summary": {
|
||||
"agent_status": "needs_review",
|
||||
"review": {"required": 1, "completed": 0, "remaining": 1, "total": 1},
|
||||
},
|
||||
"conflicts": [],
|
||||
}
|
||||
with open(os.path.join(out_dir, "conflicts.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(report, f)
|
||||
store = ReviewStore(out_dir)
|
||||
store.write_queue([_queue_item("finding:AGENT-0001")])
|
||||
monkeypatch.setattr("backend.config.OUTPUT_DIR", str(tmp_path))
|
||||
client = TestClient(app)
|
||||
|
||||
job_response = client.get(f"/jobs/{job_id}")
|
||||
assert job_response.status_code == 200
|
||||
assert job_response.json()["report"]["summary"]["review"]["required"] == 1
|
||||
|
||||
review_response = client.get(f"/jobs/{job_id}/review")
|
||||
assert review_response.status_code == 200
|
||||
body = review_response.json()
|
||||
assert [i["review_item_id"] for i in body["queue"]] == ["finding:AGENT-0001"]
|
||||
assert body["progress"]["remaining"] == 1
|
||||
assert body["decisions"] == {}
|
||||
|
||||
post = client.post(f"/jobs/{job_id}/review-decisions", json={"decisions": [
|
||||
{"review_item_id": "finding:AGENT-0001", "decision": "reject",
|
||||
"reason_code": "not_a_contradiction"},
|
||||
]})
|
||||
assert post.status_code == 200
|
||||
assert post.json()["progress"]["remaining"] == 0
|
||||
|
||||
again = client.get(f"/jobs/{job_id}/review")
|
||||
assert again.status_code == 200
|
||||
saved = again.json()["decisions"]["finding:AGENT-0001"]
|
||||
assert saved["decision"] == "reject"
|
||||
assert saved["reason_code"] == "not_a_contradiction"
|
||||
|
||||
|
||||
def _write_restart_job(tmp_path, job_id, email=None):
|
||||
"""On-disk needs_review job artifacts, as a pre-restart run would leave
|
||||
them: candidate report + memory snapshot + review queue (+ job.json)."""
|
||||
out_dir = os.path.join(str(tmp_path), job_id)
|
||||
os.makedirs(os.path.join(out_dir, "agent"))
|
||||
report = {
|
||||
"source": "set.pdf",
|
||||
"generated_at": "2026-07-28T00:00:00+00:00",
|
||||
"summary": {
|
||||
"sheets_analyzed": 0, "disciplines": [], "assertions_extracted": 0,
|
||||
"clusters_checked": 0, "conflicts_found": 0,
|
||||
"by_severity": {"high": 0, "medium": 0, "low": 0}, "by_category": {},
|
||||
"pipeline_mode": "agent", "agent_status": "needs_review",
|
||||
"review": {"required": 1, "completed": 0, "remaining": 1, "total": 1},
|
||||
},
|
||||
"conflicts": [], "sheets": [],
|
||||
"validated_issues": [{"issue_id": "AGENT-0001", "severity": "high"}],
|
||||
"suppressed_issues": [], "rfis": [],
|
||||
}
|
||||
with open(os.path.join(out_dir, "conflicts.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(report, f)
|
||||
with open(os.path.join(out_dir, "agent", "memory.json"), "w", encoding="utf-8") as f:
|
||||
json.dump({}, f)
|
||||
if email is not None:
|
||||
with open(os.path.join(out_dir, "job.json"), "w", encoding="utf-8") as f:
|
||||
json.dump({"job_id": job_id, "email": email,
|
||||
"pipeline_mode": "agent", "source": "set.pdf"}, f)
|
||||
store = ReviewStore(out_dir)
|
||||
store.write_queue([_queue_item("finding:AGENT-0001")])
|
||||
return out_dir
|
||||
|
||||
|
||||
def test_restart_recovered_needs_review_job_finalizes(monkeypatch, tmp_path):
|
||||
"""CRITICAL: after a restart, a needs_review job recovered from disk keeps
|
||||
its status (not "done"), hydrates the in-memory registry, and the whole
|
||||
decide -> finalize flow completes to done via the real get_job."""
|
||||
job_id = "jobrestart"
|
||||
_write_restart_job(tmp_path, job_id)
|
||||
monkeypatch.setattr("backend.config.OUTPUT_DIR", str(tmp_path))
|
||||
monkeypatch.setattr("backend.review.finalizer._draft_rfis", lambda kept: [])
|
||||
monkeypatch.setattr("backend.jobs._notify", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(threading, "Thread", _SyncThread)
|
||||
try:
|
||||
# Real disk fallback: simulates a fresh post-restart process.
|
||||
job = backend.jobs.get_job(job_id)
|
||||
assert job["status"] == "needs_review"
|
||||
assert job_id in backend.jobs._jobs # hydrated for _set() transitions
|
||||
|
||||
client = TestClient(app)
|
||||
post = client.post(f"/jobs/{job_id}/review-decisions", json={"decisions": [
|
||||
{"review_item_id": "finding:AGENT-0001", "decision": "confirm"},
|
||||
]})
|
||||
assert post.status_code == 200
|
||||
|
||||
fin = client.post(f"/jobs/{job_id}/finalize-review")
|
||||
assert fin.status_code == 200
|
||||
job = backend.jobs.get_job(job_id)
|
||||
assert job["status"] == "done"
|
||||
assert job["report"]["summary"]["agent_status"] == "complete"
|
||||
finally:
|
||||
backend.jobs._jobs.pop(job_id, None)
|
||||
|
||||
|
||||
def test_restart_recovered_job_final_email_uses_job_json(monkeypatch, tmp_path):
|
||||
"""CRITICAL: job.json (written at job start) restores the recipient email
|
||||
after a restart, so finalization still fires the final report email."""
|
||||
job_id = "jobemail"
|
||||
_write_restart_job(tmp_path, job_id, email="arch@example.com")
|
||||
monkeypatch.setattr("backend.config.OUTPUT_DIR", str(tmp_path))
|
||||
monkeypatch.setattr("backend.review.finalizer._draft_rfis", lambda kept: [])
|
||||
sent = []
|
||||
monkeypatch.setattr("backend.jobs.send_conflict_report",
|
||||
lambda email, report, **kw: sent.append(email))
|
||||
monkeypatch.setattr(threading, "Thread", _SyncThread)
|
||||
try:
|
||||
job = backend.jobs.get_job(job_id)
|
||||
assert job["status"] == "needs_review"
|
||||
assert job["email"] == "arch@example.com"
|
||||
|
||||
client = TestClient(app)
|
||||
post = client.post(f"/jobs/{job_id}/review-decisions", json={"decisions": [
|
||||
{"review_item_id": "finding:AGENT-0001", "decision": "confirm"},
|
||||
]})
|
||||
assert post.status_code == 200
|
||||
fin = client.post(f"/jobs/{job_id}/finalize-review")
|
||||
assert fin.status_code == 200
|
||||
assert backend.jobs.get_job(job_id)["status"] == "done"
|
||||
assert sent == ["arch@example.com"]
|
||||
finally:
|
||||
backend.jobs._jobs.pop(job_id, None)
|
||||
|
||||
|
||||
def test_review_decisions_409_for_non_review_job(monkeypatch, tmp_path):
|
||||
"""Positive state guard: only needs_review/reviewing jobs accept decisions."""
|
||||
client = TestClient(app)
|
||||
monkeypatch.setattr("backend.main.get_job", lambda job_id: {
|
||||
"job_id": job_id, "status": "done", "out_dir": str(tmp_path),
|
||||
})
|
||||
response = client.post("/jobs/job1/review-decisions", json={"decisions": [
|
||||
{"review_item_id": "finding:AGENT-0001", "decision": "confirm"}]})
|
||||
assert response.status_code == 409
|
||||
assert "done" in response.json()["detail"]["detail"]
|
||||
|
||||
|
||||
def test_review_queue_get_does_not_create_review_dir(monkeypatch, tmp_path):
|
||||
"""The read-only GET endpoint must not create review/ dirs on read."""
|
||||
client = TestClient(app)
|
||||
monkeypatch.setattr("backend.main.get_job", lambda job_id: {
|
||||
"job_id": job_id, "status": "needs_review",
|
||||
"report": {"summary": {}}, "out_dir": str(tmp_path),
|
||||
})
|
||||
response = client.get("/jobs/job1/review")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["queue"] == []
|
||||
assert response.json()["decisions"] == {}
|
||||
assert not os.path.exists(os.path.join(str(tmp_path), "review"))
|
||||
|
||||
|
||||
def _write_finalizable_job(tmp_path, decisions):
|
||||
"""Minimal review-mode artifacts: candidate report + queue + decisions."""
|
||||
out_dir = str(tmp_path)
|
||||
os.makedirs(os.path.join(out_dir, "agent"), exist_ok=True)
|
||||
report = {
|
||||
"source": "set.pdf",
|
||||
"generated_at": "2026-07-28T00:00:00+00:00",
|
||||
"summary": {
|
||||
"sheets_analyzed": 0, "disciplines": [], "assertions_extracted": 0,
|
||||
"clusters_checked": 0, "conflicts_found": 0,
|
||||
"by_severity": {"high": 0, "medium": 0, "low": 0}, "by_category": {},
|
||||
"pipeline_mode": "agent", "agent_status": "needs_review",
|
||||
"review": {"required": 1, "completed": 0, "remaining": 1, "total": 1},
|
||||
},
|
||||
"conflicts": [], "sheets": [],
|
||||
"validated_issues": [{"issue_id": "AGENT-0001", "severity": "high"}],
|
||||
"suppressed_issues": [], "rfis": [],
|
||||
}
|
||||
with open(os.path.join(out_dir, "conflicts.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(report, f)
|
||||
with open(os.path.join(out_dir, "agent", "memory.json"), "w", encoding="utf-8") as f:
|
||||
json.dump({}, f)
|
||||
store = ReviewStore(out_dir)
|
||||
store.write_queue([_queue_item("finding:AGENT-0001")])
|
||||
for decision in decisions:
|
||||
store.append_decision(decision)
|
||||
return out_dir
|
||||
|
||||
|
||||
def test_finalize_review_409_while_undecided(monkeypatch, tmp_path):
|
||||
out_dir = _write_finalizable_job(tmp_path, decisions=[])
|
||||
client = TestClient(app)
|
||||
monkeypatch.setattr("backend.main.get_job", lambda job_id: {
|
||||
"job_id": job_id, "status": "needs_review", "out_dir": out_dir,
|
||||
})
|
||||
response = client.post("/jobs/job1/finalize-review")
|
||||
assert response.status_code == 409
|
||||
body = response.json()
|
||||
assert body["detail"]["detail"] == "incomplete review"
|
||||
assert body["detail"]["progress"]["remaining"] == 1
|
||||
|
||||
|
||||
def test_finalize_review_404_for_unknown_job(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("backend.config.OUTPUT_DIR", str(tmp_path))
|
||||
client = TestClient(app)
|
||||
assert client.post("/jobs/nope/finalize-review").status_code == 404
|
||||
|
||||
|
||||
def test_finalize_review_409_when_already_done(monkeypatch, tmp_path):
|
||||
client = TestClient(app)
|
||||
monkeypatch.setattr("backend.main.get_job", lambda job_id: {
|
||||
"job_id": job_id, "status": "done", "out_dir": str(tmp_path),
|
||||
})
|
||||
assert client.post("/jobs/job1/finalize-review").status_code == 409
|
||||
|
||||
|
||||
def test_finalize_review_409_for_job_not_in_review(monkeypatch, tmp_path):
|
||||
"""A running (or otherwise non-review) job must not be finalizable: no
|
||||
finalization thread, no artifact clobbering, no final email."""
|
||||
threads = []
|
||||
notified = []
|
||||
monkeypatch.setattr(threading, "Thread",
|
||||
lambda *a, **kw: threads.append((a, kw)) or _SyncThread(*a, **kw))
|
||||
monkeypatch.setattr("backend.jobs._notify",
|
||||
lambda *a, **kw: notified.append(a))
|
||||
monkeypatch.setattr("backend.main.get_job", lambda job_id: {
|
||||
"job_id": job_id, "status": "running", "out_dir": str(tmp_path),
|
||||
})
|
||||
client = TestClient(app)
|
||||
response = client.post("/jobs/job1/finalize-review")
|
||||
assert response.status_code == 409
|
||||
assert "running" in response.json()["detail"]["detail"]
|
||||
assert threads == []
|
||||
assert notified == []
|
||||
assert not os.path.exists(os.path.join(str(tmp_path), "conflicts.json"))
|
||||
|
||||
|
||||
def test_finalize_review_happy_path_notifies_once(monkeypatch, tmp_path):
|
||||
out_dir = _write_finalizable_job(tmp_path, decisions=[{
|
||||
"review_item_id": "finding:AGENT-0001", "decision": "confirm",
|
||||
}])
|
||||
notified = []
|
||||
monkeypatch.setattr("backend.review.finalizer._draft_rfis", lambda kept: [])
|
||||
monkeypatch.setattr("backend.jobs._notify",
|
||||
lambda job_id, report, out_dir: notified.append(job_id))
|
||||
monkeypatch.setattr(threading, "Thread", _SyncThread)
|
||||
job_id = "jobfinalize"
|
||||
backend.jobs._jobs[job_id] = {
|
||||
"job_id": job_id, "status": "reviewing", "out_dir": out_dir,
|
||||
"report": None, "email": "arch@example.com",
|
||||
}
|
||||
try:
|
||||
client = TestClient(app)
|
||||
response = client.post(f"/jobs/{job_id}/finalize-review")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "finalizing"}
|
||||
job = backend.jobs.get_job(job_id)
|
||||
assert job["status"] == "done"
|
||||
assert job["report"]["summary"]["agent_status"] == "complete"
|
||||
assert notified == [job_id]
|
||||
for name in ("conflicts.json", "validated_issues.json",
|
||||
"suppressed_issues.json", "rfis.json", "report.md"):
|
||||
assert os.path.isfile(os.path.join(out_dir, name)), name
|
||||
finally:
|
||||
backend.jobs._jobs.pop(job_id, None)
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Tests for the two-phase email flow: review-required notice, then final report."""
|
||||
|
||||
import threading
|
||||
|
||||
import backend.jobs
|
||||
from backend.email_sender import send_review_required
|
||||
|
||||
|
||||
class _SyncThread:
|
||||
"""Drop-in threading.Thread replacement that runs the target inline."""
|
||||
|
||||
def __init__(self, target=None, args=(), **kwargs):
|
||||
self._target = target
|
||||
self._args = args
|
||||
|
||||
def start(self):
|
||||
self._target(*self._args)
|
||||
|
||||
|
||||
def test_review_required_email_skips_without_smtp(monkeypatch):
|
||||
monkeypatch.setattr("backend.email_sender._smtp_ready", lambda: False)
|
||||
assert send_review_required("user@example.com", {"source": "set.pdf", "summary": {}}, "http://localhost:8099/?job=abc") is False
|
||||
|
||||
|
||||
def test_review_required_email_sends_with_smtp(monkeypatch):
|
||||
"""With SMTP ready, the message goes out with recipient, review URL, and
|
||||
the required-item count (0 when the report has no review summary)."""
|
||||
sent = []
|
||||
monkeypatch.setattr("backend.email_sender._smtp_ready", lambda: True)
|
||||
monkeypatch.setattr("backend.email_sender._send",
|
||||
lambda msg: sent.append(msg) or True)
|
||||
|
||||
review_url = "http://localhost:8099/?job=abc"
|
||||
report = {"source": "set.pdf", "summary": {"review": {"required": 3}}}
|
||||
assert send_review_required("user@example.com", report, review_url) is True
|
||||
|
||||
assert len(sent) == 1
|
||||
msg = sent[0]
|
||||
assert msg["To"] == "user@example.com"
|
||||
assert "review" in msg["Subject"].lower()
|
||||
body = msg.get_content()
|
||||
assert "review" in body.lower()
|
||||
assert review_url in body
|
||||
assert "3" in body
|
||||
|
||||
# Missing review summary -> required count defaults to 0.
|
||||
sent.clear()
|
||||
assert send_review_required("user@example.com", {"source": "set.pdf", "summary": {}}, review_url) is True
|
||||
assert "0" in sent[0].get_content()
|
||||
|
||||
|
||||
def test_agent_needs_review_sends_review_email_not_report(monkeypatch, tmp_path):
|
||||
"""An agent job entering needs_review emails the review-required notice
|
||||
exactly once and never sends the final conflict report."""
|
||||
review_emails = []
|
||||
report_emails = []
|
||||
monkeypatch.setattr("backend.config.OUTPUT_DIR", str(tmp_path))
|
||||
monkeypatch.setattr("backend.jobs.run_agent_pipeline", lambda pdf_path, **kw: {
|
||||
"summary": {"agent_status": "needs_review"}, "conflicts": [],
|
||||
})
|
||||
monkeypatch.setattr("backend.jobs.send_review_required",
|
||||
lambda *a, **kw: review_emails.append((a, kw)))
|
||||
monkeypatch.setattr("backend.jobs.send_conflict_report",
|
||||
lambda *a, **kw: report_emails.append((a, kw)))
|
||||
monkeypatch.setattr(threading, "Thread", _SyncThread)
|
||||
|
||||
pdf = tmp_path / "upload.pdf"
|
||||
pdf.write_bytes(b"%PDF-1.4 dummy")
|
||||
job_id = backend.jobs.create_job(str(pdf), source_filename="set.pdf",
|
||||
email="arch@example.com", pipeline_mode="agent")
|
||||
try:
|
||||
job = backend.jobs.get_job(job_id)
|
||||
assert job["status"] == "needs_review"
|
||||
assert len(review_emails) == 1
|
||||
args, _ = review_emails[0]
|
||||
assert args[0] == "arch@example.com"
|
||||
assert f"/?job={job_id}" in args[2]
|
||||
assert report_emails == []
|
||||
finally:
|
||||
backend.jobs._jobs.pop(job_id, None)
|
||||
|
||||
|
||||
def test_classic_job_sends_only_conflict_report(monkeypatch, tmp_path):
|
||||
"""Classic pipeline is untouched: only the final report email fires."""
|
||||
review_emails = []
|
||||
report_emails = []
|
||||
monkeypatch.setattr("backend.config.OUTPUT_DIR", str(tmp_path))
|
||||
monkeypatch.setattr("backend.jobs.run_pipeline", lambda pdf_path, **kw: {
|
||||
"summary": {}, "conflicts": [],
|
||||
})
|
||||
monkeypatch.setattr("backend.jobs.send_review_required",
|
||||
lambda *a, **kw: review_emails.append((a, kw)))
|
||||
monkeypatch.setattr("backend.jobs.send_conflict_report",
|
||||
lambda *a, **kw: report_emails.append((a, kw)))
|
||||
monkeypatch.setattr(threading, "Thread", _SyncThread)
|
||||
|
||||
pdf = tmp_path / "upload.pdf"
|
||||
pdf.write_bytes(b"%PDF-1.4 dummy")
|
||||
job_id = backend.jobs.create_job(str(pdf), source_filename="set.pdf",
|
||||
email="arch@example.com", pipeline_mode="classic")
|
||||
try:
|
||||
assert backend.jobs.get_job(job_id)["status"] == "done"
|
||||
assert len(report_emails) == 1
|
||||
assert review_emails == []
|
||||
finally:
|
||||
backend.jobs._jobs.pop(job_id, None)
|
||||
Reference in New Issue
Block a user