John Wilganowski
1c1d2ff21b
Agent web jobs now stop after Brain consolidation and enter needs_review with a persisted review queue (blocking: high-severity, low-confidence, sensitive-category findings; audit sample of clean clusters). Humans decide confirm/reject/unsure/needs_clarification via new review API and frontend queue; a finalizer applies decisions (rejections suppressed with reason codes), performs bounded targeted reruns for clarifications, drafts RFIs only for kept issues, and only then marks the job done and sends the final email. Two-phase email (review-required, then final report), per-decision feedback labels with redacted aggregate metrics, restart recovery from job artifacts, and CLI --no-review bypass. Classic pipeline unchanged. 65 non-LLM tests.
66 lines
2.9 KiB
Python
66 lines
2.9 KiB
Python
from backend.agents.base import AgentResult
|
|
from backend.agents.runner import run_agent_pipeline
|
|
|
|
|
|
def _patch_brain(monkeypatch):
|
|
monkeypatch.setattr("backend.agents.runner.convert_pdf_to_images", lambda path: [{"page_number": 1, "base64": "x"}])
|
|
monkeypatch.setattr("backend.agents.runner.BrainAgent", lambda usage: type("B", (), {"run": lambda self, findings, sheet_index, jurisdiction: ([{"issue_id": "AGENT-0001", "severity": "high", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"}], [])})())
|
|
|
|
|
|
def test_agent_runner_can_enter_review_mode(monkeypatch, tmp_path):
|
|
_patch_brain(monkeypatch)
|
|
pdf = tmp_path / "dummy.pdf"
|
|
pdf.write_bytes(b"%PDF-1.4\n")
|
|
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path), require_review=True)
|
|
assert report["summary"]["agent_status"] == "needs_review"
|
|
assert report["summary"]["review"]["required"] == 1
|
|
|
|
|
|
def test_review_mode_writes_memory_snapshot(monkeypatch, tmp_path):
|
|
"""The finalizer needs agent/memory.json for targeted clarification reruns."""
|
|
_patch_brain(monkeypatch)
|
|
pdf = tmp_path / "dummy.pdf"
|
|
pdf.write_bytes(b"%PDF-1.4\n")
|
|
run_agent_pipeline(str(pdf), out_dir=str(tmp_path), require_review=True)
|
|
assert (tmp_path / "agent" / "memory.json").is_file()
|
|
|
|
|
|
def test_review_mode_summary_includes_agent_observability(monkeypatch, tmp_path):
|
|
"""Review-mode candidate reports must carry the same usage/stats block as
|
|
the wave-7 path so finalizer fix-ups and feedback labels have real data."""
|
|
_patch_brain(monkeypatch)
|
|
pdf = tmp_path / "dummy.pdf"
|
|
pdf.write_bytes(b"%PDF-1.4\n")
|
|
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path), require_review=True)
|
|
summary = report["summary"]
|
|
assert "agent_stats" in summary
|
|
assert summary["by_stage"]["rfis"] == 0
|
|
assert summary["by_stage"]["validated"] == 1
|
|
assert "conflicts" in summary["by_stage"]
|
|
assert "cost_usd" in summary
|
|
assert "llm_calls" in summary
|
|
assert "cached_calls" in summary
|
|
assert "cost_by_stage" in summary
|
|
assert "models_used" in summary
|
|
|
|
|
|
def test_agent_runner_without_review_still_writes_rfis(monkeypatch, tmp_path):
|
|
_patch_brain(monkeypatch)
|
|
monkeypatch.setattr(
|
|
"backend.agents.runner.RFIWriterAgent",
|
|
lambda usage: type("R", (), {
|
|
"name": "rfi_writer",
|
|
"run": lambda self, scope: AgentResult(
|
|
scope_id=scope.scope_id,
|
|
artifacts=[{"issue_id": "AGENT-0001", "question": "Confirm intent?"}],
|
|
),
|
|
})(),
|
|
)
|
|
pdf = tmp_path / "dummy.pdf"
|
|
pdf.write_bytes(b"%PDF-1.4\n")
|
|
report = run_agent_pipeline(str(pdf), out_dir=str(tmp_path), require_review=False)
|
|
assert report["summary"]["agent_status"] == "complete"
|
|
assert "review" not in report["summary"]
|
|
assert len(report["rfis"]) == 1
|
|
assert report["rfis"][0]["issue_id"] == "AGENT-0001"
|