Add required human review gate to the Agent pipeline.
Agent web jobs now stop after Brain consolidation and enter needs_review with a persisted review queue (blocking: high-severity, low-confidence, sensitive-category findings; audit sample of clean clusters). Humans decide confirm/reject/unsure/needs_clarification via new review API and frontend queue; a finalizer applies decisions (rejections suppressed with reason codes), performs bounded targeted reruns for clarifications, drafts RFIs only for kept issues, and only then marks the job done and sends the final email. Two-phase email (review-required, then final report), per-decision feedback labels with redacted aggregate metrics, restart recovery from job artifacts, and CLI --no-review bypass. Classic pipeline unchanged. 65 non-LLM tests.
This commit is contained in:
John Wilganowski
@@ -0,0 +1,87 @@
|
||||
"""Feedback labels and aggregate metrics for human-review decisions."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
from backend.review.feedback import decision_to_label, write_label
|
||||
from backend.review.metrics import aggregate_labels
|
||||
|
||||
|
||||
def test_aggregate_redacts_text_by_default():
|
||||
labels = [{"decision": "reject", "reason_code": "missing_evidence", "comment": "secret", "payload": {"evidence": [{"source_text": "secret"}]}}]
|
||||
summary = aggregate_labels(labels)
|
||||
assert summary["reject"] == 1
|
||||
assert "secret" not in str(summary)
|
||||
|
||||
|
||||
def test_aggregate_include_text_embeds_labels():
|
||||
labels = [{"decision": "reject", "reason_code": "missing_evidence", "comment": "secret"}]
|
||||
summary = aggregate_labels(labels, include_text=True)
|
||||
assert summary["labels"] == labels
|
||||
|
||||
|
||||
def _queue_item() -> dict:
|
||||
return {
|
||||
"review_item_id": "finding:AGENT-0007",
|
||||
"kind": "finding",
|
||||
"blocking": True,
|
||||
"reasons": ["high_severity"],
|
||||
"payload": {
|
||||
"issue_id": "AGENT-0007",
|
||||
"source_stage": "conflict",
|
||||
"category": "elevation_disagreement",
|
||||
"severity": "high",
|
||||
"confidence": "medium",
|
||||
"location": "Room 204 / Level 2",
|
||||
"disciplines": ["Architectural", "Mechanical"],
|
||||
"sheets": ["A2.1", "M2.1"],
|
||||
"drawing_type": "floor_plan",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_decision_to_label_builds_spec_shape():
|
||||
decision = {"review_item_id": "finding:AGENT-0007", "decision": "reject",
|
||||
"reason_code": "same_value_different_representation"}
|
||||
job = {"job_id": "abc123", "pipeline_mode": "agent",
|
||||
"report": {"summary": {"models_used": ["google/gemini-2.5-pro"]}}}
|
||||
label = decision_to_label(_queue_item(), decision, job)
|
||||
assert label["review_item_id"] == "finding:AGENT-0007"
|
||||
assert label["job_id"] == "abc123"
|
||||
assert label["pipeline_mode"] == "agent"
|
||||
assert label["source_stage"] == "conflict"
|
||||
assert label["category"] == "elevation_disagreement"
|
||||
assert label["severity"] == "high"
|
||||
assert label["confidence"] == "medium"
|
||||
assert label["decision"] == "reject"
|
||||
assert label["reason_code"] == "same_value_different_representation"
|
||||
assert label["location"] == "Room 204 / Level 2"
|
||||
assert label["disciplines"] == ["Architectural", "Mechanical"]
|
||||
assert label["sheets"] == ["A2.1", "M2.1"]
|
||||
assert label["drawing_type"] == "floor_plan"
|
||||
assert label["models_used"] == ["google/gemini-2.5-pro"]
|
||||
datetime.fromisoformat(label["created_at"])
|
||||
|
||||
|
||||
def test_decision_to_label_degrades_on_missing_fields():
|
||||
label = decision_to_label({"review_item_id": "finding:AGENT-0001"}, {}, {})
|
||||
assert label["review_item_id"] == "finding:AGENT-0001"
|
||||
assert label["job_id"] is None
|
||||
assert label["decision"] is None
|
||||
assert label["reason_code"] is None
|
||||
assert label["category"] is None
|
||||
assert label["source_stage"] is None
|
||||
assert label["models_used"] == []
|
||||
datetime.fromisoformat(label["created_at"])
|
||||
|
||||
|
||||
def test_write_label_appends_json_lines(tmp_path):
|
||||
label1 = {"review_item_id": "finding:AGENT-0001", "decision": "confirm"}
|
||||
label2 = {"review_item_id": "finding:AGENT-0002", "decision": "reject"}
|
||||
write_label(str(tmp_path), label1)
|
||||
write_label(str(tmp_path), label2)
|
||||
path = os.path.join(str(tmp_path), "review", "feedback_labels.jsonl")
|
||||
with open(path, encoding="utf-8") as f:
|
||||
lines = [json.loads(line) for line in f if line.strip()]
|
||||
assert lines == [label1, label2]
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Non-LLM tests for the review finalizer: decisions, reruns, final artifacts."""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.agents.base import AgentResult
|
||||
from backend.review.finalizer import (
|
||||
apply_decisions,
|
||||
finalize_review,
|
||||
rerun_clarified_scopes,
|
||||
)
|
||||
from backend.review.store import ReviewStore
|
||||
|
||||
|
||||
def test_reject_suppresses_with_reason():
|
||||
prioritized = [{"issue_id": "AGENT-0001", "severity": "high"}]
|
||||
decisions = {"finding:AGENT-0001": {"decision": "reject", "reason_code": "duplicate"}}
|
||||
kept, suppressed = apply_decisions(prioritized, decisions)
|
||||
assert kept == []
|
||||
assert suppressed[0]["review_state"] == "rejected"
|
||||
assert suppressed[0]["reason_code"] == "duplicate"
|
||||
|
||||
|
||||
def test_unsure_is_kept_but_flagged():
|
||||
prioritized = [{"issue_id": "AGENT-0002", "severity": "medium"}]
|
||||
decisions = {"finding:AGENT-0002": {"decision": "unsure"}}
|
||||
kept, suppressed = apply_decisions(prioritized, decisions)
|
||||
assert kept[0]["review_state"] == "unsure"
|
||||
assert suppressed == []
|
||||
|
||||
|
||||
def _write_job(out_dir, prioritized, queue, decisions=None, memory=None):
|
||||
"""Hand-written review-mode artifacts (conflicts.json + agent/memory.json)."""
|
||||
os.makedirs(os.path.join(out_dir, "agent"), exist_ok=True)
|
||||
report = {
|
||||
"source": "set.pdf",
|
||||
"generated_at": "2026-07-28T00:00:00+00:00",
|
||||
"summary": {
|
||||
"sheets_analyzed": 0,
|
||||
"disciplines": [],
|
||||
"assertions_extracted": 0,
|
||||
"clusters_checked": 0,
|
||||
"conflicts_found": 0,
|
||||
"by_severity": {"high": 0, "medium": 0, "low": 0},
|
||||
"by_category": {},
|
||||
"pipeline_mode": "agent",
|
||||
"agent_status": "needs_review",
|
||||
"review": {"required": 1, "completed": 0, "remaining": 1, "total": 1},
|
||||
"by_stage": {"validated": len(prioritized), "rfis": 0},
|
||||
},
|
||||
"conflicts": [],
|
||||
"sheets": [],
|
||||
"validated_issues": prioritized,
|
||||
"suppressed_issues": [],
|
||||
"rfis": [],
|
||||
}
|
||||
with open(os.path.join(out_dir, "conflicts.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(report, f)
|
||||
with open(os.path.join(out_dir, "agent", "memory.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(memory or {}, f)
|
||||
store = ReviewStore(out_dir)
|
||||
store.write_queue(queue)
|
||||
for decision in decisions or []:
|
||||
store.append_decision(decision)
|
||||
|
||||
|
||||
def _blocking_item(issue_id):
|
||||
return {"review_item_id": f"finding:{issue_id}", "kind": "finding",
|
||||
"blocking": True, "reasons": ["high_severity"], "payload": {}}
|
||||
|
||||
|
||||
def test_finalize_confirm_keeps_confirmed(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("backend.review.finalizer._draft_rfis", lambda kept: [])
|
||||
_write_job(
|
||||
str(tmp_path),
|
||||
prioritized=[{"issue_id": "AGENT-0001", "severity": "high"}],
|
||||
queue=[_blocking_item("AGENT-0001")],
|
||||
decisions=[{"review_item_id": "finding:AGENT-0001", "decision": "confirm"}],
|
||||
)
|
||||
report = finalize_review("job1", str(tmp_path))
|
||||
assert report["validated_issues"][0]["review_state"] == "confirmed"
|
||||
assert report["suppressed_issues"] == []
|
||||
assert report["summary"]["agent_status"] == "complete"
|
||||
|
||||
|
||||
def test_finalize_no_decision_keeps_unreviewed(monkeypatch, tmp_path):
|
||||
"""Non-blocking (audit) items don't need a decision; issue stays unreviewed."""
|
||||
monkeypatch.setattr("backend.review.finalizer._draft_rfis", lambda kept: [])
|
||||
item = {**_blocking_item("AGENT-0001"), "blocking": False, "kind": "audit_finding"}
|
||||
_write_job(
|
||||
str(tmp_path),
|
||||
prioritized=[{"issue_id": "AGENT-0001", "severity": "medium"}],
|
||||
queue=[item],
|
||||
)
|
||||
report = finalize_review("job1", str(tmp_path))
|
||||
assert report["validated_issues"][0]["review_state"] == "unreviewed"
|
||||
|
||||
|
||||
def test_finalize_clarification_replacement_marked_clarified(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("backend.review.finalizer._draft_rfis", lambda kept: [])
|
||||
replacement = {"issue_id": "AGENT-0001-R1", "severity": "medium",
|
||||
"clarification_of": "AGENT-0001"}
|
||||
monkeypatch.setattr(
|
||||
"backend.review.finalizer.rerun_clarified_scopes",
|
||||
lambda snapshot, decisions, prioritized=None: [replacement],
|
||||
)
|
||||
_write_job(
|
||||
str(tmp_path),
|
||||
prioritized=[{"issue_id": "AGENT-0001", "severity": "high"}],
|
||||
queue=[_blocking_item("AGENT-0001")],
|
||||
decisions=[{"review_item_id": "finding:AGENT-0001",
|
||||
"decision": "needs_clarification",
|
||||
"clarification_answer": "Ceiling is 9'-0\" AFF."}],
|
||||
)
|
||||
report = finalize_review("job1", str(tmp_path))
|
||||
kept = report["validated_issues"]
|
||||
assert [issue["issue_id"] for issue in kept] == ["AGENT-0001-R1"]
|
||||
assert kept[0]["review_state"] == "clarified"
|
||||
|
||||
|
||||
def test_finalize_failed_clarification_flagged(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("backend.review.finalizer._draft_rfis", lambda kept: [])
|
||||
monkeypatch.setattr(
|
||||
"backend.review.finalizer.rerun_clarified_scopes",
|
||||
lambda snapshot, decisions, prioritized=None: [],
|
||||
)
|
||||
_write_job(
|
||||
str(tmp_path),
|
||||
prioritized=[{"issue_id": "AGENT-0001", "severity": "high"}],
|
||||
queue=[_blocking_item("AGENT-0001")],
|
||||
decisions=[{"review_item_id": "finding:AGENT-0001",
|
||||
"decision": "needs_clarification",
|
||||
"clarification_answer": "Ceiling is 9'-0\" AFF."}],
|
||||
)
|
||||
report = finalize_review("job1", str(tmp_path))
|
||||
assert report["validated_issues"][0]["review_state"] == "clarification_failed"
|
||||
|
||||
|
||||
def test_finalize_incomplete_review_raises(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("backend.review.finalizer._draft_rfis", lambda kept: [])
|
||||
_write_job(
|
||||
str(tmp_path),
|
||||
prioritized=[{"issue_id": "AGENT-0001", "severity": "high"}],
|
||||
queue=[_blocking_item("AGENT-0001")],
|
||||
)
|
||||
with pytest.raises(ValueError, match="incomplete review"):
|
||||
finalize_review("job1", str(tmp_path))
|
||||
|
||||
|
||||
def test_finalize_writes_final_artifacts(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("backend.review.finalizer._draft_rfis",
|
||||
lambda kept: [{"issue_id": kept[0]["issue_id"], "question": "?"}])
|
||||
_write_job(
|
||||
str(tmp_path),
|
||||
prioritized=[{"issue_id": "AGENT-0001", "severity": "high"}],
|
||||
queue=[_blocking_item("AGENT-0001")],
|
||||
decisions=[{"review_item_id": "finding:AGENT-0001", "decision": "confirm"}],
|
||||
)
|
||||
report = finalize_review("job1", str(tmp_path))
|
||||
assert report["summary"]["by_stage"]["validated"] == 1
|
||||
assert report["summary"]["by_stage"]["rfis"] == 1
|
||||
for name in ("conflicts.json", "validated_issues.json",
|
||||
"suppressed_issues.json", "rfis.json", "report.md"):
|
||||
assert os.path.isfile(os.path.join(str(tmp_path), name)), name
|
||||
with open(os.path.join(str(tmp_path), "validated_issues.json"), encoding="utf-8") as f:
|
||||
assert json.load(f)[0]["review_state"] == "confirmed"
|
||||
|
||||
|
||||
def test_finalize_reject_rebuilds_conflicts_and_counts(monkeypatch, tmp_path):
|
||||
"""Rejected conflict-stage findings must not survive into the final
|
||||
report's conflicts / headline counts; suppressed_issues keeps them."""
|
||||
monkeypatch.setattr("backend.review.finalizer._draft_rfis", lambda kept: [])
|
||||
kept_finding = {
|
||||
"issue_id": "AGENT-0001", "source_stage": "conflict",
|
||||
"category": "note_or_spec_contradiction", "severity": "high",
|
||||
"location": "Grid A", "disciplines": ["A", "S"], "sheets": ["A-1"],
|
||||
"description": "kept finding", "evidence": [],
|
||||
"recommended_resolution": "fix", "confidence": "high",
|
||||
}
|
||||
rejected_finding = {
|
||||
**kept_finding, "issue_id": "AGENT-0002", "severity": "medium",
|
||||
"description": "rejected finding",
|
||||
}
|
||||
_write_job(
|
||||
str(tmp_path),
|
||||
prioritized=[kept_finding, rejected_finding],
|
||||
queue=[_blocking_item("AGENT-0001"), _blocking_item("AGENT-0002")],
|
||||
decisions=[
|
||||
{"review_item_id": "finding:AGENT-0001", "decision": "confirm"},
|
||||
{"review_item_id": "finding:AGENT-0002", "decision": "reject",
|
||||
"reason_code": "not_a_contradiction"},
|
||||
],
|
||||
)
|
||||
# Simulate the pre-review candidate values the finalizer must overwrite.
|
||||
candidate_path = os.path.join(str(tmp_path), "conflicts.json")
|
||||
with open(candidate_path, encoding="utf-8") as f:
|
||||
candidate = json.load(f)
|
||||
candidate["conflicts"] = [{"description": "kept finding", "severity": "high",
|
||||
"category": "note_or_spec_contradiction"},
|
||||
{"description": "rejected finding", "severity": "medium",
|
||||
"category": "note_or_spec_contradiction"}]
|
||||
candidate["summary"]["conflicts_found"] = 2
|
||||
candidate["summary"]["by_severity"] = {"high": 1, "medium": 1, "low": 0}
|
||||
candidate["summary"]["by_category"] = {"note_or_spec_contradiction": 2}
|
||||
with open(candidate_path, "w", encoding="utf-8") as f:
|
||||
json.dump(candidate, f)
|
||||
|
||||
report = finalize_review("job1", str(tmp_path))
|
||||
assert [c["description"] for c in report["conflicts"]] == ["kept finding"]
|
||||
assert report["summary"]["conflicts_found"] == 1
|
||||
assert report["summary"]["by_severity"] == {"high": 1, "medium": 0, "low": 0}
|
||||
assert report["summary"]["by_category"] == {"note_or_spec_contradiction": 1}
|
||||
suppressed = report["suppressed_issues"]
|
||||
assert [s["issue_id"] for s in suppressed] == ["AGENT-0002"]
|
||||
assert suppressed[0]["review_state"] == "rejected"
|
||||
assert suppressed[0]["reason_code"] == "not_a_contradiction"
|
||||
with open(os.path.join(str(tmp_path), "report.md"), encoding="utf-8") as f:
|
||||
assert "rejected finding" not in f.read()
|
||||
|
||||
|
||||
def test_rerun_missing_cluster_degrades_to_analysis_gap():
|
||||
snapshot = {"findings": [{"issue_id": "AGENT-0001", "scope_id": "conflict:link:1"}],
|
||||
"clusters": []}
|
||||
decisions = {"finding:AGENT-0001": {
|
||||
"decision": "needs_clarification", "clarification_answer": "9'-0\" AFF"}}
|
||||
findings = rerun_clarified_scopes(snapshot, decisions)
|
||||
assert len(findings) == 1
|
||||
assert findings[0]["category"] == "analysis_gap"
|
||||
assert findings[0]["source_stage"] == "qaqc"
|
||||
assert findings[0]["severity"] == "low"
|
||||
assert findings[0]["confidence"] == "high"
|
||||
|
||||
|
||||
def test_rerun_non_conflict_scope_noted_as_analysis_gap():
|
||||
"""v1 only reruns conflict scopes; other scopes get a visible gap, no raise."""
|
||||
snapshot = {"findings": [{"issue_id": "AGENT-0002", "scope_id": "code: egress"}],
|
||||
"clusters": []}
|
||||
decisions = {"finding:AGENT-0002": {
|
||||
"decision": "needs_clarification", "clarification_answer": "Corridor is 44 in."}}
|
||||
findings = rerun_clarified_scopes(snapshot, decisions)
|
||||
assert len(findings) == 1
|
||||
assert findings[0]["category"] == "analysis_gap"
|
||||
|
||||
|
||||
def test_rerun_successful_scope_prepends_clarification_and_tags(monkeypatch):
|
||||
"""Real rerun path (non-LLM): cluster lookup, pseudo-assertion injection,
|
||||
and clarification_of tagging through the real rerun_clarified_scopes."""
|
||||
captured = {}
|
||||
|
||||
class FakeCritic:
|
||||
name = "conflict_critic"
|
||||
|
||||
def __init__(self, usage):
|
||||
pass
|
||||
|
||||
def run(self, scope):
|
||||
captured["scope"] = scope
|
||||
return AgentResult(
|
||||
scope_id=scope.scope_id,
|
||||
artifacts=[{"issue_id": "AGENT-0001-R1", "severity": "medium"}],
|
||||
)
|
||||
|
||||
monkeypatch.setattr("backend.review.finalizer.ConflictCriticAgent", FakeCritic)
|
||||
snapshot = {
|
||||
"findings": [{"issue_id": "AGENT-0001", "scope_id": "conflict:link:1"}],
|
||||
"clusters": [{"key": "link:1",
|
||||
"assertions": [{"attribute": "height", "value": "10'-0\""}]}],
|
||||
}
|
||||
decisions = {"finding:AGENT-0001": {
|
||||
"decision": "needs_clarification", "clarification_answer": "9'-0\" AFF"}}
|
||||
findings = rerun_clarified_scopes(snapshot, decisions)
|
||||
|
||||
assert len(findings) == 1
|
||||
assert findings[0]["issue_id"] == "AGENT-0001-R1"
|
||||
assert findings[0]["clarification_of"] == "AGENT-0001"
|
||||
|
||||
payload = captured["scope"].payload
|
||||
assert payload["page_to_b64"] == {}
|
||||
assertions = payload["cluster"]["assertions"]
|
||||
# Prepended at index 0 so front-truncation can't drop the clarification.
|
||||
assert assertions[0]["discipline"] == "Reviewer"
|
||||
assert assertions[0]["attribute"] == "clarification"
|
||||
assert assertions[0]["value"] == "9'-0\" AFF"
|
||||
assert assertions[1]["attribute"] == "height"
|
||||
|
||||
|
||||
def test_rerun_ignores_other_decisions():
|
||||
decisions = {"finding:AGENT-0001": {"decision": "confirm"}}
|
||||
assert rerun_clarified_scopes({}, decisions) == []
|
||||
@@ -0,0 +1,28 @@
|
||||
from backend.review.gate import build_review_queue
|
||||
|
||||
|
||||
def test_gate_marks_blocking_and_audit_items():
|
||||
memory = {"clusters": [{"key": "room:101", "location": "Room 101", "assertions": [{"id": "a1"}, {"id": "a2"}]}], "findings": []}
|
||||
prioritized = [
|
||||
{"issue_id": "AGENT-0001", "severity": "high", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"},
|
||||
{"issue_id": "AGENT-0002", "severity": "low", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"},
|
||||
]
|
||||
queue = build_review_queue(memory, prioritized, [])
|
||||
by_id = {item["review_item_id"]: item for item in queue}
|
||||
assert by_id["finding:AGENT-0001"]["blocking"] is True
|
||||
assert by_id["finding:AGENT-0002"]["blocking"] is False
|
||||
assert any(item["kind"] == "clean_cluster" for item in queue)
|
||||
|
||||
|
||||
def test_gate_limit_caps_clean_cluster_items():
|
||||
memory = {
|
||||
"clusters": [
|
||||
{"key": f"room:{index}", "assertions": [{"id": "a"}, {"id": "b"}]}
|
||||
for index in range(3)
|
||||
],
|
||||
"findings": [],
|
||||
}
|
||||
queue = build_review_queue(memory, [], [], limit=1)
|
||||
clean_items = [item for item in queue if item["kind"] == "clean_cluster"]
|
||||
assert len(clean_items) == 1
|
||||
assert clean_items[0]["review_item_id"] == "clean_cluster:room:0"
|
||||
@@ -0,0 +1,130 @@
|
||||
from backend import config
|
||||
from backend.review.policy import build_audit_sample, requires_review
|
||||
from backend.review.schemas import validate_decision
|
||||
|
||||
|
||||
def test_high_severity_requires_review():
|
||||
issue = {"severity": "high", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"}
|
||||
assert "severity_high" in requires_review(issue)
|
||||
|
||||
|
||||
def test_low_confidence_requires_review():
|
||||
issue = {"severity": "low", "confidence": "low", "category": "note_or_spec_contradiction", "source_stage": "conflict"}
|
||||
assert "confidence_low" in requires_review(issue)
|
||||
|
||||
|
||||
def test_sensitive_code_category_requires_review():
|
||||
issue = {"severity": "medium", "confidence": "high", "category": "egress", "source_stage": "code"}
|
||||
assert "sensitive_category" in requires_review(issue)
|
||||
|
||||
|
||||
def test_medium_high_confidence_note_does_not_require_review():
|
||||
issue = {"severity": "medium", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"}
|
||||
assert requires_review(issue) == []
|
||||
|
||||
|
||||
def test_build_audit_sample_returns_clean_cluster_spot_check():
|
||||
memory = {
|
||||
"clusters": [
|
||||
{
|
||||
"key": "room:101",
|
||||
"location": "Room 101",
|
||||
"assertions": [{"id": "a1"}, {"id": "a2"}],
|
||||
}
|
||||
],
|
||||
"findings": [],
|
||||
}
|
||||
prioritized = []
|
||||
items = build_audit_sample(memory, prioritized)
|
||||
assert len(items) == 1
|
||||
item = items[0]
|
||||
assert item["kind"] == "clean_cluster"
|
||||
assert item["blocking"] is False
|
||||
assert item["review_item_id"] == "clean_cluster:room:101"
|
||||
|
||||
|
||||
def test_build_audit_sample_strips_base64_from_assertions():
|
||||
memory = {
|
||||
"clusters": [
|
||||
{
|
||||
"key": "room:101",
|
||||
"assertions": [
|
||||
{"id": "a1", "base64": "AAAA"},
|
||||
{"id": "a2", "base64": "BBBB"},
|
||||
],
|
||||
}
|
||||
],
|
||||
"findings": [],
|
||||
}
|
||||
items = build_audit_sample(memory, [])
|
||||
assert len(items) == 1
|
||||
assertions = items[0]["payload"]["assertions"]
|
||||
assert assertions == [{"id": "a1"}, {"id": "a2"}]
|
||||
assert all("base64" not in assertion for assertion in assertions)
|
||||
|
||||
|
||||
def test_build_audit_sample_respects_limit():
|
||||
memory = {
|
||||
"clusters": [
|
||||
{"key": f"room:{index}", "assertions": [{"id": "a"}, {"id": "b"}]}
|
||||
for index in range(4)
|
||||
],
|
||||
"findings": [],
|
||||
}
|
||||
items = build_audit_sample(memory, [], limit=2)
|
||||
assert len(items) == 2
|
||||
assert [item["review_item_id"] for item in items] == [
|
||||
"clean_cluster:room:0",
|
||||
"clean_cluster:room:1",
|
||||
]
|
||||
|
||||
|
||||
def test_build_audit_sample_excludes_implicated_clusters():
|
||||
memory = {
|
||||
"clusters": [
|
||||
{"key": "room:101", "assertions": [{"id": "a1"}, {"id": "a2"}]},
|
||||
{"key": "room:102", "assertions": [{"id": "b1"}, {"id": "b2"}]},
|
||||
],
|
||||
"findings": [{"scope_id": "conflict:room:101"}],
|
||||
}
|
||||
items = build_audit_sample(memory, [])
|
||||
assert [item["review_item_id"] for item in items] == ["clean_cluster:room:102"]
|
||||
|
||||
|
||||
def test_validate_decision_confirm_without_reason_code():
|
||||
result = validate_decision({"review_item_id": "x", "decision": "confirm"})
|
||||
assert result is not None
|
||||
assert result["decision"] == "confirm"
|
||||
assert result["reason_code"] is None
|
||||
|
||||
|
||||
def test_validate_decision_reject_with_valid_reason_code():
|
||||
result = validate_decision({"decision": "reject", "reason_code": "duplicate"})
|
||||
assert result is not None
|
||||
assert result["reason_code"] == "duplicate"
|
||||
|
||||
|
||||
def test_validate_decision_reject_with_missing_reason_code_returns_none():
|
||||
assert validate_decision({"decision": "reject"}) is None
|
||||
|
||||
|
||||
def test_validate_decision_reject_with_invalid_reason_code_returns_none():
|
||||
assert validate_decision({"decision": "reject", "reason_code": "bogus"}) is None
|
||||
|
||||
|
||||
def test_validate_decision_unknown_decision_returns_none():
|
||||
assert validate_decision({"decision": "approve"}) is None
|
||||
|
||||
|
||||
def test_validate_decision_non_dict_returns_none():
|
||||
assert validate_decision("confirm") is None
|
||||
|
||||
|
||||
def test_validate_decision_invalid_reason_code_on_non_reject_returns_none():
|
||||
assert validate_decision({"decision": "confirm", "reason_code": "bogus"}) is None
|
||||
|
||||
|
||||
def test_review_defaults():
|
||||
assert config.AGENT_REQUIRE_REVIEW is True
|
||||
assert config.AGENT_REVIEW_AUDIT_SAMPLE == 5
|
||||
assert config.REVIEW_AGGREGATE_INCLUDE_TEXT is False
|
||||
@@ -0,0 +1,24 @@
|
||||
import json
|
||||
from backend.review.store import ReviewStore
|
||||
|
||||
|
||||
def test_queue_and_decisions_round_trip(tmp_path):
|
||||
store = ReviewStore(str(tmp_path))
|
||||
queue = [{"review_item_id": "finding:1", "blocking": True}]
|
||||
store.write_queue(queue)
|
||||
assert store.read_queue() == queue
|
||||
store.append_decision({"review_item_id": "finding:1", "decision": "confirm"})
|
||||
assert store.read_decisions()["finding:1"]["decision"] == "confirm"
|
||||
|
||||
|
||||
def test_progress_counts_required_items(tmp_path):
|
||||
store = ReviewStore(str(tmp_path))
|
||||
queue = [
|
||||
{"review_item_id": "a", "blocking": True},
|
||||
{"review_item_id": "b", "blocking": False},
|
||||
]
|
||||
store.write_queue(queue)
|
||||
store.append_decision({"review_item_id": "a", "decision": "confirm"})
|
||||
progress = store.progress(queue)
|
||||
assert progress["required"] == 1
|
||||
assert progress["completed"] == 1
|
||||
Reference in New Issue
Block a user