Add required human review gate to the Agent pipeline.
Docker Release / build-and-push (push) Successful in 1m10s
Docker Release / release (push) Skipped

Agent web jobs now stop after Brain consolidation and enter needs_review
with a persisted review queue (blocking: high-severity, low-confidence,
sensitive-category findings; audit sample of clean clusters). Humans
decide confirm/reject/unsure/needs_clarification via new review API and
frontend queue; a finalizer applies decisions (rejections suppressed with
reason codes), performs bounded targeted reruns for clarifications,
drafts RFIs only for kept issues, and only then marks the job done and
sends the final email. Two-phase email (review-required, then final
report), per-decision feedback labels with redacted aggregate metrics,
restart recovery from job artifacts, and CLI --no-review bypass.
Classic pipeline unchanged. 65 non-LLM tests.
This commit is contained in:
John Wilganowski
2026-07-28 19:23:57 +00:00
parent ac328d34fd
commit 1c1d2ff21b
27 changed files with 3344 additions and 22 deletions
+130
View File
@@ -0,0 +1,130 @@
from backend import config
from backend.review.policy import build_audit_sample, requires_review
from backend.review.schemas import validate_decision
def test_high_severity_requires_review():
issue = {"severity": "high", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"}
assert "severity_high" in requires_review(issue)
def test_low_confidence_requires_review():
issue = {"severity": "low", "confidence": "low", "category": "note_or_spec_contradiction", "source_stage": "conflict"}
assert "confidence_low" in requires_review(issue)
def test_sensitive_code_category_requires_review():
issue = {"severity": "medium", "confidence": "high", "category": "egress", "source_stage": "code"}
assert "sensitive_category" in requires_review(issue)
def test_medium_high_confidence_note_does_not_require_review():
issue = {"severity": "medium", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"}
assert requires_review(issue) == []
def test_build_audit_sample_returns_clean_cluster_spot_check():
memory = {
"clusters": [
{
"key": "room:101",
"location": "Room 101",
"assertions": [{"id": "a1"}, {"id": "a2"}],
}
],
"findings": [],
}
prioritized = []
items = build_audit_sample(memory, prioritized)
assert len(items) == 1
item = items[0]
assert item["kind"] == "clean_cluster"
assert item["blocking"] is False
assert item["review_item_id"] == "clean_cluster:room:101"
def test_build_audit_sample_strips_base64_from_assertions():
memory = {
"clusters": [
{
"key": "room:101",
"assertions": [
{"id": "a1", "base64": "AAAA"},
{"id": "a2", "base64": "BBBB"},
],
}
],
"findings": [],
}
items = build_audit_sample(memory, [])
assert len(items) == 1
assertions = items[0]["payload"]["assertions"]
assert assertions == [{"id": "a1"}, {"id": "a2"}]
assert all("base64" not in assertion for assertion in assertions)
def test_build_audit_sample_respects_limit():
memory = {
"clusters": [
{"key": f"room:{index}", "assertions": [{"id": "a"}, {"id": "b"}]}
for index in range(4)
],
"findings": [],
}
items = build_audit_sample(memory, [], limit=2)
assert len(items) == 2
assert [item["review_item_id"] for item in items] == [
"clean_cluster:room:0",
"clean_cluster:room:1",
]
def test_build_audit_sample_excludes_implicated_clusters():
memory = {
"clusters": [
{"key": "room:101", "assertions": [{"id": "a1"}, {"id": "a2"}]},
{"key": "room:102", "assertions": [{"id": "b1"}, {"id": "b2"}]},
],
"findings": [{"scope_id": "conflict:room:101"}],
}
items = build_audit_sample(memory, [])
assert [item["review_item_id"] for item in items] == ["clean_cluster:room:102"]
def test_validate_decision_confirm_without_reason_code():
result = validate_decision({"review_item_id": "x", "decision": "confirm"})
assert result is not None
assert result["decision"] == "confirm"
assert result["reason_code"] is None
def test_validate_decision_reject_with_valid_reason_code():
result = validate_decision({"decision": "reject", "reason_code": "duplicate"})
assert result is not None
assert result["reason_code"] == "duplicate"
def test_validate_decision_reject_with_missing_reason_code_returns_none():
assert validate_decision({"decision": "reject"}) is None
def test_validate_decision_reject_with_invalid_reason_code_returns_none():
assert validate_decision({"decision": "reject", "reason_code": "bogus"}) is None
def test_validate_decision_unknown_decision_returns_none():
assert validate_decision({"decision": "approve"}) is None
def test_validate_decision_non_dict_returns_none():
assert validate_decision("confirm") is None
def test_validate_decision_invalid_reason_code_on_non_reject_returns_none():
assert validate_decision({"decision": "confirm", "reason_code": "bogus"}) is None
def test_review_defaults():
assert config.AGENT_REQUIRE_REVIEW is True
assert config.AGENT_REVIEW_AUDIT_SAMPLE == 5
assert config.REVIEW_AGGREGATE_INCLUDE_TEXT is False