John Wilganowski
1c1d2ff21b
Agent web jobs now stop after Brain consolidation and enter needs_review with a persisted review queue (blocking: high-severity, low-confidence, sensitive-category findings; audit sample of clean clusters). Humans decide confirm/reject/unsure/needs_clarification via new review API and frontend queue; a finalizer applies decisions (rejections suppressed with reason codes), performs bounded targeted reruns for clarifications, drafts RFIs only for kept issues, and only then marks the job done and sends the final email. Two-phase email (review-required, then final report), per-decision feedback labels with redacted aggregate metrics, restart recovery from job artifacts, and CLI --no-review bypass. Classic pipeline unchanged. 65 non-LLM tests.
131 lines
4.4 KiB
Python
131 lines
4.4 KiB
Python
from backend import config
|
|
from backend.review.policy import build_audit_sample, requires_review
|
|
from backend.review.schemas import validate_decision
|
|
|
|
|
|
def test_high_severity_requires_review():
|
|
issue = {"severity": "high", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"}
|
|
assert "severity_high" in requires_review(issue)
|
|
|
|
|
|
def test_low_confidence_requires_review():
|
|
issue = {"severity": "low", "confidence": "low", "category": "note_or_spec_contradiction", "source_stage": "conflict"}
|
|
assert "confidence_low" in requires_review(issue)
|
|
|
|
|
|
def test_sensitive_code_category_requires_review():
|
|
issue = {"severity": "medium", "confidence": "high", "category": "egress", "source_stage": "code"}
|
|
assert "sensitive_category" in requires_review(issue)
|
|
|
|
|
|
def test_medium_high_confidence_note_does_not_require_review():
|
|
issue = {"severity": "medium", "confidence": "high", "category": "note_or_spec_contradiction", "source_stage": "conflict"}
|
|
assert requires_review(issue) == []
|
|
|
|
|
|
def test_build_audit_sample_returns_clean_cluster_spot_check():
|
|
memory = {
|
|
"clusters": [
|
|
{
|
|
"key": "room:101",
|
|
"location": "Room 101",
|
|
"assertions": [{"id": "a1"}, {"id": "a2"}],
|
|
}
|
|
],
|
|
"findings": [],
|
|
}
|
|
prioritized = []
|
|
items = build_audit_sample(memory, prioritized)
|
|
assert len(items) == 1
|
|
item = items[0]
|
|
assert item["kind"] == "clean_cluster"
|
|
assert item["blocking"] is False
|
|
assert item["review_item_id"] == "clean_cluster:room:101"
|
|
|
|
|
|
def test_build_audit_sample_strips_base64_from_assertions():
|
|
memory = {
|
|
"clusters": [
|
|
{
|
|
"key": "room:101",
|
|
"assertions": [
|
|
{"id": "a1", "base64": "AAAA"},
|
|
{"id": "a2", "base64": "BBBB"},
|
|
],
|
|
}
|
|
],
|
|
"findings": [],
|
|
}
|
|
items = build_audit_sample(memory, [])
|
|
assert len(items) == 1
|
|
assertions = items[0]["payload"]["assertions"]
|
|
assert assertions == [{"id": "a1"}, {"id": "a2"}]
|
|
assert all("base64" not in assertion for assertion in assertions)
|
|
|
|
|
|
def test_build_audit_sample_respects_limit():
|
|
memory = {
|
|
"clusters": [
|
|
{"key": f"room:{index}", "assertions": [{"id": "a"}, {"id": "b"}]}
|
|
for index in range(4)
|
|
],
|
|
"findings": [],
|
|
}
|
|
items = build_audit_sample(memory, [], limit=2)
|
|
assert len(items) == 2
|
|
assert [item["review_item_id"] for item in items] == [
|
|
"clean_cluster:room:0",
|
|
"clean_cluster:room:1",
|
|
]
|
|
|
|
|
|
def test_build_audit_sample_excludes_implicated_clusters():
|
|
memory = {
|
|
"clusters": [
|
|
{"key": "room:101", "assertions": [{"id": "a1"}, {"id": "a2"}]},
|
|
{"key": "room:102", "assertions": [{"id": "b1"}, {"id": "b2"}]},
|
|
],
|
|
"findings": [{"scope_id": "conflict:room:101"}],
|
|
}
|
|
items = build_audit_sample(memory, [])
|
|
assert [item["review_item_id"] for item in items] == ["clean_cluster:room:102"]
|
|
|
|
|
|
def test_validate_decision_confirm_without_reason_code():
|
|
result = validate_decision({"review_item_id": "x", "decision": "confirm"})
|
|
assert result is not None
|
|
assert result["decision"] == "confirm"
|
|
assert result["reason_code"] is None
|
|
|
|
|
|
def test_validate_decision_reject_with_valid_reason_code():
|
|
result = validate_decision({"decision": "reject", "reason_code": "duplicate"})
|
|
assert result is not None
|
|
assert result["reason_code"] == "duplicate"
|
|
|
|
|
|
def test_validate_decision_reject_with_missing_reason_code_returns_none():
|
|
assert validate_decision({"decision": "reject"}) is None
|
|
|
|
|
|
def test_validate_decision_reject_with_invalid_reason_code_returns_none():
|
|
assert validate_decision({"decision": "reject", "reason_code": "bogus"}) is None
|
|
|
|
|
|
def test_validate_decision_unknown_decision_returns_none():
|
|
assert validate_decision({"decision": "approve"}) is None
|
|
|
|
|
|
def test_validate_decision_non_dict_returns_none():
|
|
assert validate_decision("confirm") is None
|
|
|
|
|
|
def test_validate_decision_invalid_reason_code_on_non_reject_returns_none():
|
|
assert validate_decision({"decision": "confirm", "reason_code": "bogus"}) is None
|
|
|
|
|
|
def test_review_defaults():
|
|
assert config.AGENT_REQUIRE_REVIEW is True
|
|
assert config.AGENT_REVIEW_AUDIT_SAMPLE == 5
|
|
assert config.REVIEW_AGGREGATE_INCLUDE_TEXT is False
|