John Wilganowski
1c1d2ff21b
Agent web jobs now stop after Brain consolidation and enter needs_review with a persisted review queue (blocking: high-severity, low-confidence, sensitive-category findings; audit sample of clean clusters). Humans decide confirm/reject/unsure/needs_clarification via new review API and frontend queue; a finalizer applies decisions (rejections suppressed with reason codes), performs bounded targeted reruns for clarifications, drafts RFIs only for kept issues, and only then marks the job done and sends the final email. Two-phase email (review-required, then final report), per-decision feedback labels with redacted aggregate metrics, restart recovery from job artifacts, and CLI --no-review bypass. Classic pipeline unchanged. 65 non-LLM tests.
88 lines
3.5 KiB
Python
88 lines
3.5 KiB
Python
"""Feedback labels and aggregate metrics for human-review decisions."""
|
|
|
|
import json
|
|
import os
|
|
from datetime import datetime
|
|
|
|
from backend.review.feedback import decision_to_label, write_label
|
|
from backend.review.metrics import aggregate_labels
|
|
|
|
|
|
def test_aggregate_redacts_text_by_default():
|
|
labels = [{"decision": "reject", "reason_code": "missing_evidence", "comment": "secret", "payload": {"evidence": [{"source_text": "secret"}]}}]
|
|
summary = aggregate_labels(labels)
|
|
assert summary["reject"] == 1
|
|
assert "secret" not in str(summary)
|
|
|
|
|
|
def test_aggregate_include_text_embeds_labels():
|
|
labels = [{"decision": "reject", "reason_code": "missing_evidence", "comment": "secret"}]
|
|
summary = aggregate_labels(labels, include_text=True)
|
|
assert summary["labels"] == labels
|
|
|
|
|
|
def _queue_item() -> dict:
|
|
return {
|
|
"review_item_id": "finding:AGENT-0007",
|
|
"kind": "finding",
|
|
"blocking": True,
|
|
"reasons": ["high_severity"],
|
|
"payload": {
|
|
"issue_id": "AGENT-0007",
|
|
"source_stage": "conflict",
|
|
"category": "elevation_disagreement",
|
|
"severity": "high",
|
|
"confidence": "medium",
|
|
"location": "Room 204 / Level 2",
|
|
"disciplines": ["Architectural", "Mechanical"],
|
|
"sheets": ["A2.1", "M2.1"],
|
|
"drawing_type": "floor_plan",
|
|
},
|
|
}
|
|
|
|
|
|
def test_decision_to_label_builds_spec_shape():
|
|
decision = {"review_item_id": "finding:AGENT-0007", "decision": "reject",
|
|
"reason_code": "same_value_different_representation"}
|
|
job = {"job_id": "abc123", "pipeline_mode": "agent",
|
|
"report": {"summary": {"models_used": ["google/gemini-2.5-pro"]}}}
|
|
label = decision_to_label(_queue_item(), decision, job)
|
|
assert label["review_item_id"] == "finding:AGENT-0007"
|
|
assert label["job_id"] == "abc123"
|
|
assert label["pipeline_mode"] == "agent"
|
|
assert label["source_stage"] == "conflict"
|
|
assert label["category"] == "elevation_disagreement"
|
|
assert label["severity"] == "high"
|
|
assert label["confidence"] == "medium"
|
|
assert label["decision"] == "reject"
|
|
assert label["reason_code"] == "same_value_different_representation"
|
|
assert label["location"] == "Room 204 / Level 2"
|
|
assert label["disciplines"] == ["Architectural", "Mechanical"]
|
|
assert label["sheets"] == ["A2.1", "M2.1"]
|
|
assert label["drawing_type"] == "floor_plan"
|
|
assert label["models_used"] == ["google/gemini-2.5-pro"]
|
|
datetime.fromisoformat(label["created_at"])
|
|
|
|
|
|
def test_decision_to_label_degrades_on_missing_fields():
|
|
label = decision_to_label({"review_item_id": "finding:AGENT-0001"}, {}, {})
|
|
assert label["review_item_id"] == "finding:AGENT-0001"
|
|
assert label["job_id"] is None
|
|
assert label["decision"] is None
|
|
assert label["reason_code"] is None
|
|
assert label["category"] is None
|
|
assert label["source_stage"] is None
|
|
assert label["models_used"] == []
|
|
datetime.fromisoformat(label["created_at"])
|
|
|
|
|
|
def test_write_label_appends_json_lines(tmp_path):
|
|
label1 = {"review_item_id": "finding:AGENT-0001", "decision": "confirm"}
|
|
label2 = {"review_item_id": "finding:AGENT-0002", "decision": "reject"}
|
|
write_label(str(tmp_path), label1)
|
|
write_label(str(tmp_path), label2)
|
|
path = os.path.join(str(tmp_path), "review", "feedback_labels.jsonl")
|
|
with open(path, encoding="utf-8") as f:
|
|
lines = [json.loads(line) for line in f if line.strip()]
|
|
assert lines == [label1, label2]
|