woogiandClaude Opus 5
23e19f53b2
Read-only Q&A on the review screen, per finding and per run, answered from
the job's own artifacts (evidence, cluster, extraction, verification, Brain
merge, sheet index, cover reconciliation, job.log). It never mutates findings,
decisions, or the report.
Turns are logged job-locally (review/chat_log.jsonl, transcript at
/jobs/{id}/review-chat/log) and to a cross-job feedback store
(REVIEW_FEEDBACK_DIR), which now also receives review decisions with their
category/severity corrections.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0115gGtrSxXE9DKvS9XPFSoT
119 lines
5.0 KiB
Python
119 lines
5.0 KiB
Python
"""Feedback labels and aggregate metrics for human-review decisions."""
|
|
|
|
import json
|
|
import os
|
|
from datetime import datetime
|
|
|
|
from backend.review.feedback import decision_to_label, write_label
|
|
from backend.review.metrics import aggregate_labels
|
|
|
|
|
|
def test_aggregate_redacts_text_by_default():
|
|
labels = [{"decision": "reject", "reason_code": "missing_evidence", "comment": "secret", "payload": {"evidence": [{"source_text": "secret"}]}}]
|
|
summary = aggregate_labels(labels)
|
|
assert summary["reject"] == 1
|
|
assert "secret" not in str(summary)
|
|
|
|
|
|
def test_aggregate_include_text_embeds_labels():
|
|
labels = [{"decision": "reject", "reason_code": "missing_evidence", "comment": "secret"}]
|
|
summary = aggregate_labels(labels, include_text=True)
|
|
assert summary["labels"] == labels
|
|
|
|
|
|
def _queue_item() -> dict:
|
|
return {
|
|
"review_item_id": "finding:AGENT-0007",
|
|
"kind": "finding",
|
|
"blocking": True,
|
|
"reasons": ["high_severity"],
|
|
"payload": {
|
|
"issue_id": "AGENT-0007",
|
|
"source_stage": "conflict",
|
|
"category": "elevation_disagreement",
|
|
"severity": "high",
|
|
"confidence": "medium",
|
|
"location": "Room 204 / Level 2",
|
|
"disciplines": ["Architectural", "Mechanical"],
|
|
"sheets": ["A2.1", "M2.1"],
|
|
"drawing_type": "floor_plan",
|
|
},
|
|
}
|
|
|
|
|
|
def test_decision_to_label_builds_spec_shape():
|
|
decision = {"review_item_id": "finding:AGENT-0007", "decision": "reject",
|
|
"reason_code": "same_value_different_representation"}
|
|
job = {"job_id": "abc123", "pipeline_mode": "agent",
|
|
"report": {"summary": {"models_used": ["google/gemini-2.5-pro"]}}}
|
|
label = decision_to_label(_queue_item(), decision, job)
|
|
assert label["review_item_id"] == "finding:AGENT-0007"
|
|
assert label["job_id"] == "abc123"
|
|
assert label["pipeline_mode"] == "agent"
|
|
assert label["source_stage"] == "conflict"
|
|
assert label["category"] == "elevation_disagreement"
|
|
assert label["severity"] == "high"
|
|
assert label["confidence"] == "medium"
|
|
assert label["decision"] == "reject"
|
|
assert label["reason_code"] == "same_value_different_representation"
|
|
assert label["location"] == "Room 204 / Level 2"
|
|
assert label["disciplines"] == ["Architectural", "Mechanical"]
|
|
assert label["sheets"] == ["A2.1", "M2.1"]
|
|
assert label["drawing_type"] == "floor_plan"
|
|
assert label["models_used"] == ["google/gemini-2.5-pro"]
|
|
datetime.fromisoformat(label["created_at"])
|
|
|
|
|
|
def test_decision_to_label_degrades_on_missing_fields():
|
|
label = decision_to_label({"review_item_id": "finding:AGENT-0001"}, {}, {})
|
|
assert label["review_item_id"] == "finding:AGENT-0001"
|
|
assert label["job_id"] is None
|
|
assert label["decision"] is None
|
|
assert label["reason_code"] is None
|
|
assert label["category"] is None
|
|
assert label["source_stage"] is None
|
|
assert label["models_used"] == []
|
|
datetime.fromisoformat(label["created_at"])
|
|
|
|
|
|
def test_write_label_appends_json_lines(tmp_path):
|
|
label1 = {"review_item_id": "finding:AGENT-0001", "decision": "confirm"}
|
|
label2 = {"review_item_id": "finding:AGENT-0002", "decision": "reject"}
|
|
write_label(str(tmp_path), label1)
|
|
write_label(str(tmp_path), label2)
|
|
path = os.path.join(str(tmp_path), "review", "feedback_labels.jsonl")
|
|
with open(path, encoding="utf-8") as f:
|
|
lines = [json.loads(line) for line in f if line.strip()]
|
|
assert lines == [label1, label2]
|
|
|
|
|
|
def test_decision_label_carries_reviewer_corrections():
|
|
"""category/severity corrections reach the label instead of being dropped."""
|
|
decision = {"decision": "reject", "reason_code": "extraction_misread",
|
|
"category_correction": "power floor box",
|
|
"severity_correction": "low"}
|
|
label = decision_to_label(_queue_item(), decision, {"job_id": "abc123", "pipeline_mode": "agent",
|
|
"report": {"summary": {}}})
|
|
assert label["category_correction"] == "power floor box"
|
|
assert label["severity_correction"] == "low"
|
|
assert label["kind"] == "review_decision"
|
|
|
|
|
|
def test_write_label_also_lands_in_the_cross_job_store(tmp_path):
|
|
from backend import config
|
|
from backend.review.feedback import read_shared_feedback
|
|
label = decision_to_label(_queue_item(), {"decision": "reject",
|
|
"reason_code": "extraction_misread"},
|
|
{"job_id": "abc123", "pipeline_mode": "agent",
|
|
"report": {"summary": {}}})
|
|
write_label(str(tmp_path), label)
|
|
assert os.path.isfile(os.path.join(config.REVIEW_FEEDBACK_DIR, "decisions.jsonl"))
|
|
records = read_shared_feedback("review_decision")
|
|
assert len(records) == 1
|
|
assert records[0]["reason_code"] == "extraction_misread"
|
|
|
|
|
|
def test_shared_feedback_read_is_empty_when_nothing_written():
|
|
from backend.review.feedback import read_shared_feedback
|
|
assert read_shared_feedback("review_decision") == []
|