John Wilganowski
1c1d2ff21b
Agent web jobs now stop after Brain consolidation and enter needs_review with a persisted review queue (blocking: high-severity, low-confidence, sensitive-category findings; audit sample of clean clusters). Humans decide confirm/reject/unsure/needs_clarification via new review API and frontend queue; a finalizer applies decisions (rejections suppressed with reason codes), performs bounded targeted reruns for clarifications, drafts RFIs only for kept issues, and only then marks the job done and sends the final email. Two-phase email (review-required, then final report), per-decision feedback labels with redacted aggregate metrics, restart recovery from job artifacts, and CLI --no-review bypass. Classic pipeline unchanged. 65 non-LLM tests.
52 lines
1.9 KiB
Python
52 lines
1.9 KiB
Python
"""Feedback labels: one label artifact per human-review decision, for metrics."""
|
|
|
|
import json
|
|
import os
|
|
from datetime import datetime, timezone
|
|
|
|
|
|
def _as_dict(value) -> dict:
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def decision_to_label(queue_item: dict, decision: dict, job: dict) -> dict:
|
|
"""Build one feedback label from a queue item, its decision, and the job.
|
|
|
|
All field access is defensive: missing fields degrade to None (or [] for
|
|
models_used) rather than raising.
|
|
"""
|
|
queue_item = _as_dict(queue_item)
|
|
decision = _as_dict(decision)
|
|
job = _as_dict(job)
|
|
payload = _as_dict(queue_item.get("payload"))
|
|
summary = _as_dict(_as_dict(job.get("report")).get("summary"))
|
|
return {
|
|
"review_item_id": queue_item.get("review_item_id"),
|
|
"job_id": job.get("job_id"),
|
|
"pipeline_mode": job.get("pipeline_mode"),
|
|
"source_stage": payload.get("source_stage"),
|
|
"category": payload.get("category"),
|
|
"severity": payload.get("severity"),
|
|
"confidence": payload.get("confidence"),
|
|
"decision": decision.get("decision"),
|
|
"reason_code": decision.get("reason_code"),
|
|
"location": payload.get("location"),
|
|
"disciplines": payload.get("disciplines"),
|
|
"sheets": payload.get("sheets"),
|
|
"drawing_type": payload.get("drawing_type"),
|
|
"models_used": summary.get("models_used") or [],
|
|
"created_at": datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
|
|
|
|
def write_label(out_dir: str, label: dict) -> None:
|
|
"""Append one label as a JSON line; never raises on I/O failure."""
|
|
try:
|
|
review_dir = os.path.join(out_dir, "review")
|
|
os.makedirs(review_dir, exist_ok=True)
|
|
path = os.path.join(review_dir, "feedback_labels.jsonl")
|
|
with open(path, "a", encoding="utf-8") as f:
|
|
f.write(json.dumps(label) + "\n")
|
|
except OSError as e:
|
|
print(f"[Review] feedback label write failed: {e}")
|