Verbose per-call LLM logging, raw request/response dumps, and end-of-log cost summary.

- [LLM] line per call: stage, model, prompt size, output size, cost, parsed item counts
- LLM_RAW_DUMP: full prompt/response JSON per call under outputs/<job>/llm_raw/
- Cost block at tail of job.log (per-stage, per-model, cached vs live)
- Agent mode: reset llm cost counters per job; review finalization now teed into job.log + dumps

backend/llm.py

@@ -8,6 +8,7 @@ with an image) and conflict-reasoning (Stage 3, with images) calls.
 """
 
 import os
+import re
 import json
 import hashlib
 import threading
@@ -31,6 +32,104 @@ _text_local = False
 _vision_model_override: Optional[str] = None
 _text_model_override: Optional[str] = None
 
+# Per-job raw request/response dumps (missed/hallucinated-item debugging).
+# Set by the job runner to <out_dir>/llm_raw at job start, cleared after.
+# Same module-global pattern as the model overrides (single-user tool).
+_raw_dump_dir: Optional[str] = None
+_seq_lock = threading.Lock()
+_call_seq = 0
+
+
+def set_raw_dump_dir(path: Optional[str]) -> None:
+    """Point raw LLM request/response dumps at a directory. None disables."""
+    global _raw_dump_dir, _call_seq
+    with _seq_lock:
+        _raw_dump_dir = path
+        _call_seq = 0
+
+
+def _next_seq() -> int:
+    global _call_seq
+    with _seq_lock:
+        _call_seq += 1
+        return _call_seq
+
+
+def _summarize_parsed(parsed: Any) -> str:
+    """
+    Compact digest of a parsed response for the job log. List values become
+    item counts (e.g. conflicts[3]) so a stage that returned nothing (miss)
+    or invented items (hallucination) is visible without opening the raw dump.
+    """
+    if isinstance(parsed, list):
+        return f"list[{len(parsed)}]"
+    if not isinstance(parsed, dict):
+        return type(parsed).__name__
+    parts = []
+    for k, v in parsed.items():
+        if isinstance(v, list):
+            parts.append(f"{k}[{len(v)}]")
+        elif isinstance(v, dict):
+            parts.append(f"{k}{{{len(v)}}}")
+        else:
+            s = str(v)
+            parts.append(f"{k}={s[:40]!r}{'...' if len(s) > 40 else ''}")
+    out = ", ".join(parts)
+    return out[:300] + ("..." if len(out) > 300 else "")
+
+
+def _dump_raw(seq: int, be: Dict[str, Any], usage_stage: str,
+              system_prompt: str, user_text: str,
+              images_b64: Optional[List[str]], max_tokens: int,
+              raw: str, parsed: Any) -> None:
+    """Write the full request/response for one call to the job's llm_raw dir."""
+    if not _raw_dump_dir:
+        return
+    try:
+        os.makedirs(_raw_dump_dir, exist_ok=True)
+        safe_stage = re.sub(r"[^A-Za-z0-9_.-]+", "_", usage_stage)[:40]
+        safe_model = re.sub(r"[^A-Za-z0-9_.-]+", "_", be["model"])
+        payload = {
+            "seq": seq,
+            "stage": usage_stage,
+            "model": be["model"],
+            "backend": "local" if be.get("local") else "cloud",
+            "max_tokens": max_tokens,
+            # base64 image payloads deliberately excluded (multi-MB each);
+            # the count + source.pdf in the job dir identify what was sent.
+            "n_images": len(images_b64 or []),
+            "system_prompt": system_prompt,
+            "user_text": user_text,
+            "raw_response": raw,
+            "parsed": parsed,
+        }
+        path = os.path.join(_raw_dump_dir, f"{seq:04d}_{safe_stage}_{safe_model}.json")
+        tmp = f"{path}.{threading.get_ident()}.tmp"
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(payload, f, indent=2)
+        os.replace(tmp, path)        # atomic so thread-pooled stages can't tear it
+    except OSError as e:
+        print(f"[LLM] raw dump failed: {e}")
+
+
+def _log_call(seq: int, be: Dict[str, Any], usage_stage: str,
+              user_text: str, images_b64: Optional[List[str]],
+              raw: str, parsed: Any, usd: Optional[float],
+              cached: bool = False) -> None:
+    """One verbose per-call line for the job log (tee'd by job_log.py)."""
+    if not config.LLM_VERBOSE:
+        return
+    backend = "local" if be.get("local") else "cloud"
+    if cached:
+        cost_str = "cache hit"
+    elif usd is not None:
+        cost_str = f"${usd:.4f}"
+    else:
+        cost_str = "cost n/a"
+    print(f"[LLM] #{seq:04d} {usage_stage} | {be['model']} ({backend}) | "
+          f"in {len(user_text)}ch+{len(images_b64 or [])}img | "
+          f"out {len(raw)}ch | {cost_str} | {_summarize_parsed(parsed)}")
+
 
 def set_text_backend(local: bool) -> None:
     """Choose whether text (no-image) calls go to the local endpoint this run."""
@@ -300,6 +399,7 @@ def call_json(
     """
     has_images = bool(images_b64)
     be = _resolve_backend(has_images, model)
+    seq = _next_seq()
 
     cache_key = None
     if config.LLM_CACHE:
@@ -312,6 +412,8 @@ def call_json(
                 usage_tracker.record(
                     usage_stage, be["model"], cached=True, has_images=has_images
                 )
+            _log_call(seq, be, usage_stage, user_text, images_b64,
+                      "", hit, None, cached=True)
             return hit
 
     content: List[Dict[str, Any]] = []
@@ -352,6 +454,11 @@ def call_json(
                 _record_model(be, has_images, fell_back)
                 if cache_key:
                     _cache_set(cache_key, parsed)
+                _log_call(seq, be, usage_stage, user_text, images_b64,
+                          raw, parsed, usd)
+                if config.LLM_RAW_DUMP:
+                    _dump_raw(seq, be, usage_stage, system_prompt, user_text,
+                              images_b64, max_tokens, raw, parsed)
                 return parsed
             if attempt == 0:
                 print("[LLM] JSON parse error (retrying)")