Add job run logs, OpenRouter model picker, and discipline grouping.
Docker Release / build-and-push (push) Successful in 55s
Docker Release / release (push) Skipped

- Job logs: each job's stdout/stderr is teed into outputs/<id>/job.log
  (survives restarts) and served at GET /jobs/{id}/log as text/plain, so
  full run logs can be shared for debugging and refinement.
- Model picker: GET /models proxies OpenRouter's public model list with
  per-1M-token pricing (1h cache, 502 on failure); the UI shows a model
  dropdown with costs when OpenRouter compute is selected, and the pick
  overrides vision+text models for that job (Classic and Agent modes).
- Conflicts in the report view are grouped by discipline pair
  (collapsible sections, severity-ordered within groups) instead of one
  flat severity-only list.
This commit is contained in:
John Wilganowski
2026-07-28 21:23:42 +00:00
parent 4ecc7c5cef
commit afa1089311
8 changed files with 441 additions and 55 deletions
+71 -12
View File
@@ -12,7 +12,9 @@ not. No external queue/DB.
"""
import json
import contextlib
import os
import sys
import time
import uuid
import shutil
@@ -20,6 +22,7 @@ import threading
from typing import Dict, Optional
from backend import config
from backend import llm
from backend.agents.runner import run_agent_pipeline
from backend.pipeline.runner import run_pipeline
from backend.email_sender import send_conflict_report, send_review_required
@@ -29,6 +32,40 @@ _lock = threading.Lock()
PIPELINE_MODES = {"classic", "agent"}
class _Tee:
"""Write to both the real stream and the job log file."""
def __init__(self, stream, log_file) -> None:
self._stream = stream
self._log = log_file
def write(self, data):
self._stream.write(data)
self._log.write(data)
def flush(self):
self._stream.flush()
self._log.flush()
@contextlib.contextmanager
def _tee_log(log_path: str, header: str):
"""Mirror stdout/stderr into a per-job log file for the duration of a run.
sys.stdout is process-global, so two concurrent jobs would interleave in
each other's logs - acceptable for this single-user tool (same tradeoff as
the LLM cost globals in llm.py).
"""
with open(log_path, "a", encoding="utf-8") as log_file:
log_file.write(header + "\n")
real_out, real_err = sys.stdout, sys.stderr
sys.stdout, sys.stderr = _Tee(real_out, log_file), _Tee(real_err, log_file)
try:
yield
finally:
sys.stdout, sys.stderr = real_out, real_err
def _set(job_id: str, **fields) -> None:
with _lock:
_jobs[job_id].update(fields)
@@ -36,13 +73,14 @@ def _set(job_id: str, **fields) -> None:
def create_job(pdf_path: str, source_filename: str, email: Optional[str] = None,
project_input: Optional[Dict] = None, text_local: bool = False,
pipeline_mode: str = "classic") -> str:
pipeline_mode: str = "classic", model: Optional[str] = None) -> str:
"""Register a job and kick off its background thread. Returns the job_id."""
pipeline_mode = pipeline_mode.strip().lower()
if pipeline_mode not in PIPELINE_MODES:
raise ValueError(f"Unsupported pipeline mode: {pipeline_mode!r}")
# Agent mode v1 is OpenRouter-only.
text_local = bool(text_local and pipeline_mode == "classic")
model = (model or "").strip() or None
job_id = uuid.uuid4().hex[:12]
with _lock:
_jobs[job_id] = {
@@ -53,6 +91,7 @@ def create_job(pdf_path: str, source_filename: str, email: Optional[str] = None,
"project_input": project_input or {},
"text_local": text_local,
"pipeline_mode": pipeline_mode,
"model": model,
"stage": None,
"created_at": time.time(),
"finished_at": None,
@@ -60,19 +99,41 @@ def create_job(pdf_path: str, source_filename: str, email: Optional[str] = None,
"error": None,
}
threading.Thread(target=_run, args=(
job_id, pdf_path, project_input, text_local, pipeline_mode,
job_id, pdf_path, project_input, text_local, pipeline_mode, model,
),
daemon=True).start()
return job_id
def _run(job_id: str, pdf_path: str, project_input: Optional[Dict] = None,
text_local: bool = False, pipeline_mode: str = "classic") -> None:
text_local: bool = False, pipeline_mode: str = "classic",
model: Optional[str] = None) -> None:
out_dir = os.path.join(config.OUTPUT_DIR, job_id)
try:
_set(job_id, status="running")
# Keep a copy of the source PDF so its sheets can be viewed later.
os.makedirs(out_dir, exist_ok=True)
header = (f"=== Job {job_id} | {pipeline_mode} | {_jobs[job_id].get('source')} | "
f"model={model or 'default'} | "
f"started {time.strftime('%Y-%m-%d %H:%M:%S %Z', time.gmtime())} UTC ===")
with _tee_log(os.path.join(out_dir, "job.log"), header):
_run_pipeline(job_id, pdf_path, out_dir, project_input, text_local,
pipeline_mode, model)
except Exception as e:
print(f"[Jobs] Job {job_id} failed: {e}")
_set(job_id, status="error", error=str(e), finished_at=time.time())
_notify_error(job_id)
finally:
try:
os.remove(pdf_path)
except OSError:
pass
def _run_pipeline(job_id: str, pdf_path: str, out_dir: str,
project_input: Optional[Dict], text_local: bool,
pipeline_mode: str, model: Optional[str]) -> None:
"""The body of a job run; executes inside the job's tee'd log capture."""
# Persist minimal job metadata so the disk fallback in get_job can
# recover the recipient email / pipeline mode after a server restart
# (plain json.dump, matching the _dump style used elsewhere).
@@ -95,7 +156,14 @@ def _run(job_id: str, pdf_path: str, project_input: Optional[Dict] = None,
runner_kwargs["text_local"] = text_local
else:
runner_kwargs["require_review"] = config.AGENT_REQUIRE_REVIEW
if model:
print(f"[Jobs] Model override for this run: {model}")
llm.set_model_override(model)
try:
report = runner(pdf_path, **runner_kwargs)
finally:
if model:
llm.set_model_override(None)
report.setdefault("summary", {})["pipeline_mode"] = pipeline_mode
if report["summary"].get("agent_status") == "needs_review":
# Human-review gate: hold the job, don't email the unreviewed report.
@@ -108,15 +176,6 @@ def _run(job_id: str, pdf_path: str, project_input: Optional[Dict] = None,
else:
_set(job_id, status="done", report=report, finished_at=time.time(), stage=None)
_notify(job_id, report, out_dir)
except Exception as e:
print(f"[Jobs] Job {job_id} failed: {e}")
_set(job_id, status="error", error=str(e), finished_at=time.time())
_notify_error(job_id)
finally:
try:
os.remove(pdf_path)
except OSError:
pass
def _notify(job_id: str, report: Dict, out_dir: str) -> None:
+13 -2
View File
@@ -24,6 +24,16 @@ _clients: Dict[str, OpenAI] = {}
# set_stage/cost pattern (single-user tool).
_text_local = False
# Per-job model override (user picked a model in the UI). Same module-global
# pattern: set by the job runner before the pipeline starts, cleared after.
_model_override: Optional[str] = None
def set_model_override(model: Optional[str]) -> None:
"""Override the model for all OpenRouter calls (vision + text), or None to clear."""
global _model_override
_model_override = (model or "").strip() or None
def set_text_backend(local: bool) -> None:
"""Choose whether text (no-image) calls go to the local endpoint this run."""
@@ -171,12 +181,13 @@ def _resolve_backend(has_images: bool, model_override: Optional[str]) -> Dict[st
"usage": False, # local has no OpenRouter usage accounting
"local": True,
}
# Vision, or text-on-OpenRouter (default / fallback).
# Vision, or text-on-OpenRouter (default / fallback). A per-job override
# (user's UI model pick) wins over per-call and env defaults.
default_model = config.MODEL if has_images else config.TEXT_MODEL
return {
"base_url": config.AI_BASE_URL,
"api_key": config.AI_API_KEY,
"model": model_override or default_model,
"model": _model_override or model_override or default_model,
"usage": True,
"local": False,
}
+23 -1
View File
@@ -41,6 +41,27 @@ def health():
"email_configured": bool(config.SMTP_HOST and config.SMTP_USER and config.SMTP_PASSWORD)}
@app.get("/models")
def list_models():
"""Available OpenRouter models with per-1M-token pricing for the UI picker."""
from backend.models import fetch_models
models = fetch_models()
if models is None:
raise HTTPException(status_code=502,
detail="Could not fetch the model list from OpenRouter")
return {"models": models, "default": config.MODEL, "default_text": config.TEXT_MODEL}
@app.get("/jobs/{job_id}/log")
def job_log(job_id: str):
"""The full captured stdout/stderr log of a job run (persists on disk)."""
path = os.path.join(config.OUTPUT_DIR, job_id, "job.log")
if not os.path.isfile(path):
raise HTTPException(status_code=404, detail="Log not found for this job")
with open(path, encoding="utf-8", errors="replace") as f:
return Response(content=f.read(), media_type="text/plain")
@app.post("/check")
async def check(
file: UploadFile = File(...),
@@ -51,6 +72,7 @@ async def check(
work_type: Optional[str] = Form(None),
text_local: bool = Form(False),
pipeline_mode: str = Form("classic"),
model: Optional[str] = Form(None),
):
"""
Accept a PDF, start a background conflict check, and return a job_id
@@ -86,7 +108,7 @@ async def check(
}
job_id = create_job(tmp_path, source_filename=file.filename, email=email,
project_input=project_input, text_local=text_local,
pipeline_mode=pipeline_mode)
pipeline_mode=pipeline_mode, model=model)
return JSONResponse({
"job_id": job_id,
"status": "queued",
+65
View File
@@ -0,0 +1,65 @@
"""
models.py - Fetch the available OpenRouter model list with pricing (cached).
The /models endpoint is public (no API key needed). Results are normalized to
per-1M-token USD costs for display and cached in memory for an hour; callers
degrade gracefully when OpenRouter is unreachable.
"""
import time
from typing import List, Optional
import httpx
from backend import config
_CACHE_TTL_SECONDS = 3600
_cache = {"at": 0.0, "models": None}
def _per_mtok(rate) -> float:
"""OpenRouter pricing is USD per token (as a string); display is per 1M."""
try:
return round(float(rate) * 1_000_000, 4)
except (TypeError, ValueError):
return 0.0
def _fetch_openrouter_models() -> Optional[List[dict]]:
"""Raw GET of the OpenRouter model list; None on any failure."""
try:
response = httpx.get(f"{config.AI_BASE_URL.rstrip('/')}/models", timeout=10)
response.raise_for_status()
data = response.json().get("data")
return data if isinstance(data, list) else None
except Exception as e:
print(f"[Models] OpenRouter /models fetch failed: {e}")
return None
def fetch_models(force: bool = False) -> Optional[List[dict]]:
"""Normalized model list for the UI picker, or None when unavailable."""
if (
not force
and _cache["models"] is not None
and time.time() - _cache["at"] < _CACHE_TTL_SECONDS
):
return _cache["models"]
data = _fetch_openrouter_models()
if data is None:
return None
models = [
{
"id": item.get("id") or "",
"name": item.get("name") or item.get("id") or "",
"prompt_usd_per_mtok": _per_mtok((item.get("pricing") or {}).get("prompt")),
"completion_usd_per_mtok": _per_mtok((item.get("pricing") or {}).get("completion")),
"context_length": item.get("context_length"),
}
for item in data
if item.get("id")
]
models.sort(key=lambda m: m["id"])
_cache["models"] = models
_cache["at"] = time.time()
return models
+76 -14
View File
@@ -120,6 +120,10 @@
<input type="radio" name="compute" value="openrouter" checked> OpenRouter &mdash; all stages (fastest, paid)</label>
<label style="display:block;font-weight:400;margin-top:6px">
<input type="radio" name="compute" value="local"> Hybrid &mdash; text stages on local LLM (cheaper, slower)</label>
<div id="modelPick" style="margin-top:10px">
<label for="model" style="font-weight:400">Model <span class="opt" id="modelNote">loading...</span></label>
<select id="model" style="width:100%;margin-top:6px;padding:10px;border-radius:8px;border:1px solid var(--line);background:#0c0e13;color:var(--text)"></select>
</div>
</div>
<button class="btn full" id="run" disabled>Run conflict check</button>
<div class="status" id="status"></div>
@@ -163,6 +167,10 @@ runBtn.addEventListener('click',async e=>{
});
const compute=(document.querySelector('input[name="compute"]:checked')||{}).value;
fd.append('text_local', compute==='local' ? 'true' : 'false');
if(compute==='openrouter'){
const modelSel=document.getElementById('model');
if(modelSel.value) fd.append('model', modelSel.value);
}
const pipelineMode=(document.querySelector('input[name="pipeline_mode"]:checked')||{}).value||'classic';
fd.append('pipeline_mode',pipelineMode);
try{
@@ -222,6 +230,33 @@ function syncPipelineOptions(){
document.querySelectorAll('input[name="pipeline_mode"]').forEach(el=>el.addEventListener('change',syncPipelineOptions));
syncPipelineOptions();
// --- model picker (OpenRouter compute only) ---
let modelList=null;
async function loadModels(){
const note=document.getElementById('modelNote'), sel=document.getElementById('model');
try{
const res=await fetch('/models');
if(!res.ok) throw new Error('list unavailable');
const data=await res.json();
modelList=data.models||[];
sel.innerHTML=modelList.map(m=>
'<option value="'+escAttr(m.id)+'"'+(m.id===data.default?' selected':'')+'>'+
esc(m.name||m.id)+' &mdash; $'+esc(m.prompt_usd_per_mtok)+' / $'+esc(m.completion_usd_per_mtok)+
' per 1M tok</option>').join('');
note.textContent='('+modelList.length+' available)';
}catch(e){
sel.innerHTML='';
note.textContent='using configured default (list unavailable)';
}
}
function syncCompute(){
const openrouter=(document.querySelector('input[name="compute"]:checked')||{}).value==='openrouter';
document.getElementById('modelPick').style.display=openrouter?'block':'none';
if(openrouter&&!modelList) loadModels();
}
document.querySelectorAll('input[name="compute"]').forEach(el=>el.addEventListener('change',syncCompute));
syncCompute();
// --- sheet viewer ---
function pageFor(num){ return sheetPage[num] || sheetPage[(num||'').toUpperCase()] || null; }
function sheetSpan(num){
@@ -248,6 +283,43 @@ function closeSheet(){ document.getElementById('viewer').classList.remove('open'
document.getElementById('viewer').addEventListener('click',e=>{ if(e.target.id==='viewer') closeSheet(); });
document.addEventListener('keydown',e=>{ if(e.key==='Escape') closeSheet(); });
// --- conflicts grouped by discipline pair ---
const SEV_RANK={critical:0,high:1,medium:2,low:3};
function sevRank(c){ const r=SEV_RANK[(c.severity||'').toLowerCase()]; return r==null?4:r; }
function groupConflicts(conflicts){
// Group key: disciplines sorted alphabetically, joined ' vs ' (order-independent
// pair). Missing disciplines -> 'General'. Groups ordered by their most severe
// conflict, then name; items within a group ordered critical->high->medium->low.
const groups={};
for(const c of conflicts||[]){
const ds=(c.disciplines||[]).map(d=>String(d)).filter(Boolean).sort();
const key=ds.length?ds.join(' vs '):'General';
(groups[key]=groups[key]||[]).push(c);
}
const names=Object.keys(groups).sort((a,b)=>{
const ra=Math.min.apply(null,groups[a].map(sevRank)),
rb=Math.min.apply(null,groups[b].map(sevRank));
return (ra-rb)||a.localeCompare(b);
});
return names.map(name=>({name:name,
items:groups[name].slice().sort((x,y)=>sevRank(x)-sevRank(y))}));
}
function conflictCard(c){
let html='<div class="conflict '+esc(c.severity)+'">'+
'<div class="row"><span class="cat">'+esc(c.category)+'</span>'+
'<span class="pill '+esc(c.severity)+'">'+esc(c.severity)+'</span></div>'+
'<div class="loc">'+esc(c.location)+'</div>'+
'<div class="meta">'+esc((c.disciplines||[]).join(' vs '))+
' &middot; sheets '+sheetList(c.sheets)+'</div>'+
'<div class="desc">'+esc(c.description)+'</div>';
if(c.evidence&&c.evidence.length){
html+='<div class="ev">'+c.evidence.map(e=>
'<div><span class="d">'+esc(e.discipline)+'</span> ('+sheetSpan(e.sheet)+'): "'+esc(e.source_text)+'"</div>').join('')+'</div>';
}
if(c.recommended_resolution){ html+='<div class="reso">Resolution: '+esc(c.recommended_resolution)+'</div>'; }
return html+'</div>';
}
function render(rep){
const s=rep.summary;
if(!currentJobId) currentJobId=new URLSearchParams(location.search).get('job');
@@ -279,20 +351,10 @@ function render(rep){
} else if(!rep.conflicts.length){
html+='<div class="empty">No cross-discipline conflicts detected.</div>';
}
for(const c of rep.conflicts){
html+='<div class="conflict '+esc(c.severity)+'">'+
'<div class="row"><span class="cat">'+esc(c.category)+'</span>'+
'<span class="pill '+esc(c.severity)+'">'+esc(c.severity)+'</span></div>'+
'<div class="loc">'+esc(c.location)+'</div>'+
'<div class="meta">'+esc((c.disciplines||[]).join(' vs '))+
' &middot; sheets '+sheetList(c.sheets)+'</div>'+
'<div class="desc">'+esc(c.description)+'</div>';
if(c.evidence&&c.evidence.length){
html+='<div class="ev">'+c.evidence.map(e=>
'<div><span class="d">'+esc(e.discipline)+'</span> ('+sheetSpan(e.sheet)+'): "'+esc(e.source_text)+'"</div>').join('')+'</div>';
}
if(c.recommended_resolution){ html+='<div class="reso">Resolution: '+esc(c.recommended_resolution)+'</div>'; }
html+='</div>';
for(const g of groupConflicts(rep.conflicts)){
html+='<details open style="margin-top:16px"><summary><b>'+esc(g.name)+' ('+g.items.length+')</b></summary>';
for(const c of g.items){ html+=conflictCard(c); }
html+='</details>';
}
const issues=rep.validated_issues||[];
+77
View File
@@ -0,0 +1,77 @@
import threading
import pytest
from fastapi.testclient import TestClient
import backend.jobs as jobs
from backend.main import app
class _SyncThread:
"""Drop-in threading.Thread replacement that runs the target inline."""
def __init__(self, target=None, args=(), kwargs=None, **_ignored):
self._target = target
self._args = args
self._kwargs = kwargs or {}
def start(self):
self._target(*self._args, **self._kwargs)
@pytest.fixture
def job_env(monkeypatch, tmp_path):
monkeypatch.setattr("backend.config.OUTPUT_DIR", str(tmp_path))
monkeypatch.setattr(threading, "Thread", _SyncThread)
monkeypatch.setattr("backend.jobs.send_conflict_report", lambda *a, **k: True)
pdf = tmp_path / "set.pdf"
pdf.write_bytes(b"%PDF-1.4\n")
yield tmp_path
jobs._jobs.clear()
def test_job_log_captures_pipeline_output(job_env, monkeypatch):
def fake_runner(pdf_path, **kwargs):
print("STAGE banner: fake wave ran")
return {"source": "set.pdf", "summary": {"conflicts_found": 0}}
monkeypatch.setattr("backend.jobs.run_pipeline", fake_runner)
job_id = jobs.create_job(str(job_env / "set.pdf"), "set.pdf", pipeline_mode="classic")
log_path = job_env / job_id / "job.log"
assert log_path.is_file()
content = log_path.read_text()
assert "STAGE banner: fake wave ran" in content
assert job_id in content # header line
def test_job_log_endpoint_serves_log_and_404s(job_env, monkeypatch):
monkeypatch.setattr(
"backend.jobs.run_pipeline",
lambda pdf_path, **kw: {"source": "s", "summary": {}},
)
job_id = jobs.create_job(str(job_env / "set.pdf"), "set.pdf", pipeline_mode="classic")
client = TestClient(app)
ok = client.get(f"/jobs/{job_id}/log")
assert ok.status_code == 200
assert ok.headers["content-type"].startswith("text/plain")
assert "Job " + job_id in ok.text
assert client.get("/jobs/nope/log").status_code == 404
def test_model_override_set_and_cleared_around_run(job_env, monkeypatch):
from backend import llm
seen = {}
def fake_runner(pdf_path, **kwargs):
seen["override"] = llm._model_override
return {"source": "set.pdf", "summary": {}}
monkeypatch.setattr("backend.jobs.run_pipeline", fake_runner)
jobs.create_job(str(job_env / "set.pdf"), "set.pdf",
pipeline_mode="classic", model="openai/gpt-4o")
assert seen["override"] == "openai/gpt-4o"
assert llm._model_override is None # cleared after the run
+64
View File
@@ -0,0 +1,64 @@
from fastapi.testclient import TestClient
import backend.models as models
from backend import config
from backend.main import app
_PAYLOAD = {
"data": [
{
"id": "openai/gpt-4o",
"name": "GPT-4o",
"pricing": {"prompt": "0.0000025", "completion": "0.00001"},
"context_length": 128000,
},
{
"id": "google/gemini-2.5-pro",
"name": "Gemini 2.5 Pro",
"pricing": {"prompt": "0.00000125", "completion": "0.00001"},
"context_length": 1000000,
},
]
}
def _reset_cache():
models._cache["models"] = None
models._cache["at"] = 0.0
def test_models_endpoint_normalizes_pricing(monkeypatch):
_reset_cache()
monkeypatch.setattr(models, "_fetch_openrouter_models", lambda: _PAYLOAD["data"])
client = TestClient(app)
response = client.get("/models")
assert response.status_code == 200
body = response.json()
assert body["default"] == config.MODEL
assert body["default_text"] == config.TEXT_MODEL
by_id = {m["id"]: m for m in body["models"]}
assert by_id["openai/gpt-4o"]["prompt_usd_per_mtok"] == 2.5
assert by_id["openai/gpt-4o"]["completion_usd_per_mtok"] == 10.0
assert by_id["openai/gpt-4o"]["context_length"] == 128000
def test_models_endpoint_caches(monkeypatch):
_reset_cache()
calls = []
def fake_fetch():
calls.append(1)
return _PAYLOAD["data"]
monkeypatch.setattr(models, "_fetch_openrouter_models", fake_fetch)
client = TestClient(app)
assert client.get("/models").status_code == 200
assert client.get("/models").status_code == 200
assert len(calls) == 1
def test_models_endpoint_502_on_fetch_failure(monkeypatch):
_reset_cache()
monkeypatch.setattr(models, "_fetch_openrouter_models", lambda: None)
client = TestClient(app)
assert client.get("/models").status_code == 502
+26
View File
@@ -0,0 +1,26 @@
from backend import config
from backend.llm import _resolve_backend, set_model_override
def test_override_wins_for_vision_and_text():
set_model_override("openai/gpt-4o")
try:
assert _resolve_backend(has_images=True, model_override=None)["model"] == "openai/gpt-4o"
assert _resolve_backend(has_images=False, model_override=None)["model"] == "openai/gpt-4o"
finally:
set_model_override(None)
def test_override_beats_per_call_model_arg():
set_model_override("openai/gpt-4o")
try:
# Agents pass their AGENT_*_MODEL per call; the user's job pick wins.
assert _resolve_backend(has_images=False, model_override="other/model")["model"] == "openai/gpt-4o"
finally:
set_model_override(None)
def test_no_override_keeps_defaults():
set_model_override(None)
assert _resolve_backend(has_images=True, model_override=None)["model"] == config.MODEL
assert _resolve_backend(has_images=False, model_override=None)["model"] == config.TEXT_MODEL