Add job run logs, OpenRouter model picker, and discipline grouping.
- Job logs: each job's stdout/stderr is teed into outputs/<id>/job.log
(survives restarts) and served at GET /jobs/{id}/log as text/plain, so
full run logs can be shared for debugging and refinement.
- Model picker: GET /models proxies OpenRouter's public model list with
per-1M-token pricing (1h cache, 502 on failure); the UI shows a model
dropdown with costs when OpenRouter compute is selected, and the pick
overrides vision+text models for that job (Classic and Agent modes).
- Conflicts in the report view are grouped by discipline pair
(collapsible sections, severity-ordered within groups) instead of one
flat severity-only list.
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import backend.jobs as jobs
|
||||
from backend.main import app
|
||||
|
||||
|
||||
class _SyncThread:
|
||||
"""Drop-in threading.Thread replacement that runs the target inline."""
|
||||
|
||||
def __init__(self, target=None, args=(), kwargs=None, **_ignored):
|
||||
self._target = target
|
||||
self._args = args
|
||||
self._kwargs = kwargs or {}
|
||||
|
||||
def start(self):
|
||||
self._target(*self._args, **self._kwargs)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def job_env(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("backend.config.OUTPUT_DIR", str(tmp_path))
|
||||
monkeypatch.setattr(threading, "Thread", _SyncThread)
|
||||
monkeypatch.setattr("backend.jobs.send_conflict_report", lambda *a, **k: True)
|
||||
pdf = tmp_path / "set.pdf"
|
||||
pdf.write_bytes(b"%PDF-1.4\n")
|
||||
yield tmp_path
|
||||
jobs._jobs.clear()
|
||||
|
||||
|
||||
def test_job_log_captures_pipeline_output(job_env, monkeypatch):
|
||||
def fake_runner(pdf_path, **kwargs):
|
||||
print("STAGE banner: fake wave ran")
|
||||
return {"source": "set.pdf", "summary": {"conflicts_found": 0}}
|
||||
|
||||
monkeypatch.setattr("backend.jobs.run_pipeline", fake_runner)
|
||||
job_id = jobs.create_job(str(job_env / "set.pdf"), "set.pdf", pipeline_mode="classic")
|
||||
|
||||
log_path = job_env / job_id / "job.log"
|
||||
assert log_path.is_file()
|
||||
content = log_path.read_text()
|
||||
assert "STAGE banner: fake wave ran" in content
|
||||
assert job_id in content # header line
|
||||
|
||||
|
||||
def test_job_log_endpoint_serves_log_and_404s(job_env, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"backend.jobs.run_pipeline",
|
||||
lambda pdf_path, **kw: {"source": "s", "summary": {}},
|
||||
)
|
||||
job_id = jobs.create_job(str(job_env / "set.pdf"), "set.pdf", pipeline_mode="classic")
|
||||
|
||||
client = TestClient(app)
|
||||
ok = client.get(f"/jobs/{job_id}/log")
|
||||
assert ok.status_code == 200
|
||||
assert ok.headers["content-type"].startswith("text/plain")
|
||||
assert "Job " + job_id in ok.text
|
||||
assert client.get("/jobs/nope/log").status_code == 404
|
||||
|
||||
|
||||
def test_model_override_set_and_cleared_around_run(job_env, monkeypatch):
|
||||
from backend import llm
|
||||
|
||||
seen = {}
|
||||
|
||||
def fake_runner(pdf_path, **kwargs):
|
||||
seen["override"] = llm._model_override
|
||||
return {"source": "set.pdf", "summary": {}}
|
||||
|
||||
monkeypatch.setattr("backend.jobs.run_pipeline", fake_runner)
|
||||
jobs.create_job(str(job_env / "set.pdf"), "set.pdf",
|
||||
pipeline_mode="classic", model="openai/gpt-4o")
|
||||
|
||||
assert seen["override"] == "openai/gpt-4o"
|
||||
assert llm._model_override is None # cleared after the run
|
||||
@@ -0,0 +1,64 @@
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import backend.models as models
|
||||
from backend import config
|
||||
from backend.main import app
|
||||
|
||||
_PAYLOAD = {
|
||||
"data": [
|
||||
{
|
||||
"id": "openai/gpt-4o",
|
||||
"name": "GPT-4o",
|
||||
"pricing": {"prompt": "0.0000025", "completion": "0.00001"},
|
||||
"context_length": 128000,
|
||||
},
|
||||
{
|
||||
"id": "google/gemini-2.5-pro",
|
||||
"name": "Gemini 2.5 Pro",
|
||||
"pricing": {"prompt": "0.00000125", "completion": "0.00001"},
|
||||
"context_length": 1000000,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def _reset_cache():
|
||||
models._cache["models"] = None
|
||||
models._cache["at"] = 0.0
|
||||
|
||||
|
||||
def test_models_endpoint_normalizes_pricing(monkeypatch):
|
||||
_reset_cache()
|
||||
monkeypatch.setattr(models, "_fetch_openrouter_models", lambda: _PAYLOAD["data"])
|
||||
client = TestClient(app)
|
||||
response = client.get("/models")
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["default"] == config.MODEL
|
||||
assert body["default_text"] == config.TEXT_MODEL
|
||||
by_id = {m["id"]: m for m in body["models"]}
|
||||
assert by_id["openai/gpt-4o"]["prompt_usd_per_mtok"] == 2.5
|
||||
assert by_id["openai/gpt-4o"]["completion_usd_per_mtok"] == 10.0
|
||||
assert by_id["openai/gpt-4o"]["context_length"] == 128000
|
||||
|
||||
|
||||
def test_models_endpoint_caches(monkeypatch):
|
||||
_reset_cache()
|
||||
calls = []
|
||||
|
||||
def fake_fetch():
|
||||
calls.append(1)
|
||||
return _PAYLOAD["data"]
|
||||
|
||||
monkeypatch.setattr(models, "_fetch_openrouter_models", fake_fetch)
|
||||
client = TestClient(app)
|
||||
assert client.get("/models").status_code == 200
|
||||
assert client.get("/models").status_code == 200
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_models_endpoint_502_on_fetch_failure(monkeypatch):
|
||||
_reset_cache()
|
||||
monkeypatch.setattr(models, "_fetch_openrouter_models", lambda: None)
|
||||
client = TestClient(app)
|
||||
assert client.get("/models").status_code == 502
|
||||
@@ -0,0 +1,26 @@
|
||||
from backend import config
|
||||
from backend.llm import _resolve_backend, set_model_override
|
||||
|
||||
|
||||
def test_override_wins_for_vision_and_text():
|
||||
set_model_override("openai/gpt-4o")
|
||||
try:
|
||||
assert _resolve_backend(has_images=True, model_override=None)["model"] == "openai/gpt-4o"
|
||||
assert _resolve_backend(has_images=False, model_override=None)["model"] == "openai/gpt-4o"
|
||||
finally:
|
||||
set_model_override(None)
|
||||
|
||||
|
||||
def test_override_beats_per_call_model_arg():
|
||||
set_model_override("openai/gpt-4o")
|
||||
try:
|
||||
# Agents pass their AGENT_*_MODEL per call; the user's job pick wins.
|
||||
assert _resolve_backend(has_images=False, model_override="other/model")["model"] == "openai/gpt-4o"
|
||||
finally:
|
||||
set_model_override(None)
|
||||
|
||||
|
||||
def test_no_override_keeps_defaults():
|
||||
set_model_override(None)
|
||||
assert _resolve_backend(has_images=True, model_override=None)["model"] == config.MODEL
|
||||
assert _resolve_backend(has_images=False, model_override=None)["model"] == config.TEXT_MODEL
|
||||
Reference in New Issue
Block a user