John Wilganowski
1c1d2ff21b
Agent web jobs now stop after Brain consolidation and enter needs_review with a persisted review queue (blocking: high-severity, low-confidence, sensitive-category findings; audit sample of clean clusters). Humans decide confirm/reject/unsure/needs_clarification via new review API and frontend queue; a finalizer applies decisions (rejections suppressed with reason codes), performs bounded targeted reruns for clarifications, drafts RFIs only for kept issues, and only then marks the job done and sends the final email. Two-phase email (review-required, then final report), per-decision feedback labels with redacted aggregate metrics, restart recovery from job artifacts, and CLI --no-review bypass. Classic pipeline unchanged. 65 non-LLM tests.
138 lines
8.1 KiB
Python
138 lines
8.1 KiB
Python
"""
|
|
config.py - Environment-driven configuration for the Conflict Checker.
|
|
|
|
All knobs come from backend/.env (see .env.example). Paths are derived from this
|
|
file's location so the app runs regardless of the current working directory.
|
|
"""
|
|
|
|
import os
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env"))
|
|
|
|
_BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
# -- AI Backend (OpenRouter) ----------------------------------------
|
|
# One multimodal model does both extraction (Stage 1) and conflict
|
|
# reasoning (Stage 3). Override MODEL per-stage if you ever split them.
|
|
AI_BASE_URL = os.getenv("AI_BASE_URL", "https://openrouter.ai/api/v1")
|
|
AI_API_KEY = os.getenv("AI_API_KEY", "")
|
|
MODEL = os.getenv("MODEL", "google/gemini-2.5-pro")
|
|
# Text model when text stages run on OpenRouter (non-hybrid). Falls back to
|
|
# MODEL when unset.
|
|
TEXT_MODEL = os.getenv("TEXT_MODEL", "") or MODEL
|
|
|
|
# Agent-mode model overrides (OpenRouter IDs). Empty values inherit the
|
|
# matching general-purpose model so the skeleton requires no extra config.
|
|
AGENT_EXTRACT_MODEL = os.getenv("AGENT_EXTRACT_MODEL", "") or MODEL
|
|
AGENT_INDEX_MODEL = os.getenv("AGENT_INDEX_MODEL", "") or TEXT_MODEL
|
|
AGENT_JURISDICTION_MODEL = os.getenv("AGENT_JURISDICTION_MODEL", "") or TEXT_MODEL
|
|
AGENT_LINKER_MODEL = os.getenv("AGENT_LINKER_MODEL", "") or TEXT_MODEL
|
|
AGENT_CONFLICT_MODEL = os.getenv("AGENT_CONFLICT_MODEL", "") or MODEL
|
|
AGENT_CODE_MODEL = os.getenv("AGENT_CODE_MODEL", "") or TEXT_MODEL
|
|
AGENT_CONSTRUCT_MODEL = os.getenv("AGENT_CONSTRUCT_MODEL", "") or TEXT_MODEL
|
|
AGENT_COMPLETENESS_MODEL = os.getenv("AGENT_COMPLETENESS_MODEL", "") or TEXT_MODEL
|
|
AGENT_BRAIN_MODEL = os.getenv("AGENT_BRAIN_MODEL", "") or TEXT_MODEL
|
|
AGENT_RFI_MODEL = os.getenv("AGENT_RFI_MODEL", "") or TEXT_MODEL
|
|
|
|
# Agent-mode hard scope limits. These are intentionally independent of Classic
|
|
# batching so Agent workers can never grow into whole-set reasoning calls.
|
|
AGENT_LINK_MAX_ASSERTIONS = int(os.getenv("AGENT_LINK_MAX_ASSERTIONS", "60"))
|
|
AGENT_CLUSTER_MAX_ASSERTIONS = int(os.getenv("AGENT_CLUSTER_MAX_ASSERTIONS", "24"))
|
|
AGENT_CONFLICT_MAX_IMAGES = int(os.getenv("AGENT_CONFLICT_MAX_IMAGES", "6"))
|
|
AGENT_CODE_BATCH_SIZE = int(os.getenv("AGENT_CODE_BATCH_SIZE", "60"))
|
|
AGENT_BRAIN_MAX_TOKENS = int(os.getenv("AGENT_BRAIN_MAX_TOKENS", "16384"))
|
|
AGENT_LINK_CONCURRENCY = int(os.getenv("AGENT_LINK_CONCURRENCY", "4"))
|
|
AGENT_CONFLICT_CONCURRENCY = int(os.getenv("AGENT_CONFLICT_CONCURRENCY", "4"))
|
|
AGENT_SPECIALIST_CONCURRENCY = int(os.getenv("AGENT_SPECIALIST_CONCURRENCY", "4"))
|
|
AGENT_RFI_CONCURRENCY = int(os.getenv("AGENT_RFI_CONCURRENCY", "4"))
|
|
|
|
# Agent-mode human-review gate. When on (default), Agent runs stop after the
|
|
# Brain merge and wait for human decisions before RFIs/final report/email go
|
|
# out. AGENT_REVIEW_AUDIT_SAMPLE caps how many clean clusters get added to the
|
|
# queue as non-blocking spot-checks. REVIEW_AGGREGATE_INCLUDE_TEXT controls
|
|
# whether future cross-job review feedback aggregation may include verbatim
|
|
# source_text/images/comments (off by default = privacy-preserving).
|
|
AGENT_REQUIRE_REVIEW = os.getenv("AGENT_REQUIRE_REVIEW", "true").strip().lower() in ("1", "true", "yes")
|
|
AGENT_REVIEW_AUDIT_SAMPLE = int(os.getenv("AGENT_REVIEW_AUDIT_SAMPLE", "5"))
|
|
# NOTE: currently unwired - reserved for future cross-job aggregation tooling.
|
|
REVIEW_AGGREGATE_INCLUDE_TEXT = os.getenv("REVIEW_AGGREGATE_INCLUDE_TEXT", "false").strip().lower() in ("1", "true", "yes")
|
|
|
|
# -- Hybrid (local text LLM) ----------------------------------------
|
|
# Optional OpenAI-compatible local endpoint (e.g. a vLLM box) for the text-only
|
|
# QAQC stages. Vision stages ALWAYS use OpenRouter. The user picks hybrid per
|
|
# job in the UI; these say WHERE local is + the toggle's default state. Empty
|
|
# LOCAL_BASE_URL = hybrid disabled (falls back to OpenRouter).
|
|
LOCAL_BASE_URL = os.getenv("LOCAL_BASE_URL", "")
|
|
LOCAL_API_KEY = os.getenv("LOCAL_API_KEY", "local")
|
|
LOCAL_TEXT_MODEL = os.getenv("LOCAL_TEXT_MODEL", "")
|
|
HYBRID_DEFAULT = os.getenv("HYBRID_DEFAULT", "false").strip().lower() in ("1", "true", "yes")
|
|
|
|
# -- Pipeline -------------------------------------------------------
|
|
PDF_DPI = int(os.getenv("PDF_DPI", "100"))
|
|
MAX_PAGES = int(os.getenv("MAX_PAGES", "60"))
|
|
MAX_DIMENSION = int(os.getenv("MAX_DIMENSION", "2400")) # px cap on the long edge
|
|
LLM_TIMEOUT = int(os.getenv("LLM_TIMEOUT", "180")) # seconds per call
|
|
EXTRACT_MAX_TOKENS = int(os.getenv("EXTRACT_MAX_TOKENS", "16384"))
|
|
REASON_MAX_TOKENS = int(os.getenv("REASON_MAX_TOKENS", "4096"))
|
|
|
|
# -- QAQC stage knobs (Stages 0-1, 3, 6-11) -------------------------
|
|
# Token caps per stage. Most are single whole-set calls, so they need more
|
|
# headroom than a per-cluster reason call.
|
|
JURISDICTION_MAX_TOKENS = int(os.getenv("JURISDICTION_MAX_TOKENS", "4096"))
|
|
SHEET_INDEX_MAX_TOKENS = int(os.getenv("SHEET_INDEX_MAX_TOKENS", "16384"))
|
|
NORMALIZE_MAX_TOKENS = int(os.getenv("NORMALIZE_MAX_TOKENS", "16384"))
|
|
QAQC_MAX_TOKENS = int(os.getenv("QAQC_MAX_TOKENS", "16384"))
|
|
CODE_MAX_TOKENS = int(os.getenv("CODE_MAX_TOKENS", "16384"))
|
|
CONSTRUCT_MAX_TOKENS = int(os.getenv("CONSTRUCT_MAX_TOKENS", "16384"))
|
|
VALIDATE_MAX_TOKENS = int(os.getenv("VALIDATE_MAX_TOKENS", "16384"))
|
|
RISK_MAX_TOKENS = int(os.getenv("RISK_MAX_TOKENS", "16384"))
|
|
RFI_MAX_TOKENS = int(os.getenv("RFI_MAX_TOKENS", "16384"))
|
|
|
|
# Stage 4 clustering engine: "llm" (semantic, fuzzy matches + catches more) or
|
|
# "deterministic" (clusterer.py, free/reproducible). Swap via env to A/B.
|
|
CLUSTERER = os.getenv("CLUSTERER", "llm").strip().lower()
|
|
CLUSTER_MAX_TOKENS = int(os.getenv("CLUSTER_MAX_TOKENS", "16384"))
|
|
CLUSTER_MAX = int(os.getenv("CLUSTER_MAX", "120"))
|
|
|
|
# Disk-backed LLM response cache for the testing loop. When on, identical calls
|
|
# (same model/prompt/images/params) replay the saved response at zero API cost,
|
|
# so re-running a set only pays for stages whose input actually changed. Off by
|
|
# default so production never serves stale results. Clear: rm -rf the dir.
|
|
LLM_CACHE = os.getenv("LLM_CACHE", "false").strip().lower() in ("1", "true", "yes")
|
|
LLM_CACHE_DIR = os.getenv("LLM_CACHE_DIR", os.path.join(_BASE_DIR, ".llm_cache"))
|
|
|
|
# Parallelism (ThreadPoolExecutor workers)
|
|
EXTRACT_CONCURRENCY = int(os.getenv("EXTRACT_CONCURRENCY", "4"))
|
|
REASON_CONCURRENCY = int(os.getenv("REASON_CONCURRENCY", "4"))
|
|
# Batched stages (normalization, per-sheet code/constructability) reuse this.
|
|
NORMALIZE_CONCURRENCY = int(os.getenv("NORMALIZE_CONCURRENCY", "4"))
|
|
CODE_CONCURRENCY = int(os.getenv("CODE_CONCURRENCY", "4"))
|
|
# Max assertions per code-review call. Code review batches sheets to keep each
|
|
# call small (avoids the truncation that zeroed out a whole-set call).
|
|
CODE_BATCH_SIZE = int(os.getenv("CODE_BATCH_SIZE", "80"))
|
|
CONSTRUCT_CONCURRENCY = int(os.getenv("CONSTRUCT_CONCURRENCY", "4"))
|
|
# Assertions per batch for the normalization stage.
|
|
NORMALIZE_BATCH_SIZE = int(os.getenv("NORMALIZE_BATCH_SIZE", "60"))
|
|
|
|
# -- App ------------------------------------------------------------
|
|
UPLOAD_DIR = os.path.join(_BASE_DIR, "uploads")
|
|
OUTPUT_DIR = os.path.join(_BASE_DIR, "outputs")
|
|
APP_TITLE = "Conflict Checker"
|
|
APP_VERSION = "0.1.0"
|
|
# Public base URL used to build the "view results" link in notification
|
|
# emails. Set to whatever address users reach this server on (e.g. the
|
|
# Tailscale/LAN URL) so the link in the email actually resolves.
|
|
APP_BASE_URL = os.getenv("APP_BASE_URL", "http://localhost:8099")
|
|
|
|
# -- Email / SMTP (optional notification on completion) -------------
|
|
# If unset, the app still works; it just logs "SMTP not configured" and
|
|
# skips the email. Mirrors IronBid's graceful behavior.
|
|
SMTP_HOST = os.getenv("SMTP_HOST", "")
|
|
SMTP_PORT = int(os.getenv("SMTP_PORT", "587"))
|
|
SMTP_USER = os.getenv("SMTP_USER", "")
|
|
SMTP_PASSWORD = os.getenv("SMTP_PASSWORD", "")
|
|
SMTP_FROM = os.getenv("SMTP_FROM", "")
|
|
SMTP_USE_TLS = os.getenv("SMTP_USE_TLS", "true").lower() == "true"
|
|
SMTP_USE_SSL = os.getenv("SMTP_USE_SSL", "false").lower() == "true"
|