Fix sheet-extraction page loss: bare-list wrap, compact retry, reasoning cap

- SheetExtractorAgent accepts top-level array responses as the objects
  array instead of discarding them (recovered the failure mode behind
  16/38 failed sheets on job 475a6f184dd1)
- Second-chance compact retry per page before declaring extraction failed
- call_json: reasoning_effort param (cloud-only extra_body), finish_reason
  capture + explicit max_tokens log line, finish_reason in raw dumps,
  cache key covers reasoning_effort
- EXTRACT_MAX_TOKENS default 16384 -> 32768 (Gemini thinking tokens count
  against the cap); new EXTRACT_REASONING_EFFORT=low default for extract
- tests: 5 new fallback-ladder tests

backend/llm.py

@@ -81,7 +81,8 @@ def _summarize_parsed(parsed: Any) -> str:
 def _dump_raw(seq: int, be: Dict[str, Any], usage_stage: str,
               system_prompt: str, user_text: str,
               images_b64: Optional[List[str]], max_tokens: int,
-              raw: str, parsed: Any) -> None:
+              raw: str, parsed: Any,
+              finish_reason: Optional[str] = None) -> None:
     """Write the full request/response for one call to the job's llm_raw dir."""
     if not _raw_dump_dir:
         return
@@ -100,6 +101,7 @@ def _dump_raw(seq: int, be: Dict[str, Any], usage_stage: str,
             "n_images": len(images_b64 or []),
             "system_prompt": system_prompt,
             "user_text": user_text,
+            "finish_reason": finish_reason,
             "raw_response": raw,
             "parsed": parsed,
         }
@@ -228,9 +230,10 @@ def _add_cached() -> None:
 
 def _cache_key(model: str, system_prompt: str, user_text: str,
                images_b64: Optional[List[str]], max_tokens: int,
-               json_mode: bool) -> str:
+               json_mode: bool, reasoning_effort: Optional[str] = None) -> str:
     h = hashlib.sha256()
-    parts = [model, str(max_tokens), str(json_mode), system_prompt, user_text]
+    parts = [model, str(max_tokens), str(json_mode), str(reasoning_effort),
+             system_prompt, user_text]
     for b in (images_b64 or []):
         parts.append(b)
     for p in parts:
@@ -388,6 +391,7 @@ def call_json(
     model: Optional[str] = None,
     usage_tracker: Optional[Any] = None,
     usage_stage: str = "?",
+    reasoning_effort: Optional[str] = None,
 ) -> Optional[Dict[str, Any]]:
     """
     Send one chat completion expecting a JSON object back.
@@ -395,7 +399,10 @@ def call_json(
     Uses the provider's JSON mode (response_format) so the model returns a bare
     JSON object instead of prose/empty text, and recovers from max_tokens
     truncation. images_b64: optional base64 JPEGs attached as high-detail image
-    parts. Returns the parsed dict, or None on a hard failure (caller degrades).
+    parts. reasoning_effort: optional OpenRouter reasoning knob ("low"/"medium"/
+    "high") - keeps thinking models from burning the output budget on hidden
+    reasoning. Returns the parsed dict, or None on a hard failure (caller
+    degrades).
     """
     has_images = bool(images_b64)
     be = _resolve_backend(has_images, model)
@@ -404,7 +411,8 @@ def call_json(
     cache_key = None
     if config.LLM_CACHE:
         cache_key = _cache_key(be["model"], system_prompt, user_text,
-                               images_b64, max_tokens, json_mode=True)
+                               images_b64, max_tokens, json_mode=True,
+                               reasoning_effort=reasoning_effort)
         hit = _cache_get(cache_key)
         if hit is not None:
             _add_cached()
@@ -434,10 +442,17 @@ def call_json(
     for attempt in range(2):
         try:
             client = get_client(be["base_url"], be["api_key"])
-            kwargs = dict(model=be["model"], messages=messages,
-                          max_tokens=max_tokens, timeout=config.LLM_TIMEOUT)
+            kwargs: Dict[str, Any] = dict(model=be["model"], messages=messages,
+                                          max_tokens=max_tokens, timeout=config.LLM_TIMEOUT)
+            extra_body: Dict[str, Any] = {}
             if be["usage"]:
-                kwargs["extra_body"] = {"usage": {"include": True}}
+                extra_body["usage"] = {"include": True}
+            # OpenRouter reasoning knob; only sent to cloud backends (local
+            # servers reject unknown fields). Skipped when effort is blank.
+            if reasoning_effort and not be.get("local"):
+                extra_body["reasoning"] = {"effort": reasoning_effort}
+            if extra_body:
+                kwargs["extra_body"] = extra_body
             if use_json_mode:
                 kwargs["response_format"] = {"type": "json_object"}
             response = client.chat.completions.create(**kwargs)
@@ -448,7 +463,15 @@ def call_json(
                 usage_tracker.record(
                     usage_stage, be["model"], usd=usd or 0.0, has_images=has_images
                 )
-            raw = _strip_fences(response.choices[0].message.content or "")
+            choice = response.choices[0]
+            finish_reason = getattr(choice, "finish_reason", None)
+            raw = _strip_fences(choice.message.content or "")
+            if finish_reason == "length":
+                # Hit max_tokens (thinking tokens included on reasoning models).
+                # Logged explicitly so silent truncation isn't mistaken for a
+                # parse problem; _parse below still salvages what it can.
+                print(f"[LLM] output hit max_tokens (finish_reason=length, "
+                      f"{len(raw)}ch returned)")
             parsed = _parse(raw)
             if parsed is not None:
                 _record_model(be, has_images, fell_back)
@@ -458,7 +481,8 @@ def call_json(
                           raw, parsed, usd)
                 if config.LLM_RAW_DUMP:
                     _dump_raw(seq, be, usage_stage, system_prompt, user_text,
-                              images_b64, max_tokens, raw, parsed)
+                              images_b64, max_tokens, raw, parsed,
+                              finish_reason=finish_reason)
                 return parsed
             if attempt == 0:
                 print("[LLM] JSON parse error (retrying)")