Fix sheet-extraction page loss: bare-list wrap, compact retry, reasoning cap

- SheetExtractorAgent accepts top-level array responses as the objects
  array instead of discarding them (recovered the failure mode behind
  16/38 failed sheets on job 475a6f184dd1)
- Second-chance compact retry per page before declaring extraction failed
- call_json: reasoning_effort param (cloud-only extra_body), finish_reason
  capture + explicit max_tokens log line, finish_reason in raw dumps,
  cache key covers reasoning_effort
- EXTRACT_MAX_TOKENS default 16384 -> 32768 (Gemini thinking tokens count
  against the cap); new EXTRACT_REASONING_EFFORT=low default for extract
- tests: 5 new fallback-ladder tests

backend/agents/extractors.py

@@ -18,27 +18,64 @@ from backend.prompts import (
 )
 
 
+# Appended to the extractor instruction on the second-chance retry. Dense plan
+# sheets blow the output budget on the full schema; compact mode trades
+# per-object verbosity for actually finishing the page.
+_COMPACT_RETRY_SUFFIX = """
+IMPORTANT - COMPACT RETRY: the first pass did not complete. Keep the SAME JSON
+schema, but extract at most 40 objects, prioritizing coordination-relevant
+items (equipment, fixtures, devices, keynotes, dimensions, markers/callouts,
+schedule rows). Keep descriptions/attributes short; skip review_uses entries
+you are unsure about. Finish the JSON - a smaller complete answer beats a
+larger truncated one."""
+
+
+def _wrap_bare_list(parsed, page_number: int):
+    """Models sometimes skip the {sheet, objects} wrapper and return a bare
+    objects array (especially after truncation repair). Accept it - the sheet
+    header falls back to title-block deduction downstream."""
+    if isinstance(parsed, list):
+        print(f"[Extract] Page {page_number}: wrapping bare objects array "
+              f"({len(parsed)} items, no sheet header)")
+        return {"sheet": {}, "objects": parsed}
+    return parsed
+
+
 class SheetExtractorAgent:
     name = "sheet_extractor"
 
     def __init__(self, usage: AgentUsage) -> None:
         self.usage = usage
 
+    def _call(self, instruction: str, page: Dict):
+        return call_json(
+            system_prompt=EXTRACTOR_SYSTEM_PROMPT,
+            user_text=instruction,
+            images_b64=[page["base64"]],
+            max_tokens=config.EXTRACT_MAX_TOKENS,
+            model=config.AGENT_EXTRACT_MODEL,
+            usage_tracker=self.usage,
+            usage_stage="agent.extract",
+            reasoning_effort=config.EXTRACT_REASONING_EFFORT or None,
+        )
+
     def run(self, scope: AgentScope) -> AgentResult:
         try:
             page = scope.payload["page"]
             instruction = EXTRACTOR_USER_INSTRUCTION.replace(
                 "{sheet_hint}", str(scope.payload.get("sheet_hint") or "")
             )
-            parsed = call_json(
-                system_prompt=EXTRACTOR_SYSTEM_PROMPT,
-                user_text=instruction,
-                images_b64=[page["base64"]],
-                max_tokens=config.EXTRACT_MAX_TOKENS,
-                model=config.AGENT_EXTRACT_MODEL,
-                usage_tracker=self.usage,
-                usage_stage="agent.extract",
-            )
+            parsed = _wrap_bare_list(self._call(instruction, page),
+                                     page["page_number"])
+            if not isinstance(parsed, dict):
+                # Second chance: same page, compact instructions. Runs only
+                # when the full-schema pass returned nothing usable.
+                print(f"[Extract] Page {page['page_number']}: full extraction "
+                      f"failed, retrying compact")
+                parsed = _wrap_bare_list(
+                    self._call(instruction + _COMPACT_RETRY_SUFFIX, page),
+                    page["page_number"],
+                )
             if not isinstance(parsed, dict):
                 raise ValueError("no structured extraction returned")
             sheet = _normalize_sheet(parsed, page["page_number"])