Add complex invoice (MiniCPM vision OCR) + complex multi-step web automation tab

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+backend/evals/datasets/complex_invoice_messy.png filter=lfs diff=lfs merge=lfs -text

backend/app/browser/agent.py

@@ -52,6 +52,20 @@ def _parse_decision(text: str) -> dict:
 
 def _build_plan(scenario: str, base_url: str, order: dict | None) -> list[dict]:
     """The recorded plan the offline provider replays (ignored by real LLMs)."""
+    if scenario == "complex_order":
+        # intricate multi-step interaction: dashboard → Procurement → +Create Order
+        # → read the complex order-form fields.
+        return [
+            {"tool": "navigate", "args": {"url": f"{base_url}/erp/"},
+             "reason": "Open the ERP dashboard."},
+            {"tool": "click", "args": {"selector": "#tile-procurement"},
+             "reason": "Click the Procurement module tile."},
+            {"tool": "click", "args": {"selector": "#create-order"},
+             "reason": "Click '+ Create Order' to open the order-form modal."},
+            {"tool": "extract", "args": {},
+             "reason": "Read the complex order fields (vendor, terms, ship-to, line items, totals, approver)."},
+            {"tool": "done", "args": {"result": None}, "reason": "Captured the order; finish."},
+        ]
     if scenario == "order_fill":
         order = order or {}
         plan = [{"tool": "navigate", "args": {"url": f"{base_url}/orders/new"},
@@ -87,10 +101,11 @@ def run_browser_agent(
     order: dict | None = None,
     base_url: str | None = None,
     headless: bool = True,
+    prefer_simulated: bool | None = None,
 ) -> dict:
     base_url = base_url or settings.demo_portal_url
     run_id = uuid.uuid4().hex
-    session = get_session(headless=headless)
+    session = get_session(headless=headless, prefer_simulated=prefer_simulated)
     registry = browser_registry()
     registry.bind("navigate", lambda url: session.navigate(url))
     registry.bind("click", lambda selector: session.click(selector))
@@ -101,6 +116,17 @@ def run_browser_agent(
     plan = _build_plan(scenario, base_url, order)
     tool_defs = _tool_defs_block(registry)
 
+    # Decision "brain": a capable frontier agent LLM (Claude/Gemini) drives autonomously
+    # when configured; otherwise we replay a recorded plan deterministically (reliable RPA).
+    # (MiniCPM-V is used for OCR/extraction, not as the browser-agent controller.)
+    reg = router.registry
+    if reg.anthropic and reg.anthropic.available():
+        agent_provider, agent_model, agent_mode = reg.anthropic, settings.anthropic_model_smart, "llm:claude"
+    elif reg.gemini and reg.gemini.available():
+        agent_provider, agent_model, agent_mode = reg.gemini, settings.gemini_model, "llm:gemini"
+    else:
+        agent_provider, agent_model, agent_mode = reg.mock, "mock", "deterministic-plan"
+
     trace: list[dict] = []
     final_result = None
     last_extract = None
@@ -117,7 +143,8 @@ def run_browser_agent(
             max_tokens=400,
             context={"plan": plan, "step": step},
         )
-        resp = router.run(req, run_id)
+        resp = agent_provider.complete(req, agent_model)
+        metrics.record_call(run_id, resp, "agent")
         decision = _parse_decision(resp.text)
         tool = decision.get("tool", "done")
         args = decision.get("args", {}) or {}
@@ -158,6 +185,7 @@ def run_browser_agent(
     agg = metrics.call_aggregates(run_id)
     return {
         "mode": "agentic",
+        "agent_mode": agent_mode,
         "backend": session.backend,
         "goal": goal,
         "scenario": scenario,

backend/app/browser/session.py

@@ -33,6 +33,28 @@ ORDER_FORM_FIELDS = [
 ]
 
 
+# Complex order the ERP "Create Order" modal exposes (read by the agent).
+COMPLEX_ORDER = {
+    "po_number": "PO-2026-88440",
+    "vendor": "Meridian Industrial Components Ltd",
+    "contact": "j.harlow@meridian.example",
+    "incoterms": "DAP",
+    "payment_terms": "Net 45",
+    "priority": "Expedited",
+    "ship_to": {"facility": "Aperture DC-West (Bay 14)",
+                "address": "4400 Logistics Pkwy, Reno NV 89506, USA",
+                "need_by": "2026-07-09"},
+    "line_items": [
+        {"sku": "HX-220", "description": "Hydraulic pump HX-220", "qty": 4, "unit_price": 1250.00, "amount": 5000.00},
+        {"sku": "SK-12", "description": "Seal kit (set of 12)", "qty": 10, "unit_price": 85.50, "amount": 855.00},
+        {"sku": "PG-300", "description": "Pressure gauge 0-300psi", "qty": 6, "unit_price": 142.75, "amount": 856.50},
+        {"sku": "LAB-1", "description": "Installation labor", "qty": 1, "unit_price": 2400.00, "amount": 2400.00},
+    ],
+    "subtotal": 9111.50, "tax": 751.70, "freight": 180.00, "total": 10043.20, "currency": "USD",
+    "approver": "Priya Anand (Procurement Lead)",
+}
+
+
 @dataclass
 class ActionRecord:
     tool: str
@@ -52,18 +74,39 @@ class SimulatedBrowser:
     screenshots: int = 0
     log: list = field(default_factory=list)
     backend: str = "simulated"
+    erp_stage: str | None = None  # None | dashboard | procurement | form
 
     # --- tool primitives ---
     def navigate(self, url: str) -> ActionRecord:
         self.url = url
-        rec = ActionRecord("navigate", {"url": url}, True, f"loaded {url}")
+        note = f"loaded {url}"
+        if "/erp" in url:
+            self.erp_stage = "dashboard"
+            note = f"loaded ERP dashboard ({url})"
+        rec = ActionRecord("navigate", {"url": url}, True, note)
         self.log.append(rec)
         return rec
 
     def click(self, selector: str) -> ActionRecord:
         ok = True
+        sel = selector.lower()
         note = f"clicked {selector}"
-        if "submit" in selector.lower():
+        # --- multi-step ERP flow: dashboard → procurement → order modal ---
+        if self.erp_stage is not None:
+            if "procurement" in sel:
+                self.erp_stage = "procurement"
+                note = "opened Procurement module"
+            elif "create-order" in sel or "create order" in sel:
+                self.erp_stage = "form"
+                note = "clicked '+ Create Order' → order form modal opened"
+            elif "submit" in sel:
+                self.submitted = True
+                self.confirmation_id = f"PO-CONF-{uuid.uuid4().hex[:6].upper()}"
+                note = f"submitted order for approval → {self.confirmation_id}"
+            rec = ActionRecord("click", {"selector": selector}, ok, note)
+            self.log.append(rec)
+            return rec
+        if "submit" in sel:
             self.submitted = True
             self.confirmation_id = f"CONF-{uuid.uuid4().hex[:6].upper()}"
             note = f"submitted form → confirmation {self.confirmation_id}"
@@ -84,6 +127,9 @@ class SimulatedBrowser:
         return rec
 
     def extract(self) -> dict:
+        if self.erp_stage == "form":
+            self.log.append(ActionRecord("extract", {}, True, "read complex order form fields"))
+            return dict(COMPLEX_ORDER)
         if "orders" in self.url and "new" not in self.url:
             data = {"orders": PENDING_ORDERS}
         elif self.submitted:
@@ -101,6 +147,28 @@ class SimulatedBrowser:
 
     # --- sanitized page state for the LLM (the 'compact markdown DOM') ---
     def get_state(self) -> str:
+        # multi-step ERP flow states
+        if self.erp_stage == "dashboard":
+            return ("# Acme ERP — Dashboard\nURL: " + self.url +
+                    "\n\nModules (tiles):\n"
+                    "- Procurement (link `#tile-procurement` → procurement/) — create & manage POs\n"
+                    "- Invoices `#tile-invoices` · Suppliers `#tile-suppliers` · Reports `#tile-reports`\n"
+                    "Goal hint: open Procurement to create/read an order.")
+        if self.erp_stage == "procurement":
+            return ("# Acme ERP — Procurement\nURL: " + self.url +
+                    "\n\nA table of purchase orders is shown.\n"
+                    "Button: '+ Create Order' (selector `#create-order`) — opens the new-order form modal.")
+        if self.erp_stage == "form":
+            o = COMPLEX_ORDER
+            lines = "\n".join(f"  - {li['sku']} {li['description']} x{li['qty']} @ ${li['unit_price']:.2f} = ${li['amount']:.2f}"
+                              for li in o["line_items"])
+            return (f"# New Purchase Order (modal)\nURL: {self.url}\n\n"
+                    f"PO: {o['po_number']} · Vendor: {o['vendor']} · Terms: {o['payment_terms']} · Priority: {o['priority']}\n"
+                    f"Ship to: {o['ship_to']['facility']}, {o['ship_to']['address']} (need-by {o['ship_to']['need_by']})\n"
+                    f"Line items:\n{lines}\n"
+                    f"Subtotal ${o['subtotal']:.2f} · Tax ${o['tax']:.2f} · Freight ${o['freight']:.2f} · "
+                    f"Total ${o['total']:.2f} {o['currency']} · Approver {o['approver']}\n"
+                    f"Buttons: 'Submit for approval' (`#submit-order`). Use extract() to read the fields.")
         if self.submitted:
             return (f"# Order Portal — Confirmation\nURL: {self.url}\n\n"
                     f"Order submitted. Confirmation ID: **{self.confirmation_id}**\n"

backend/app/pipeline/nodes.py

@@ -44,7 +44,31 @@ def _parse_json(text: str) -> dict:
     return {}
 
 
+_NUM_FIELDS = {"subtotal", "tax_amount", "tax", "total", "shipping", "freight", "amount",
+               "unit_price", "line_total", "quantity", "contract_value", "selected_total",
+               "grand_total", "net_total", "vat_amount"}
+
+
+def _clean_numbers(data):
+    """Convert money-formatted strings ('9,111.50', '$1,250.00') to floats anywhere in
+    the structure — models often return formatted numbers regardless of instructions."""
+    from .. import extraction_heuristics as H
+    if isinstance(data, dict):
+        out = {}
+        for k, v in data.items():
+            if isinstance(v, str) and k in _NUM_FIELDS:
+                m = H.parse_money(v)
+                out[k] = m if m is not None else v
+            else:
+                out[k] = _clean_numbers(v)
+        return out
+    if isinstance(data, list):
+        return [_clean_numbers(x) for x in data]
+    return data
+
+
 def _coerce_to_schema(data: dict, doc_type: str) -> dict:
+    data = _clean_numbers(data)
     schema = SCHEMA_BY_TYPE.get(doc_type)
     if schema is None:
         return data
@@ -126,6 +150,24 @@ def classify_node(state: dict, ctx: PipelineContext) -> dict:
     }
 
 
+def _image_bytes(file_path: str) -> bytes:
+    """First-page PNG bytes for vision extraction (image file as-is; PDF → rasterize)."""
+    p = Path(file_path)
+    if p.suffix.lower() in {".png", ".jpg", ".jpeg", ".bmp", ".tif", ".tiff"}:
+        return p.read_bytes()
+    import io
+
+    import fitz  # PyMuPDF
+    doc = fitz.open(str(p))
+    pix = doc.load_page(0).get_pixmap(matrix=fitz.Matrix(2, 2))
+    from PIL import Image
+    img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
+    doc.close()
+    buf = io.BytesIO()
+    img.save(buf, format="PNG")
+    return buf.getvalue()
+
+
 def extract_node(state: dict, ctx: PipelineContext) -> dict:
     doc_type = state["doc_type"]
 
@@ -149,6 +191,34 @@ def extract_node(state: dict, ctx: PipelineContext) -> dict:
 
     schema = SCHEMA_BY_TYPE[doc_type]
     schema_json = json.dumps(schema.model_json_schema().get("properties", {}), indent=0)[:1500]
+
+    # VISION EXTRACTION: for scanned / complex-layout docs, send the IMAGE directly to
+    # the VLM (MiniCPM-V) for structured JSON — it sees the layout (rotated text,
+    # scattered values, watermarks) that linear OCR text loses.
+    reg = getattr(ctx.router, "registry", None)
+    vlm = getattr(reg, "minicpm", None)
+    if vlm and vlm.available() and state.get("channel") == "scanned":
+        try:
+            img = _image_bytes(state["file_path"])
+            fields = list(schema.model_json_schema().get("properties", {}).keys())
+            data, usage = vlm.vision_extract(img, json.dumps(fields), doc_type)
+            if data:
+                from ..providers.base import LLMResponse, Usage
+                resp = LLMResponse.build(
+                    text=json.dumps(data), model=f"minicpm-vision:{ctx.settings.minicpm_model}",
+                    provider="minicpm", usage=Usage(
+                        input_tokens=usage.get("prompt_tokens", 800),
+                        output_tokens=usage.get("completion_tokens", 200)),
+                    latency_ms=0.0, routing_reason="vision extraction (image→JSON)")
+                ctx.metrics.record_call(ctx.run_id, resp, "extract")
+                data.setdefault("doc_type", doc_type)
+                extracted = _coerce_to_schema(data, doc_type)
+                return {"extracted": extracted, "extract_model": resp.model,
+                        "_summary": f"Vision extraction (image→JSON) via {resp.model} — "
+                                    "reads complex layout directly"}
+        except Exception:
+            pass  # fall back to text-LLM extraction below
+
     text = (state.get("text") or "")[:MAX_TEXT_FOR_LLM]
     req = LLMRequest(
         # cacheable prefix: instructions + schema (stable across all docs of this type)
@@ -250,12 +320,15 @@ def validate_node(state: dict, ctx: PipelineContext) -> dict:
     doc_type = state["doc_type"]
     data = state.get("normalized") or state.get("extracted") or {}
     vr = validate_financials(data, doc_type)
-    # Blend in OCR coverage: low-confidence OCR should lower overall confidence.
+    # Effective input quality: a digital text layer is 1.0; a scanned doc uses its OCR
+    # engine's confidence (a high-confidence VLM like MiniCPM is NOT treated as 0).
     coverage = float(state.get("coverage", 1.0))
-    blended = round(0.8 * vr.confidence + 0.2 * coverage, 3)
+    ocr_conf = float(((state.get("ocr") or {}).get("ocr_channel") or {}).get("confidence", 0.0) or 0.0)
+    quality = coverage if coverage >= 0.5 else max(coverage, ocr_conf)
+    blended = round(0.8 * vr.confidence + 0.2 * quality, 3)
     flags = list(vr.flags)
-    if coverage < 0.5:
-        flags.append(f"low_ocr_coverage:{coverage}")
+    if quality < 0.5:
+        flags.append(f"low_ocr_quality:{round(quality,2)}")
     return {
         "validation": {"confidence": vr.confidence, "checks": vr.checks, "flags": vr.flags},
         "confidence": blended,

backend/app/providers/__init__.py

@@ -41,13 +41,20 @@ class ProviderRegistry:
         self.local = LocalProvider(
             settings.ollama_base_url, settings.local_model, settings.local_backend
         )
+        # MiniCPM-V as an LLM (text reasoning) — a capable small model.
+        self.minicpm = None
+        if settings.minicpm_base_url:
+            from .minicpm_llm import MiniCPMLLMProvider
+            self.minicpm = MiniCPMLLMProvider(
+                settings.minicpm_base_url, settings.minicpm_api_key, settings.minicpm_model)
 
     def capabilities(self) -> dict:
         """Report which tiers are live — surfaced at /api/capabilities."""
         anthropic_ok = bool(self.anthropic) and self.anthropic.available()
         gemini_ok = bool(self.gemini) and self.gemini.available()
         local_ok = bool(self.local) and self.local.available()
-        if anthropic_ok or gemini_ok:
+        minicpm_ok = bool(self.minicpm) and self.minicpm.available()
+        if anthropic_ok or gemini_ok or minicpm_ok:
             active = "hosted"
         elif local_ok:
             active = "local"
@@ -65,6 +72,8 @@ class ProviderRegistry:
                 "local": {"available": local_ok, "tier": "local",
                           "model": self.settings.local_model,
                           "backend": self.settings.local_backend},
+                "minicpm": {"available": minicpm_ok, "tier": "hosted",
+                            "model": self.settings.minicpm_model},
             },
             "ocr": _ocr_capabilities(),
             "browser": {"playwright": _has("playwright")},

backend/app/providers/minicpm_llm.py

@@ -0,0 +1,113 @@
+"""MiniCPM-V as an LLM provider (text reasoning) — not just OCR.
+
+MiniCPM-V-4.6 (~8B) is a capable small model. Beyond image OCR, we use it for the
+text reasoning steps (classify / extract / normalize) via the same OpenAI-compatible
+ModelBest endpoint. This makes the whole pipeline run on a single small model — ideal
+for the Build Small hackathon — instead of falling back to regex heuristics.
+
+For complex/scattered documents it can also extract structured JSON directly from the
+*image* (vision extraction); see `vision_extract`.
+"""
+from __future__ import annotations
+
+import base64
+import json
+import urllib.request
+
+from .base import LLMProvider, LLMRequest, LLMResponse, Usage, estimate_tokens
+from ..ocr.backends.minicpm import _ssl_context
+
+
+class MiniCPMLLMProvider(LLMProvider):
+    name = "minicpm"
+    tier = "hosted"
+
+    def __init__(self, base_url: str | None, api_key: str | None, model: str,
+                 timeout: int = 90) -> None:
+        self.base_url = (base_url or "").rstrip("/")
+        self.api_key = api_key
+        self.model = model
+        self.timeout = timeout
+
+    def available(self) -> bool:
+        return bool(self.base_url)
+
+    def _endpoint(self) -> str:
+        b = self.base_url
+        return b + ("/chat/completions" if b.endswith("/v1") else "/v1/chat/completions")
+
+    def _post(self, messages: list, max_tokens: int, temperature: float) -> dict:
+        payload = json.dumps({"model": self.model, "messages": messages,
+                              "temperature": temperature, "max_tokens": max_tokens}).encode()
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        req = urllib.request.Request(self._endpoint(), data=payload, headers=headers)
+        ctx = _ssl_context() if self._endpoint().startswith("https") else None
+        with urllib.request.urlopen(req, timeout=self.timeout, context=ctx) as r:
+            return json.loads(r.read().decode())
+
+    def complete(self, req: LLMRequest, model: str | None = None) -> LLMResponse:
+        t0 = self._now()
+        model = model or self.model
+        try:
+            system = req.cacheable_prefix() + "\n" + "\n".join(
+                b.text for b in req.system_blocks if not b.cacheable)
+            user = req.user_content
+            if req.json_schema:
+                user += "\n\nReturn ONLY a single valid JSON object — no prose, no markdown fences."
+            messages = []
+            if system.strip():
+                messages.append({"role": "system", "content": system.strip()})
+            messages.append({"role": "user", "content": user})
+            body = self._post(messages, req.max_tokens, req.temperature)
+            text = body["choices"][0]["message"]["content"]
+            u = body.get("usage", {}) or {}
+            usage = Usage(
+                input_tokens=u.get("prompt_tokens", estimate_tokens(system + user)),
+                output_tokens=u.get("completion_tokens", estimate_tokens(text)),
+            )
+            return LLMResponse.build(text=text, model=f"minicpm:{model}", provider=self.name,
+                                     usage=usage, latency_ms=(self._now() - t0) * 1000,
+                                     routing_reason=f"minicpm-llm:{model}")
+        except Exception as e:
+            return LLMResponse.build(text="", model=f"minicpm:{model}", provider=self.name,
+                                     usage=Usage(input_tokens=estimate_tokens(req.full_prompt())),
+                                     latency_ms=(self._now() - t0) * 1000,
+                                     routing_reason="minicpm-llm (error)", error=str(e))
+
+    # --- direct vision extraction (image -> structured JSON), best for messy layouts ---
+    def vision_extract(self, image_bytes: bytes, schema_hint: str, doc_type: str) -> dict:
+        b64 = base64.b64encode(image_bytes).decode()
+        prompt = (
+            f"You are extracting fields from a {doc_type} that may have a complex layout, "
+            "rotated text, watermarks, and values scattered across the page. Read the WHOLE "
+            "image carefully and return ONLY a JSON object with these fields (use null if "
+            f"truly absent):\n{schema_hint}\n"
+            "Amounts are numbers (strip currency symbols); dates as ISO-8601 (YYYY-MM-DD). "
+            "For line_items return an array of {description, quantity, unit_price, line_total}. "
+            "No prose, no markdown fences."
+        )
+        messages = [{"role": "user", "content": [
+            {"type": "text", "text": prompt},
+            {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}]}]
+        body = self._post(messages, 2048, 0.0)
+        text = body["choices"][0]["message"]["content"]
+        return _parse_json(text), body.get("usage", {}) or {}
+
+
+def _parse_json(text: str) -> dict:
+    import re
+    text = (text or "").strip()
+    text = re.sub(r"^```(?:json)?", "", text).strip()
+    text = re.sub(r"```$", "", text).strip()
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        m = re.search(r"\{.*\}", text, re.DOTALL)
+        if m:
+            try:
+                return json.loads(m.group(0))
+            except json.JSONDecodeError:
+                return {}
+    return {}

backend/app/providers/pricing.py

@@ -31,6 +31,7 @@ PRICES: dict[str, Price] = {
     "claude-haiku": Price.standard(0.80, 4.00),
     "gemini-flash": Price(input=0.30, output=2.50, cached_read=0.075, cache_write=0.30),
     "gemma-local": Price(input=0.0, output=0.0, cached_read=0.0, cache_write=0.0),
+    "minicpm": Price(input=0.10, output=0.30, cached_read=0.03, cache_write=0.12),  # small VLM
     "mock": Price(input=0.0, output=0.0, cached_read=0.0, cache_write=0.0),
 }
 
@@ -46,6 +47,8 @@ def price_family(model: str) -> str:
         return "claude-haiku"
     if "gemini" in m:
         return "gemini-flash"
+    if "minicpm" in m:
+        return "minicpm"
     if "gemma" in m or "mistral" in m or "llama" in m or m.startswith("local"):
         return "gemma-local"
     return "mock"

backend/app/router.py

@@ -121,6 +121,9 @@ class ModelRouter:
         if self.registry.gemini and self.registry.gemini.available():
             chain.append((self.registry.gemini, self.settings.gemini_model,
                           "cheap: Gemini Flash"))
+        if getattr(self.registry, "minicpm", None) and self.registry.minicpm.available():
+            chain.append((self.registry.minicpm, self.settings.minicpm_model,
+                          "cheap: MiniCPM-V (small model)"))
         return chain
 
     def _smart_chain(self) -> list[tuple[LLMProvider, str, str]]:
@@ -131,6 +134,9 @@ class ModelRouter:
         if self.registry.gemini and self.registry.gemini.available():
             chain.append((self.registry.gemini, self.settings.gemini_model,
                           "smart: Gemini Flash"))
+        if getattr(self.registry, "minicpm", None) and self.registry.minicpm.available():
+            chain.append((self.registry.minicpm, self.settings.minicpm_model,
+                          "smart: MiniCPM-V (small model)"))
         if self.registry.local and self.registry.local.available():
             chain.append((self.registry.local, self.settings.local_model,
                           "smart: local Gemma (no frontier available)"))

backend/app/schemas.py

@@ -42,6 +42,7 @@ class InvoiceSchema(BaseModel):
     currency: Optional[str] = Field(default=None, description="ISO-4217, e.g. USD.")
     subtotal: Optional[float] = None
     tax_amount: Optional[float] = None
+    shipping: Optional[float] = None
     total: Optional[float] = None
     line_items: List[LineItem] = Field(default_factory=list)
     notes: Optional[str] = None
@@ -316,12 +317,13 @@ def validate_financials(data: dict, doc_type: str) -> ValidationResult:
     if missing:
         flags.append(f"missing_required:{','.join(missing)}")
 
-    # 2) Financial balance: net + tax ≈ total (where present).
+    # 2) Financial balance: net + tax (+ shipping) ≈ total (where present).
     subtotal, tax, total = num(data.get("subtotal")), num(data.get("tax_amount")), num(
         data.get("total")
     )
+    shipping = num(data.get("shipping")) or num(data.get("freight")) or 0.0
     if subtotal is not None and total is not None:
-        expected = subtotal + (tax or 0.0)
+        expected = subtotal + (tax or 0.0) + shipping
         ok = abs(expected - total) <= max(MONEY_TOL, 0.01 * total)
         checks["totals_balance"] = ok
         if not ok:

backend/evals/datasets/complex_invoice_messy.gt.json

@@ -0,0 +1,44 @@
+{
+  "doc_type": "invoice",
+  "invoice_number": "INV-9X-44821",
+  "issue_date": "2026-04-18",
+  "due_date": "2026-05-18",
+  "vendor_name": "Meridian Industrial Components Ltd",
+  "bill_to_name": "Aperture Retail Group",
+  "currency": "USD",
+  "subtotal": 9111.5,
+  "tax_amount": 751.7,
+  "total": 10043.2,
+  "line_items": [
+    {
+      "description": "Hydraulic pump HX-220",
+      "quantity": 4,
+      "unit_price": 1250.0,
+      "line_total": 5000.0
+    },
+    {
+      "description": "Seal kit (set of 12)",
+      "quantity": 10,
+      "unit_price": 85.5,
+      "line_total": 855.0
+    },
+    {
+      "description": "Pressure gauge 0-300psi",
+      "quantity": 6,
+      "unit_price": 142.75,
+      "line_total": 856.5
+    },
+    {
+      "description": "Installation labor",
+      "quantity": 1,
+      "unit_price": 2400.0,
+      "line_total": 2400.0
+    }
+  ],
+  "_meta": {
+    "doc_type": "invoice",
+    "channel": "scanned",
+    "difficulty": "complex_layout",
+    "skip_eval": true
+  }
+}

gradio_app.py

@@ -101,6 +101,22 @@ def search(query: str):
             for r in RAG.search(query, k=8)]
 
 
+def run_complex_web_automation():
+    """Intricate multi-step browser automation: ERP dashboard → Procurement →
+    +Create Order → read the complex order-form fields."""
+    from app.browser import run_browser_agent
+    # The Gradio app doesn't host the ERP portal, so drive the simulated browser
+    # (the same multi-step flow runs on real Chromium in the full web app).
+    res = run_browser_agent(
+        "Open the ERP dashboard, click Procurement, click '+ Create Order', and read all order fields",
+        router=ROUTER, settings=S, metrics=METRICS, scenario="complex_order",
+        base_url="https://portal.local/portal", prefer_simulated=True)
+    trace_md = f"**{res['backend']} browser · {res['agent_mode']} · {res['steps']} steps**\n\n"
+    for t in res["trace"]:
+        trace_md += f"- **step {t['step']}** `{t['tool']}` {t.get('args') or ''} — {t.get('note','')[:90]}\n"
+    return trace_md, res.get("result")
+
+
 with gr.Blocks(title="Aperture — Retail Document Intelligence") as demo:
     gr.Markdown("# 📄 Aperture — Retail Document Intelligence\n"
                 "Open-source agentic OCR + IDP (orders · receipts · invoices · contracts · subscription memos). "
@@ -125,6 +141,17 @@ with gr.Blocks(title="Aperture — Retail Document Intelligence") as demo:
         search_btn = gr.Button("🔍 Search")
         results = gr.Dataframe(headers=["ref", "score", "text"], label="Results")
         search_btn.click(search, [q], [results])
+    with gr.Tab("Web Automation"):
+        gr.Markdown("### Complex multi-step browser automation\n"
+                    "Replaces UiPath Studio Web: navigate the ERP dashboard → click the **Procurement** "
+                    "tile → click **+ Create Order** → open the order-form modal → **read the complex "
+                    "nested fields** (vendor & terms, ship-to, line items, totals, approver). Runs on real "
+                    "Chromium when Playwright is installed; replays on the simulated browser here.")
+        cx_btn = gr.Button("▶ Run: Procurement → Create Order → read fields", variant="primary")
+        with gr.Row():
+            cx_trace = gr.Markdown()
+            cx_result = gr.JSON(label="Order fields read from the form")
+        cx_btn.click(run_complex_web_automation, [], [cx_trace, cx_result])
 
 if __name__ == "__main__":
     try:

scripts/generate_complex_invoice.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Generate a DELIBERATELY HARD invoice image to stress the OCR backends:
+  • rotated vertical "INVOICE" banner + sideways Terms block (orientation problems)
+  • a diagonal translucent "ORIGINAL COPY" watermark overlapping text
+  • scattered header fields (invoice #, dates, vendor, bill-to in offset boxes)
+  • a misaligned line-item table (inconsistent column alignment)
+  • totals scattered across corners (subtotal / tax / shipping / grand total / balance due)
+
+This is the kind of layout where classic OCR (Tesseract) struggles but a vision
+LLM (MiniCPM-V) reads it correctly. Writes:
+  backend/evals/datasets/complex_invoice_messy.png
+  backend/evals/datasets/complex_invoice_messy.gt.json   (skip_eval — showcase only)
+No .txt sidecar on purpose — this document REQUIRES a real OCR engine.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from PIL import Image, ImageDraw, ImageFont
+
+ROOT = Path(__file__).resolve().parent.parent
+OUT = ROOT / "backend" / "evals" / "datasets"
+W, H = 1240, 1600
+
+GT = {
+    "doc_type": "invoice",
+    "invoice_number": "INV-9X-44821",
+    "issue_date": "2026-04-18",
+    "due_date": "2026-05-18",
+    "vendor_name": "Meridian Industrial Components Ltd",
+    "bill_to_name": "Aperture Retail Group",
+    "currency": "USD",
+    "subtotal": 9111.50,
+    "tax_amount": 751.70,
+    "total": 10043.20,
+    "line_items": [
+        {"description": "Hydraulic pump HX-220", "quantity": 4, "unit_price": 1250.00, "line_total": 5000.00},
+        {"description": "Seal kit (set of 12)", "quantity": 10, "unit_price": 85.50, "line_total": 855.00},
+        {"description": "Pressure gauge 0-300psi", "quantity": 6, "unit_price": 142.75, "line_total": 856.50},
+        {"description": "Installation labor", "quantity": 1, "unit_price": 2400.00, "line_total": 2400.00},
+    ],
+    "_meta": {"doc_type": "invoice", "channel": "scanned", "difficulty": "complex_layout", "skip_eval": True},
+}
+
+
+def font(sz, bold=False):
+    for p in ([
+        "/System/Library/Fonts/Supplemental/Arial Bold.ttf" if bold else "/System/Library/Fonts/Supplemental/Arial.ttf",
+        "/System/Library/Fonts/Helvetica.ttc",
+        "/Library/Fonts/Arial.ttf",
+    ]):
+        try:
+            return ImageFont.truetype(p, sz)
+        except Exception:
+            continue
+    return ImageFont.load_default()
+
+
+def rotated(base, text, xy, angle, fnt, fill=(20, 20, 20)):
+    tmp = Image.new("RGBA", (max(20, len(text) * fnt.size), fnt.size + 16), (0, 0, 0, 0))
+    ImageDraw.Draw(tmp).text((2, 2), text, font=fnt, fill=fill)
+    tmp = tmp.rotate(angle, expand=True)
+    base.paste(tmp, xy, tmp)
+
+
+def main():
+    OUT.mkdir(parents=True, exist_ok=True)
+    img = Image.new("RGB", (W, H), "white")
+    d = ImageDraw.Draw(img)
+
+    # --- diagonal translucent watermark overlapping content ---
+    wm = Image.new("RGBA", (W, H), (0, 0, 0, 0))
+    wd = ImageDraw.Draw(wm)
+    wd.text((140, 120), "ORIGINAL COPY", font=font(120, True), fill=(150, 150, 150, 60))
+    wm = wm.rotate(28, center=(W // 2, H // 2))
+    img.paste(wm, (0, 0), wm)
+
+    # --- left vertical INVOICE banner (rotated 90) ---
+    d.rectangle([18, 60, 96, 760], fill=(28, 40, 80))
+    rotated(img, "INVOICE", (24, 470), 90, font(54, True), fill=(255, 255, 255))
+
+    # --- scattered header fields (offset boxes, inconsistent placement) ---
+    d.text((900, 70), "Invoice No.", font=font(20, True), fill=(90, 90, 90))
+    d.text((900, 98), GT["invoice_number"], font=font(26, True), fill=(10, 10, 10))
+    d.text((690, 150), f"Issued: {GT['issue_date']}", font=font(20), fill=(10, 10, 10))
+    d.text((980, 200), f"Due {GT['due_date']}", font=font(20), fill=(160, 30, 30))  # offset, different spot
+    # Balance due repeated near top in red (inconsistent)
+    d.text((620, 60), "BALANCE DUE  $10,043.20", font=font(26, True), fill=(190, 20, 20))
+
+    # vendor block (top-left, after banner) and bill-to (offset right-middle)
+    d.rectangle([130, 90, 560, 220], outline=(120, 120, 120), width=2)
+    d.text((145, 100), "FROM / Remit to:", font=font(18, True), fill=(70, 70, 70))
+    d.text((145, 128), GT["vendor_name"], font=font(22, True), fill=(10, 10, 10))
+    d.text((145, 160), "Unit 7, Kvaerner Estate", font=font(18), fill=(40, 40, 40))
+    d.text((145, 184), "VAT GB-882-114", font=font(18), fill=(40, 40, 40))
+
+    d.rectangle([640, 300, 1080, 420], outline=(120, 120, 120), width=2)
+    d.text((655, 310), "Bill To", font=font(18, True), fill=(70, 70, 70))
+    d.text((655, 338), GT["bill_to_name"], font=font(22, True), fill=(10, 10, 10))
+    d.text((655, 372), "Accounts Payable, Floor 3", font=font(18), fill=(40, 40, 40))
+    d.text((655, 396), "PO ref: PO-77-3391", font=font(18), fill=(40, 40, 40))
+
+    # --- misaligned line-item table ---
+    ty = 470
+    d.line([130, ty - 10, 1110, ty - 10], fill=(40, 40, 40), width=2)
+    # headers placed inconsistently (not above their columns)
+    d.text((150, ty), "Item / Description", font=font(20, True), fill=(20, 20, 20))
+    d.text((720, ty), "Unit", font=font(20, True), fill=(20, 20, 20))
+    d.text((640, ty), "Qty", font=font(20, True), fill=(20, 20, 20))
+    d.text((980, ty), "Amount", font=font(20, True), fill=(20, 20, 20))
+    ry = ty + 44
+    for i, it in enumerate(GT["line_items"], 1):
+        d.text((150, ry), f"{i}.  {it['description']}", font=font(20), fill=(15, 15, 15))
+        # qty/unit/amount with inconsistent alignment (left vs right vs centered)
+        d.text((648, ry), str(int(it["quantity"])), font=font(20), fill=(15, 15, 15))
+        d.text((700, ry), f"${it['unit_price']:,.2f}", font=font(20), fill=(15, 15, 15))
+        amt = f"${it['line_total']:,.2f}"
+        d.text((1090 - d.textlength(amt, font=font(20)), ry), amt, font=font(20), fill=(15, 15, 15))
+        ry += 46
+    d.line([130, ry + 6, 1110, ry + 6], fill=(40, 40, 40), width=1)
+
+    # --- totals scattered across corners ---
+    d.text((180, ry + 60), "Tax @ 8.25%", font=font(20), fill=(15, 15, 15))   # bottom-left
+    d.text((320, ry + 60), "$751.70", font=font(20), fill=(15, 15, 15))
+    d.text((640, ry + 30), "Sub-total", font=font(20), fill=(15, 15, 15))      # middle
+    d.text((780, ry + 30), "$9,111.50", font=font(20), fill=(15, 15, 15))
+    d.text((640, ry + 90), "Freight/Shipping", font=font(20), fill=(15, 15, 15))
+    d.text((820, ry + 90), "$180.00", font=font(20), fill=(15, 15, 15))
+    # grand total in a bold box, bottom-right
+    d.rectangle([800, ry + 140, 1110, ry + 210], fill=(28, 40, 80))
+    d.text((820, ry + 150), "GRAND TOTAL", font=font(20, True), fill=(255, 255, 255))
+    d.text((820, ry + 178), "$10,043.20  USD", font=font(24, True), fill=(255, 255, 255))
+
+    # --- sideways Terms & Conditions block (rotated 90) on the right edge ---
+    rotated(img, "Terms: Net 30 days. Late fee 1.5%/mo. Goods remain property of seller until paid.",
+            (1150, 360), 90, font(18), fill=(90, 90, 90))
+
+    # footer note (two-column-ish)
+    d.text((150, H - 120), "Notes: Partial back-order on item 3 may apply.", font=font(18), fill=(60, 60, 60))
+    d.text((150, H - 92), "Remittance: SWIFT MERIGB2L  /  IBAN GB22 MERI 0099 8812", font=font(18), fill=(60, 60, 60))
+
+    out_png = OUT / "complex_invoice_messy.png"
+    img.save(out_png)
+    (OUT / "complex_invoice_messy.gt.json").write_text(json.dumps(GT, indent=2))
+    print(f"✓ wrote {out_png} ({W}x{H})")
+    print(f"✓ wrote {OUT/'complex_invoice_messy.gt.json'}")
+
+
+if __name__ == "__main__":
+    main()