A fast, helpful, and open-source document parser
fix(suryaocr): run Surya 2 on GPU in a single llama.cpp container
Fu Dai6d05bd022ca4
diff --git a/ocr/suryaocr/Dockerfile b/ocr/suryaocr/Dockerfileindex 3b620ff..ddadccf 100644--- a/ocr/suryaocr/Dockerfile+++ b/ocr/suryaocr/Dockerfile@@ -1,25 +1,51 @@-FROM ghcr.io/astral-sh/uv:python3.12-trixie+# Single image: GPU-enabled llama.cpp `llama-server` + the Surya OCR API.+#+# Based on the official CUDA llama.cpp server image, which already provides a+# `qwen35`-capable `llama-server` (with libggml-cuda), the CUDA runtime, and+# Python 3.12. Surya's llamacpp backend finds `llama-server` on PATH and spawns+# it INSIDE this same container, so one image runs both the API and the model+# server. Run with `--gpus all`.+#+# (Stock Ollama's bundled llama.cpp cannot load Surya 2's custom qwen35 GGUF;+# this upstream build can.)+FROM ghcr.io/ggml-org/llama.cpp:server-cuda-# System libraries needed by torch / Pillow / Surya-RUN apt-get update && apt-get install -y \- libgomp1 \- libglib2.0-0 \- libsm6 \- libxext6 \- libxrender-dev \+# Bring uv in from its official image (no external install script needed).+COPY --from=ghcr.io/astral-sh/uv:0.11.21 /uv /usr/local/bin/uv++# Python/runtime libs for Surya, torch, and Pillow.+RUN apt-get update && apt-get install -y --no-install-recommends \+ python3-venv \libgl1 \+ libglib2.0-0 \+ libgomp1 \+ libcurl4 \&& rm -rf /var/lib/apt/lists/*-WORKDIR /app+# llama-server and its shared libraries live in /app.+ENV PATH="/app:${PATH}"+ENV LD_LIBRARY_PATH="/app:${LD_LIBRARY_PATH}"++# Surya: llama.cpp backend, all layers on GPU. Surya spawns llama-server and+# downloads the GGUF weights from Hugging Face on the first OCR request.+ENV SURYA_INFERENCE_BACKEND=llamacpp+ENV LLAMA_CPP_NGL=99+# Surya's layout step is the only request that sends a grammar; its regex+# `pattern` breaks the upstream llama.cpp grammar parser. Run layout+# unconstrained so the OCR pipeline returns results on this build.+ENV SURYA_GUIDED_LAYOUT=false+# Cache model weights (Surya torch models + the GGUF) under /models.+ENV HF_HOME=/models++WORKDIR /opt/app# Copy project metadata, lockfile, and server codeCOPY pyproject.toml uv.lock README.md server.py test_server.py ./-# Install dependencies+# Install dependencies (uses the committed lockfile)RUN uv sync --frozen-# Expose port+# The base image's entrypoint is llama-server; clear it and run the API.+ENTRYPOINT []EXPOSE 8830--# Run serverCMD ["uv", "run", "python3", "server.py"]diff --git a/ocr/suryaocr/server.py b/ocr/suryaocr/server.pyindex 7d3fda6..95defd2 100644--- a/ocr/suryaocr/server.py+++ b/ocr/suryaocr/server.py@@ -1,11 +1,28 @@import ioimport logging+import osimport reimport tracebackfrom html import unescapefrom html.parser import HTMLParserfrom typing import Any+# Surya 2 is a VLM-backed model: text recognition runs through an inference+# backend. Default to the CPU-friendly llama.cpp backend, which spawns a+# `llama-server` binary (bundled in the Docker image; install locally with+# `brew install llama.cpp` or from github.com/ggml-org/llama.cpp/releases) and+# downloads the GGUF weights on first use. Override for GPU with+# SURYA_INFERENCE_BACKEND=vllm, or attach to an already-running inference+# server with SURYA_INFERENCE_URL. These must be set before importing surya.+os.environ.setdefault("SURYA_INFERENCE_BACKEND", "llamacpp")+os.environ.setdefault("LLAMA_CPP_NGL", "0") # 0 = CPU; set 99 for full GPU offload+# Surya only sends a grammar for the layout step (LAYOUT_JSON_SCHEMA), whose+# regex `pattern` the upstream llama.cpp json-schema→GBNF converter cannot+# parse ("failed to parse grammar"). Disable guided layout so layout runs as+# free generation (Surya parses the output itself); block/full-page OCR never+# use a grammar. Required for the llama.cpp backend to return results.+os.environ.setdefault("SURYA_GUIDED_LAYOUT", "false")+import uvicornfrom fastapi import FastAPI, HTTPExceptionfrom fastapi.datastructures import UploadFile