Skip to content

A fast, helpful, and open-source document parser

fix(suryaocr): run Surya 2 on GPU in a single llama.cpp container

Fu Dai6d05bd022ca4
diff --git a/ocr/suryaocr/Dockerfile b/ocr/suryaocr/Dockerfile
index 3b620ff..ddadccf 100644
--- a/ocr/suryaocr/Dockerfile
+++ b/ocr/suryaocr/Dockerfile
@@ -1,25 +1,51 @@
-FROM ghcr.io/astral-sh/uv:python3.12-trixie
+# Single image: GPU-enabled llama.cpp `llama-server` + the Surya OCR API.
+#
+# Based on the official CUDA llama.cpp server image, which already provides a
+# `qwen35`-capable `llama-server` (with libggml-cuda), the CUDA runtime, and
+# Python 3.12. Surya's llamacpp backend finds `llama-server` on PATH and spawns
+# it INSIDE this same container, so one image runs both the API and the model
+# server. Run with `--gpus all`.
+#
+# (Stock Ollama's bundled llama.cpp cannot load Surya 2's custom qwen35 GGUF;
+# this upstream build can.)
+FROM ghcr.io/ggml-org/llama.cpp:server-cuda
-# System libraries needed by torch / Pillow / Surya
-RUN apt-get update && apt-get install -y \
- libgomp1 \
- libglib2.0-0 \
- libsm6 \
- libxext6 \
- libxrender-dev \
+# Bring uv in from its official image (no external install script needed).
+COPY --from=ghcr.io/astral-sh/uv:0.11.21 /uv /usr/local/bin/uv
+
+# Python/runtime libs for Surya, torch, and Pillow.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ python3-venv \
libgl1 \
+ libglib2.0-0 \
+ libgomp1 \
+ libcurl4 \
&& rm -rf /var/lib/apt/lists/*
-WORKDIR /app
+# llama-server and its shared libraries live in /app.
+ENV PATH="/app:${PATH}"
+ENV LD_LIBRARY_PATH="/app:${LD_LIBRARY_PATH}"
+
+# Surya: llama.cpp backend, all layers on GPU. Surya spawns llama-server and
+# downloads the GGUF weights from Hugging Face on the first OCR request.
+ENV SURYA_INFERENCE_BACKEND=llamacpp
+ENV LLAMA_CPP_NGL=99
+# Surya's layout step is the only request that sends a grammar; its regex
+# `pattern` breaks the upstream llama.cpp grammar parser. Run layout
+# unconstrained so the OCR pipeline returns results on this build.
+ENV SURYA_GUIDED_LAYOUT=false
+# Cache model weights (Surya torch models + the GGUF) under /models.
+ENV HF_HOME=/models
+
+WORKDIR /opt/app
# Copy project metadata, lockfile, and server code
COPY pyproject.toml uv.lock README.md server.py test_server.py ./
-# Install dependencies
+# Install dependencies (uses the committed lockfile)
RUN uv sync --frozen
-# Expose port
+# The base image's entrypoint is llama-server; clear it and run the API.
+ENTRYPOINT []
EXPOSE 8830
-
-# Run server
CMD ["uv", "run", "python3", "server.py"]
diff --git a/ocr/suryaocr/server.py b/ocr/suryaocr/server.py
index 7d3fda6..95defd2 100644
--- a/ocr/suryaocr/server.py
+++ b/ocr/suryaocr/server.py
@@ -1,11 +1,28 @@
import io
import logging
+import os
import re
import traceback
from html import unescape
from html.parser import HTMLParser
from typing import Any
+# Surya 2 is a VLM-backed model: text recognition runs through an inference
+# backend. Default to the CPU-friendly llama.cpp backend, which spawns a
+# `llama-server` binary (bundled in the Docker image; install locally with
+# `brew install llama.cpp` or from github.com/ggml-org/llama.cpp/releases) and
+# downloads the GGUF weights on first use. Override for GPU with
+# SURYA_INFERENCE_BACKEND=vllm, or attach to an already-running inference
+# server with SURYA_INFERENCE_URL. These must be set before importing surya.
+os.environ.setdefault("SURYA_INFERENCE_BACKEND", "llamacpp")
+os.environ.setdefault("LLAMA_CPP_NGL", "0") # 0 = CPU; set 99 for full GPU offload
+# Surya only sends a grammar for the layout step (LAYOUT_JSON_SCHEMA), whose
+# regex `pattern` the upstream llama.cpp json-schema→GBNF converter cannot
+# parse ("failed to parse grammar"). Disable guided layout so layout runs as
+# free generation (Surya parses the output itself); block/full-page OCR never
+# use a grammar. Required for the llama.cpp backend to return results.
+os.environ.setdefault("SURYA_GUIDED_LAYOUT", "false")
+
import uvicorn
from fastapi import FastAPI, HTTPException
from fastapi.datastructures import UploadFile