RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
ci : add ttf font to ragflow_deps image (#21461)
maoyifeng1731e54b3aa0
diff --git a/Dockerfile b/Dockerfileindex c4820f08f2..0d15c07785 100644--- a/Dockerfile+++ b/Dockerfile@@ -85,6 +85,14 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.coENV DEBIAN_FRONTEND=noninteractive+# The Noto Sans CJK SC font is now pre-baked into infiniflow/ragflow_deps:latest+# (downloaded + verified by deps/download_deps.py) instead of being curl'd from+# GitHub at main-image build time. gopdf reads it at runtime from+# /usr/local/share/fonts/truetype/noto/NotoSansCJKsc-VF.ttf (pdf_writer.go).+RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/NotoSansCJKsc-VF.ttf,target=/tmp/NotoSansCJKsc-VF.ttf \+ mkdir -p /usr/local/share/fonts/truetype/noto && \+ cp /tmp/NotoSansCJKsc-VF.ttf /usr/local/share/fonts/truetype/noto/NotoSansCJKsc-VF.ttf+# Setup aptRUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \if [ "$NEED_MIRROR" == "1" ]; then \@@ -99,10 +107,6 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \chmod 1777 /tmp && \apt update && \apt --no-install-recommends install -y ca-certificates curl vim unzip iproute2 fonts-dejavu fontconfig && \- mkdir -p /usr/local/share/fonts/truetype/noto && \- curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL \- -o /usr/local/share/fonts/truetype/noto/NotoSansCJKsc-VF.ttf \- https://github.com/notofonts/noto-cjk/raw/main/Sans/Variable/TTF/NotoSansCJKsc-VF.ttf && \fc-cache -f && \rm -rf /var/lib/apt/lists/* && apt-get cleandiff --git a/deps/Dockerfile b/deps/Dockerfileindex e2ed64e5b7..e45e64158e 100644--- a/deps/Dockerfile+++ b/deps/Dockerfile@@ -2,6 +2,6 @@# docker build -f Dockerfile -t infiniflow/ragflow_deps:latest .FROM scratch-COPY cl100k_base.tiktoken stagehand-server-v3-linux-x64 stagehand-server-v3-linux-arm64 /+COPY cl100k_base.tiktoken stagehand-server-v3-linux-x64 stagehand-server-v3-linux-arm64 NotoSansCJKsc-VF.ttf /COPY huggingface.co /huggingface.codiff --git a/deps/download_deps.py b/deps/download_deps.pyindex 69f3c2c97f..2cfbe621de 100755--- a/deps/download_deps.py+++ b/deps/download_deps.py@@ -45,6 +45,7 @@# tokenizer cannot be ingested (see internal/tokenizer/embedding_token_limits.md).import argparse+import hashlibimport osimport platformimport re@@ -71,6 +72,16 @@ REPO_ROOT = os.path.dirname(SCRIPT_DIR)STAGEHAND_SERVER_VERSION = "v3.7.2"+# Noto Sans CJK SC variable TTF, rendered into PDFs by the Go backend+# (internal/agent/component/io/pdf_writer.go). When non-empty, it is the pinned+# SHA-256 every downloaded/kept font must match. When empty, the hash recorded in+# deps/NotoSansCJKsc-VF.ttf.sha256 is the baseline: a present non-empty sidecar is+# verified (drift rejected), while a missing/empty sidecar is bootstrapped by+# recording the downloaded hash so the next run detects changes. Both first and+# later runs therefore run the download directly without manual setup.+NOTO_CJK_FONT_SHA256 = ""++def gh_proxy() -> str:"""GitHub release proxy prefix (gh-proxy.com behind the GFW), else empty."""return os.environ.get("GH_PROXY") or ""@@ -101,6 +112,13 @@ def stagehand_url(arch: str) -> str:return f"{gh_proxy()}https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/{STAGEHAND_SERVER_VERSION}/stagehand-server-v3-linux-{arch}"+def noto_cjk_font_url() -> str:+ """Noto Sans CJK SC variable TTF, rendered into PDFs by the Go backend+ (internal/agent/component/io/pdf_writer.go). It is a GitHub raw URL, so the+ gh-proxy prefix applies behind the GFW."""+ return f"{gh_proxy()}https://github.com/notofonts/noto-cjk/raw/main/Sans/Variable/TTF/NotoSansCJKsc-VF.ttf"++# Mirrors internal/common.DeepDocModelFiles (Go in-process DeepDoc backend).# These are the ONLY weights the Go backend loads; the full InfiniFlow/deepdoc# repo also ships .onnx (Python-only), which this Go-only script deliberately@@ -167,6 +185,59 @@ def _valid_stagehand_binary(path):return file.read(4) == b"\x7fELF"+def _valid_font(path):+ """True when PATH is a TrueType font with the sfnt signature b"\\x00\\x01\\x00\\x00".++ gopdf's AddTTFFont accepts ONLY this signature; OpenType (OTTO), TrueType+ collections (ttcf), and WOFF/WOFF2 are rejected, so we must not let them+ through. A raw GitHub non-font response (e.g. a 404 HTML page) also fails this+ check, so we never copy a non-font into the dependency image."""+ if not os.path.isfile(path) or os.path.getsize(path) == 0:+ return False+ with open(path, "rb") as file:+ return file.read(4) == b"\x00\x01\x00\x00"+++def _sha256_of(path):+ """SHA-256 of PATH, streamed in chunks so large files don't blow memory."""+ h = hashlib.sha256()+ with open(path, "rb") as file:+ for chunk in iter(lambda: file.read(1 << 20), b""):+ h.update(chunk)+ return h.hexdigest()+++def _verify_font_checksum(font):+ """Validate FONT's SHA-256, removing it and failing on any mismatch.++ Precedence: a non-empty NOTO_CJK_FONT_SHA256 pin wins and is verified strictly.+ Otherwise the hash recorded in deps/NotoSansCJKsc-VF.ttf.sha256 is the baseline:+ a present non-empty sidecar is verified (drift is rejected), while a missing or+ empty sidecar is bootstrapped by recording the downloaded hash so subsequent runs+ detect changes. The caller runs the magic-number (_valid_font) check beforehand."""+ actual = _sha256_of(font)+ sidecar = os.path.join(SCRIPT_DIR, "NotoSansCJKsc-VF.ttf.sha256")+ if NOTO_CJK_FONT_SHA256:+ if actual != NOTO_CJK_FONT_SHA256:+ os.unlink(font)+ raise RuntimeError(f"NotoSansCJKsc-VF.ttf checksum mismatch (got {actual}, pinned {NOTO_CJK_FONT_SHA256})")+ return+ # No pinned constant: verify against the sidecar baseline, or bootstrap it.+ if os.path.isfile(sidecar):+ with open(sidecar) as file:+ content = file.read().strip()+ expected = content.split()[0] if content else ""+ if expected:+ if actual != expected:+ os.unlink(font)+ raise RuntimeError(f"NotoSansCJKsc-VF.ttf checksum changed vs recorded {expected} (got {actual})")+ return+ # Empty sidecar: fall through to (re)record the baseline below.+ with open(sidecar, "w") as file:+ file.write(actual + "\n")+ print(f" >>> PIN THIS HASH in download_deps.py NOTO_CJK_FONT_SHA256: {actual}")++def load_tokenizer_assets(assets=None):"""Validate and return the inlined tokenizer asset list as (repo, file, kind)tuples. When ASSETS is None, uses the module-level TOKENIZER_ASSETS.@@ -361,6 +432,29 @@ def main():raise RuntimeError(f"Invalid downloaded dependency: {filename}")_install_stagehand()+ # ── Noto Sans CJK SC font (rendered into PDFs by the Go backend) ─────────+ # Verified by a font magic-number check (only the sfnt signature+ # b"\x00\x01\x00\x00" that gopdf's AddTTFFont accepts) plus a SHA-256 check+ # (_verify_font_checksum). A present file is NEVER reused unverified; a bad+ # download is removed and the build fails loudly instead of shipping a PDF+ # backend that silently loses CJK rendering.+ print("== NotoSansCJKsc-VF.ttf ==")+ font = os.path.join(SCRIPT_DIR, "NotoSansCJKsc-VF.ttf")+ if os.path.isfile(font) and os.path.getsize(font) > 0:+ if not _valid_font(font):+ print(" WARNING: existing NotoSansCJKsc-VF.ttf is not a valid font; re-downloading", file=sys.stderr)+ os.unlink(font)+ else:+ _verify_font_checksum(font)+ if not (os.path.isfile(font) and os.path.getsize(font) > 0):+ print(f"Downloading NotoSansCJKsc-VF.ttf from {noto_cjk_font_url()}...")+ download_with_progress(noto_cjk_font_url(), font)+ if not _valid_font(font):+ os.unlink(font)+ raise RuntimeError("Downloaded NotoSansCJKsc-VF.ttf is not a valid font; refusing to keep it")+ _verify_font_checksum(font)+ print(" ✓ NotoSansCJKsc-VF.ttf ready")+# Download the Go DeepDoc `.ort` weights so this script is a one-stop for a# Go dev's runtime resources: models (below) from a single invocation.print("== DeepDoc models ==")