Skip to content

World's first open-source enterprise world model.

Fix GBK-encoded PDF text extraction (#761)

Daniel67e20ee2f35b
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 535d2c97..22fc4f96 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -23,12 +23,18 @@ jobs:
with:
components: rustfmt, clippy
- uses: Swatinem/rust-cache@v2
+ - name: Install PDF text extraction tools
+ run: sudo apt-get update && sudo apt-get install -y poppler-utils poppler-data
- name: Format
run: cargo fmt --all --check
- name: Clippy
run: cargo clippy --workspace --all-targets -- -D warnings
- name: Test
run: cargo test --workspace
+ # 装了 Poppler 就必须真的用它测:没有它时 PDF 回退的那个测试会跳过,
+ # 而一个静静跳过的测试等于没写(#248)
+ env:
+ UTOPIA_TEST_REQUIRE_PDFTOTEXT: "1"
- name: Build
run: cargo build --workspace
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 5df71ea1..fd970a14 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -43,6 +43,8 @@ Both branches are protected: pull request required, CI (`backend` and `web`) mus
Requires Docker, Rust 1.85+, Node 20+, pnpm.
+PDF text extraction also uses `pdftotext` when the Rust parser cannot read a file. Source installs need Poppler and its CJK CMap data (`poppler-utils poppler-data` on Debian); the Docker image includes both. Without them the PDF fallback test skips, and a PDF that draws text we cannot read reports a reader failure rather than claiming the file is a scan.
+
```bash
docker compose up -d db # Postgres with pgvector
cargo run -p utopia-server # runs migrations, :1516
diff --git a/CONTRIBUTING.zh-CN.md b/CONTRIBUTING.zh-CN.md
index 1d2258a2..f148b8bd 100644
--- a/CONTRIBUTING.zh-CN.md
+++ b/CONTRIBUTING.zh-CN.md
@@ -43,6 +43,8 @@ git push -u origin fix/some-thing
依赖:Docker、Rust 1.85+、Node 20+、pnpm。
+Rust 解析器读不了的 PDF 会交给 `pdftotext`。源码部署还需要安装 Poppler 及中文 CMap 数据(Debian:`poppler-utils poppler-data`);Docker 镜像已包含两者。没装的话那个 PDF 回退测试会跳过;这时一份画了字却读不出来的 PDF 会报读取失败,而不会谎称它是扫描件。
+
```bash
docker compose up -d db # pgvector 版 Postgres
cargo run -p utopia-server # 自动跑迁移,:1516
diff --git a/Cargo.lock b/Cargo.lock
index e09af1e8..7256b74c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5431,6 +5431,7 @@ dependencies = [
"flate2",
"htmd",
"infer",
+ "lopdf",
"oxrdf",
"oxrdfxml",
"oxttl",
@@ -5439,6 +5440,7 @@ dependencies = [
"pulldown-cmark",
"quick-xml 0.42.0",
"serde_json",
+ "tempfile",
"text-splitter",
"thiserror 2.0.20",
"tiktoken-rs",
diff --git a/crates/utopia-ingest/Cargo.toml b/crates/utopia-ingest/Cargo.toml
index 12e97412..6ced9b90 100644
--- a/crates/utopia-ingest/Cargo.toml
+++ b/crates/utopia-ingest/Cargo.toml
@@ -10,6 +10,10 @@ text-splitter.workspace = true
tiktoken-rs.workspace = true
pulldown-cmark.workspace = true
pdf-extract.workspace = true
+tempfile.workspace = true
+# 只用来回答一个问题:这份 PDF 的内容流里有没有画出字形(见 parsers::draws_text)。
+# pdf-extract 本来就依赖它,所以这里不添新的编译负担
+lopdf = { version = "0.42", default-features = false }
calamine.workspace = true
htmd.workspace = true
dom_smoothie.workspace = true
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rs
index b481a90c..8f92fe14 100644
--- a/crates/utopia-ingest/src/parsers.rs
+++ b/crates/utopia-ingest/src/parsers.rs
@@ -3,7 +3,8 @@
use anyhow::Context;
use quick_xml::events::Event;
use quick_xml::Reader;
-use std::io::{Cursor, Read};
+use std::io::{Cursor, Read, Write};
+use std::process::Command;
/// 文本解码:chardetng 探测编码(覆盖 GBK/GB18030/BIG5 等中文常见编码)。
pub fn plain_text(bytes: &[u8]) -> String {
@@ -15,8 +16,111 @@ pub fn plain_text(bytes: &[u8]) -> String {
text.into_owned()
}
+/// 这份 PDF 的内容流里有没有画出字形:任何一个画字的算子(`Tj`、`TJ`、`'`、`"`)
+/// 带着非空的串就算。
+///
+/// **这是「扫描件」与「我们读不了」之间唯一靠得住的分界。** 两种情况下 `pdftotext` 都是
+/// 空输出、退出码 0:一张扫描的图本来就没有字可取,而一份用 `GBK-EUC-H` 这类预定义 CMap
+/// 的文件是取字的人不认那张表(#739)。读不出结构就算作没有——宁可让一份文件多走一趟 OCR,
+/// 也不要把它挡在库外
+fn draws_text(bytes: &[u8]) -> bool {
+ let Ok(doc) = lopdf::Document::load_mem(bytes) else {
+ return false;
+ };
+ let drawn = |object: &lopdf::Object| match object {
+ lopdf::Object::String(s, _) => !s.is_empty(),
+ lopdf::Object::Array(items) => items
+ .iter()
+ .any(|i| matches!(i, lopdf::Object::String(s, _) if !s.is_empty())),
+ _ => false,
+ };
+ let pages: Vec<lopdf::ObjectId> = doc.page_iter().collect();
+ pages.into_iter().any(|id| {
+ doc.get_page_content(id)
+ .ok()
+ .and_then(|data| lopdf::content::Content::decode(&data).ok())
+ .is_some_and(|content| {
+ content.operations.iter().any(|op| {
+ matches!(op.operator.as_str(), "Tj" | "TJ" | "'" | "\"")
+ && op.operands.iter().any(drawn)
+ })
+ })
+ })
+}
+
+/// PDF 的文字层。取不出来时交给外面的 `pdftotext`——它是另一个进程,所以那边再崩也带不走
+/// 一个工作线程。
+///
+/// 回退也空手而归时,[`draws_text`] 决定该说哪句话:这份文件没画过字,那是扫描件,交给
+/// OCR 那条路(`NeedsReader`);画了字却一个都没取到,那是我们读不了它,带着第一个解析器
+/// 的原话往外抛。报错的那句话值得较真:说成「没有文字层」会把人支去配一个 OCR 服务,而
+/// 这份文件的文字层好端端地在那儿(#739:同名的 `pdftotext` 有两个实现,Xpdf 那个和缺了
+/// CJK CMap 数据的 Poppler 都会静静地返回空)
pub fn pdf(bytes: &[u8]) -> anyhow::Result<String> {
- pdf_extract::extract_text_from_mem(bytes).context("PDF text-layer extraction failed")
+ let extracted = std::panic::catch_unwind(|| pdf_extract::extract_text_from_mem(bytes));
+ let original_error = match extracted {
+ Ok(Ok(text)) if !text.trim().is_empty() => return Ok(text),
+ Ok(Ok(_)) => None,
+ Ok(Err(error)) => Some(error.to_string()),
+ Err(panic) => Some(
+ panic
+ .downcast_ref::<String>()
+ .cloned()
+ .or_else(|| {
+ panic
+ .downcast_ref::<&str>()
+ .map(|message| (*message).to_owned())
+ })
+ .unwrap_or_else(|| "PDF parser panicked".to_owned()),
+ ),
+ };
+
+ let fallback = pdf_with_poppler(bytes);
+ if let Ok(text) = &fallback {
+ if !text.trim().is_empty() {
+ return Ok(text.clone());
+ }
+ }
+ // The first extractor read the file and found no text layer: the OCR path takes it from here.
+ let Some(original) = original_error else {
+ return Ok(String::new());
+ };
+ // It could not read the file. A file that draws no glyphs is a scan even so.
+ if !draws_text(bytes) {
+ return Ok(String::new());
+ }
+ match fallback {
+ Ok(_) => anyhow::bail!(
+ "PDF text-layer extraction failed: {original}; the fallback read no text from a file \
+that draws it (is pdftotext Poppler, with its CJK CMap data?)"
+ ),
+ Err(error) => anyhow::bail!(
+ "PDF text-layer extraction failed: {original}; Poppler fallback failed: {error:#}"
+ ),
+ }
+}
+
+fn pdf_with_poppler(bytes: &[u8]) -> anyhow::Result<String> {
+ let mut input = tempfile::NamedTempFile::new().context("Could not create a temporary PDF")?;
+ input
+ .write_all(bytes)
+ .context("Could not write the temporary PDF")?;
+ input.flush().context("Could not flush the temporary PDF")?;
+
+ let output = Command::new("pdftotext")
+ .args(["-enc", "UTF-8", "-nopgbrk"])
+ .arg(input.path())
+ .arg("-")
+ .output()
+ .context("Could not run pdftotext")?;
+ if !output.status.success() {
+ anyhow::bail!(
+ "pdftotext exited with {}: {}",
+ output.status,
+ String::from_utf8_lossy(&output.stderr).trim()
+ );
+ }
+ String::from_utf8(output.stdout).context("pdftotext returned invalid UTF-8")
}
/// docx:解压 word/document.xml。正文 w:t 取字、w:p 分段;表格(w:tbl)收成网格交给
diff --git a/crates/utopia-ingest/tests/pdf_gbk.rs b/crates/utopia-ingest/tests/pdf_gbk.rs
new file mode 100644
index 00000000..0c734f61
--- /dev/null
+++ b/crates/utopia-ingest/tests/pdf_gbk.rs
@@ -0,0 +1,118 @@
+use lopdf::content::{Content, Operation};
+use lopdf::{dictionary, Document, Object, Stream, StringFormat};
+
+// Build a Type0 PDF with a named CMap and no ToUnicode map, using only test text.
+fn gbk_pdf(text: &str) -> Vec<u8> {
+ let mut document = Document::with_version("1.4");
+ let pages_id = document.new_object_id();
+ let descriptor_id = document.add_object(dictionary! {
+ "Type" => "FontDescriptor",
+ "FontName" => "STSong-Light",
+ "Flags" => 4,
+ "FontBBox" => vec![0.into(), (-200).into(), 1000.into(), 900.into()],
+ "ItalicAngle" => 0,
+ "Ascent" => 880,
+ "Descent" => -120,
+ "CapHeight" => 880,
+ "StemV" => 80,
+ });
+ let cid_font_id = document.add_object(dictionary! {
+ "Type" => "Font",
+ "Subtype" => "CIDFontType0",
+ "BaseFont" => "STSong-Light",
+ "CIDSystemInfo" => dictionary! {
+ "Registry" => Object::string_literal("Adobe"),
+ "Ordering" => Object::string_literal("GB1"),
+ "Supplement" => 2,
+ },
+ "FontDescriptor" => descriptor_id,
+ "DW" => 1000,
+ });
+ let font_id = document.add_object(dictionary! {
+ "Type" => "Font",
+ "Subtype" => "Type0",
+ "BaseFont" => "STSong-Light",
+ "Encoding" => "GBK-EUC-H",
+ "DescendantFonts" => vec![Object::Reference(cid_font_id)],
+ });
+ let resources_id = document.add_object(dictionary! {
+ "Font" => dictionary! { "F1" => font_id },
+ });
+ let (encoded, _, had_errors) = encoding_rs::GBK.encode(text);
+ assert!(!had_errors);
+ let content = Content {
+ operations: vec![
+ Operation::new("BT", vec![]),
+ Operation::new("Tf", vec!["F1".into(), 16.into()]),
+ Operation::new("Td", vec![50.into(), 750.into()]),
+ Operation::new(
+ "Tj",
+ vec![Object::String(encoded.into_owned(), StringFormat::Literal)],
+ ),
+ Operation::new("ET", vec![]),
+ ],
+ };
+ let content_id = document.add_object(Stream::new(dictionary! {}, content.encode().unwrap()));
+ let page_id = document.add_object(dictionary! {
+ "Type" => "Page",
+ "Parent" => pages_id,
+ "Contents" => content_id,
+ });
+ document.objects.insert(
+ pages_id,
+ Object::Dictionary(dictionary! {
+ "Type" => "Pages",
+ "Kids" => vec![Object::Reference(page_id)],
+ "Count" => 1,
+ "Resources" => resources_id,
+ "MediaBox" => vec![0.into(), 0.into(), 595.into(), 842.into()],
+ }),
+ );
+ let catalog_id = document.add_object(dictionary! {
+ "Type" => "Catalog",
+ "Pages" => pages_id,
+ });
+ document.trailer.set("Root", catalog_id);
+ let mut bytes = Vec::new();
+ document.save_to(&mut bytes).unwrap();
+ bytes
+}
+
+/// 回退用的 `pdftotext` 是不是 Poppler。同名的程序有两个实现——Git for Windows 带的是
+/// Xpdf 4.00,它对这份文件退出码 0、输出为空——只有 Poppler 连着 `poppler-data` 的 CJK
+/// CMap 表读得了 `GBK-EUC-H`。
+fn fallback_is_poppler() -> bool {
+ let Ok(out) = std::process::Command::new("pdftotext").arg("-v").output() else {
+ return false;
+ };
+ let said = format!(
+ "{}{}",
+ String::from_utf8_lossy(&out.stdout),
+ String::from_utf8_lossy(&out.stderr)
+ );
+ said.to_lowercase().contains("poppler")
+}
+
+#[test]
+fn gbk_euc_h_pdf_extracts_chinese_text() {
+ if !fallback_is_poppler() {
+ // CI 装了 Poppler 并设了这个变量:那里不许跳过,否则绿色是假的
+ assert!(
+ std::env::var("UTOPIA_TEST_REQUIRE_PDFTOTEXT").is_err(),
+ "this run requires Poppler: install poppler-utils and poppler-data"
+ );
+ eprintln!("skipped: pdftotext on PATH is not Poppler");
+ return;
+ }
+ let expected = "中文测试";
+ let bytes = gbk_pdf(expected);
+ let parsed = utopia_ingest::parse("sample.pdf", &bytes).unwrap();
+ assert!(parsed.text.contains(expected), "{}", parsed.text);
+}
+
+#[test]
+fn empty_gbk_euc_h_pdf_still_requests_ocr() {
+ let bytes = gbk_pdf("");
+ let error = utopia_ingest::parse("empty.pdf", &bytes).unwrap_err();
+ assert!(error.downcast_ref::<utopia_ingest::NeedsReader>().is_some());
+}
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 7c5f4a71..788535f8 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -24,7 +24,7 @@ RUN cargo build --release -p utopia-server
# ---- 运行 ----
FROM debian:bookworm-slim
-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates poppler-utils poppler-data && rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY --from=build /src/target/release/utopia-server /usr/local/bin/utopia-server
COPY --from=web /src/dist /app/web-dist