World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Fix GBK-encoded PDF text extraction (#761)
Daniel67e20ee2f35b
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 535d2c97..22fc4f96 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -23,12 +23,18 @@ jobs:with:components: rustfmt, clippy- uses: Swatinem/rust-cache@v2+ - name: Install PDF text extraction tools+ run: sudo apt-get update && sudo apt-get install -y poppler-utils poppler-data- name: Formatrun: cargo fmt --all --check- name: Clippyrun: cargo clippy --workspace --all-targets -- -D warnings- name: Testrun: cargo test --workspace+ # 装了 Poppler 就必须真的用它测:没有它时 PDF 回退的那个测试会跳过,+ # 而一个静静跳过的测试等于没写(#248)+ env:+ UTOPIA_TEST_REQUIRE_PDFTOTEXT: "1"- name: Buildrun: cargo build --workspacediff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 5df71ea1..fd970a14 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -43,6 +43,8 @@ Both branches are protected: pull request required, CI (`backend` and `web`) musRequires Docker, Rust 1.85+, Node 20+, pnpm.+PDF text extraction also uses `pdftotext` when the Rust parser cannot read a file. Source installs need Poppler and its CJK CMap data (`poppler-utils poppler-data` on Debian); the Docker image includes both. Without them the PDF fallback test skips, and a PDF that draws text we cannot read reports a reader failure rather than claiming the file is a scan.+```bashdocker compose up -d db # Postgres with pgvectorcargo run -p utopia-server # runs migrations, :1516diff --git a/CONTRIBUTING.zh-CN.md b/CONTRIBUTING.zh-CN.mdindex 1d2258a2..f148b8bd 100644--- a/CONTRIBUTING.zh-CN.md+++ b/CONTRIBUTING.zh-CN.md@@ -43,6 +43,8 @@ git push -u origin fix/some-thing依赖:Docker、Rust 1.85+、Node 20+、pnpm。+Rust 解析器读不了的 PDF 会交给 `pdftotext`。源码部署还需要安装 Poppler 及中文 CMap 数据(Debian:`poppler-utils poppler-data`);Docker 镜像已包含两者。没装的话那个 PDF 回退测试会跳过;这时一份画了字却读不出来的 PDF 会报读取失败,而不会谎称它是扫描件。+```bashdocker compose up -d db # pgvector 版 Postgrescargo run -p utopia-server # 自动跑迁移,:1516diff --git a/Cargo.lock b/Cargo.lockindex e09af1e8..7256b74c 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5431,6 +5431,7 @@ dependencies = ["flate2","htmd","infer",+ "lopdf","oxrdf","oxrdfxml","oxttl",@@ -5439,6 +5440,7 @@ dependencies = ["pulldown-cmark","quick-xml 0.42.0","serde_json",+ "tempfile","text-splitter","thiserror 2.0.20","tiktoken-rs",diff --git a/crates/utopia-ingest/Cargo.toml b/crates/utopia-ingest/Cargo.tomlindex 12e97412..6ced9b90 100644--- a/crates/utopia-ingest/Cargo.toml+++ b/crates/utopia-ingest/Cargo.toml@@ -10,6 +10,10 @@ text-splitter.workspace = truetiktoken-rs.workspace = truepulldown-cmark.workspace = truepdf-extract.workspace = true+tempfile.workspace = true+# 只用来回答一个问题:这份 PDF 的内容流里有没有画出字形(见 parsers::draws_text)。+# pdf-extract 本来就依赖它,所以这里不添新的编译负担+lopdf = { version = "0.42", default-features = false }calamine.workspace = truehtmd.workspace = truedom_smoothie.workspace = truediff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rsindex b481a90c..8f92fe14 100644--- a/crates/utopia-ingest/src/parsers.rs+++ b/crates/utopia-ingest/src/parsers.rs@@ -3,7 +3,8 @@use anyhow::Context;use quick_xml::events::Event;use quick_xml::Reader;-use std::io::{Cursor, Read};+use std::io::{Cursor, Read, Write};+use std::process::Command;/// 文本解码:chardetng 探测编码(覆盖 GBK/GB18030/BIG5 等中文常见编码)。pub fn plain_text(bytes: &[u8]) -> String {@@ -15,8 +16,111 @@ pub fn plain_text(bytes: &[u8]) -> String {text.into_owned()}+/// 这份 PDF 的内容流里有没有画出字形:任何一个画字的算子(`Tj`、`TJ`、`'`、`"`)+/// 带着非空的串就算。+///+/// **这是「扫描件」与「我们读不了」之间唯一靠得住的分界。** 两种情况下 `pdftotext` 都是+/// 空输出、退出码 0:一张扫描的图本来就没有字可取,而一份用 `GBK-EUC-H` 这类预定义 CMap+/// 的文件是取字的人不认那张表(#739)。读不出结构就算作没有——宁可让一份文件多走一趟 OCR,+/// 也不要把它挡在库外+fn draws_text(bytes: &[u8]) -> bool {+ let Ok(doc) = lopdf::Document::load_mem(bytes) else {+ return false;+ };+ let drawn = |object: &lopdf::Object| match object {+ lopdf::Object::String(s, _) => !s.is_empty(),+ lopdf::Object::Array(items) => items+ .iter()+ .any(|i| matches!(i, lopdf::Object::String(s, _) if !s.is_empty())),+ _ => false,+ };+ let pages: Vec<lopdf::ObjectId> = doc.page_iter().collect();+ pages.into_iter().any(|id| {+ doc.get_page_content(id)+ .ok()+ .and_then(|data| lopdf::content::Content::decode(&data).ok())+ .is_some_and(|content| {+ content.operations.iter().any(|op| {+ matches!(op.operator.as_str(), "Tj" | "TJ" | "'" | "\"")+ && op.operands.iter().any(drawn)+ })+ })+ })+}++/// PDF 的文字层。取不出来时交给外面的 `pdftotext`——它是另一个进程,所以那边再崩也带不走+/// 一个工作线程。+///+/// 回退也空手而归时,[`draws_text`] 决定该说哪句话:这份文件没画过字,那是扫描件,交给+/// OCR 那条路(`NeedsReader`);画了字却一个都没取到,那是我们读不了它,带着第一个解析器+/// 的原话往外抛。报错的那句话值得较真:说成「没有文字层」会把人支去配一个 OCR 服务,而+/// 这份文件的文字层好端端地在那儿(#739:同名的 `pdftotext` 有两个实现,Xpdf 那个和缺了+/// CJK CMap 数据的 Poppler 都会静静地返回空)pub fn pdf(bytes: &[u8]) -> anyhow::Result<String> {- pdf_extract::extract_text_from_mem(bytes).context("PDF text-layer extraction failed")+ let extracted = std::panic::catch_unwind(|| pdf_extract::extract_text_from_mem(bytes));+ let original_error = match extracted {+ Ok(Ok(text)) if !text.trim().is_empty() => return Ok(text),+ Ok(Ok(_)) => None,+ Ok(Err(error)) => Some(error.to_string()),+ Err(panic) => Some(+ panic+ .downcast_ref::<String>()+ .cloned()+ .or_else(|| {+ panic+ .downcast_ref::<&str>()+ .map(|message| (*message).to_owned())+ })+ .unwrap_or_else(|| "PDF parser panicked".to_owned()),+ ),+ };++ let fallback = pdf_with_poppler(bytes);+ if let Ok(text) = &fallback {+ if !text.trim().is_empty() {+ return Ok(text.clone());+ }+ }+ // The first extractor read the file and found no text layer: the OCR path takes it from here.+ let Some(original) = original_error else {+ return Ok(String::new());+ };+ // It could not read the file. A file that draws no glyphs is a scan even so.+ if !draws_text(bytes) {+ return Ok(String::new());+ }+ match fallback {+ Ok(_) => anyhow::bail!(+ "PDF text-layer extraction failed: {original}; the fallback read no text from a file \+that draws it (is pdftotext Poppler, with its CJK CMap data?)"+ ),+ Err(error) => anyhow::bail!(+ "PDF text-layer extraction failed: {original}; Poppler fallback failed: {error:#}"+ ),+ }+}++fn pdf_with_poppler(bytes: &[u8]) -> anyhow::Result<String> {+ let mut input = tempfile::NamedTempFile::new().context("Could not create a temporary PDF")?;+ input+ .write_all(bytes)+ .context("Could not write the temporary PDF")?;+ input.flush().context("Could not flush the temporary PDF")?;++ let output = Command::new("pdftotext")+ .args(["-enc", "UTF-8", "-nopgbrk"])+ .arg(input.path())+ .arg("-")+ .output()+ .context("Could not run pdftotext")?;+ if !output.status.success() {+ anyhow::bail!(+ "pdftotext exited with {}: {}",+ output.status,+ String::from_utf8_lossy(&output.stderr).trim()+ );+ }+ String::from_utf8(output.stdout).context("pdftotext returned invalid UTF-8")}/// docx:解压 word/document.xml。正文 w:t 取字、w:p 分段;表格(w:tbl)收成网格交给diff --git a/crates/utopia-ingest/tests/pdf_gbk.rs b/crates/utopia-ingest/tests/pdf_gbk.rsnew file mode 100644index 00000000..0c734f61--- /dev/null+++ b/crates/utopia-ingest/tests/pdf_gbk.rs@@ -0,0 +1,118 @@+use lopdf::content::{Content, Operation};+use lopdf::{dictionary, Document, Object, Stream, StringFormat};++// Build a Type0 PDF with a named CMap and no ToUnicode map, using only test text.+fn gbk_pdf(text: &str) -> Vec<u8> {+ let mut document = Document::with_version("1.4");+ let pages_id = document.new_object_id();+ let descriptor_id = document.add_object(dictionary! {+ "Type" => "FontDescriptor",+ "FontName" => "STSong-Light",+ "Flags" => 4,+ "FontBBox" => vec![0.into(), (-200).into(), 1000.into(), 900.into()],+ "ItalicAngle" => 0,+ "Ascent" => 880,+ "Descent" => -120,+ "CapHeight" => 880,+ "StemV" => 80,+ });+ let cid_font_id = document.add_object(dictionary! {+ "Type" => "Font",+ "Subtype" => "CIDFontType0",+ "BaseFont" => "STSong-Light",+ "CIDSystemInfo" => dictionary! {+ "Registry" => Object::string_literal("Adobe"),+ "Ordering" => Object::string_literal("GB1"),+ "Supplement" => 2,+ },+ "FontDescriptor" => descriptor_id,+ "DW" => 1000,+ });+ let font_id = document.add_object(dictionary! {+ "Type" => "Font",+ "Subtype" => "Type0",+ "BaseFont" => "STSong-Light",+ "Encoding" => "GBK-EUC-H",+ "DescendantFonts" => vec![Object::Reference(cid_font_id)],+ });+ let resources_id = document.add_object(dictionary! {+ "Font" => dictionary! { "F1" => font_id },+ });+ let (encoded, _, had_errors) = encoding_rs::GBK.encode(text);+ assert!(!had_errors);+ let content = Content {+ operations: vec![+ Operation::new("BT", vec![]),+ Operation::new("Tf", vec!["F1".into(), 16.into()]),+ Operation::new("Td", vec![50.into(), 750.into()]),+ Operation::new(+ "Tj",+ vec![Object::String(encoded.into_owned(), StringFormat::Literal)],+ ),+ Operation::new("ET", vec![]),+ ],+ };+ let content_id = document.add_object(Stream::new(dictionary! {}, content.encode().unwrap()));+ let page_id = document.add_object(dictionary! {+ "Type" => "Page",+ "Parent" => pages_id,+ "Contents" => content_id,+ });+ document.objects.insert(+ pages_id,+ Object::Dictionary(dictionary! {+ "Type" => "Pages",+ "Kids" => vec![Object::Reference(page_id)],+ "Count" => 1,+ "Resources" => resources_id,+ "MediaBox" => vec![0.into(), 0.into(), 595.into(), 842.into()],+ }),+ );+ let catalog_id = document.add_object(dictionary! {+ "Type" => "Catalog",+ "Pages" => pages_id,+ });+ document.trailer.set("Root", catalog_id);+ let mut bytes = Vec::new();+ document.save_to(&mut bytes).unwrap();+ bytes+}++/// 回退用的 `pdftotext` 是不是 Poppler。同名的程序有两个实现——Git for Windows 带的是+/// Xpdf 4.00,它对这份文件退出码 0、输出为空——只有 Poppler 连着 `poppler-data` 的 CJK+/// CMap 表读得了 `GBK-EUC-H`。+fn fallback_is_poppler() -> bool {+ let Ok(out) = std::process::Command::new("pdftotext").arg("-v").output() else {+ return false;+ };+ let said = format!(+ "{}{}",+ String::from_utf8_lossy(&out.stdout),+ String::from_utf8_lossy(&out.stderr)+ );+ said.to_lowercase().contains("poppler")+}++#[test]+fn gbk_euc_h_pdf_extracts_chinese_text() {+ if !fallback_is_poppler() {+ // CI 装了 Poppler 并设了这个变量:那里不许跳过,否则绿色是假的+ assert!(+ std::env::var("UTOPIA_TEST_REQUIRE_PDFTOTEXT").is_err(),+ "this run requires Poppler: install poppler-utils and poppler-data"+ );+ eprintln!("skipped: pdftotext on PATH is not Poppler");+ return;+ }+ let expected = "中文测试";+ let bytes = gbk_pdf(expected);+ let parsed = utopia_ingest::parse("sample.pdf", &bytes).unwrap();+ assert!(parsed.text.contains(expected), "{}", parsed.text);+}++#[test]+fn empty_gbk_euc_h_pdf_still_requests_ocr() {+ let bytes = gbk_pdf("");+ let error = utopia_ingest::parse("empty.pdf", &bytes).unwrap_err();+ assert!(error.downcast_ref::<utopia_ingest::NeedsReader>().is_some());+}diff --git a/docker/Dockerfile b/docker/Dockerfileindex 7c5f4a71..788535f8 100644--- a/docker/Dockerfile+++ b/docker/Dockerfile@@ -24,7 +24,7 @@ RUN cargo build --release -p utopia-server# ---- 运行 ----FROM debian:bookworm-slim-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates poppler-utils poppler-data && rm -rf /var/lib/apt/lists/*WORKDIR /appCOPY --from=build /src/target/release/utopia-server /usr/local/bin/utopia-serverCOPY --from=web /src/dist /app/web-dist