Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

fix: switch PDF extraction from pdf-extract to pdfium-render

nash_su1ba8fc17a92f
diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock
index b724f252..31f5be65 100644
--- a/src-tauri/Cargo.lock
+++ b/src-tauri/Cargo.lock
@@ -8,15 +8,6 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
-[[package]]
-name = "adobe-cmap-parser"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae8abfa9a4688de8fc9f42b3f013b6fffec18ed8a554f5f113577e0b9b3212a3"
-dependencies = [
- "pom",
-]
-
[[package]]
name = "aes"
version = "0.8.4"
@@ -660,15 +651,6 @@ dependencies = [
"generic-array",
]
-[[package]]
-name = "block-padding"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
-dependencies = [
- "generic-array",
-]
-
[[package]]
name = "block2"
version = "0.6.2"
@@ -743,12 +725,6 @@ version = "3.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
-[[package]]
-name = "bytecount"
-version = "0.6.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
-
[[package]]
name = "bytemuck"
version = "1.25.0"
@@ -879,15 +855,6 @@ dependencies = [
"toml 0.9.12+spec-1.1.0",
]
-[[package]]
-name = "cbc"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
-dependencies = [
- "cipher",
-]
-
[[package]]
name = "cc"
version = "1.2.59"
@@ -923,12 +890,6 @@ dependencies = [
"uuid",
]
-[[package]]
-name = "cff-parser"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31f5b6e9141c036f3ff4ce7b2f7e432b0f00dee416ddcd4f17741d189ddc2e9d"
-
[[package]]
name = "cfg-expr"
version = "0.15.8"
@@ -1052,6 +1013,26 @@ dependencies = [
"crossbeam-utils",
]
+[[package]]
+name = "console_error_panic_hook"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "console_log"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be8aed40e4edbf4d3b4431ab260b63fdc40f5780a4766824329ea0f1eefe3c0f"
+dependencies = [
+ "log",
+ "web-sys",
+]
+
[[package]]
name = "const-random"
version = "0.1.18"
@@ -2211,15 +2192,6 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
-[[package]]
-name = "ecb"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7"
-dependencies = [
- "cipher",
-]
-
[[package]]
name = "either"
version = "1.15.0"
@@ -2315,15 +2287,6 @@ version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca81e6b4777c89fd810c25a4be2b1bd93ea034fbe58e6a75216a34c6b82c539b"
-[[package]]
-name = "euclid"
-version = "0.20.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bb7ef65b3777a325d1eeefefab5b6d4959da54747e33bd6258e789640f307ad"
-dependencies = [
- "num-traits",
-]
-
[[package]]
name = "event-listener"
version = "5.4.1"
@@ -3435,7 +3398,6 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
dependencies = [
- "block-padding",
"generic-array",
]
@@ -4408,7 +4370,7 @@ dependencies = [
"docx-rs",
"futures",
"lancedb",
- "pdf-extract",
+ "pdfium-render",
"serde",
"serde_json",
"tauri",
@@ -4449,34 +4411,6 @@ dependencies = [
"tracing-subscriber",
]
-[[package]]
-name = "lopdf"
-version = "0.38.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7184fdea2bc3cd272a1acec4030c321a8f9875e877b3f92a53f2f6033fdc289"
-dependencies = [
- "aes",
- "bitflags 2.11.0",
- "cbc",
- "ecb",
- "encoding_rs",
- "flate2",
- "getrandom 0.3.4",
- "indexmap 2.13.1",
- "itoa",
- "log",
- "md-5",
- "nom 8.0.0",
- "nom_locate",
- "rand 0.9.2",
- "rangemap",
- "sha2",
- "stringprep",
- "thiserror 2.0.18",
- "ttf-parser",
- "weezl",
-]
-
[[package]]
name = "lru"
version = "0.12.5"
@@ -4617,6 +4551,12 @@ dependencies = [
"thread-tree",
]
+[[package]]
+name = "maybe-owned"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4"
+
[[package]]
name = "md-5"
version = "0.10.6"
@@ -4848,17 +4788,6 @@ dependencies = [
"memchr",
]
-[[package]]
-name = "nom_locate"
-version = "5.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d"
-dependencies = [
- "bytecount",
- "memchr",
- "nom 8.0.0",
-]
-
[[package]]
name = "nu-ansi-term"
version = "0.50.3"
@@ -5245,20 +5174,29 @@ dependencies = [
]
[[package]]
-name = "pdf-extract"
-version = "0.10.0"
+name = "pdfium-render"
+version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e28ba1758a3d3f361459645780e09570b573fc3c82637449e9963174c813a98"
+checksum = "671afc8522e8f36c5854a8231bdba50c4144be0138521329629b8f102e55f65a"
dependencies = [
- "adobe-cmap-parser",
- "cff-parser",
- "encoding_rs",
- "euclid",
+ "bitflags 2.11.0",
+ "bytemuck",
+ "bytes",
+ "chrono",
+ "console_error_panic_hook",
+ "console_log",
+ "image",
+ "itertools 0.14.0",
+ "js-sys",
+ "libloading",
"log",
- "lopdf",
- "postscript",
- "type1-encoding-parser",
- "unicode-normalization",
+ "maybe-owned",
+ "once_cell",
+ "utf16string",
+ "vecmath",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
]
[[package]]
@@ -5527,6 +5465,12 @@ dependencies = [
"futures-io",
]
+[[package]]
+name = "piston-float"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad78bf43dcf80e8f950c92b84f938a0fc7590b7f6866fbcbeca781609c115590"
+
[[package]]
name = "pkg-config"
version = "0.3.32"
@@ -5586,12 +5530,6 @@ dependencies = [
"windows-sys 0.61.2",
]
-[[package]]
-name = "pom"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6"
-
[[package]]
name = "portable-atomic"
version = "1.13.1"
@@ -5607,12 +5545,6 @@ dependencies = [
"portable-atomic",
]
-[[package]]
-name = "postscript"
-version = "0.14.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306"
-
[[package]]
name = "potential_utf"
version = "0.1.5"
@@ -7007,17 +6939,6 @@ dependencies = [
"quote",
]
-[[package]]
-name = "stringprep"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
-dependencies = [
- "unicode-bidi",
- "unicode-normalization",
- "unicode-properties",
-]
-
[[package]]
name = "strsim"
version = "0.11.1"
@@ -8132,12 +8053,6 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
-[[package]]
-name = "ttf-parser"
-version = "0.25.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31"
-
[[package]]
name = "twox-hash"
version = "2.1.2"
@@ -8147,15 +8062,6 @@ dependencies = [
"rand 0.9.2",
]
-[[package]]
-name = "type1-encoding-parser"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa10c302f5a53b7ad27fd42a3996e23d096ba39b5b8dd6d9e683a05b01bee749"
-dependencies = [
- "pom",
-]
-
[[package]]
name = "typed-path"
version = "0.12.3"
@@ -8232,33 +8138,12 @@ version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
-[[package]]
-name = "unicode-bidi"
-version = "0.3.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
-
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
-[[package]]
-name = "unicode-normalization"
-version = "0.1.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
-dependencies = [
- "tinyvec",
-]
-
-[[package]]
-name = "unicode-properties"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
-
[[package]]
name = "unicode-segmentation"
version = "1.13.2"
@@ -8314,6 +8199,15 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+[[package]]
+name = "utf16string"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b62a1e85e12d5d712bf47a85f426b73d303e2d00a90de5f3004df3596e9d216"
+dependencies = [
+ "byteorder",
+]
+
[[package]]
name = "utf8-ranges"
version = "1.0.5"
@@ -8344,6 +8238,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
+[[package]]
+name = "vecmath"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "956ae1e0d85bca567dee1dcf87fb1ca2e792792f66f87dced8381f99cd91156a"
+dependencies = [
+ "piston-float",
+]
+
[[package]]
name = "version-compare"
version = "0.2.1"
diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml
index ea691269..44399d6b 100644
--- a/src-tauri/Cargo.toml
+++ b/src-tauri/Cargo.toml
@@ -25,7 +25,7 @@ serde = { version = "1", features = ["derive"] }
serde_json = "1"
chrono = { version = "0.4", features = ["clock"] }
tauri-plugin-dialog = "2.7.0"
-pdf-extract = "0.10.0"
+pdfium-render = "0.9"
tauri-plugin-store = "2.4.2"
tiny_http = "0.12"
zip = "2"
diff --git a/src-tauri/src/commands/fs.rs b/src-tauri/src/commands/fs.rs
index e8dccdcf..02619c41 100644
--- a/src-tauri/src/commands/fs.rs
+++ b/src-tauri/src/commands/fs.rs
@@ -115,11 +115,60 @@ fn write_cache(original: &Path, text: &str) -> Result<(), String> {
.map_err(|e| format!("Failed to write cache: {}", e))
}
+/// Global PDFium instance — the library prefers a single binding shared
+/// across threads over repeatedly binding/unbinding.
+static PDFIUM: std::sync::OnceLock<Result<pdfium_render::prelude::Pdfium, String>> =
+ std::sync::OnceLock::new();
+
+fn pdfium() -> Result<&'static pdfium_render::prelude::Pdfium, String> {
+ PDFIUM
+ .get_or_init(|| {
+ use pdfium_render::prelude::*;
+ // Resolution order:
+ // 1. PDFIUM_DYNAMIC_LIB_PATH env var (absolute path to dylib)
+ // 2. ./libpdfium.<ext> next to the binary
+ // 3. The OS dynamic loader's default search path
+ let bindings = if let Ok(path) = std::env::var("PDFIUM_DYNAMIC_LIB_PATH") {
+ Pdfium::bind_to_library(path)
+ } else {
+ Pdfium::bind_to_library(
+ Pdfium::pdfium_platform_library_name_at_path("./"),
+ )
+ .or_else(|_| Pdfium::bind_to_system_library())
+ };
+ bindings
+ .map(Pdfium::new)
+ .map_err(|e| format!("Failed to bind to Pdfium library: {e}"))
+ })
+ .as_ref()
+ .map_err(|e| e.clone())
+}
+
fn extract_pdf_text(path: &str) -> Result<String, String> {
- let bytes =
- fs::read(path).map_err(|e| format!("Failed to read PDF '{}': {}", path, e))?;
- pdf_extract::extract_text_from_mem(&bytes)
- .map_err(|e| format!("Failed to extract text from PDF '{}': {}", path, e))
+ use pdfium_render::prelude::*;
+ let pdfium = pdfium()?;
+
+ let doc = pdfium
+ .load_pdf_from_file(path, None)
+ .map_err(|e| match e {
+ PdfiumError::PdfiumLibraryInternalError(
+ PdfiumInternalError::PasswordError,
+ ) => format!(
+ "PDF is password-protected and cannot be read: '{}'",
+ path
+ ),
+ _ => format!("Failed to open PDF '{}': {}", path, e),
+ })?;
+
+ let mut out = String::new();
+ for (idx, page) in doc.pages().iter().enumerate() {
+ let page_text = page
+ .text()
+ .map_err(|e| format!("Page {} text extraction failed in '{}': {}", idx + 1, path, e))?;
+ out.push_str(&page_text.all());
+ out.push('\n');
+ }
+ Ok(out)
}
/// Extract text from Office Open XML formats, converting to Markdown.
@@ -975,4 +1024,81 @@ mod tests {
let result = read_file("/nonexistent/path/that/does/not/exist.pdf".to_string());
assert!(result.is_err() || result.is_ok()); // must at least return
}
+
+ /// Ad-hoc probe: run the production PDF extraction path against every
+ /// .pdf under a user-provided directory and print a per-file report of
+ /// Ok / Err (library returned an error) / Panic (library panicked and
+ /// was caught by panic_guard). Gated with #[ignore] so it never runs
+ /// in CI; execute locally with:
+ ///
+ /// PDF_PROBE_DIR=/path/to/pdfs cargo test --lib \
+ /// -- --ignored --nocapture pdf_probe
+ #[test]
+ #[ignore = "local probe; set PDF_PROBE_DIR"]
+ fn pdf_probe() {
+ let dir = std::env::var("PDF_PROBE_DIR")
+ .unwrap_or_else(|_| "/Users/nash_su/Downloads/pdftests".to_string());
+ let root = std::path::Path::new(&dir);
+ if !root.exists() {
+ eprintln!("[pdf_probe] dir not found: {}", root.display());
+ return;
+ }
+
+ let mut pdfs: Vec<std::path::PathBuf> = Vec::new();
+ fn walk(d: &std::path::Path, out: &mut Vec<std::path::PathBuf>) {
+ if let Ok(entries) = fs::read_dir(d) {
+ for entry in entries.flatten() {
+ let p = entry.path();
+ if p.is_dir() {
+ walk(&p, out);
+ } else if p.extension()
+ .and_then(|e| e.to_str())
+ .map(|e| e.eq_ignore_ascii_case("pdf"))
+ .unwrap_or(false)
+ {
+ out.push(p);
+ }
+ }
+ }
+ }
+ walk(root, &mut pdfs);
+ pdfs.sort();
+
+ eprintln!("\n[pdf_probe] found {} PDFs under {}\n", pdfs.len(), root.display());
+
+ let mut ok = 0usize;
+ let mut err = 0usize;
+ let mut panicked = 0usize;
+
+ for (idx, path) in pdfs.iter().enumerate() {
+ let display = path.display().to_string();
+ // Call extract_pdf_text directly (not read_file) so we bypass
+ // the .cache sibling dir and always exercise the parser.
+ let path_str = path.to_string_lossy().to_string();
+ let result = std::panic::catch_unwind(|| extract_pdf_text(&path_str));
+ match result {
+ Ok(Ok(text)) => {
+ ok += 1;
+ eprintln!("[{:>3}/{}] OK ({:>7} chars) {}", idx + 1, pdfs.len(), text.len(), display);
+ }
+ Ok(Err(e)) => {
+ err += 1;
+ eprintln!("[{:>3}/{}] ERR {} → {}", idx + 1, pdfs.len(), display, e);
+ }
+ Err(payload) => {
+ panicked += 1;
+ let msg = if let Some(s) = payload.downcast_ref::<String>() {
+ s.clone()
+ } else if let Some(s) = payload.downcast_ref::<&str>() {
+ (*s).to_string()
+ } else {
+ "(non-string panic)".to_string()
+ };
+ eprintln!("[{:>3}/{}] PANIC {} → {}", idx + 1, pdfs.len(), display, msg);
+ }
+ }
+ }
+
+ eprintln!("\n[pdf_probe] summary: {} OK / {} ERR / {} PANIC (total {})", ok, err, panicked, pdfs.len());
+ }
}