Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): parse PDFs in isolated Unicode workers

buger19f0dd404138
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index c50e18b30..54a160078 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -8,6 +8,17 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+[[package]]
+name = "aes"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138"
+dependencies = [
+ "cipher",
+ "cpubits",
+ "cpufeatures 0.3.0",
+]
+
[[package]]
name = "ahash"
version = "0.8.12"
@@ -31,6 +42,21 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "alloc-no-stdlib"
+version = "2.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
+
+[[package]]
+name = "alloc-stdlib"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195"
+dependencies = [
+ "alloc-no-stdlib",
+]
+
[[package]]
name = "allocator-api2"
version = "0.2.21"
@@ -126,6 +152,12 @@ dependencies = [
"rustversion",
]
+[[package]]
+name = "arrayvec"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
+
[[package]]
name = "asn1-rs"
version = "0.6.2"
@@ -135,7 +167,7 @@ dependencies = [
"asn1-rs-derive",
"asn1-rs-impl",
"displaydoc",
- "nom",
+ "nom 7.1.3",
"num-traits",
"rusticata-macros",
"thiserror 1.0.69",
@@ -198,6 +230,16 @@ dependencies = [
"syn",
]
+[[package]]
+name = "atoi_simd"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44"
+dependencies = [
+ "debug_unsafe",
+ "rustversion",
+]
+
[[package]]
name = "atomic-waker"
version = "1.1.2"
@@ -373,6 +415,12 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
[[package]]
name = "bitflags"
version = "2.11.1"
@@ -397,6 +445,45 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "block-buffer"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
+dependencies = [
+ "hybrid-array",
+]
+
+[[package]]
+name = "block-padding"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b"
+dependencies = [
+ "hybrid-array",
+]
+
+[[package]]
+name = "brotli"
+version = "8.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor",
+]
+
+[[package]]
+name = "brotli-decompressor"
+version = "5.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+]
+
[[package]]
name = "bumpalo"
version = "3.20.2"
@@ -429,6 +516,12 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+[[package]]
+name = "byteorder-lite"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
+
[[package]]
name = "bytes"
version = "1.11.1"
@@ -508,6 +601,15 @@ dependencies = [
"rustversion",
]
+[[package]]
+name = "cbc"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896"
+dependencies = [
+ "cipher",
+]
+
[[package]]
name = "cc"
version = "1.2.60"
@@ -554,7 +656,7 @@ checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e"
dependencies = [
"chrono",
"chrono-tz-build",
- "phf",
+ "phf 0.11.3",
]
[[package]]
@@ -564,7 +666,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f"
dependencies = [
"parse-zoneinfo",
- "phf",
+ "phf 0.11.3",
"phf_codegen",
]
@@ -595,6 +697,16 @@ dependencies = [
"half",
]
+[[package]]
+name = "cipher"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c"
+dependencies = [
+ "crypto-common 0.2.2",
+ "inout",
+]
+
[[package]]
name = "clap"
version = "4.6.1"
@@ -679,6 +791,12 @@ dependencies = [
"windows-sys 0.59.0",
]
+[[package]]
+name = "const-oid"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
+
[[package]]
name = "cookie"
version = "0.18.1"
@@ -716,6 +834,12 @@ version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+[[package]]
+name = "cpubits"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae"
+
[[package]]
name = "cpufeatures"
version = "0.2.17"
@@ -725,6 +849,15 @@ dependencies = [
"libc",
]
+[[package]]
+name = "cpufeatures"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "crc32fast"
version = "1.5.0"
@@ -842,6 +975,15 @@ dependencies = [
"typenum",
]
+[[package]]
+name = "crypto-common"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
+dependencies = [
+ "hybrid-array",
+]
+
[[package]]
name = "csv"
version = "1.4.0"
@@ -927,6 +1069,43 @@ version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
+[[package]]
+name = "debug_unsafe"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2"
+
+[[package]]
+name = "defmt"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1"
+dependencies = [
+ "bitflags 1.3.2",
+ "defmt-macros",
+]
+
+[[package]]
+name = "defmt-macros"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8"
+dependencies = [
+ "defmt-parser",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "defmt-parser"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e"
+dependencies = [
+ "thiserror 2.0.18",
+]
+
[[package]]
name = "der-parser"
version = "9.0.0"
@@ -935,7 +1114,7 @@ checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553"
dependencies = [
"asn1-rs",
"displaydoc",
- "nom",
+ "nom 7.1.3",
"num-bigint",
"num-traits",
"rusticata-macros",
@@ -999,11 +1178,22 @@ version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
- "block-buffer",
- "crypto-common",
+ "block-buffer 0.10.4",
+ "crypto-common 0.1.7",
"subtle",
]
+[[package]]
+name = "digest"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
+dependencies = [
+ "block-buffer 0.12.1",
+ "const-oid",
+ "crypto-common 0.2.2",
+]
+
[[package]]
name = "dirs"
version = "6.0.0"
@@ -1085,6 +1275,29 @@ dependencies = [
"syn",
]
+[[package]]
+name = "env_filter"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217"
+dependencies = [
+ "log",
+ "regex",
+]
+
+[[package]]
+name = "env_logger"
+version = "0.11.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "env_filter",
+ "jiff",
+ "log",
+]
+
[[package]]
name = "equivalent"
version = "1.0.2"
@@ -1121,6 +1334,15 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
+[[package]]
+name = "euclid"
+version = "0.22.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06"
+dependencies = [
+ "num-traits",
+]
+
[[package]]
name = "fallible-iterator"
version = "0.3.0"
@@ -1155,12 +1377,39 @@ dependencies = [
"regex-syntax",
]
+[[package]]
+name = "fast-float2"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
+
[[package]]
name = "fastrand"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
+[[package]]
+name = "fax"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a"
+
+[[package]]
+name = "fax"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ed8646f3993cad0585c6669ffba6f7e5440a0bfb0a1f16600cae33ce7cf307"
+
+[[package]]
+name = "fdeflate"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
+dependencies = [
+ "simd-adler32",
+]
+
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
@@ -1181,6 +1430,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
dependencies = [
"crc32fast",
"miniz_oxide",
+ "zlib-rs",
]
[[package]]
@@ -1222,6 +1472,15 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+[[package]]
+name = "font-types"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b38ad915f6dadd993ced50848a8291a543bd41ca62bc10740d5e64e2ab4cfd7"
+dependencies = [
+ "bytemuck",
+]
+
[[package]]
name = "foreign-types"
version = "0.3.2"
@@ -1513,10 +1772,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
dependencies = [
"cfg-if",
+ "js-sys",
"libc",
"r-efi 6.0.0",
"wasip2",
"wasip3",
+ "wasm-bindgen",
]
[[package]]
@@ -1525,6 +1786,12 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+[[package]]
+name = "grid"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b40ca9252762c466af32d0b1002e91e4e1bc5398f77455e55474deb466355ff5"
+
[[package]]
name = "h2"
version = "0.4.13"
@@ -1665,7 +1932,7 @@ version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
dependencies = [
- "digest",
+ "digest 0.10.7",
]
[[package]]
@@ -1719,6 +1986,15 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+[[package]]
+name = "hybrid-array"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c"
+dependencies = [
+ "typenum",
+]
+
[[package]]
name = "hyper"
version = "1.9.0"
@@ -1950,6 +2226,22 @@ dependencies = [
"icu_properties",
]
+[[package]]
+name = "image"
+version = "0.25.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
+dependencies = [
+ "bytemuck",
+ "byteorder-lite",
+ "moxcms",
+ "num-traits",
+ "png",
+ "tiff",
+ "zune-core",
+ "zune-jpeg",
+]
+
[[package]]
name = "indexmap"
version = "1.9.3"
@@ -1985,6 +2277,16 @@ dependencies = [
"web-time",
]
+[[package]]
+name = "inout"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7"
+dependencies = [
+ "block-padding",
+ "hybrid-array",
+]
+
[[package]]
name = "ipnet"
version = "2.12.0"
@@ -2042,6 +2344,42 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+[[package]]
+name = "jiff"
+version = "0.2.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e184d09547b80eb7e20d141ba2fb1fbac843ca53f4cf1b31210adc4c1adc6e16"
+dependencies = [
+ "defmt",
+ "jiff-core",
+ "jiff-static",
+ "log",
+ "portable-atomic",
+ "portable-atomic-util",
+ "serde_core",
+]
+
+[[package]]
+name = "jiff-core"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09"
+dependencies = [
+ "defmt",
+]
+
+[[package]]
+name = "jiff-static"
+version = "0.2.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "323da076b7a6faf914dc677cb05a4b907742ff7375c8322c9e7f5061e5e0e9de"
+dependencies = [
+ "jiff-core",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "jobserver"
version = "0.1.34"
@@ -2052,6 +2390,15 @@ dependencies = [
"libc",
]
+[[package]]
+name = "jpeg-decoder"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
+dependencies = [
+ "rayon",
+]
+
[[package]]
name = "js-sys"
version = "0.3.95"
@@ -2064,6 +2411,18 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "kurbo"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2"
+dependencies = [
+ "arrayvec",
+ "euclid",
+ "polycool",
+ "smallvec",
+]
+
[[package]]
name = "lazy_static"
version = "1.5.0"
@@ -2230,6 +2589,16 @@ dependencies = [
"rawpointer",
]
+[[package]]
+name = "md-5"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98"
+dependencies = [
+ "cfg-if",
+ "digest 0.11.3",
+]
+
[[package]]
name = "memchr"
version = "2.8.0"
@@ -2323,6 +2692,16 @@ dependencies = [
"syn",
]
+[[package]]
+name = "moxcms"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
+dependencies = [
+ "num-traits",
+ "pxfm",
+]
+
[[package]]
name = "multer"
version = "3.1.0"
@@ -2388,6 +2767,15 @@ dependencies = [
"minimal-lexical",
]
+[[package]]
+name = "nom"
+version = "8.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "nu-ansi-term"
version = "0.50.3"
@@ -2458,6 +2846,24 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+[[package]]
+name = "office_oxide"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "317c254cebe5b87c74a0fbd84e889d51ec0c4db885fa5aca98700ad85fd9ec5f"
+dependencies = [
+ "atoi_simd",
+ "encoding_rs",
+ "fast-float2",
+ "libc",
+ "log",
+ "quick-xml 0.41.0",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.18",
+ "zip 8.6.0",
+]
+
[[package]]
name = "oid-registry"
version = "0.7.1"
@@ -2491,7 +2897,7 @@ version = "0.10.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfe4646e360ec77dff7dde40ed3d6c5fee52d156ef4a62f53973d38294dad87f"
dependencies = [
- "bitflags",
+ "bitflags 2.11.1",
"cfg-if",
"foreign-types",
"libc",
@@ -2613,6 +3019,52 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec"
+[[package]]
+name = "pdf_oxide"
+version = "0.3.75"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e00ca5f52780bb764da7fc1783c0c6f549fcf960d69734c8d7cf7996bdbb49d"
+dependencies = [
+ "aes",
+ "base64 0.22.1",
+ "bitflags 2.11.1",
+ "brotli",
+ "byteorder",
+ "bytes",
+ "cbc",
+ "chrono",
+ "encoding_rs",
+ "env_logger",
+ "fax 0.3.0",
+ "flate2",
+ "getrandom 0.4.2",
+ "image",
+ "jpeg-decoder",
+ "libc",
+ "log",
+ "md-5",
+ "memchr",
+ "nom 8.0.0",
+ "office_oxide",
+ "phf 0.14.0",
+ "quick-xml 0.41.0",
+ "regex",
+ "serde",
+ "serde_json",
+ "sha2 0.11.0",
+ "smallvec",
+ "stringprep",
+ "subsetter",
+ "taffy",
+ "thiserror 2.0.18",
+ "ttf-parser",
+ "unicode-bidi",
+ "unicode-linebreak",
+ "unicode-normalization",
+ "uuid",
+ "weezl 0.2.1",
+]
+
[[package]]
name = "pem"
version = "3.0.6"
@@ -2645,7 +3097,18 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
dependencies = [
- "phf_shared",
+ "phf_shared 0.11.3",
+]
+
+[[package]]
+name = "phf"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
+dependencies = [
+ "phf_macros",
+ "phf_shared 0.14.0",
+ "serde",
]
[[package]]
@@ -2654,8 +3117,8 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
dependencies = [
- "phf_generator",
- "phf_shared",
+ "phf_generator 0.11.3",
+ "phf_shared 0.11.3",
]
[[package]]
@@ -2664,10 +3127,33 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
dependencies = [
- "phf_shared",
+ "phf_shared 0.11.3",
"rand 0.8.5",
]
+[[package]]
+name = "phf_generator"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"
+dependencies = [
+ "fastrand",
+ "phf_shared 0.14.0",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085"
+dependencies = [
+ "phf_generator 0.14.0",
+ "phf_shared 0.14.0",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "phf_shared"
version = "0.11.3"
@@ -2677,6 +3163,15 @@ dependencies = [
"siphasher",
]
+[[package]]
+name = "phf_shared"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89"
+dependencies = [
+ "siphasher",
+]
+
[[package]]
name = "pin-project"
version = "1.1.11"
@@ -2737,6 +3232,28 @@ dependencies = [
"plotters-backend",
]
+[[package]]
+name = "png"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
+dependencies = [
+ "bitflags 2.11.1",
+ "crc32fast",
+ "fdeflate",
+ "flate2",
+ "miniz_oxide",
+]
+
+[[package]]
+name = "polycool"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6"
+dependencies = [
+ "arrayvec",
+]
+
[[package]]
name = "portable-atomic"
version = "1.13.1"
@@ -2931,6 +3448,18 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740"
+[[package]]
+name = "pxfm"
+version = "0.1.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
+
+[[package]]
+name = "quick-error"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
+
[[package]]
name = "quick-xml"
version = "0.36.2"
@@ -2940,6 +3469,16 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "quick-xml"
+version = "0.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
[[package]]
name = "quinn"
version = "0.11.9"
@@ -3091,7 +3630,7 @@ version = "11.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
dependencies = [
- "bitflags",
+ "bitflags 2.11.1",
]
[[package]]
@@ -3144,6 +3683,16 @@ dependencies = [
"yasna",
]
+[[package]]
+name = "read-fonts"
+version = "0.39.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4ed38b89c2c77ff968c524145ad65fb010f38af5c7a224b53b81d47ac2daa81"
+dependencies = [
+ "bytemuck",
+ "font-types",
+]
+
[[package]]
name = "reborrow"
version = "0.5.5"
@@ -3156,7 +3705,7 @@ version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
- "bitflags",
+ "bitflags 2.11.1",
]
[[package]]
@@ -3295,7 +3844,7 @@ version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e"
dependencies = [
- "bitflags",
+ "bitflags 2.11.1",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
@@ -3325,7 +3874,7 @@ version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"
dependencies = [
- "nom",
+ "nom 7.1.3",
]
[[package]]
@@ -3334,7 +3883,7 @@ version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
dependencies = [
- "bitflags",
+ "bitflags 2.11.1",
"errno",
"libc",
"linux-raw-sys",
@@ -3439,7 +3988,7 @@ version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
dependencies = [
- "bitflags",
+ "bitflags 2.11.1",
"core-foundation 0.10.1",
"core-foundation-sys",
"libc",
@@ -3583,8 +4132,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
- "cpufeatures",
- "digest",
+ "cpufeatures 0.2.17",
+ "digest 0.10.7",
+]
+
+[[package]]
+name = "sha2"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.3.0",
+ "digest 0.11.3",
]
[[package]]
@@ -3645,12 +4205,31 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
+[[package]]
+name = "skrifa"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c34617370ae968efb7161bb2beb517d9084659aae19e24b89e3db25b46e4564"
+dependencies = [
+ "bytemuck",
+ "read-fonts",
+]
+
[[package]]
name = "slab"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
+[[package]]
+name = "slotmap"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038"
+dependencies = [
+ "version_check",
+]
+
[[package]]
name = "smallvec"
version = "1.15.1"
@@ -3701,7 +4280,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
dependencies = [
"base64 0.13.1",
- "nom",
+ "nom 7.1.3",
"serde",
"unicode-segmentation",
]
@@ -3718,12 +4297,35 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+[[package]]
+name = "stringprep"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+ "unicode-properties",
+]
+
[[package]]
name = "strsim"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+[[package]]
+name = "subsetter"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38803281d1c23166c5ebcb455439a5d2afe711cc909cf88af72448c297756ad6"
+dependencies = [
+ "kurbo",
+ "rustc-hash",
+ "skrifa",
+ "write-fonts",
+]
+
[[package]]
name = "subtle"
version = "2.6.1"
@@ -3767,7 +4369,7 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc"
dependencies = [
- "bitflags",
+ "bitflags 2.11.1",
"byteorder",
"enum-as-inner",
"libc",
@@ -3781,7 +4383,7 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
dependencies = [
- "bitflags",
+ "bitflags 2.11.1",
"core-foundation 0.9.4",
"system-configuration-sys",
]
@@ -3796,6 +4398,18 @@ dependencies = [
"libc",
]
+[[package]]
+name = "taffy"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "340a09581f29809fc0df82a3955501dc7f2a21f887e5d1c13dbe288fe1c0bef4"
+dependencies = [
+ "arrayvec",
+ "grid",
+ "serde",
+ "slotmap",
+]
+
[[package]]
name = "tempfile"
version = "3.27.0"
@@ -3858,6 +4472,20 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "tiff"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52"
+dependencies = [
+ "fax 0.2.7",
+ "flate2",
+ "half",
+ "quick-error",
+ "weezl 0.1.12",
+ "zune-jpeg",
+]
+
[[package]]
name = "tikv-jemalloc-sys"
version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7"
@@ -4176,7 +4804,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
dependencies = [
- "bitflags",
+ "bitflags 2.11.1",
"bytes",
"http",
"http-body",
@@ -4193,7 +4821,7 @@ version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
dependencies = [
- "bitflags",
+ "bitflags 2.11.1",
"bytes",
"futures-util",
"http",
@@ -4298,6 +4926,12 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+[[package]]
+name = "ttf-parser"
+version = "0.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31"
+
[[package]]
name = "twox-hash"
version = "2.1.2"
@@ -4316,12 +4950,24 @@ version = "1.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
+[[package]]
+name = "unicode-bidi"
+version = "0.3.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
+
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+[[package]]
+name = "unicode-linebreak"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
+
[[package]]
name = "unicode-normalization"
version = "0.1.25"
@@ -4340,6 +4986,12 @@ dependencies = [
"smallvec",
]
+[[package]]
+name = "unicode-properties"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
+
[[package]]
name = "unicode-segmentation"
version = "1.13.2"
@@ -4629,7 +5281,7 @@ version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
dependencies = [
- "bitflags",
+ "bitflags 2.11.1",
"hashbrown 0.15.5",
"indexmap 2.14.0",
"semver",
@@ -4693,7 +5345,7 @@ dependencies = [
"base64urlsafedata",
"der-parser",
"hex",
- "nom",
+ "nom 7.1.3",
"openssl",
"openssl-sys",
"rand 0.9.4",
@@ -4741,6 +5393,18 @@ dependencies = [
"rustls-pki-types",
]
+[[package]]
+name = "weezl"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
+
+[[package]]
+name = "weezl"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2"
+
[[package]]
name = "winapi"
version = "0.3.9"
@@ -5074,7 +5738,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
dependencies = [
"anyhow",
- "bitflags",
+ "bitflags 2.11.1",
"indexmap 2.14.0",
"log",
"serde",
@@ -5104,6 +5768,19 @@ dependencies = [
"wasmparser",
]
+[[package]]
+name = "write-fonts"
+version = "0.48.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb731d4c4d93eacc69a1ad2f270f905788a98e4a3438267bcafbe08d3431c8d8"
+dependencies = [
+ "font-types",
+ "indexmap 2.14.0",
+ "kurbo",
+ "log",
+ "read-fonts",
+]
+
[[package]]
name = "writeable"
version = "0.6.3"
@@ -5120,7 +5797,7 @@ dependencies = [
"data-encoding",
"der-parser",
"lazy_static",
- "nom",
+ "nom 7.1.3",
"oid-registry",
"rusticata-macros",
"thiserror 1.0.69",
@@ -5140,7 +5817,7 @@ dependencies = [
"reqwest",
"serde",
"serde_json",
- "sha2",
+ "sha2 0.10.9",
"thiserror 2.0.18",
"tokenizers",
"tokio",
@@ -5186,8 +5863,10 @@ dependencies = [
"csv",
"encoding_rs",
"flate2",
+ "libc",
"memchr",
- "quick-xml",
+ "pdf_oxide",
+ "quick-xml 0.36.2",
"rayon",
"regex",
"reqwest",
@@ -5273,7 +5952,7 @@ dependencies = [
"rand 0.8.5",
"serde",
"serde_json",
- "sha2",
+ "sha2 0.10.9",
"subtle",
"tempfile",
"thiserror 2.0.18",
@@ -5309,7 +5988,7 @@ dependencies = [
"rustc-hash",
"serde",
"serde_json",
- "sha2",
+ "sha2 0.10.9",
"simd-json",
"tempfile",
"thiserror 2.0.18",
@@ -5650,6 +6329,26 @@ dependencies = [
"typed-path",
]
+[[package]]
+name = "zip"
+version = "8.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
+dependencies = [
+ "crc32fast",
+ "flate2",
+ "indexmap 2.14.0",
+ "memchr",
+ "typed-path",
+ "zopfli",
+]
+
+[[package]]
+name = "zlib-rs"
+version = "0.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5"
+
[[package]]
name = "zmij"
version = "1.0.21"
@@ -5695,3 +6394,18 @@ dependencies = [
"cc",
"pkg-config",
]
+
+[[package]]
+name = "zune-core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9"
+
+[[package]]
+name = "zune-jpeg"
+version = "0.5.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
+dependencies = [
+ "zune-core",
+]
diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.toml
index 07ed1157e..731df6fa7 100644
--- a/engine/crates/xerj-autoindex/Cargo.toml
+++ b/engine/crates/xerj-autoindex/Cargo.toml
@@ -17,12 +17,14 @@ encoding_rs = "0.8"
xxhash-rust.workspace = true
memchr.workspace = true
rayon.workspace = true
+libc = "0.2"
# ES-compat HTTP client. Blocking mode: all of autoindex is synchronous
# (std threads), invoked from the server binary via spawn_blocking.
reqwest = { workspace = true, features = ["blocking"] }
# Format-family extractors (new deps, isolated to this crate):
flate2 = "1" # gzip streaming (rust backend)
+pdf_oxide = { version = "=0.3.75", default-features = false, features = ["legacy-crypto"] }
csv = "1" # streaming CSV with sniffed dialect
quick-xml = "0.36" # XML pull parser + DOCX internals
zip = { version = "2", default-features = false, features = ["deflate"] } # DOCX container
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index c046d6fe6..13f902386 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -8,6 +8,8 @@ pub struct IndexCfg {
pub url: String,
pub api_key: Option<String>,
pub workers: usize,
+ pub pdf_workers: usize,
+ pub pdf_timeout_secs: u64,
pub bulk_mb: usize,
pub prefix: String,
pub state_dir: Option<PathBuf>,
@@ -59,6 +61,8 @@ pub fn print_help() {
--url <U> ES-compat endpoint (default http://localhost:9200)\n\
--api-key <K> Authorization header (or env XERJ_API_KEY)\n\
--workers <N> extract workers (default min(cores,8))\n\
+ --pdf-workers <N> concurrent PDF parser processes (default min(cores,4); max 4)\n\
+ --pdf-timeout-secs <N> per-PDF parser timeout (default 120; max 3600)\n\
--bulk-mb <N> bulk cut size in MB (default 8)\n\
--prefix <P> index prefix (default ax)\n\
--state-dir <PATH> resume journal location (default ~/.xerj/autoindex/<hash>/)\n\
@@ -73,6 +77,17 @@ pub fn print_help() {
--dataset <SLUG> (map) show a single dataset\n\
--help, -h this help\n\
\n\
+ PDF EXTRACTION:\n\
+ Each PDF uses a fresh process. Limits: 512 MiB input, 32 MiB worker output,\n\
+ 100,000 pages, and 1.5 GiB address space on Unix. This is crash/resource\n\
+ isolation, not a security sandbox; non-Unix platforms have no OS memory cap.\n\
+ Image-only pages need OCR. Page parse failures reject the whole PDF instead\n\
+ of silently creating a partial index. XERJ_PDF_WORKER_BIN is a trusted,\n\
+ developer-only executable override.\n\
+ Current cost: phase-A sampling parses/materializes each complete PDF, and\n\
+ phase-B indexing parses it again. A framed, early-stop protocol is planned;\n\
+ use fewer --pdf-workers when parent memory is constrained.\n\
+ \n\
EMBEDDINGS:\n\
autoindex sends semantic_text to the running server; it does not choose the\n\
server's embedding backend. The default is lexical (not neural). For the\n\
@@ -98,6 +113,11 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
.map(|n| n.get())
.unwrap_or(8)
.min(8);
+ let mut pdf_workers = std::thread::available_parallelism()
+ .map(|n| n.get())
+ .unwrap_or(4)
+ .min(4);
+ let mut pdf_timeout_secs = 120u64;
let mut bulk_mb = 8usize;
let mut prefix = "ax".to_string();
let mut state_dir: Option<PathBuf> = None;
@@ -121,6 +141,20 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
.and_then(|s| s.parse().ok())
.ok_or("--workers needs a number")?
}
+ "--pdf-workers" => {
+ pdf_workers = it
+ .next()
+ .and_then(|s| s.parse().ok())
+ .filter(|n| (1..=4).contains(n))
+ .ok_or("--pdf-workers needs a number from 1 to 4")?
+ }
+ "--pdf-timeout-secs" => {
+ pdf_timeout_secs = it
+ .next()
+ .and_then(|s| s.parse().ok())
+ .filter(|n| (1..=3600).contains(n))
+ .ok_or("--pdf-timeout-secs needs a number from 1 to 3600")?
+ }
"--bulk-mb" => {
bulk_mb = it
.next()
@@ -186,6 +220,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
url,
api_key,
workers: workers.max(1),
+ pdf_workers,
+ pdf_timeout_secs,
bulk_mb: bulk_mb.clamp(1, 24),
prefix,
state_dir,
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rs
index d657dd79a..7c79edea4 100644
--- a/engine/crates/xerj-autoindex/src/extract/mod.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs
@@ -17,12 +17,13 @@ pub mod yaml_x;
use crate::sniff::{Family, Sniffed};
use anyhow::Result;
+use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use std::io::{BufRead, BufReader, Read};
use std::path::Path;
/// One extracted record before coercion.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RawRecord {
pub fields: Map<String, Value>,
/// Canonical, content-positional locator (byte offset / ordinal /
diff --git a/engine/crates/xerj-autoindex/src/extract/pdf.rs b/engine/crates/xerj-autoindex/src/extract/pdf.rs
index 24158244a..51e44b0f3 100644
--- a/engine/crates/xerj-autoindex/src/extract/pdf.rs
+++ b/engine/crates/xerj-autoindex/src/extract/pdf.rs
@@ -1,324 +1,643 @@
-//! PDF text extraction — a self-contained content-stream scanner (no
-//! external PDF crate: the workspace release profile is panic=abort, so a
-//! panicky dependency would take the whole process down; this scanner is
-//! Result-only by construction).
+//! PDF text extraction backed by a real PDF parser.
//!
-//! Strategy: locate `stream…endstream` ranges, inflate FlateDecode bodies
-//! (or use raw bodies that already look like content streams), then scan for
-//! text-showing operators: `(…) Tj`, `(…) '`, `[…] TJ`, with PDF string
-//! escapes and hex strings. Newlines derive from Td/TD/T*/ET ops. `/Title`
-//! is pulled from the document info dictionary when present.
+//! The old implementation searched raw streams and interpreted font character
+//! codes as Latin-1. `pdf_oxide` resolves the object graph and font mappings
+//! before returning Unicode text. Each document runs in a fresh worker process
+//! so parser state or failure cannot contaminate the server. On Unix the worker
+//! also gets a process group and an address-space limit. This is crash/resource
+//! isolation, not a security sandbox: the worker retains the user's authority.
-use super::{emit_document, ExtractStats, Sink};
-use anyhow::Result;
-use std::io::Read;
-use std::path::Path;
+use super::{split_sections, ExtractStats, RawRecord, Sink};
+use anyhow::{anyhow, Context, Result};
+use pdf_oxide::PdfDocument;
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value};
+use std::io::{Read, Write};
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Condvar, Mutex, OnceLock};
+use std::time::{Duration, Instant};
const PDF_CAP: u64 = 512 << 20;
-const STREAM_INFLATE_CAP: u64 = 64 << 20;
+const MAX_PAGES: usize = 100_000;
+const MAX_PAGE_TEXT: usize = 16 << 20;
+const MAX_EXTRACTED_TEXT: usize = 64 << 20;
+const MAX_WORKER_OUTPUT: usize = 32 << 20;
+const MAX_WORKER_STDERR: u64 = 64 << 10;
+const WORKER_ADDRESS_SPACE: u64 = 1536 << 20;
pub fn extract(path: &Path, sink: Sink) -> Result<ExtractStats> {
+ let _permit = worker_gate().acquire();
+ let response = spawn_worker(path)?;
let mut stats = ExtractStats::default();
- let size = std::fs::metadata(path)?.len();
- if size > PDF_CAP {
- stats.junk += 1;
- return Ok(stats);
- }
- let bytes = std::fs::read(path)?;
- let mut text = String::new();
- for body in stream_bodies(&bytes) {
- let content: Vec<u8> = match inflate(body) {
- Some(d) => d,
- None => {
- if looks_like_content(body) {
- body.to_vec()
- } else {
- continue;
- }
- }
- };
- if !looks_like_content(&content) {
- continue;
+ for record in response.records {
+ stats.records += 1;
+ if !sink(record) {
+ break;
}
- scan_text_ops(&content, &mut text);
- }
- let title = doc_title(&bytes).unwrap_or_else(|| {
- text.lines()
- .find(|l| !l.trim().is_empty())
- .map(|l| l.trim().chars().take(200).collect())
- .unwrap_or_else(|| {
- path.file_stem()
- .map(|s| s.to_string_lossy().to_string())
- .unwrap_or_else(|| "untitled".into())
- })
- });
- let body = text.trim();
- if body.is_empty() {
- stats.junk += 1;
- return Ok(stats);
- }
- emit_document(&title, &[], body, sink, &mut stats);
+ }
Ok(stats)
}
-fn stream_bodies(bytes: &[u8]) -> Vec<&[u8]> {
- let mut out = Vec::new();
- let mut i = 0usize;
- while let Some(p) = find(bytes, i, b"stream") {
- // must be the keyword, not part of "endstream"
- let word_ok = p == 0 || !bytes[p - 1].is_ascii_alphanumeric();
- let mut start = p + b"stream".len();
- if start < bytes.len() && bytes[start] == b'\r' {
- start += 1;
+#[derive(Debug, Serialize, Deserialize)]
+struct WorkerResponse {
+ schema: u32,
+ extractor: String,
+ parser: String,
+ containment: String,
+ records: Vec<RawRecord>,
+}
+
+pub fn configure_workers(workers: usize) {
+ worker_gate().set_limit(workers.clamp(1, 4));
+}
+
+pub fn configure_timeout(seconds: u64) {
+ WORKER_TIMEOUT_SECS.store(seconds.clamp(1, 3600), Ordering::Relaxed);
+}
+
+/// Hidden same-binary worker entry point. One invocation parses one document.
+pub fn run_worker_cli() -> i32 {
+ let mut args = std::env::args_os().skip(2);
+ let Some(path) = args.next().map(PathBuf::from) else {
+ eprintln!("PDF worker protocol error: missing input path");
+ return 2;
+ };
+ if args.next().is_some() {
+ eprintln!("PDF worker protocol error: unexpected arguments");
+ return 2;
+ }
+ match extract_in_process(&path) {
+ Ok(records) => {
+ let response = WorkerResponse {
+ schema: 1,
+ extractor: format!("xerj-autoindex/{}", env!("CARGO_PKG_VERSION")),
+ parser: format!("pdf_oxide/{}", pdf_oxide::VERSION),
+ containment: containment_description().to_string(),
+ records,
+ };
+ let result = (|| -> Result<()> {
+ let mut stdout = std::io::stdout().lock();
+ serde_json::to_writer(&mut stdout, &response)?;
+ stdout.flush()?;
+ Ok(())
+ })();
+ if let Err(error) = result {
+ eprintln!("PDF worker could not write its bounded result: {error}");
+ return 1;
+ }
+ 0
}
- if start < bytes.len() && bytes[start] == b'\n' {
- start += 1;
+ Err(error) => {
+ eprintln!("{error:#}");
+ 1
}
- match find(bytes, start, b"endstream") {
- Some(e) if word_ok => {
- out.push(&bytes[start..e]);
- i = e + b"endstream".len();
- }
- Some(e) => {
- i = e + b"endstream".len();
+ }
+}
+
+fn spawn_worker(path: &Path) -> Result<WorkerResponse> {
+ // Trusted developer/test hook. The selected executable runs with the
+ // invoking user's authority and must implement this private protocol.
+ let executable = std::env::var_os("XERJ_PDF_WORKER_BIN")
+ .map(PathBuf::from)
+ .or_else(|| std::env::current_exe().ok())
+ .context("cannot locate the xerj executable for isolated PDF extraction")?;
+ let mut command = Command::new(executable);
+ command
+ .arg("__extract-pdf")
+ .arg(path)
+ .stdin(Stdio::null())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped());
+ set_worker_memory_limit(&mut command);
+ let mut child = command.spawn().with_context(|| {
+ format!(
+ "could not start the isolated PDF parser for {}; check process/resource limits",
+ path.display()
+ )
+ })?;
+ let stdout = child.stdout.take().expect("piped stdout");
+ let stderr = child.stderr.take().expect("piped stderr");
+ let out_reader = std::thread::spawn(move || read_capped(stdout, MAX_WORKER_OUTPUT));
+ let err_reader = std::thread::spawn(move || read_capped(stderr, MAX_WORKER_STDERR as usize));
+
+ let started = Instant::now();
+ let timeout = Duration::from_secs(WORKER_TIMEOUT_SECS.load(Ordering::Relaxed));
+ let status = loop {
+ match child.try_wait() {
+ Ok(Some(status)) => break status,
+ Ok(None) => {}
+ Err(error) => {
+ terminate_worker_tree(&mut child);
+ let _ = out_reader.join();
+ let _ = err_reader.join();
+ return Err(error).with_context(|| {
+ format!("could not monitor PDF parser for {}", path.display())
+ });
}
- None => break,
}
+ if started.elapsed() >= timeout {
+ terminate_worker_tree(&mut child);
+ let _ = out_reader.join();
+ let _ = err_reader.join();
+ return Err(anyhow!(
+ "PDF extraction timed out after {} seconds for {}; repair/split the PDF, or run OCR if it is image-only",
+ timeout.as_secs(),
+ path.display()
+ ));
+ }
+ std::thread::sleep(Duration::from_millis(20));
+ };
+ terminate_worker_descendants(child.id());
+ let stdout = out_reader
+ .join()
+ .map_err(|_| anyhow!("PDF parser output reader panicked"))??;
+ let stderr = err_reader
+ .join()
+ .map_err(|_| anyhow!("PDF parser error reader panicked"))??;
+ if stdout.overflow {
+ return Err(anyhow!(
+ "PDF parser output exceeded the {} MiB parent-memory safety limit for {}; split the document before indexing",
+ MAX_WORKER_OUTPUT >> 20,
+ path.display()
+ ));
+ }
+ if stderr.overflow {
+ return Err(anyhow!(
+ "PDF parser error output exceeded the {} KiB safety limit for {}; the worker was excessively noisy and its truncated diagnostic is: {}",
+ MAX_WORKER_STDERR >> 10,
+ path.display(),
+ String::from_utf8_lossy(&stderr.prefix)
+ ));
+ }
+ let stderr = String::from_utf8_lossy(&stderr.prefix);
+ if !status.success() {
+ return Err(anyhow!(
+ "isolated PDF parser failed for {}{}{}; verify/decrypt the PDF, repair it, or run OCR for image-only input",
+ path.display(),
+ if stderr.trim().is_empty() { "" } else { ": " },
+ stderr.trim()
+ ));
+ }
+ let response: WorkerResponse = serde_json::from_slice(&stdout.prefix).with_context(|| {
+ format!(
+ "isolated PDF parser returned an invalid result for {}; this is an internal worker-protocol error",
+ path.display()
+ )
+ })?;
+ if response.schema != 1 {
+ return Err(anyhow!(
+ "PDF parser returned unsupported extraction schema {}; update parent and worker together",
+ response.schema
+ ));
+ }
+ let expected_extractor = format!("xerj-autoindex/{}", env!("CARGO_PKG_VERSION"));
+ if response.extractor != expected_extractor {
+ return Err(anyhow!(
+ "PDF extractor version mismatch: expected {expected_extractor}, worker reported {}; update parent and worker together",
+ response.extractor
+ ));
+ }
+ if response.parser != format!("pdf_oxide/{}", pdf_oxide::VERSION) {
+ return Err(anyhow!(
+ "PDF parser version mismatch: expected pdf_oxide/{}, worker reported {}; update parent and worker together",
+ pdf_oxide::VERSION, response.parser
+ ));
+ }
+ Ok(response)
+}
+
+static WORKER_TIMEOUT_SECS: AtomicU64 = AtomicU64::new(120);
+
+struct CappedRead {
+ prefix: Vec<u8>,
+ overflow: bool,
+}
+
+fn read_capped(mut input: impl Read, cap: usize) -> Result<CappedRead> {
+ let mut prefix = Vec::with_capacity(cap.min(64 << 10));
+ let mut overflow = false;
+ let mut chunk = [0u8; 64 << 10];
+ loop {
+ let read = input
+ .read(&mut chunk)
+ .context("could not read isolated PDF parser output")?;
+ if read == 0 {
+ break;
+ }
+ let remaining = cap.saturating_sub(prefix.len());
+ let retain = remaining.min(read);
+ prefix.extend_from_slice(&chunk[..retain]);
+ overflow |= retain < read;
+ // Keep draining after overflow so the child can exit instead of
+ // blocking forever on a full pipe.
}
- out
+ Ok(CappedRead { prefix, overflow })
}
-fn find(hay: &[u8], from: usize, needle: &[u8]) -> Option<usize> {
- if from >= hay.len() {
- return None;
+#[cfg(unix)]
+fn set_worker_memory_limit(command: &mut Command) {
+ use std::os::unix::process::CommandExt;
+ unsafe {
+ command.pre_exec(|| {
+ if libc::setpgid(0, 0) != 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ let limit = libc::rlimit {
+ rlim_cur: WORKER_ADDRESS_SPACE,
+ rlim_max: WORKER_ADDRESS_SPACE,
+ };
+ if libc::setrlimit(libc::RLIMIT_AS, &limit) != 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ Ok(())
+ });
}
- memchr::memmem::find(&hay[from..], needle).map(|p| p + from)
}
-fn inflate(body: &[u8]) -> Option<Vec<u8>> {
- let mut d = flate2::read::ZlibDecoder::new(body).take(STREAM_INFLATE_CAP);
- let mut out = Vec::new();
- d.read_to_end(&mut out).ok()?;
- if out.is_empty() {
- None
- } else {
- Some(out)
+#[cfg(not(unix))]
+fn set_worker_memory_limit(_command: &mut Command) {}
+
+#[cfg(unix)]
+fn terminate_worker_tree(child: &mut std::process::Child) {
+ unsafe {
+ libc::kill(-(child.id() as i32), libc::SIGKILL);
}
+ let _ = child.kill();
+ let _ = child.wait();
}
-fn looks_like_content(b: &[u8]) -> bool {
- let head = &b[..b.len().min(4096)];
- memchr::memmem::find(head, b"BT").is_some()
- || memchr::memmem::find(head, b"Tj").is_some()
- || memchr::memmem::find(head, b"TJ").is_some()
+#[cfg(unix)]
+fn terminate_worker_descendants(process_group: u32) {
+ unsafe {
+ libc::kill(-(process_group as i32), libc::SIGKILL);
+ }
}
-/// PDFDocEncoding ≈ latin-1 for the printable range — good enough for
-/// generated PDFs; unknown bytes are passed through as latin-1.
-fn scan_text_ops(content: &[u8], out: &mut String) {
- let mut i = 0usize;
- let mut pending: Vec<String> = Vec::new(); // strings awaiting an operator
- let mut line = String::new();
- let flush_line = |line: &mut String, out: &mut String| {
- let t = line.trim();
- if !t.is_empty() {
- out.push_str(t);
- out.push('\n');
- }
- line.clear();
- };
- while i < content.len() {
- match content[i] {
- b'(' => {
- let (s, ni) = parse_literal_string(content, i);
- pending.push(s);
- i = ni;
- }
- b'<' if i + 1 < content.len() && content[i + 1] != b'<' => {
- let (s, ni) = parse_hex_string(content, i);
- pending.push(s);
- i = ni;
- }
- b'[' => {
- // TJ array: collect strings until ]
- let mut j = i + 1;
- let mut parts: Vec<String> = Vec::new();
- while j < content.len() && content[j] != b']' {
- match content[j] {
- b'(' => {
- let (s, nj) = parse_literal_string(content, j);
- parts.push(s);
- j = nj;
- }
- b'<' => {
- let (s, nj) = parse_hex_string(content, j);
- parts.push(s);
- j = nj;
- }
- _ => j += 1,
- }
- }
- pending.push(parts.join(""));
- i = j + 1;
- }
- b'%' => {
- // comment to EOL
- i = memchr::memchr(b'\n', &content[i..])
- .map(|p| i + p + 1)
- .unwrap_or(content.len());
+#[cfg(not(unix))]
+fn terminate_worker_tree(child: &mut std::process::Child) {
+ let _ = child.kill();
+ let _ = child.wait();
+}
+
+#[cfg(not(unix))]
+fn terminate_worker_descendants(_process_group: u32) {}
+
+#[cfg(unix)]
+fn containment_description() -> &'static str {
+ "fresh process; Unix process-group cleanup; 1536 MiB RLIMIT_AS; not a security sandbox"
+}
+
+#[cfg(not(unix))]
+fn containment_description() -> &'static str {
+ "fresh process; no OS memory cap on this platform; not a security sandbox"
+}
+
+fn extract_in_process(path: &Path) -> Result<Vec<RawRecord>> {
+ let size = std::fs::metadata(path)?.len();
+ if size > PDF_CAP {
+ return Err(anyhow!(
+ "PDF is {:.1} MiB, above the 512 MiB safety limit; split or compress it before indexing",
+ size as f64 / (1 << 20) as f64
+ ));
+ }
+
+ pdf_oxide::fonts::global_cache::clear_global_font_cache();
+ let doc = PdfDocument::open(path).with_context(|| {
+ "PDF parser could not open the document; verify/repair damaged input, decrypt password-protected input, or run OCR for image-only input"
+ })?;
+ let pages = doc.page_count().with_context(|| {
+ "PDF parser could not read the page tree; verify the file or decrypt it first"
+ })?;
+ if pages == 0 {
+ return Err(anyhow!("PDF contains no pages"));
+ }
+ if pages > MAX_PAGES {
+ return Err(anyhow!(
+ "PDF declares {pages} pages, above the {MAX_PAGES}-page safety limit; split it before indexing"
+ ));
+ }
+
+ let title = path
+ .file_stem()
+ .map(|s| s.to_string_lossy().into_owned())
+ .filter(|s| !s.trim().is_empty())
+ .unwrap_or_else(|| "untitled".to_string());
+ let mut records = Vec::new();
+ let mut extracted_bytes = 0usize;
+ let mut failed_pages = Vec::new();
+ let mut omitted_pages = Vec::new();
+ let mut pages_with_text = 0usize;
+
+ for page_index in 0..pages {
+ let page = match doc.extract_text(page_index) {
+ Ok(text) => text,
+ Err(error) => {
+ failed_pages.push(format!("{} ({error})", page_index + 1));
+ continue;
}
- c if c.is_ascii_alphabetic() || c == b'\'' || c == b'"' || c == b'*' => {
- let start = i;
- while i < content.len()
- && (content[i].is_ascii_alphanumeric()
- || matches!(content[i], b'\'' | b'"' | b'*'))
- {
- i += 1;
- }
- let op = &content[start..i];
- match op {
- b"Tj" | b"TJ" | b"'" | b"\"" => {
- for s in pending.drain(..) {
- if !line.is_empty() && !line.ends_with(' ') {
- line.push(' ');
- }
- line.push_str(&s);
- }
- if op == b"'" || op == b"\"" {
- flush_line(&mut line, out);
- }
- }
- b"Td" | b"TD" | b"T*" | b"ET" => {
- pending.clear();
- flush_line(&mut line, out);
- }
- b"BT" => {
- pending.clear();
- }
- _ => {
- pending.clear();
- }
- }
+ };
+ let page = normalize_legacy_symbol_glyphs(&page);
+ let body = page.trim();
+ if body.is_empty() {
+ omitted_pages.push(page_index + 1);
+ continue;
+ }
+ if body.len() > MAX_PAGE_TEXT {
+ return Err(anyhow!(
+ "PDF page {} produced more than 16 MiB of text; split or repair the document before indexing",
+ page_index + 1
+ ));
+ }
+ if body.chars().filter(|ch| !ch.is_whitespace()).count() < 8 {
+ omitted_pages.push(page_index + 1);
+ continue;
+ }
+ validate_text_quality(body).with_context(|| {
+ format!(
+ "PDF page {} produced untrusted text; refusing to index possible binary/font garbage",
+ page_index + 1
+ )
+ })?;
+ extracted_bytes = extracted_bytes.saturating_add(body.len());
+ pages_with_text += 1;
+ if extracted_bytes > MAX_EXTRACTED_TEXT {
+ return Err(anyhow!(
+ "PDF extraction exceeded the 64 MiB text safety limit; split the document before indexing"
+ ));
+ }
+
+ for (section, text) in split_sections(body).into_iter().enumerate() {
+ let mut fields = Map::new();
+ fields.insert("title".into(), Value::String(title.clone()));
+ fields.insert(
+ "page".into(),
+ Value::Number(((page_index + 1) as u64).into()),
+ );
+ if section > 0 {
+ fields.insert("section".into(), Value::Number((section as u64).into()));
}
- _ => i += 1,
+ fields.insert("body".into(), Value::String(text));
+ records.push(RawRecord {
+ fields,
+ locator: format!("p{}-s{}", page_index + 1, section),
+ group: None,
+ });
+ }
+ }
+
+ if records.is_empty() {
+ if failed_pages.is_empty() {
+ return Err(anyhow!(
+ "PDF has no extractable text; it may be an image-only scan. Run OCR first, then autoindex the searchable PDF"
+ ));
}
- if out.len() > 32 << 20 {
- break; // hard safety cap
+ return Err(anyhow!(
+ "PDF text extraction failed on every non-empty page ({}); verify/decrypt the PDF or run OCR before indexing",
+ failed_pages.iter().take(3).cloned().collect::<Vec<_>>().join(", ")
+ ));
+ }
+ if !failed_pages.is_empty() {
+ return Err(anyhow!(
+ "PDF extraction failed on {}/{} pages (first failures: {}); refusing a partial index. Repair/decrypt the PDF or run OCR first",
+ failed_pages.len(),
+ pages,
+ failed_pages.iter().take(3).cloned().collect::<Vec<_>>().join(", ")
+ ));
+ }
+ for record in &mut records {
+ record.fields.insert(
+ "pdf_pages_total".into(),
+ Value::Number((pages as u64).into()),
+ );
+ record.fields.insert(
+ "pdf_pages_with_text".into(),
+ Value::Number((pages_with_text as u64).into()),
+ );
+ record.fields.insert(
+ "pdf_pages_omitted".into(),
+ Value::Number((omitted_pages.len() as u64).into()),
+ );
+ if !omitted_pages.is_empty() {
+ record.fields.insert(
+ "pdf_ocr_warning".into(),
+ Value::String(format!(
+ "{} page(s) had no usable text (first: {}); intentional blanks or scanned pages may need OCR",
+ omitted_pages.len(),
+ omitted_pages
+ .iter()
+ .take(8)
+ .map(usize::to_string)
+ .collect::<Vec<_>>()
+ .join(", ")
+ )),
+ );
}
}
- flush_line(&mut line, out);
+ Ok(records)
}
-fn parse_literal_string(b: &[u8], open: usize) -> (String, usize) {
- let mut s = String::new();
- let mut i = open + 1;
- let mut depth = 1usize;
- while i < b.len() {
- match b[i] {
- b'\\' if i + 1 < b.len() => {
- let c = b[i + 1];
- match c {
- b'n' => s.push('\n'),
- b'r' => s.push('\r'),
- b't' => s.push('\t'),
- b'(' => s.push('('),
- b')' => s.push(')'),
- b'\\' => s.push('\\'),
- b'0'..=b'7' => {
- // up to 3 octal digits
- let mut v = 0u32;
- let mut n = 0;
- while n < 3 && i + 1 + n < b.len() && (b'0'..=b'7').contains(&b[i + 1 + n])
- {
- v = v * 8 + (b[i + 1 + n] - b'0') as u32;
- n += 1;
- }
- if let Some(ch) = char::from_u32(v) {
- s.push(ch);
- }
- i += n + 1;
- continue;
- }
- b'\n' => {} // line continuation
- other => s.push(other as char),
- }
- i += 2;
- }
- b'(' => {
- depth += 1;
- s.push('(');
- i += 1;
- }
- b')' => {
- depth -= 1;
- if depth == 0 {
- return (s, i + 1);
- }
- s.push(')');
- i += 1;
- }
- c => {
- s.push(c as char); // latin-1 passthrough
- i += 1;
- }
- }
- if s.len() > 1 << 20 {
- break;
+struct WorkerGate {
+ state: Mutex<(usize, usize)>,
+ ready: Condvar,
+}
+
+impl WorkerGate {
+ fn set_limit(&self, limit: usize) {
+ self.state.lock().expect("PDF worker gate poisoned").1 = limit;
+ self.ready.notify_all();
+ }
+
+ fn acquire(&self) -> WorkerPermit<'_> {
+ let mut state = self.state.lock().expect("PDF worker gate poisoned");
+ while state.0 >= state.1 {
+ state = self.ready.wait(state).expect("PDF worker gate poisoned");
}
+ state.0 += 1;
+ WorkerPermit(self)
}
- (s, i)
}
-fn parse_hex_string(b: &[u8], open: usize) -> (String, usize) {
- let mut i = open + 1;
- let mut nibbles: Vec<u8> = Vec::new();
- while i < b.len() && b[i] != b'>' {
- let c = b[i];
- if c.is_ascii_hexdigit() {
- nibbles.push(c);
+struct WorkerPermit<'a>(&'a WorkerGate);
+
+impl Drop for WorkerPermit<'_> {
+ fn drop(&mut self) {
+ let mut state = self.0.state.lock().expect("PDF worker gate poisoned");
+ state.0 -= 1;
+ self.0.ready.notify_one();
+ }
+}
+
+fn worker_gate() -> &'static WorkerGate {
+ static GATE: OnceLock<WorkerGate> = OnceLock::new();
+ GATE.get_or_init(|| WorkerGate {
+ state: Mutex::new((
+ 0,
+ std::thread::available_parallelism()
+ .map(|n| n.get())
+ .unwrap_or(4)
+ .min(4),
+ )),
+ ready: Condvar::new(),
+ })
+}
+
+fn normalize_legacy_symbol_glyphs(text: &str) -> String {
+ text.chars()
+ .map(|ch| match ch {
+ '\u{f0b7}' => '•',
+ '\u{f02d}' => '-',
+ '\u{f078}' | '\u{f0a8}' | '\u{f06f}' | '\u{f052}' | '\u{f0a3}' | '\u{f020}' => '□',
+ '\u{f0d2}' | '\u{f0d4}' | '\u{f0be}' => ' ',
+ other => other,
+ })
+ .collect()
+}
+
+fn validate_text_quality(text: &str) -> Result<()> {
+ // Fail closed rather than embed broken font codes. On the pinned
+ // 368-document FinanceBench PDF corpus this 0.5% threshold accepted 367
+ // documents and rejected one; keep that false-rejection cost visible when
+ // recalibrating against broader labeled corpora.
+ let mut visible = 0usize;
+ let mut semantic = 0usize;
+ let mut suspicious = 0usize;
+ for ch in text.chars() {
+ if ch.is_whitespace() {
+ continue;
}
- i += 1;
- if nibbles.len() > 1 << 20 {
- break;
+ visible += 1;
+ if ch.is_alphanumeric() {
+ semantic += 1;
+ }
+ if ch == '\u{fffd}'
+ || ch.is_control()
+ || ('\u{e000}'..='\u{f8ff}').contains(&ch)
+ || ('\u{f0000}'..='\u{ffffd}').contains(&ch)
+ || ('\u{100000}'..='\u{10fffd}').contains(&ch)
+ {
+ suspicious += 1;
}
}
- if nibbles.len() % 2 == 1 {
- nibbles.push(b'0');
+ if visible < 8 {
+ return Err(anyhow!("too little visible text"));
}
- let mut s = String::new();
- for pair in nibbles.chunks(2) {
- let hi = (pair[0] as char).to_digit(16).unwrap_or(0);
- let lo = (pair[1] as char).to_digit(16).unwrap_or(0);
- if let Some(ch) = char::from_u32(hi * 16 + lo) {
- s.push(ch);
- }
+ if suspicious * 200 > visible {
+ return Err(anyhow!(
+ "more than 0.5% of visible characters are controls, replacement glyphs, or private-use codes"
+ ));
}
- (s, i + 1)
+ if semantic * 20 < visible {
+ return Err(anyhow!(
+ "fewer than 5% of visible characters are letters or numbers"
+ ));
+ }
+ Ok(())
}
-fn doc_title(bytes: &[u8]) -> Option<String> {
- let p = memchr::memmem::find(bytes, b"/Title")?;
- let after = &bytes[p + 6..(p + 4096).min(bytes.len())];
- let start = after.iter().position(|&c| !c.is_ascii_whitespace())?;
- match after[start] {
- b'(' => {
- let (s, _) = parse_literal_string(after, start);
- let t = s.trim();
- if t.is_empty() {
- None
- } else {
- Some(t.to_string())
- }
+#[cfg(test)]
+mod tests {
+ use super::{extract_in_process, normalize_legacy_symbol_glyphs, validate_text_quality};
+ use crate::infer::{infer_fields, FieldAcc};
+ use serde_json::Value;
+ use std::collections::HashMap;
+
+ fn write_prose_pdf(path: &std::path::Path) {
+ let prose = "Quarterly revenue increased because subscription demand remained strong. \
+ Operating income improved as cloud infrastructure costs declined. Management expects \
+ cash flow to support continued investment throughout the next fiscal year.";
+ let stream = format!("BT /F1 12 Tf 72 720 Td ({prose}) Tj ET");
+ let objects = [
+ "<< /Type /Catalog /Pages 2 0 R >>".to_string(),
+ "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(),
+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>".to_string(),
+ format!("<< /Length {} >>\nstream\n{stream}\nendstream", stream.len()),
+ "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(),
+ ];
+ let mut pdf = b"%PDF-1.4\n".to_vec();
+ let mut offsets = vec![0usize];
+ for (index, object) in objects.iter().enumerate() {
+ offsets.push(pdf.len());
+ pdf.extend_from_slice(format!("{} 0 obj\n{}\nendobj\n", index + 1, object).as_bytes());
}
- b'<' => {
- let (s, _) = parse_hex_string(after, start);
- let t = s.trim();
- if t.is_empty() {
- None
- } else {
- Some(t.to_string())
+ let xref = pdf.len();
+ pdf.extend_from_slice(format!("xref\n0 {}\n", objects.len() + 1).as_bytes());
+ pdf.extend_from_slice(b"0000000000 65535 f \n");
+ for offset in offsets.iter().skip(1) {
+ pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
+ }
+ pdf.extend_from_slice(
+ format!(
+ "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",
+ objects.len() + 1
+ )
+ .as_bytes(),
+ );
+ std::fs::write(path, pdf).unwrap();
+ }
+
+ #[test]
+ fn quality_gate_accepts_financial_and_international_text() {
+ validate_text_quality(
+ "Consolidated revenue — Q4 2025\n€1,234.5 million (增长率 12%; прибыль 8%).",
+ )
+ .unwrap();
+ }
+
+ #[test]
+ fn quality_gate_rejects_binary_and_font_garbage() {
+ let garbage = "abc\u{0000}\u{0001}\u{fffd}\u{e123}\u{0002}xyz";
+ assert!(validate_text_quality(garbage).is_err());
+ }
+
+ #[test]
+ fn quality_gate_rejects_operator_noise() {
+ assert!(validate_text_quality("//// <<>> [] () -- === +++").is_err());
+ }
+
+ #[test]
+ fn known_legacy_symbol_glyphs_are_normalized_but_unknown_pua_is_rejected() {
+ let normalized = normalize_legacy_symbol_glyphs(
+ "Yes \u{f078} No \u{f0a8}\n\u{f0b7} Revenue\nrisk\u{f02d}adjusted",
+ );
+ assert_eq!(normalized, "Yes □ No □\n• Revenue\nrisk-adjusted");
+ validate_text_quality(&normalized).unwrap();
+ assert!(validate_text_quality("Revenue \u{e123} remains unknown").is_err());
+ }
+
+ #[test]
+ fn reproducible_pdf_extracts_pages_and_elects_semantic_text() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("quarterly-report.pdf");
+ write_prose_pdf(&path);
+
+ let first = extract_in_process(&path).unwrap();
+ let second = extract_in_process(&path).unwrap();
+ assert_eq!(
+ serde_json::to_value(&first).unwrap(),
+ serde_json::to_value(&second).unwrap()
+ );
+ assert!(!first.is_empty());
+ assert_eq!(first[0].fields["page"], 1);
+ assert!(first[0].locator.starts_with("p1-s"));
+ assert!(first[0].fields["body"]
+ .as_str()
+ .unwrap()
+ .contains("Quarterly revenue increased"));
+
+ let mut fields: HashMap<String, FieldAcc> = HashMap::new();
+ for record in &first {
+ for (name, value) in &record.fields {
+ fields.entry(name.clone()).or_default().add(value);
}
}
- _ => None,
+ let specs = infer_fields(&fields, first.len() as u64, false);
+ let body = specs.iter().find(|spec| spec.name == "body").unwrap();
+ assert_eq!(body.es_type, "semantic_text");
+ assert!(matches!(
+ first[0].fields.get("body"),
+ Some(Value::String(_))
+ ));
}
}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 643c6f6d7..9e1103ce4 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -189,6 +189,8 @@ fn build_mapping(specs: &[infer::FieldSpec]) -> Value {
// ─── the main run ────────────────────────────────────────────────────────
fn run_index(cfg: IndexCfg) -> Result<i32> {
+ extract::pdf::configure_workers(cfg.pdf_workers);
+ extract::pdf::configure_timeout(cfg.pdf_timeout_secs);
let t0 = Instant::now();
let es = Es::new(&cfg.url, cfg.api_key.clone())?;
es.ping()?;
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index d68995fd8..1f35e6d60 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -1137,6 +1137,10 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {
/// Override with `TOKIO_WORKER_THREADS` (operators on very small or very large
/// hosts, or wanting the stock `ncpus` behaviour, can set it explicitly).
fn main() -> Result<()> {
+ // Dispatch before creating Tokio's pool so each PDF worker stays small.
+ if matches!(std::env::args().nth(1).as_deref(), Some("__extract-pdf")) {
+ std::process::exit(xerj_autoindex::extract::pdf::run_worker_cli());
+ }
let cores = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(8);
diff --git a/engine/crates/xerj-server/tests/pdf_worker.rs b/engine/crates/xerj-server/tests/pdf_worker.rs
new file mode 100644
index 000000000..5c53085ff
--- /dev/null
+++ b/engine/crates/xerj-server/tests/pdf_worker.rs
@@ -0,0 +1,236 @@
+use serde_json::Value;
+use std::fmt::Write as _;
+use std::fs;
+use std::process::Command;
+use std::time::{Duration, Instant};
+
+fn write_prose_pdf(path: &std::path::Path) {
+ let prose = "Quarterly financial results show revenue increased twelve percent while operating margin improved. International customers and subscription renewals drove durable growth across the reporting period.";
+ let stream = format!("BT /F1 12 Tf 72 720 Td ({prose}) Tj ET");
+ let objects = [
+ "<< /Type /Catalog /Pages 2 0 R >>".to_string(),
+ "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(),
+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>".to_string(),
+ format!("<< /Length {} >>\nstream\n{stream}\nendstream", stream.len()),
+ "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(),
+ ];
+
+ let mut pdf = String::from("%PDF-1.4\n");
+ let mut offsets = Vec::new();
+ for (index, object) in objects.iter().enumerate() {
+ offsets.push(pdf.len());
+ writeln!(pdf, "{} 0 obj\n{}\nendobj", index + 1, object).unwrap();
+ }
+ let xref = pdf.len();
+ write!(pdf, "xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).unwrap();
+ for offset in offsets {
+ writeln!(pdf, "{offset:010} 00000 n ").unwrap();
+ }
+ write!(
+ pdf,
+ "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",
+ objects.len() + 1
+ )
+ .unwrap();
+ fs::write(path, pdf).unwrap();
+}
+
+fn write_mixed_text_blank_pdf(path: &std::path::Path) {
+ let prose =
+ "Financial statement prose appears on this page while the next page is intentionally blank.";
+ let stream = format!("BT /F1 12 Tf 72 720 Td ({prose}) Tj ET");
+ let objects = [
+ "<< /Type /Catalog /Pages 2 0 R >>".to_string(),
+ "<< /Type /Pages /Kids [3 0 R 6 0 R] /Count 2 >>".to_string(),
+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>".to_string(),
+ format!("<< /Length {} >>\nstream\n{stream}\nendstream", stream.len()),
+ "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(),
+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>".to_string(),
+ ];
+ let mut pdf = String::from("%PDF-1.4\n");
+ let mut offsets = Vec::new();
+ for (index, object) in objects.iter().enumerate() {
+ offsets.push(pdf.len());
+ writeln!(pdf, "{} 0 obj\n{}\nendobj", index + 1, object).unwrap();
+ }
+ let xref = pdf.len();
+ write!(pdf, "xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).unwrap();
+ for offset in offsets {
+ writeln!(pdf, "{offset:010} 00000 n ").unwrap();
+ }
+ write!(
+ pdf,
+ "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",
+ objects.len() + 1
+ )
+ .unwrap();
+ fs::write(path, pdf).unwrap();
+}
+
+fn run_worker(path: &std::path::Path) -> std::process::Output {
+ Command::new(env!("CARGO_BIN_EXE_xerj"))
+ .arg("__extract-pdf")
+ .arg(path)
+ .output()
+ .expect("start PDF worker")
+}
+
+#[test]
+fn pdf_worker_is_fresh_process_deterministic_and_preserves_page_provenance() {
+ let temp = tempfile::tempdir().unwrap();
+ let pdf = temp.path().join("quarterly-report.pdf");
+ write_prose_pdf(&pdf);
+
+ let first = run_worker(&pdf);
+ let second = run_worker(&pdf);
+ assert!(
+ first.status.success(),
+ "{}",
+ String::from_utf8_lossy(&first.stderr)
+ );
+ assert!(
+ second.status.success(),
+ "{}",
+ String::from_utf8_lossy(&second.stderr)
+ );
+ assert_eq!(first.stdout, second.stdout);
+
+ let response: Value = serde_json::from_slice(&first.stdout).unwrap();
+ assert_eq!(response["extractor"], "xerj-autoindex/1.0.0-rc.4");
+ assert_eq!(response["parser"], "pdf_oxide/0.3.75");
+ assert!(response["containment"]
+ .as_str()
+ .unwrap()
+ .contains("not a security sandbox"));
+ let record = &response["records"][0];
+ assert_eq!(record["fields"]["page"], 1);
+ assert_eq!(record["locator"], "p1-s0");
+ assert!(record["fields"]["body"]
+ .as_str()
+ .unwrap()
+ .contains("operating margin improved"));
+ assert_eq!(record["fields"]["pdf_pages_total"], 1);
+ assert_eq!(record["fields"]["pdf_pages_with_text"], 1);
+ assert_eq!(record["fields"]["pdf_pages_omitted"], 0);
+}
+
+#[test]
+fn corrupt_pdf_failure_is_actionable_and_emits_no_records() {
+ let temp = tempfile::tempdir().unwrap();
+ let pdf = temp.path().join("corrupt.pdf");
+ fs::write(&pdf, b"%PDF-1.4\nnot a PDF").unwrap();
+
+ let output = run_worker(&pdf);
+ assert!(!output.status.success());
+ assert!(output.stdout.is_empty());
+ let stderr = String::from_utf8_lossy(&output.stderr);
+ assert!(stderr.contains("damaged"), "{stderr}");
+ assert!(stderr.contains("password"), "{stderr}");
+ assert!(stderr.contains("OCR"), "{stderr}");
+}
+
+#[test]
+fn mixed_text_and_empty_pages_disclose_ocr_coverage_without_dropping_good_text() {
+ let temp = tempfile::tempdir().unwrap();
+ let pdf = temp.path().join("mixed.pdf");
+ write_mixed_text_blank_pdf(&pdf);
+ let output = run_worker(&pdf);
+ assert!(
+ output.status.success(),
+ "{}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let response: Value = serde_json::from_slice(&output.stdout).unwrap();
+ let fields = &response["records"][0]["fields"];
+ assert_eq!(fields["pdf_pages_total"], 2);
+ assert_eq!(fields["pdf_pages_with_text"], 1);
+ assert_eq!(fields["pdf_pages_omitted"], 1);
+ assert!(fields["pdf_ocr_warning"]
+ .as_str()
+ .unwrap()
+ .contains("may need OCR"));
+}
+
+#[cfg(unix)]
+#[test]
+fn parent_drains_overflow_and_timeout_kills_worker_process_group() {
+ use std::os::unix::fs::PermissionsExt;
+
+ let temp = tempfile::tempdir().unwrap();
+ let input = temp.path().join("input.pdf");
+ fs::write(&input, b"unused by fake worker").unwrap();
+
+ let overflow = temp.path().join("overflow-worker");
+ fs::write(
+ &overflow,
+ b"#!/bin/sh\nhead -c 34603008 /dev/zero\nexit 0\n",
+ )
+ .unwrap();
+ fs::set_permissions(&overflow, fs::Permissions::from_mode(0o700)).unwrap();
+ std::env::set_var("XERJ_PDF_WORKER_BIN", &overflow);
+ let started = Instant::now();
+ let mut sink = |_record| true;
+ let error = xerj_autoindex::extract::pdf::extract(&input, &mut sink).unwrap_err();
+ assert!(started.elapsed() < Duration::from_secs(10));
+ assert!(error.to_string().contains("exceeded"), "{error:#}");
+
+ let stderr_overflow = temp.path().join("stderr-overflow-worker");
+ fs::write(
+ &stderr_overflow,
+ b"#!/bin/sh\nhead -c 131072 /dev/zero >&2\nexit 1\n",
+ )
+ .unwrap();
+ fs::set_permissions(&stderr_overflow, fs::Permissions::from_mode(0o700)).unwrap();
+ std::env::set_var("XERJ_PDF_WORKER_BIN", &stderr_overflow);
+ let started = Instant::now();
+ let error = xerj_autoindex::extract::pdf::extract(&input, &mut sink).unwrap_err();
+ assert!(started.elapsed() < Duration::from_secs(10));
+ assert!(
+ error.to_string().contains("error output exceeded"),
+ "{error:#}"
+ );
+
+ let timeout = temp.path().join("timeout-worker");
+ fs::write(
+ &timeout,
+ b"#!/bin/sh\nsleep 60 &\necho $! > \"$2.pid\"\nwait\n",
+ )
+ .unwrap();
+ fs::set_permissions(&timeout, fs::Permissions::from_mode(0o700)).unwrap();
+ std::env::set_var("XERJ_PDF_WORKER_BIN", &timeout);
+ xerj_autoindex::extract::pdf::configure_timeout(1);
+ let error = xerj_autoindex::extract::pdf::extract(&input, &mut sink).unwrap_err();
+ xerj_autoindex::extract::pdf::configure_timeout(120);
+ std::env::remove_var("XERJ_PDF_WORKER_BIN");
+ assert!(error.to_string().contains("timed out"), "{error:#}");
+
+ let descendant = fs::read_to_string(format!("{}.pid", input.display())).unwrap();
+ let proc_entry = std::path::PathBuf::from("/proc").join(descendant.trim());
+ for _ in 0..50 {
+ if !proc_entry.exists() {
+ return;
+ }
+ std::thread::sleep(Duration::from_millis(20));
+ }
+ panic!("worker descendant {} survived timeout", descendant.trim());
+}
+
+#[cfg(unix)]
+#[test]
+fn pdf_worker_accepts_non_utf8_unix_paths() {
+ use std::os::unix::ffi::OsStringExt;
+
+ let temp = tempfile::tempdir().unwrap();
+ let name = std::ffi::OsString::from_vec(b"quarterly-\xff.pdf".to_vec());
+ let pdf = temp.path().join(name);
+ write_prose_pdf(&pdf);
+
+ let output = run_worker(&pdf);
+ assert!(
+ output.status.success(),
+ "{}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let response: Value = serde_json::from_slice(&output.stdout).unwrap();
+ assert_eq!(response["records"][0]["fields"]["page"], 1);
+}