AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): parse PDFs in isolated Unicode workers
buger19f0dd404138
diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex c50e18b30..54a160078 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -8,6 +8,17 @@ version = "2.0.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"+[[package]]+name = "aes"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138"+dependencies = [+ "cipher",+ "cpubits",+ "cpufeatures 0.3.0",+]+[[package]]name = "ahash"version = "0.8.12"@@ -31,6 +42,21 @@ dependencies = ["memchr",]+[[package]]+name = "alloc-no-stdlib"+version = "2.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"++[[package]]+name = "alloc-stdlib"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195"+dependencies = [+ "alloc-no-stdlib",+]+[[package]]name = "allocator-api2"version = "0.2.21"@@ -126,6 +152,12 @@ dependencies = ["rustversion",]+[[package]]+name = "arrayvec"+version = "0.7.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"+[[package]]name = "asn1-rs"version = "0.6.2"@@ -135,7 +167,7 @@ dependencies = ["asn1-rs-derive","asn1-rs-impl","displaydoc",- "nom",+ "nom 7.1.3","num-traits","rusticata-macros","thiserror 1.0.69",@@ -198,6 +230,16 @@ dependencies = ["syn",]+[[package]]+name = "atoi_simd"+version = "0.18.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44"+dependencies = [+ "debug_unsafe",+ "rustversion",+]+[[package]]name = "atomic-waker"version = "1.1.2"@@ -373,6 +415,12 @@ version = "0.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"+[[package]]+name = "bitflags"+version = "1.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"+[[package]]name = "bitflags"version = "2.11.1"@@ -397,6 +445,45 @@ dependencies = ["generic-array",]+[[package]]+name = "block-buffer"+version = "0.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"+dependencies = [+ "hybrid-array",+]++[[package]]+name = "block-padding"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b"+dependencies = [+ "hybrid-array",+]++[[package]]+name = "brotli"+version = "8.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3"+dependencies = [+ "alloc-no-stdlib",+ "alloc-stdlib",+ "brotli-decompressor",+]++[[package]]+name = "brotli-decompressor"+version = "5.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583"+dependencies = [+ "alloc-no-stdlib",+ "alloc-stdlib",+]+[[package]]name = "bumpalo"version = "3.20.2"@@ -429,6 +516,12 @@ version = "1.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"+[[package]]+name = "byteorder-lite"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"+[[package]]name = "bytes"version = "1.11.1"@@ -508,6 +601,15 @@ dependencies = ["rustversion",]+[[package]]+name = "cbc"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896"+dependencies = [+ "cipher",+]+[[package]]name = "cc"version = "1.2.60"@@ -554,7 +656,7 @@ checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e"dependencies = ["chrono","chrono-tz-build",- "phf",+ "phf 0.11.3",][[package]]@@ -564,7 +666,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f"dependencies = ["parse-zoneinfo",- "phf",+ "phf 0.11.3","phf_codegen",]@@ -595,6 +697,16 @@ dependencies = ["half",]+[[package]]+name = "cipher"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c"+dependencies = [+ "crypto-common 0.2.2",+ "inout",+]+[[package]]name = "clap"version = "4.6.1"@@ -679,6 +791,12 @@ dependencies = ["windows-sys 0.59.0",]+[[package]]+name = "const-oid"+version = "0.10.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"+[[package]]name = "cookie"version = "0.18.1"@@ -716,6 +834,12 @@ version = "0.8.7"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"+[[package]]+name = "cpubits"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae"+[[package]]name = "cpufeatures"version = "0.2.17"@@ -725,6 +849,15 @@ dependencies = ["libc",]+[[package]]+name = "cpufeatures"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"+dependencies = [+ "libc",+]+[[package]]name = "crc32fast"version = "1.5.0"@@ -842,6 +975,15 @@ dependencies = ["typenum",]+[[package]]+name = "crypto-common"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"+dependencies = [+ "hybrid-array",+]+[[package]]name = "csv"version = "1.4.0"@@ -927,6 +1069,43 @@ version = "2.11.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"+[[package]]+name = "debug_unsafe"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2"++[[package]]+name = "defmt"+version = "1.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1"+dependencies = [+ "bitflags 1.3.2",+ "defmt-macros",+]++[[package]]+name = "defmt-macros"+version = "1.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8"+dependencies = [+ "defmt-parser",+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "defmt-parser"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e"+dependencies = [+ "thiserror 2.0.18",+]+[[package]]name = "der-parser"version = "9.0.0"@@ -935,7 +1114,7 @@ checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553"dependencies = ["asn1-rs","displaydoc",- "nom",+ "nom 7.1.3","num-bigint","num-traits","rusticata-macros",@@ -999,11 +1178,22 @@ version = "0.10.7"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"dependencies = [- "block-buffer",- "crypto-common",+ "block-buffer 0.10.4",+ "crypto-common 0.1.7","subtle",]+[[package]]+name = "digest"+version = "0.11.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"+dependencies = [+ "block-buffer 0.12.1",+ "const-oid",+ "crypto-common 0.2.2",+]+[[package]]name = "dirs"version = "6.0.0"@@ -1085,6 +1275,29 @@ dependencies = ["syn",]+[[package]]+name = "env_filter"+version = "2.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217"+dependencies = [+ "log",+ "regex",+]++[[package]]+name = "env_logger"+version = "0.11.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6"+dependencies = [+ "anstream",+ "anstyle",+ "env_filter",+ "jiff",+ "log",+]+[[package]]name = "equivalent"version = "1.0.2"@@ -1121,6 +1334,15 @@ version = "0.1.10"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"+[[package]]+name = "euclid"+version = "0.22.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06"+dependencies = [+ "num-traits",+]+[[package]]name = "fallible-iterator"version = "0.3.0"@@ -1155,12 +1377,39 @@ dependencies = ["regex-syntax",]+[[package]]+name = "fast-float2"+version = "0.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"+[[package]]name = "fastrand"version = "2.4.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"+[[package]]+name = "fax"+version = "0.2.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a"++[[package]]+name = "fax"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "70ed8646f3993cad0585c6669ffba6f7e5440a0bfb0a1f16600cae33ce7cf307"++[[package]]+name = "fdeflate"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"+dependencies = [+ "simd-adler32",+]+[[package]]name = "find-msvc-tools"version = "0.1.9"@@ -1181,6 +1430,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"dependencies = ["crc32fast","miniz_oxide",+ "zlib-rs",][[package]]@@ -1222,6 +1472,15 @@ version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"+[[package]]+name = "font-types"+version = "0.11.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5b38ad915f6dadd993ced50848a8291a543bd41ca62bc10740d5e64e2ab4cfd7"+dependencies = [+ "bytemuck",+]+[[package]]name = "foreign-types"version = "0.3.2"@@ -1513,10 +1772,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"dependencies = ["cfg-if",+ "js-sys","libc","r-efi 6.0.0","wasip2","wasip3",+ "wasm-bindgen",][[package]]@@ -1525,6 +1786,12 @@ version = "0.3.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"+[[package]]+name = "grid"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b40ca9252762c466af32d0b1002e91e4e1bc5398f77455e55474deb466355ff5"+[[package]]name = "h2"version = "0.4.13"@@ -1665,7 +1932,7 @@ version = "0.12.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"dependencies = [- "digest",+ "digest 0.10.7",][[package]]@@ -1719,6 +1986,15 @@ version = "1.0.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"+[[package]]+name = "hybrid-array"+version = "0.4.13"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c"+dependencies = [+ "typenum",+]+[[package]]name = "hyper"version = "1.9.0"@@ -1950,6 +2226,22 @@ dependencies = ["icu_properties",]+[[package]]+name = "image"+version = "0.25.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"+dependencies = [+ "bytemuck",+ "byteorder-lite",+ "moxcms",+ "num-traits",+ "png",+ "tiff",+ "zune-core",+ "zune-jpeg",+]+[[package]]name = "indexmap"version = "1.9.3"@@ -1985,6 +2277,16 @@ dependencies = ["web-time",]+[[package]]+name = "inout"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7"+dependencies = [+ "block-padding",+ "hybrid-array",+]+[[package]]name = "ipnet"version = "2.12.0"@@ -2042,6 +2344,42 @@ version = "1.0.18"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"+[[package]]+name = "jiff"+version = "0.2.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e184d09547b80eb7e20d141ba2fb1fbac843ca53f4cf1b31210adc4c1adc6e16"+dependencies = [+ "defmt",+ "jiff-core",+ "jiff-static",+ "log",+ "portable-atomic",+ "portable-atomic-util",+ "serde_core",+]++[[package]]+name = "jiff-core"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09"+dependencies = [+ "defmt",+]++[[package]]+name = "jiff-static"+version = "0.2.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "323da076b7a6faf914dc677cb05a4b907742ff7375c8322c9e7f5061e5e0e9de"+dependencies = [+ "jiff-core",+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "jobserver"version = "0.1.34"@@ -2052,6 +2390,15 @@ dependencies = ["libc",]+[[package]]+name = "jpeg-decoder"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"+dependencies = [+ "rayon",+]+[[package]]name = "js-sys"version = "0.3.95"@@ -2064,6 +2411,18 @@ dependencies = ["wasm-bindgen",]+[[package]]+name = "kurbo"+version = "0.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2"+dependencies = [+ "arrayvec",+ "euclid",+ "polycool",+ "smallvec",+]+[[package]]name = "lazy_static"version = "1.5.0"@@ -2230,6 +2589,16 @@ dependencies = ["rawpointer",]+[[package]]+name = "md-5"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98"+dependencies = [+ "cfg-if",+ "digest 0.11.3",+]+[[package]]name = "memchr"version = "2.8.0"@@ -2323,6 +2692,16 @@ dependencies = ["syn",]+[[package]]+name = "moxcms"+version = "0.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"+dependencies = [+ "num-traits",+ "pxfm",+]+[[package]]name = "multer"version = "3.1.0"@@ -2388,6 +2767,15 @@ dependencies = ["minimal-lexical",]+[[package]]+name = "nom"+version = "8.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"+dependencies = [+ "memchr",+]+[[package]]name = "nu-ansi-term"version = "0.50.3"@@ -2458,6 +2846,24 @@ version = "0.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"+[[package]]+name = "office_oxide"+version = "0.1.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "317c254cebe5b87c74a0fbd84e889d51ec0c4db885fa5aca98700ad85fd9ec5f"+dependencies = [+ "atoi_simd",+ "encoding_rs",+ "fast-float2",+ "libc",+ "log",+ "quick-xml 0.41.0",+ "serde",+ "serde_json",+ "thiserror 2.0.18",+ "zip 8.6.0",+]+[[package]]name = "oid-registry"version = "0.7.1"@@ -2491,7 +2897,7 @@ version = "0.10.77"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bfe4646e360ec77dff7dde40ed3d6c5fee52d156ef4a62f53973d38294dad87f"dependencies = [- "bitflags",+ "bitflags 2.11.1","cfg-if","foreign-types","libc",@@ -2613,6 +3019,52 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec"+[[package]]+name = "pdf_oxide"+version = "0.3.75"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2e00ca5f52780bb764da7fc1783c0c6f549fcf960d69734c8d7cf7996bdbb49d"+dependencies = [+ "aes",+ "base64 0.22.1",+ "bitflags 2.11.1",+ "brotli",+ "byteorder",+ "bytes",+ "cbc",+ "chrono",+ "encoding_rs",+ "env_logger",+ "fax 0.3.0",+ "flate2",+ "getrandom 0.4.2",+ "image",+ "jpeg-decoder",+ "libc",+ "log",+ "md-5",+ "memchr",+ "nom 8.0.0",+ "office_oxide",+ "phf 0.14.0",+ "quick-xml 0.41.0",+ "regex",+ "serde",+ "serde_json",+ "sha2 0.11.0",+ "smallvec",+ "stringprep",+ "subsetter",+ "taffy",+ "thiserror 2.0.18",+ "ttf-parser",+ "unicode-bidi",+ "unicode-linebreak",+ "unicode-normalization",+ "uuid",+ "weezl 0.2.1",+]+[[package]]name = "pem"version = "3.0.6"@@ -2645,7 +3097,18 @@ version = "0.11.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"dependencies = [- "phf_shared",+ "phf_shared 0.11.3",+]++[[package]]+name = "phf"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"+dependencies = [+ "phf_macros",+ "phf_shared 0.14.0",+ "serde",][[package]]@@ -2654,8 +3117,8 @@ version = "0.11.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"dependencies = [- "phf_generator",- "phf_shared",+ "phf_generator 0.11.3",+ "phf_shared 0.11.3",][[package]]@@ -2664,10 +3127,33 @@ version = "0.11.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"dependencies = [- "phf_shared",+ "phf_shared 0.11.3","rand 0.8.5",]+[[package]]+name = "phf_generator"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"+dependencies = [+ "fastrand",+ "phf_shared 0.14.0",+]++[[package]]+name = "phf_macros"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085"+dependencies = [+ "phf_generator 0.14.0",+ "phf_shared 0.14.0",+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "phf_shared"version = "0.11.3"@@ -2677,6 +3163,15 @@ dependencies = ["siphasher",]+[[package]]+name = "phf_shared"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89"+dependencies = [+ "siphasher",+]+[[package]]name = "pin-project"version = "1.1.11"@@ -2737,6 +3232,28 @@ dependencies = ["plotters-backend",]+[[package]]+name = "png"+version = "0.18.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"+dependencies = [+ "bitflags 2.11.1",+ "crc32fast",+ "fdeflate",+ "flate2",+ "miniz_oxide",+]++[[package]]+name = "polycool"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6"+dependencies = [+ "arrayvec",+]+[[package]]name = "portable-atomic"version = "1.13.1"@@ -2931,6 +3448,18 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740"+[[package]]+name = "pxfm"+version = "0.1.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"++[[package]]+name = "quick-error"+version = "2.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"+[[package]]name = "quick-xml"version = "0.36.2"@@ -2940,6 +3469,16 @@ dependencies = ["memchr",]+[[package]]+name = "quick-xml"+version = "0.41.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"+dependencies = [+ "memchr",+ "serde",+]+[[package]]name = "quinn"version = "0.11.9"@@ -3091,7 +3630,7 @@ version = "11.6.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"dependencies = [- "bitflags",+ "bitflags 2.11.1",][[package]]@@ -3144,6 +3683,16 @@ dependencies = ["yasna",]+[[package]]+name = "read-fonts"+version = "0.39.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c4ed38b89c2c77ff968c524145ad65fb010f38af5c7a224b53b81d47ac2daa81"+dependencies = [+ "bytemuck",+ "font-types",+]+[[package]]name = "reborrow"version = "0.5.5"@@ -3156,7 +3705,7 @@ version = "0.5.18"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"dependencies = [- "bitflags",+ "bitflags 2.11.1",][[package]]@@ -3295,7 +3844,7 @@ version = "0.32.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e"dependencies = [- "bitflags",+ "bitflags 2.11.1","fallible-iterator","fallible-streaming-iterator","hashlink",@@ -3325,7 +3874,7 @@ version = "4.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"dependencies = [- "nom",+ "nom 7.1.3",][[package]]@@ -3334,7 +3883,7 @@ version = "1.1.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"dependencies = [- "bitflags",+ "bitflags 2.11.1","errno","libc","linux-raw-sys",@@ -3439,7 +3988,7 @@ version = "3.7.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"dependencies = [- "bitflags",+ "bitflags 2.11.1","core-foundation 0.10.1","core-foundation-sys","libc",@@ -3583,8 +4132,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"dependencies = ["cfg-if",- "cpufeatures",- "digest",+ "cpufeatures 0.2.17",+ "digest 0.10.7",+]++[[package]]+name = "sha2"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"+dependencies = [+ "cfg-if",+ "cpufeatures 0.3.0",+ "digest 0.11.3",][[package]]@@ -3645,12 +4205,31 @@ version = "1.0.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"+[[package]]+name = "skrifa"+version = "0.42.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0c34617370ae968efb7161bb2beb517d9084659aae19e24b89e3db25b46e4564"+dependencies = [+ "bytemuck",+ "read-fonts",+]+[[package]]name = "slab"version = "0.4.12"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"+[[package]]+name = "slotmap"+version = "1.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038"+dependencies = [+ "version_check",+]+[[package]]name = "smallvec"version = "1.15.1"@@ -3701,7 +4280,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"dependencies = ["base64 0.13.1",- "nom",+ "nom 7.1.3","serde","unicode-segmentation",]@@ -3718,12 +4297,35 @@ version = "1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"+[[package]]+name = "stringprep"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"+dependencies = [+ "unicode-bidi",+ "unicode-normalization",+ "unicode-properties",+]+[[package]]name = "strsim"version = "0.11.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"+[[package]]+name = "subsetter"+version = "0.2.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "38803281d1c23166c5ebcb455439a5d2afe711cc909cf88af72448c297756ad6"+dependencies = [+ "kurbo",+ "rustc-hash",+ "skrifa",+ "write-fonts",+]+[[package]]name = "subtle"version = "2.6.1"@@ -3767,7 +4369,7 @@ version = "0.6.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc"dependencies = [- "bitflags",+ "bitflags 2.11.1","byteorder","enum-as-inner","libc",@@ -3781,7 +4383,7 @@ version = "0.7.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"dependencies = [- "bitflags",+ "bitflags 2.11.1","core-foundation 0.9.4","system-configuration-sys",]@@ -3796,6 +4398,18 @@ dependencies = ["libc",]+[[package]]+name = "taffy"+version = "0.12.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "340a09581f29809fc0df82a3955501dc7f2a21f887e5d1c13dbe288fe1c0bef4"+dependencies = [+ "arrayvec",+ "grid",+ "serde",+ "slotmap",+]+[[package]]name = "tempfile"version = "3.27.0"@@ -3858,6 +4472,20 @@ dependencies = ["cfg-if",]+[[package]]+name = "tiff"+version = "0.11.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52"+dependencies = [+ "fax 0.2.7",+ "flate2",+ "half",+ "quick-error",+ "weezl 0.1.12",+ "zune-jpeg",+]+[[package]]name = "tikv-jemalloc-sys"version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7"@@ -4176,7 +4804,7 @@ version = "0.5.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"dependencies = [- "bitflags",+ "bitflags 2.11.1","bytes","http","http-body",@@ -4193,7 +4821,7 @@ version = "0.6.8"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"dependencies = [- "bitflags",+ "bitflags 2.11.1","bytes","futures-util","http",@@ -4298,6 +4926,12 @@ version = "0.2.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"+[[package]]+name = "ttf-parser"+version = "0.25.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31"+[[package]]name = "twox-hash"version = "2.1.2"@@ -4316,12 +4950,24 @@ version = "1.20.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"+[[package]]+name = "unicode-bidi"+version = "0.3.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"+[[package]]name = "unicode-ident"version = "1.0.24"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"+[[package]]+name = "unicode-linebreak"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"+[[package]]name = "unicode-normalization"version = "0.1.25"@@ -4340,6 +4986,12 @@ dependencies = ["smallvec",]+[[package]]+name = "unicode-properties"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"+[[package]]name = "unicode-segmentation"version = "1.13.2"@@ -4629,7 +5281,7 @@ version = "0.244.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"dependencies = [- "bitflags",+ "bitflags 2.11.1","hashbrown 0.15.5","indexmap 2.14.0","semver",@@ -4693,7 +5345,7 @@ dependencies = ["base64urlsafedata","der-parser","hex",- "nom",+ "nom 7.1.3","openssl","openssl-sys","rand 0.9.4",@@ -4741,6 +5393,18 @@ dependencies = ["rustls-pki-types",]+[[package]]+name = "weezl"+version = "0.1.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"++[[package]]+name = "weezl"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2"+[[package]]name = "winapi"version = "0.3.9"@@ -5074,7 +5738,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"dependencies = ["anyhow",- "bitflags",+ "bitflags 2.11.1","indexmap 2.14.0","log","serde",@@ -5104,6 +5768,19 @@ dependencies = ["wasmparser",]+[[package]]+name = "write-fonts"+version = "0.48.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cb731d4c4d93eacc69a1ad2f270f905788a98e4a3438267bcafbe08d3431c8d8"+dependencies = [+ "font-types",+ "indexmap 2.14.0",+ "kurbo",+ "log",+ "read-fonts",+]+[[package]]name = "writeable"version = "0.6.3"@@ -5120,7 +5797,7 @@ dependencies = ["data-encoding","der-parser","lazy_static",- "nom",+ "nom 7.1.3","oid-registry","rusticata-macros","thiserror 1.0.69",@@ -5140,7 +5817,7 @@ dependencies = ["reqwest","serde","serde_json",- "sha2",+ "sha2 0.10.9","thiserror 2.0.18","tokenizers","tokio",@@ -5186,8 +5863,10 @@ dependencies = ["csv","encoding_rs","flate2",+ "libc","memchr",- "quick-xml",+ "pdf_oxide",+ "quick-xml 0.36.2","rayon","regex","reqwest",@@ -5273,7 +5952,7 @@ dependencies = ["rand 0.8.5","serde","serde_json",- "sha2",+ "sha2 0.10.9","subtle","tempfile","thiserror 2.0.18",@@ -5309,7 +5988,7 @@ dependencies = ["rustc-hash","serde","serde_json",- "sha2",+ "sha2 0.10.9","simd-json","tempfile","thiserror 2.0.18",@@ -5650,6 +6329,26 @@ dependencies = ["typed-path",]+[[package]]+name = "zip"+version = "8.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"+dependencies = [+ "crc32fast",+ "flate2",+ "indexmap 2.14.0",+ "memchr",+ "typed-path",+ "zopfli",+]++[[package]]+name = "zlib-rs"+version = "0.6.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5"+[[package]]name = "zmij"version = "1.0.21"@@ -5695,3 +6394,18 @@ dependencies = ["cc","pkg-config",]++[[package]]+name = "zune-core"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9"++[[package]]+name = "zune-jpeg"+version = "0.5.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"+dependencies = [+ "zune-core",+]diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.tomlindex 07ed1157e..731df6fa7 100644--- a/engine/crates/xerj-autoindex/Cargo.toml+++ b/engine/crates/xerj-autoindex/Cargo.toml@@ -17,12 +17,14 @@ encoding_rs = "0.8"xxhash-rust.workspace = truememchr.workspace = truerayon.workspace = true+libc = "0.2"# ES-compat HTTP client. Blocking mode: all of autoindex is synchronous# (std threads), invoked from the server binary via spawn_blocking.reqwest = { workspace = true, features = ["blocking"] }# Format-family extractors (new deps, isolated to this crate):flate2 = "1" # gzip streaming (rust backend)+pdf_oxide = { version = "=0.3.75", default-features = false, features = ["legacy-crypto"] }csv = "1" # streaming CSV with sniffed dialectquick-xml = "0.36" # XML pull parser + DOCX internalszip = { version = "2", default-features = false, features = ["deflate"] } # DOCX containerdiff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rsindex c046d6fe6..13f902386 100644--- a/engine/crates/xerj-autoindex/src/cli.rs+++ b/engine/crates/xerj-autoindex/src/cli.rs@@ -8,6 +8,8 @@ pub struct IndexCfg {pub url: String,pub api_key: Option<String>,pub workers: usize,+ pub pdf_workers: usize,+ pub pdf_timeout_secs: u64,pub bulk_mb: usize,pub prefix: String,pub state_dir: Option<PathBuf>,@@ -59,6 +61,8 @@ pub fn print_help() {--url <U> ES-compat endpoint (default http://localhost:9200)\n\--api-key <K> Authorization header (or env XERJ_API_KEY)\n\--workers <N> extract workers (default min(cores,8))\n\+ --pdf-workers <N> concurrent PDF parser processes (default min(cores,4); max 4)\n\+ --pdf-timeout-secs <N> per-PDF parser timeout (default 120; max 3600)\n\--bulk-mb <N> bulk cut size in MB (default 8)\n\--prefix <P> index prefix (default ax)\n\--state-dir <PATH> resume journal location (default ~/.xerj/autoindex/<hash>/)\n\@@ -73,6 +77,17 @@ pub fn print_help() {--dataset <SLUG> (map) show a single dataset\n\--help, -h this help\n\\n\+ PDF EXTRACTION:\n\+ Each PDF uses a fresh process. Limits: 512 MiB input, 32 MiB worker output,\n\+ 100,000 pages, and 1.5 GiB address space on Unix. This is crash/resource\n\+ isolation, not a security sandbox; non-Unix platforms have no OS memory cap.\n\+ Image-only pages need OCR. Page parse failures reject the whole PDF instead\n\+ of silently creating a partial index. XERJ_PDF_WORKER_BIN is a trusted,\n\+ developer-only executable override.\n\+ Current cost: phase-A sampling parses/materializes each complete PDF, and\n\+ phase-B indexing parses it again. A framed, early-stop protocol is planned;\n\+ use fewer --pdf-workers when parent memory is constrained.\n\+ \n\EMBEDDINGS:\n\autoindex sends semantic_text to the running server; it does not choose the\n\server's embedding backend. The default is lexical (not neural). For the\n\@@ -98,6 +113,11 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {.map(|n| n.get()).unwrap_or(8).min(8);+ let mut pdf_workers = std::thread::available_parallelism()+ .map(|n| n.get())+ .unwrap_or(4)+ .min(4);+ let mut pdf_timeout_secs = 120u64;let mut bulk_mb = 8usize;let mut prefix = "ax".to_string();let mut state_dir: Option<PathBuf> = None;@@ -121,6 +141,20 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {.and_then(|s| s.parse().ok()).ok_or("--workers needs a number")?}+ "--pdf-workers" => {+ pdf_workers = it+ .next()+ .and_then(|s| s.parse().ok())+ .filter(|n| (1..=4).contains(n))+ .ok_or("--pdf-workers needs a number from 1 to 4")?+ }+ "--pdf-timeout-secs" => {+ pdf_timeout_secs = it+ .next()+ .and_then(|s| s.parse().ok())+ .filter(|n| (1..=3600).contains(n))+ .ok_or("--pdf-timeout-secs needs a number from 1 to 3600")?+ }"--bulk-mb" => {bulk_mb = it.next()@@ -186,6 +220,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {url,api_key,workers: workers.max(1),+ pdf_workers,+ pdf_timeout_secs,bulk_mb: bulk_mb.clamp(1, 24),prefix,state_dir,diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rsindex d657dd79a..7c79edea4 100644--- a/engine/crates/xerj-autoindex/src/extract/mod.rs+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs@@ -17,12 +17,13 @@ pub mod yaml_x;use crate::sniff::{Family, Sniffed};use anyhow::Result;+use serde::{Deserialize, Serialize};use serde_json::{Map, Value};use std::io::{BufRead, BufReader, Read};use std::path::Path;/// One extracted record before coercion.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct RawRecord {pub fields: Map<String, Value>,/// Canonical, content-positional locator (byte offset / ordinal /diff --git a/engine/crates/xerj-autoindex/src/extract/pdf.rs b/engine/crates/xerj-autoindex/src/extract/pdf.rsindex 24158244a..51e44b0f3 100644--- a/engine/crates/xerj-autoindex/src/extract/pdf.rs+++ b/engine/crates/xerj-autoindex/src/extract/pdf.rs@@ -1,324 +1,643 @@-//! PDF text extraction — a self-contained content-stream scanner (no-//! external PDF crate: the workspace release profile is panic=abort, so a-//! panicky dependency would take the whole process down; this scanner is-//! Result-only by construction).+//! PDF text extraction backed by a real PDF parser.//!-//! Strategy: locate `stream…endstream` ranges, inflate FlateDecode bodies-//! (or use raw bodies that already look like content streams), then scan for-//! text-showing operators: `(…) Tj`, `(…) '`, `[…] TJ`, with PDF string-//! escapes and hex strings. Newlines derive from Td/TD/T*/ET ops. `/Title`-//! is pulled from the document info dictionary when present.+//! The old implementation searched raw streams and interpreted font character+//! codes as Latin-1. `pdf_oxide` resolves the object graph and font mappings+//! before returning Unicode text. Each document runs in a fresh worker process+//! so parser state or failure cannot contaminate the server. On Unix the worker+//! also gets a process group and an address-space limit. This is crash/resource+//! isolation, not a security sandbox: the worker retains the user's authority.-use super::{emit_document, ExtractStats, Sink};-use anyhow::Result;-use std::io::Read;-use std::path::Path;+use super::{split_sections, ExtractStats, RawRecord, Sink};+use anyhow::{anyhow, Context, Result};+use pdf_oxide::PdfDocument;+use serde::{Deserialize, Serialize};+use serde_json::{Map, Value};+use std::io::{Read, Write};+use std::path::{Path, PathBuf};+use std::process::{Command, Stdio};+use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::{Condvar, Mutex, OnceLock};+use std::time::{Duration, Instant};const PDF_CAP: u64 = 512 << 20;-const STREAM_INFLATE_CAP: u64 = 64 << 20;+const MAX_PAGES: usize = 100_000;+const MAX_PAGE_TEXT: usize = 16 << 20;+const MAX_EXTRACTED_TEXT: usize = 64 << 20;+const MAX_WORKER_OUTPUT: usize = 32 << 20;+const MAX_WORKER_STDERR: u64 = 64 << 10;+const WORKER_ADDRESS_SPACE: u64 = 1536 << 20;pub fn extract(path: &Path, sink: Sink) -> Result<ExtractStats> {+ let _permit = worker_gate().acquire();+ let response = spawn_worker(path)?;let mut stats = ExtractStats::default();- let size = std::fs::metadata(path)?.len();- if size > PDF_CAP {- stats.junk += 1;- return Ok(stats);- }- let bytes = std::fs::read(path)?;- let mut text = String::new();- for body in stream_bodies(&bytes) {- let content: Vec<u8> = match inflate(body) {- Some(d) => d,- None => {- if looks_like_content(body) {- body.to_vec()- } else {- continue;- }- }- };- if !looks_like_content(&content) {- continue;+ for record in response.records {+ stats.records += 1;+ if !sink(record) {+ break;}- scan_text_ops(&content, &mut text);- }- let title = doc_title(&bytes).unwrap_or_else(|| {- text.lines()- .find(|l| !l.trim().is_empty())- .map(|l| l.trim().chars().take(200).collect())- .unwrap_or_else(|| {- path.file_stem()- .map(|s| s.to_string_lossy().to_string())- .unwrap_or_else(|| "untitled".into())- })- });- let body = text.trim();- if body.is_empty() {- stats.junk += 1;- return Ok(stats);- }- emit_document(&title, &[], body, sink, &mut stats);+ }Ok(stats)}-fn stream_bodies(bytes: &[u8]) -> Vec<&[u8]> {- let mut out = Vec::new();- let mut i = 0usize;- while let Some(p) = find(bytes, i, b"stream") {- // must be the keyword, not part of "endstream"- let word_ok = p == 0 || !bytes[p - 1].is_ascii_alphanumeric();- let mut start = p + b"stream".len();- if start < bytes.len() && bytes[start] == b'\r' {- start += 1;+#[derive(Debug, Serialize, Deserialize)]+struct WorkerResponse {+ schema: u32,+ extractor: String,+ parser: String,+ containment: String,+ records: Vec<RawRecord>,+}++pub fn configure_workers(workers: usize) {+ worker_gate().set_limit(workers.clamp(1, 4));+}++pub fn configure_timeout(seconds: u64) {+ WORKER_TIMEOUT_SECS.store(seconds.clamp(1, 3600), Ordering::Relaxed);+}++/// Hidden same-binary worker entry point. One invocation parses one document.+pub fn run_worker_cli() -> i32 {+ let mut args = std::env::args_os().skip(2);+ let Some(path) = args.next().map(PathBuf::from) else {+ eprintln!("PDF worker protocol error: missing input path");+ return 2;+ };+ if args.next().is_some() {+ eprintln!("PDF worker protocol error: unexpected arguments");+ return 2;+ }+ match extract_in_process(&path) {+ Ok(records) => {+ let response = WorkerResponse {+ schema: 1,+ extractor: format!("xerj-autoindex/{}", env!("CARGO_PKG_VERSION")),+ parser: format!("pdf_oxide/{}", pdf_oxide::VERSION),+ containment: containment_description().to_string(),+ records,+ };+ let result = (|| -> Result<()> {+ let mut stdout = std::io::stdout().lock();+ serde_json::to_writer(&mut stdout, &response)?;+ stdout.flush()?;+ Ok(())+ })();+ if let Err(error) = result {+ eprintln!("PDF worker could not write its bounded result: {error}");+ return 1;+ }+ 0}- if start < bytes.len() && bytes[start] == b'\n' {- start += 1;+ Err(error) => {+ eprintln!("{error:#}");+ 1}- match find(bytes, start, b"endstream") {- Some(e) if word_ok => {- out.push(&bytes[start..e]);- i = e + b"endstream".len();- }- Some(e) => {- i = e + b"endstream".len();+ }+}++fn spawn_worker(path: &Path) -> Result<WorkerResponse> {+ // Trusted developer/test hook. The selected executable runs with the+ // invoking user's authority and must implement this private protocol.+ let executable = std::env::var_os("XERJ_PDF_WORKER_BIN")+ .map(PathBuf::from)+ .or_else(|| std::env::current_exe().ok())+ .context("cannot locate the xerj executable for isolated PDF extraction")?;+ let mut command = Command::new(executable);+ command+ .arg("__extract-pdf")+ .arg(path)+ .stdin(Stdio::null())+ .stdout(Stdio::piped())+ .stderr(Stdio::piped());+ set_worker_memory_limit(&mut command);+ let mut child = command.spawn().with_context(|| {+ format!(+ "could not start the isolated PDF parser for {}; check process/resource limits",+ path.display()+ )+ })?;+ let stdout = child.stdout.take().expect("piped stdout");+ let stderr = child.stderr.take().expect("piped stderr");+ let out_reader = std::thread::spawn(move || read_capped(stdout, MAX_WORKER_OUTPUT));+ let err_reader = std::thread::spawn(move || read_capped(stderr, MAX_WORKER_STDERR as usize));++ let started = Instant::now();+ let timeout = Duration::from_secs(WORKER_TIMEOUT_SECS.load(Ordering::Relaxed));+ let status = loop {+ match child.try_wait() {+ Ok(Some(status)) => break status,+ Ok(None) => {}+ Err(error) => {+ terminate_worker_tree(&mut child);+ let _ = out_reader.join();+ let _ = err_reader.join();+ return Err(error).with_context(|| {+ format!("could not monitor PDF parser for {}", path.display())+ });}- None => break,}+ if started.elapsed() >= timeout {+ terminate_worker_tree(&mut child);+ let _ = out_reader.join();+ let _ = err_reader.join();+ return Err(anyhow!(+ "PDF extraction timed out after {} seconds for {}; repair/split the PDF, or run OCR if it is image-only",+ timeout.as_secs(),+ path.display()+ ));+ }+ std::thread::sleep(Duration::from_millis(20));+ };+ terminate_worker_descendants(child.id());+ let stdout = out_reader+ .join()+ .map_err(|_| anyhow!("PDF parser output reader panicked"))??;+ let stderr = err_reader+ .join()+ .map_err(|_| anyhow!("PDF parser error reader panicked"))??;+ if stdout.overflow {+ return Err(anyhow!(+ "PDF parser output exceeded the {} MiB parent-memory safety limit for {}; split the document before indexing",+ MAX_WORKER_OUTPUT >> 20,+ path.display()+ ));+ }+ if stderr.overflow {+ return Err(anyhow!(+ "PDF parser error output exceeded the {} KiB safety limit for {}; the worker was excessively noisy and its truncated diagnostic is: {}",+ MAX_WORKER_STDERR >> 10,+ path.display(),+ String::from_utf8_lossy(&stderr.prefix)+ ));+ }+ let stderr = String::from_utf8_lossy(&stderr.prefix);+ if !status.success() {+ return Err(anyhow!(+ "isolated PDF parser failed for {}{}{}; verify/decrypt the PDF, repair it, or run OCR for image-only input",+ path.display(),+ if stderr.trim().is_empty() { "" } else { ": " },+ stderr.trim()+ ));+ }+ let response: WorkerResponse = serde_json::from_slice(&stdout.prefix).with_context(|| {+ format!(+ "isolated PDF parser returned an invalid result for {}; this is an internal worker-protocol error",+ path.display()+ )+ })?;+ if response.schema != 1 {+ return Err(anyhow!(+ "PDF parser returned unsupported extraction schema {}; update parent and worker together",+ response.schema+ ));+ }+ let expected_extractor = format!("xerj-autoindex/{}", env!("CARGO_PKG_VERSION"));+ if response.extractor != expected_extractor {+ return Err(anyhow!(+ "PDF extractor version mismatch: expected {expected_extractor}, worker reported {}; update parent and worker together",+ response.extractor+ ));+ }+ if response.parser != format!("pdf_oxide/{}", pdf_oxide::VERSION) {+ return Err(anyhow!(+ "PDF parser version mismatch: expected pdf_oxide/{}, worker reported {}; update parent and worker together",+ pdf_oxide::VERSION, response.parser+ ));+ }+ Ok(response)+}++static WORKER_TIMEOUT_SECS: AtomicU64 = AtomicU64::new(120);++struct CappedRead {+ prefix: Vec<u8>,+ overflow: bool,+}++fn read_capped(mut input: impl Read, cap: usize) -> Result<CappedRead> {+ let mut prefix = Vec::with_capacity(cap.min(64 << 10));+ let mut overflow = false;+ let mut chunk = [0u8; 64 << 10];+ loop {+ let read = input+ .read(&mut chunk)+ .context("could not read isolated PDF parser output")?;+ if read == 0 {+ break;+ }+ let remaining = cap.saturating_sub(prefix.len());+ let retain = remaining.min(read);+ prefix.extend_from_slice(&chunk[..retain]);+ overflow |= retain < read;+ // Keep draining after overflow so the child can exit instead of+ // blocking forever on a full pipe.}- out+ Ok(CappedRead { prefix, overflow })}-fn find(hay: &[u8], from: usize, needle: &[u8]) -> Option<usize> {- if from >= hay.len() {- return None;+#[cfg(unix)]+fn set_worker_memory_limit(command: &mut Command) {+ use std::os::unix::process::CommandExt;+ unsafe {+ command.pre_exec(|| {+ if libc::setpgid(0, 0) != 0 {+ return Err(std::io::Error::last_os_error());+ }+ let limit = libc::rlimit {+ rlim_cur: WORKER_ADDRESS_SPACE,+ rlim_max: WORKER_ADDRESS_SPACE,+ };+ if libc::setrlimit(libc::RLIMIT_AS, &limit) != 0 {+ return Err(std::io::Error::last_os_error());+ }+ Ok(())+ });}- memchr::memmem::find(&hay[from..], needle).map(|p| p + from)}-fn inflate(body: &[u8]) -> Option<Vec<u8>> {- let mut d = flate2::read::ZlibDecoder::new(body).take(STREAM_INFLATE_CAP);- let mut out = Vec::new();- d.read_to_end(&mut out).ok()?;- if out.is_empty() {- None- } else {- Some(out)+#[cfg(not(unix))]+fn set_worker_memory_limit(_command: &mut Command) {}++#[cfg(unix)]+fn terminate_worker_tree(child: &mut std::process::Child) {+ unsafe {+ libc::kill(-(child.id() as i32), libc::SIGKILL);}+ let _ = child.kill();+ let _ = child.wait();}-fn looks_like_content(b: &[u8]) -> bool {- let head = &b[..b.len().min(4096)];- memchr::memmem::find(head, b"BT").is_some()- || memchr::memmem::find(head, b"Tj").is_some()- || memchr::memmem::find(head, b"TJ").is_some()+#[cfg(unix)]+fn terminate_worker_descendants(process_group: u32) {+ unsafe {+ libc::kill(-(process_group as i32), libc::SIGKILL);+ }}-/// PDFDocEncoding ≈ latin-1 for the printable range — good enough for-/// generated PDFs; unknown bytes are passed through as latin-1.-fn scan_text_ops(content: &[u8], out: &mut String) {- let mut i = 0usize;- let mut pending: Vec<String> = Vec::new(); // strings awaiting an operator- let mut line = String::new();- let flush_line = |line: &mut String, out: &mut String| {- let t = line.trim();- if !t.is_empty() {- out.push_str(t);- out.push('\n');- }- line.clear();- };- while i < content.len() {- match content[i] {- b'(' => {- let (s, ni) = parse_literal_string(content, i);- pending.push(s);- i = ni;- }- b'<' if i + 1 < content.len() && content[i + 1] != b'<' => {- let (s, ni) = parse_hex_string(content, i);- pending.push(s);- i = ni;- }- b'[' => {- // TJ array: collect strings until ]- let mut j = i + 1;- let mut parts: Vec<String> = Vec::new();- while j < content.len() && content[j] != b']' {- match content[j] {- b'(' => {- let (s, nj) = parse_literal_string(content, j);- parts.push(s);- j = nj;- }- b'<' => {- let (s, nj) = parse_hex_string(content, j);- parts.push(s);- j = nj;- }- _ => j += 1,- }- }- pending.push(parts.join(""));- i = j + 1;- }- b'%' => {- // comment to EOL- i = memchr::memchr(b'\n', &content[i..])- .map(|p| i + p + 1)- .unwrap_or(content.len());+#[cfg(not(unix))]+fn terminate_worker_tree(child: &mut std::process::Child) {+ let _ = child.kill();+ let _ = child.wait();+}++#[cfg(not(unix))]+fn terminate_worker_descendants(_process_group: u32) {}++#[cfg(unix)]+fn containment_description() -> &'static str {+ "fresh process; Unix process-group cleanup; 1536 MiB RLIMIT_AS; not a security sandbox"+}++#[cfg(not(unix))]+fn containment_description() -> &'static str {+ "fresh process; no OS memory cap on this platform; not a security sandbox"+}++fn extract_in_process(path: &Path) -> Result<Vec<RawRecord>> {+ let size = std::fs::metadata(path)?.len();+ if size > PDF_CAP {+ return Err(anyhow!(+ "PDF is {:.1} MiB, above the 512 MiB safety limit; split or compress it before indexing",+ size as f64 / (1 << 20) as f64+ ));+ }++ pdf_oxide::fonts::global_cache::clear_global_font_cache();+ let doc = PdfDocument::open(path).with_context(|| {+ "PDF parser could not open the document; verify/repair damaged input, decrypt password-protected input, or run OCR for image-only input"+ })?;+ let pages = doc.page_count().with_context(|| {+ "PDF parser could not read the page tree; verify the file or decrypt it first"+ })?;+ if pages == 0 {+ return Err(anyhow!("PDF contains no pages"));+ }+ if pages > MAX_PAGES {+ return Err(anyhow!(+ "PDF declares {pages} pages, above the {MAX_PAGES}-page safety limit; split it before indexing"+ ));+ }++ let title = path+ .file_stem()+ .map(|s| s.to_string_lossy().into_owned())+ .filter(|s| !s.trim().is_empty())+ .unwrap_or_else(|| "untitled".to_string());+ let mut records = Vec::new();+ let mut extracted_bytes = 0usize;+ let mut failed_pages = Vec::new();+ let mut omitted_pages = Vec::new();+ let mut pages_with_text = 0usize;++ for page_index in 0..pages {+ let page = match doc.extract_text(page_index) {+ Ok(text) => text,+ Err(error) => {+ failed_pages.push(format!("{} ({error})", page_index + 1));+ continue;}- c if c.is_ascii_alphabetic() || c == b'\'' || c == b'"' || c == b'*' => {- let start = i;- while i < content.len()- && (content[i].is_ascii_alphanumeric()- || matches!(content[i], b'\'' | b'"' | b'*'))- {- i += 1;- }- let op = &content[start..i];- match op {- b"Tj" | b"TJ" | b"'" | b"\"" => {- for s in pending.drain(..) {- if !line.is_empty() && !line.ends_with(' ') {- line.push(' ');- }- line.push_str(&s);- }- if op == b"'" || op == b"\"" {- flush_line(&mut line, out);- }- }- b"Td" | b"TD" | b"T*" | b"ET" => {- pending.clear();- flush_line(&mut line, out);- }- b"BT" => {- pending.clear();- }- _ => {- pending.clear();- }- }+ };+ let page = normalize_legacy_symbol_glyphs(&page);+ let body = page.trim();+ if body.is_empty() {+ omitted_pages.push(page_index + 1);+ continue;+ }+ if body.len() > MAX_PAGE_TEXT {+ return Err(anyhow!(+ "PDF page {} produced more than 16 MiB of text; split or repair the document before indexing",+ page_index + 1+ ));+ }+ if body.chars().filter(|ch| !ch.is_whitespace()).count() < 8 {+ omitted_pages.push(page_index + 1);+ continue;+ }+ validate_text_quality(body).with_context(|| {+ format!(+ "PDF page {} produced untrusted text; refusing to index possible binary/font garbage",+ page_index + 1+ )+ })?;+ extracted_bytes = extracted_bytes.saturating_add(body.len());+ pages_with_text += 1;+ if extracted_bytes > MAX_EXTRACTED_TEXT {+ return Err(anyhow!(+ "PDF extraction exceeded the 64 MiB text safety limit; split the document before indexing"+ ));+ }++ for (section, text) in split_sections(body).into_iter().enumerate() {+ let mut fields = Map::new();+ fields.insert("title".into(), Value::String(title.clone()));+ fields.insert(+ "page".into(),+ Value::Number(((page_index + 1) as u64).into()),+ );+ if section > 0 {+ fields.insert("section".into(), Value::Number((section as u64).into()));}- _ => i += 1,+ fields.insert("body".into(), Value::String(text));+ records.push(RawRecord {+ fields,+ locator: format!("p{}-s{}", page_index + 1, section),+ group: None,+ });+ }+ }++ if records.is_empty() {+ if failed_pages.is_empty() {+ return Err(anyhow!(+ "PDF has no extractable text; it may be an image-only scan. Run OCR first, then autoindex the searchable PDF"+ ));}- if out.len() > 32 << 20 {- break; // hard safety cap+ return Err(anyhow!(+ "PDF text extraction failed on every non-empty page ({}); verify/decrypt the PDF or run OCR before indexing",+ failed_pages.iter().take(3).cloned().collect::<Vec<_>>().join(", ")+ ));+ }+ if !failed_pages.is_empty() {+ return Err(anyhow!(+ "PDF extraction failed on {}/{} pages (first failures: {}); refusing a partial index. Repair/decrypt the PDF or run OCR first",+ failed_pages.len(),+ pages,+ failed_pages.iter().take(3).cloned().collect::<Vec<_>>().join(", ")+ ));+ }+ for record in &mut records {+ record.fields.insert(+ "pdf_pages_total".into(),+ Value::Number((pages as u64).into()),+ );+ record.fields.insert(+ "pdf_pages_with_text".into(),+ Value::Number((pages_with_text as u64).into()),+ );+ record.fields.insert(+ "pdf_pages_omitted".into(),+ Value::Number((omitted_pages.len() as u64).into()),+ );+ if !omitted_pages.is_empty() {+ record.fields.insert(+ "pdf_ocr_warning".into(),+ Value::String(format!(+ "{} page(s) had no usable text (first: {}); intentional blanks or scanned pages may need OCR",+ omitted_pages.len(),+ omitted_pages+ .iter()+ .take(8)+ .map(usize::to_string)+ .collect::<Vec<_>>()+ .join(", ")+ )),+ );}}- flush_line(&mut line, out);+ Ok(records)}-fn parse_literal_string(b: &[u8], open: usize) -> (String, usize) {- let mut s = String::new();- let mut i = open + 1;- let mut depth = 1usize;- while i < b.len() {- match b[i] {- b'\\' if i + 1 < b.len() => {- let c = b[i + 1];- match c {- b'n' => s.push('\n'),- b'r' => s.push('\r'),- b't' => s.push('\t'),- b'(' => s.push('('),- b')' => s.push(')'),- b'\\' => s.push('\\'),- b'0'..=b'7' => {- // up to 3 octal digits- let mut v = 0u32;- let mut n = 0;- while n < 3 && i + 1 + n < b.len() && (b'0'..=b'7').contains(&b[i + 1 + n])- {- v = v * 8 + (b[i + 1 + n] - b'0') as u32;- n += 1;- }- if let Some(ch) = char::from_u32(v) {- s.push(ch);- }- i += n + 1;- continue;- }- b'\n' => {} // line continuation- other => s.push(other as char),- }- i += 2;- }- b'(' => {- depth += 1;- s.push('(');- i += 1;- }- b')' => {- depth -= 1;- if depth == 0 {- return (s, i + 1);- }- s.push(')');- i += 1;- }- c => {- s.push(c as char); // latin-1 passthrough- i += 1;- }- }- if s.len() > 1 << 20 {- break;+struct WorkerGate {+ state: Mutex<(usize, usize)>,+ ready: Condvar,+}++impl WorkerGate {+ fn set_limit(&self, limit: usize) {+ self.state.lock().expect("PDF worker gate poisoned").1 = limit;+ self.ready.notify_all();+ }++ fn acquire(&self) -> WorkerPermit<'_> {+ let mut state = self.state.lock().expect("PDF worker gate poisoned");+ while state.0 >= state.1 {+ state = self.ready.wait(state).expect("PDF worker gate poisoned");}+ state.0 += 1;+ WorkerPermit(self)}- (s, i)}-fn parse_hex_string(b: &[u8], open: usize) -> (String, usize) {- let mut i = open + 1;- let mut nibbles: Vec<u8> = Vec::new();- while i < b.len() && b[i] != b'>' {- let c = b[i];- if c.is_ascii_hexdigit() {- nibbles.push(c);+struct WorkerPermit<'a>(&'a WorkerGate);++impl Drop for WorkerPermit<'_> {+ fn drop(&mut self) {+ let mut state = self.0.state.lock().expect("PDF worker gate poisoned");+ state.0 -= 1;+ self.0.ready.notify_one();+ }+}++fn worker_gate() -> &'static WorkerGate {+ static GATE: OnceLock<WorkerGate> = OnceLock::new();+ GATE.get_or_init(|| WorkerGate {+ state: Mutex::new((+ 0,+ std::thread::available_parallelism()+ .map(|n| n.get())+ .unwrap_or(4)+ .min(4),+ )),+ ready: Condvar::new(),+ })+}++fn normalize_legacy_symbol_glyphs(text: &str) -> String {+ text.chars()+ .map(|ch| match ch {+ '\u{f0b7}' => '•',+ '\u{f02d}' => '-',+ '\u{f078}' | '\u{f0a8}' | '\u{f06f}' | '\u{f052}' | '\u{f0a3}' | '\u{f020}' => '□',+ '\u{f0d2}' | '\u{f0d4}' | '\u{f0be}' => ' ',+ other => other,+ })+ .collect()+}++fn validate_text_quality(text: &str) -> Result<()> {+ // Fail closed rather than embed broken font codes. On the pinned+ // 368-document FinanceBench PDF corpus this 0.5% threshold accepted 367+ // documents and rejected one; keep that false-rejection cost visible when+ // recalibrating against broader labeled corpora.+ let mut visible = 0usize;+ let mut semantic = 0usize;+ let mut suspicious = 0usize;+ for ch in text.chars() {+ if ch.is_whitespace() {+ continue;}- i += 1;- if nibbles.len() > 1 << 20 {- break;+ visible += 1;+ if ch.is_alphanumeric() {+ semantic += 1;+ }+ if ch == '\u{fffd}'+ || ch.is_control()+ || ('\u{e000}'..='\u{f8ff}').contains(&ch)+ || ('\u{f0000}'..='\u{ffffd}').contains(&ch)+ || ('\u{100000}'..='\u{10fffd}').contains(&ch)+ {+ suspicious += 1;}}- if nibbles.len() % 2 == 1 {- nibbles.push(b'0');+ if visible < 8 {+ return Err(anyhow!("too little visible text"));}- let mut s = String::new();- for pair in nibbles.chunks(2) {- let hi = (pair[0] as char).to_digit(16).unwrap_or(0);- let lo = (pair[1] as char).to_digit(16).unwrap_or(0);- if let Some(ch) = char::from_u32(hi * 16 + lo) {- s.push(ch);- }+ if suspicious * 200 > visible {+ return Err(anyhow!(+ "more than 0.5% of visible characters are controls, replacement glyphs, or private-use codes"+ ));}- (s, i + 1)+ if semantic * 20 < visible {+ return Err(anyhow!(+ "fewer than 5% of visible characters are letters or numbers"+ ));+ }+ Ok(())}-fn doc_title(bytes: &[u8]) -> Option<String> {- let p = memchr::memmem::find(bytes, b"/Title")?;- let after = &bytes[p + 6..(p + 4096).min(bytes.len())];- let start = after.iter().position(|&c| !c.is_ascii_whitespace())?;- match after[start] {- b'(' => {- let (s, _) = parse_literal_string(after, start);- let t = s.trim();- if t.is_empty() {- None- } else {- Some(t.to_string())- }+#[cfg(test)]+mod tests {+ use super::{extract_in_process, normalize_legacy_symbol_glyphs, validate_text_quality};+ use crate::infer::{infer_fields, FieldAcc};+ use serde_json::Value;+ use std::collections::HashMap;++ fn write_prose_pdf(path: &std::path::Path) {+ let prose = "Quarterly revenue increased because subscription demand remained strong. \+ Operating income improved as cloud infrastructure costs declined. Management expects \+ cash flow to support continued investment throughout the next fiscal year.";+ let stream = format!("BT /F1 12 Tf 72 720 Td ({prose}) Tj ET");+ let objects = [+ "<< /Type /Catalog /Pages 2 0 R >>".to_string(),+ "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(),+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>".to_string(),+ format!("<< /Length {} >>\nstream\n{stream}\nendstream", stream.len()),+ "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(),+ ];+ let mut pdf = b"%PDF-1.4\n".to_vec();+ let mut offsets = vec![0usize];+ for (index, object) in objects.iter().enumerate() {+ offsets.push(pdf.len());+ pdf.extend_from_slice(format!("{} 0 obj\n{}\nendobj\n", index + 1, object).as_bytes());}- b'<' => {- let (s, _) = parse_hex_string(after, start);- let t = s.trim();- if t.is_empty() {- None- } else {- Some(t.to_string())+ let xref = pdf.len();+ pdf.extend_from_slice(format!("xref\n0 {}\n", objects.len() + 1).as_bytes());+ pdf.extend_from_slice(b"0000000000 65535 f \n");+ for offset in offsets.iter().skip(1) {+ pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());+ }+ pdf.extend_from_slice(+ format!(+ "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",+ objects.len() + 1+ )+ .as_bytes(),+ );+ std::fs::write(path, pdf).unwrap();+ }++ #[test]+ fn quality_gate_accepts_financial_and_international_text() {+ validate_text_quality(+ "Consolidated revenue — Q4 2025\n€1,234.5 million (增长率 12%; прибыль 8%).",+ )+ .unwrap();+ }++ #[test]+ fn quality_gate_rejects_binary_and_font_garbage() {+ let garbage = "abc\u{0000}\u{0001}\u{fffd}\u{e123}\u{0002}xyz";+ assert!(validate_text_quality(garbage).is_err());+ }++ #[test]+ fn quality_gate_rejects_operator_noise() {+ assert!(validate_text_quality("//// <<>> [] () -- === +++").is_err());+ }++ #[test]+ fn known_legacy_symbol_glyphs_are_normalized_but_unknown_pua_is_rejected() {+ let normalized = normalize_legacy_symbol_glyphs(+ "Yes \u{f078} No \u{f0a8}\n\u{f0b7} Revenue\nrisk\u{f02d}adjusted",+ );+ assert_eq!(normalized, "Yes □ No □\n• Revenue\nrisk-adjusted");+ validate_text_quality(&normalized).unwrap();+ assert!(validate_text_quality("Revenue \u{e123} remains unknown").is_err());+ }++ #[test]+ fn reproducible_pdf_extracts_pages_and_elects_semantic_text() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("quarterly-report.pdf");+ write_prose_pdf(&path);++ let first = extract_in_process(&path).unwrap();+ let second = extract_in_process(&path).unwrap();+ assert_eq!(+ serde_json::to_value(&first).unwrap(),+ serde_json::to_value(&second).unwrap()+ );+ assert!(!first.is_empty());+ assert_eq!(first[0].fields["page"], 1);+ assert!(first[0].locator.starts_with("p1-s"));+ assert!(first[0].fields["body"]+ .as_str()+ .unwrap()+ .contains("Quarterly revenue increased"));++ let mut fields: HashMap<String, FieldAcc> = HashMap::new();+ for record in &first {+ for (name, value) in &record.fields {+ fields.entry(name.clone()).or_default().add(value);}}- _ => None,+ let specs = infer_fields(&fields, first.len() as u64, false);+ let body = specs.iter().find(|spec| spec.name == "body").unwrap();+ assert_eq!(body.es_type, "semantic_text");+ assert!(matches!(+ first[0].fields.get("body"),+ Some(Value::String(_))+ ));}}diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 643c6f6d7..9e1103ce4 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -189,6 +189,8 @@ fn build_mapping(specs: &[infer::FieldSpec]) -> Value {// ─── the main run ────────────────────────────────────────────────────────fn run_index(cfg: IndexCfg) -> Result<i32> {+ extract::pdf::configure_workers(cfg.pdf_workers);+ extract::pdf::configure_timeout(cfg.pdf_timeout_secs);let t0 = Instant::now();let es = Es::new(&cfg.url, cfg.api_key.clone())?;es.ping()?;diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rsindex d68995fd8..1f35e6d60 100644--- a/engine/crates/xerj-server/src/main.rs+++ b/engine/crates/xerj-server/src/main.rs@@ -1137,6 +1137,10 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {/// Override with `TOKIO_WORKER_THREADS` (operators on very small or very large/// hosts, or wanting the stock `ncpus` behaviour, can set it explicitly).fn main() -> Result<()> {+ // Dispatch before creating Tokio's pool so each PDF worker stays small.+ if matches!(std::env::args().nth(1).as_deref(), Some("__extract-pdf")) {+ std::process::exit(xerj_autoindex::extract::pdf::run_worker_cli());+ }let cores = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(8);diff --git a/engine/crates/xerj-server/tests/pdf_worker.rs b/engine/crates/xerj-server/tests/pdf_worker.rsnew file mode 100644index 000000000..5c53085ff--- /dev/null+++ b/engine/crates/xerj-server/tests/pdf_worker.rs@@ -0,0 +1,236 @@+use serde_json::Value;+use std::fmt::Write as _;+use std::fs;+use std::process::Command;+use std::time::{Duration, Instant};++fn write_prose_pdf(path: &std::path::Path) {+ let prose = "Quarterly financial results show revenue increased twelve percent while operating margin improved. International customers and subscription renewals drove durable growth across the reporting period.";+ let stream = format!("BT /F1 12 Tf 72 720 Td ({prose}) Tj ET");+ let objects = [+ "<< /Type /Catalog /Pages 2 0 R >>".to_string(),+ "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(),+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>".to_string(),+ format!("<< /Length {} >>\nstream\n{stream}\nendstream", stream.len()),+ "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(),+ ];++ let mut pdf = String::from("%PDF-1.4\n");+ let mut offsets = Vec::new();+ for (index, object) in objects.iter().enumerate() {+ offsets.push(pdf.len());+ writeln!(pdf, "{} 0 obj\n{}\nendobj", index + 1, object).unwrap();+ }+ let xref = pdf.len();+ write!(pdf, "xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).unwrap();+ for offset in offsets {+ writeln!(pdf, "{offset:010} 00000 n ").unwrap();+ }+ write!(+ pdf,+ "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",+ objects.len() + 1+ )+ .unwrap();+ fs::write(path, pdf).unwrap();+}++fn write_mixed_text_blank_pdf(path: &std::path::Path) {+ let prose =+ "Financial statement prose appears on this page while the next page is intentionally blank.";+ let stream = format!("BT /F1 12 Tf 72 720 Td ({prose}) Tj ET");+ let objects = [+ "<< /Type /Catalog /Pages 2 0 R >>".to_string(),+ "<< /Type /Pages /Kids [3 0 R 6 0 R] /Count 2 >>".to_string(),+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>".to_string(),+ format!("<< /Length {} >>\nstream\n{stream}\nendstream", stream.len()),+ "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(),+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>".to_string(),+ ];+ let mut pdf = String::from("%PDF-1.4\n");+ let mut offsets = Vec::new();+ for (index, object) in objects.iter().enumerate() {+ offsets.push(pdf.len());+ writeln!(pdf, "{} 0 obj\n{}\nendobj", index + 1, object).unwrap();+ }+ let xref = pdf.len();+ write!(pdf, "xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).unwrap();+ for offset in offsets {+ writeln!(pdf, "{offset:010} 00000 n ").unwrap();+ }+ write!(+ pdf,+ "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",+ objects.len() + 1+ )+ .unwrap();+ fs::write(path, pdf).unwrap();+}++fn run_worker(path: &std::path::Path) -> std::process::Output {+ Command::new(env!("CARGO_BIN_EXE_xerj"))+ .arg("__extract-pdf")+ .arg(path)+ .output()+ .expect("start PDF worker")+}++#[test]+fn pdf_worker_is_fresh_process_deterministic_and_preserves_page_provenance() {+ let temp = tempfile::tempdir().unwrap();+ let pdf = temp.path().join("quarterly-report.pdf");+ write_prose_pdf(&pdf);++ let first = run_worker(&pdf);+ let second = run_worker(&pdf);+ assert!(+ first.status.success(),+ "{}",+ String::from_utf8_lossy(&first.stderr)+ );+ assert!(+ second.status.success(),+ "{}",+ String::from_utf8_lossy(&second.stderr)+ );+ assert_eq!(first.stdout, second.stdout);++ let response: Value = serde_json::from_slice(&first.stdout).unwrap();+ assert_eq!(response["extractor"], "xerj-autoindex/1.0.0-rc.4");+ assert_eq!(response["parser"], "pdf_oxide/0.3.75");+ assert!(response["containment"]+ .as_str()+ .unwrap()+ .contains("not a security sandbox"));+ let record = &response["records"][0];+ assert_eq!(record["fields"]["page"], 1);+ assert_eq!(record["locator"], "p1-s0");+ assert!(record["fields"]["body"]+ .as_str()+ .unwrap()+ .contains("operating margin improved"));+ assert_eq!(record["fields"]["pdf_pages_total"], 1);+ assert_eq!(record["fields"]["pdf_pages_with_text"], 1);+ assert_eq!(record["fields"]["pdf_pages_omitted"], 0);+}++#[test]+fn corrupt_pdf_failure_is_actionable_and_emits_no_records() {+ let temp = tempfile::tempdir().unwrap();+ let pdf = temp.path().join("corrupt.pdf");+ fs::write(&pdf, b"%PDF-1.4\nnot a PDF").unwrap();++ let output = run_worker(&pdf);+ assert!(!output.status.success());+ assert!(output.stdout.is_empty());+ let stderr = String::from_utf8_lossy(&output.stderr);+ assert!(stderr.contains("damaged"), "{stderr}");+ assert!(stderr.contains("password"), "{stderr}");+ assert!(stderr.contains("OCR"), "{stderr}");+}++#[test]+fn mixed_text_and_empty_pages_disclose_ocr_coverage_without_dropping_good_text() {+ let temp = tempfile::tempdir().unwrap();+ let pdf = temp.path().join("mixed.pdf");+ write_mixed_text_blank_pdf(&pdf);+ let output = run_worker(&pdf);+ assert!(+ output.status.success(),+ "{}",+ String::from_utf8_lossy(&output.stderr)+ );+ let response: Value = serde_json::from_slice(&output.stdout).unwrap();+ let fields = &response["records"][0]["fields"];+ assert_eq!(fields["pdf_pages_total"], 2);+ assert_eq!(fields["pdf_pages_with_text"], 1);+ assert_eq!(fields["pdf_pages_omitted"], 1);+ assert!(fields["pdf_ocr_warning"]+ .as_str()+ .unwrap()+ .contains("may need OCR"));+}++#[cfg(unix)]+#[test]+fn parent_drains_overflow_and_timeout_kills_worker_process_group() {+ use std::os::unix::fs::PermissionsExt;++ let temp = tempfile::tempdir().unwrap();+ let input = temp.path().join("input.pdf");+ fs::write(&input, b"unused by fake worker").unwrap();++ let overflow = temp.path().join("overflow-worker");+ fs::write(+ &overflow,+ b"#!/bin/sh\nhead -c 34603008 /dev/zero\nexit 0\n",+ )+ .unwrap();+ fs::set_permissions(&overflow, fs::Permissions::from_mode(0o700)).unwrap();+ std::env::set_var("XERJ_PDF_WORKER_BIN", &overflow);+ let started = Instant::now();+ let mut sink = |_record| true;+ let error = xerj_autoindex::extract::pdf::extract(&input, &mut sink).unwrap_err();+ assert!(started.elapsed() < Duration::from_secs(10));+ assert!(error.to_string().contains("exceeded"), "{error:#}");++ let stderr_overflow = temp.path().join("stderr-overflow-worker");+ fs::write(+ &stderr_overflow,+ b"#!/bin/sh\nhead -c 131072 /dev/zero >&2\nexit 1\n",+ )+ .unwrap();+ fs::set_permissions(&stderr_overflow, fs::Permissions::from_mode(0o700)).unwrap();+ std::env::set_var("XERJ_PDF_WORKER_BIN", &stderr_overflow);+ let started = Instant::now();+ let error = xerj_autoindex::extract::pdf::extract(&input, &mut sink).unwrap_err();+ assert!(started.elapsed() < Duration::from_secs(10));+ assert!(+ error.to_string().contains("error output exceeded"),+ "{error:#}"+ );++ let timeout = temp.path().join("timeout-worker");+ fs::write(+ &timeout,+ b"#!/bin/sh\nsleep 60 &\necho $! > \"$2.pid\"\nwait\n",+ )+ .unwrap();+ fs::set_permissions(&timeout, fs::Permissions::from_mode(0o700)).unwrap();+ std::env::set_var("XERJ_PDF_WORKER_BIN", &timeout);+ xerj_autoindex::extract::pdf::configure_timeout(1);+ let error = xerj_autoindex::extract::pdf::extract(&input, &mut sink).unwrap_err();+ xerj_autoindex::extract::pdf::configure_timeout(120);+ std::env::remove_var("XERJ_PDF_WORKER_BIN");+ assert!(error.to_string().contains("timed out"), "{error:#}");++ let descendant = fs::read_to_string(format!("{}.pid", input.display())).unwrap();+ let proc_entry = std::path::PathBuf::from("/proc").join(descendant.trim());+ for _ in 0..50 {+ if !proc_entry.exists() {+ return;+ }+ std::thread::sleep(Duration::from_millis(20));+ }+ panic!("worker descendant {} survived timeout", descendant.trim());+}++#[cfg(unix)]+#[test]+fn pdf_worker_accepts_non_utf8_unix_paths() {+ use std::os::unix::ffi::OsStringExt;++ let temp = tempfile::tempdir().unwrap();+ let name = std::ffi::OsString::from_vec(b"quarterly-\xff.pdf".to_vec());+ let pdf = temp.path().join(name);+ write_prose_pdf(&pdf);++ let output = run_worker(&pdf);+ assert!(+ output.status.success(),+ "{}",+ String::from_utf8_lossy(&output.stderr)+ );+ let response: Value = serde_json::from_slice(&output.stdout).unwrap();+ assert_eq!(response["records"][0]["fields"]["page"], 1);+}