Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(rust): real fastembed-rs EmbeddingScorer (BAAI/bge-small-en-v1.5)

chopratejas1945e5f55bdd
diff --git a/.gitignore b/.gitignore
index af93df6fe..cdf391e73 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,8 @@
+# fastembed model cache (auto-downloaded ONNX weights, ~30 MB+).
+# Should NEVER be committed — bloats the repo significantly.
+.fastembed_cache/
+**/.fastembed_cache/
+
# Private scripts (contain credentials). Allowlist checked-in helpers below.
scripts/
!scripts/
diff --git a/Cargo.lock b/Cargo.lock
index a3f46fc97..36765ede5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -31,6 +31,30 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "aligned"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685"
+dependencies = [
+ "as-slice",
+]
+
+[[package]]
+name = "aligned-vec"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b"
+dependencies = [
+ "equator",
+]
+
+[[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
[[package]]
name = "anes"
version = "0.1.6"
@@ -93,6 +117,38 @@ version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
+[[package]]
+name = "arbitrary"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
+
+[[package]]
+name = "arg_enum_proc_macro"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "arrayvec"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
+
+[[package]]
+name = "as-slice"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516"
+dependencies = [
+ "stable_deref_trait",
+]
+
[[package]]
name = "assert-json-diff"
version = "2.0.2"
@@ -126,6 +182,49 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
+[[package]]
+name = "av-scenechange"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394"
+dependencies = [
+ "aligned",
+ "anyhow",
+ "arg_enum_proc_macro",
+ "arrayvec",
+ "log",
+ "num-rational",
+ "num-traits",
+ "pastey",
+ "rayon",
+ "thiserror 2.0.18",
+ "v_frame",
+ "y4m",
+]
+
+[[package]]
+name = "av1-grain"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8"
+dependencies = [
+ "anyhow",
+ "arrayvec",
+ "log",
+ "nom 8.0.0",
+ "num-rational",
+ "v_frame",
+]
+
+[[package]]
+name = "avif-serialize"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "375082f007bd67184fb9c0374614b29f9aaa604ec301635f72338bb65386a53d"
+dependencies = [
+ "arrayvec",
+]
+
[[package]]
name = "axum"
version = "0.7.9"
@@ -208,6 +307,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+[[package]]
+name = "base64ct"
+version = "1.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
+
[[package]]
name = "bit-set"
version = "0.8.0"
@@ -223,12 +328,27 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
+[[package]]
+name = "bit_field"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6"
+
[[package]]
name = "bitflags"
version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
+[[package]]
+name = "bitstream-io"
+version = "4.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f"
+dependencies = [
+ "no_std_io2",
+]
+
[[package]]
name = "block-buffer"
version = "0.10.4"
@@ -249,18 +369,36 @@ dependencies = [
"serde",
]
+[[package]]
+name = "built"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64"
+
[[package]]
name = "bumpalo"
version = "3.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
+[[package]]
+name = "bytemuck"
+version = "1.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
+
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+[[package]]
+name = "byteorder-lite"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
+
[[package]]
name = "bytes"
version = "1.11.1"
@@ -295,6 +433,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20"
dependencies = [
"find-msvc-tools",
+ "jobserver",
+ "libc",
"shlex",
]
@@ -377,6 +517,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
+[[package]]
+name = "color_quant"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
+
[[package]]
name = "colorchoice"
version = "1.0.5"
@@ -411,6 +557,73 @@ dependencies = [
"windows-sys 0.59.0",
]
+[[package]]
+name = "console"
+version = "0.16.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "unicode-width",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "cookie"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"
+dependencies = [
+ "percent-encoding",
+ "time",
+ "version_check",
+]
+
+[[package]]
+name = "cookie_store"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206"
+dependencies = [
+ "cookie",
+ "document-features",
+ "idna",
+ "indexmap",
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "time",
+ "url",
+]
+
+[[package]]
+name = "core-foundation"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+
[[package]]
name = "cpufeatures"
version = "0.2.17"
@@ -574,6 +787,25 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"
+[[package]]
+name = "der"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b"
+dependencies = [
+ "pem-rfc7468",
+ "zeroize",
+]
+
+[[package]]
+name = "deranged"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
+dependencies = [
+ "powerfmt",
+]
+
[[package]]
name = "derive_builder"
version = "0.20.2"
@@ -647,6 +879,15 @@ dependencies = [
"syn",
]
+[[package]]
+name = "document-features"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61"
+dependencies = [
+ "litrs",
+]
+
[[package]]
name = "either"
version = "1.15.0"
@@ -659,6 +900,35 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+[[package]]
+name = "encoding_rs"
+version = "0.8.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "equator"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc"
+dependencies = [
+ "equator-macro",
+]
+
+[[package]]
+name = "equator-macro"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "equivalent"
version = "1.0.2"
@@ -684,6 +954,21 @@ dependencies = [
"cc",
]
+[[package]]
+name = "exr"
+version = "1.74.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4300e043a56aa2cb633c01af81ca8f699a321879a7854d3896a0ba89056363be"
+dependencies = [
+ "bit_field",
+ "half",
+ "lebe",
+ "miniz_oxide",
+ "rayon-core",
+ "smallvec",
+ "zune-inflate",
+]
+
[[package]]
name = "fancy-regex"
version = "0.17.0"
@@ -695,12 +980,58 @@ dependencies = [
"regex-syntax",
]
+[[package]]
+name = "fastembed"
+version = "5.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58d74247f8cb93f94459e6f3599391f30c3f434f167f7109bd01a288db1bbe67"
+dependencies = [
+ "anyhow",
+ "hf-hub 0.5.0",
+ "image",
+ "ndarray",
+ "ort",
+ "safetensors",
+ "serde",
+ "serde_json",
+ "tokenizers 0.22.2",
+]
+
[[package]]
name = "fastrand"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
+[[package]]
+name = "fax"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f05de7d48f37cd6730705cbca900770cab77a89f413d23e100ad7fad7795a0ab"
+dependencies = [
+ "fax_derive",
+]
+
+[[package]]
+name = "fax_derive"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "fdeflate"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
+dependencies = [
+ "simd-adler32",
+]
+
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
@@ -729,6 +1060,27 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+[[package]]
+name = "foldhash"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
[[package]]
name = "form_urlencoded"
version = "1.2.2"
@@ -876,6 +1228,16 @@ dependencies = [
"wasip3",
]
+[[package]]
+name = "gif"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159"
+dependencies = [
+ "color_quant",
+ "weezl",
+]
+
[[package]]
name = "h2"
version = "0.4.13"
@@ -912,7 +1274,20 @@ version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
- "foldhash",
+ "foldhash 0.1.5",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+ "serde",
+ "serde_core",
]
[[package]]
@@ -927,8 +1302,9 @@ version = "0.1.0"
dependencies = [
"bytes",
"criterion",
+ "fastembed",
"flate2",
- "hf-hub",
+ "hf-hub 0.4.3",
"md-5",
"proptest",
"regex",
@@ -937,7 +1313,7 @@ dependencies = [
"sha2",
"thiserror 1.0.69",
"tiktoken-rs",
- "tokenizers",
+ "tokenizers 0.21.4",
"tracing",
]
@@ -1015,7 +1391,7 @@ checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97"
dependencies = [
"dirs",
"http",
- "indicatif",
+ "indicatif 0.17.11",
"libc",
"log",
"rand 0.9.4",
@@ -1023,10 +1399,37 @@ dependencies = [
"serde",
"serde_json",
"thiserror 2.0.18",
- "ureq",
+ "ureq 2.12.1",
"windows-sys 0.60.2",
]
+[[package]]
+name = "hf-hub"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aef3982638978efa195ff11b305f51f1f22f4f0a6cabee7af79b383ebee6a213"
+dependencies = [
+ "dirs",
+ "http",
+ "indicatif 0.18.4",
+ "libc",
+ "log",
+ "native-tls",
+ "rand 0.9.4",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.18",
+ "ureq 3.3.0",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "hmac-sha256"
+version = "1.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f"
+
[[package]]
name = "http"
version = "1.4.0"
@@ -1116,6 +1519,22 @@ dependencies = [
"webpki-roots 1.0.7",
]
+[[package]]
+name = "hyper-tls"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
+dependencies = [
+ "bytes",
+ "http-body-util",
+ "hyper",
+ "hyper-util",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+ "tower-service",
+]
+
[[package]]
name = "hyper-util"
version = "0.1.20"
@@ -1134,9 +1553,11 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"socket2",
+ "system-configuration",
"tokio",
"tower-service",
"tracing",
+ "windows-registry",
]
[[package]]
@@ -1254,6 +1675,46 @@ dependencies = [
"icu_properties",
]
+[[package]]
+name = "image"
+version = "0.25.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
+dependencies = [
+ "bytemuck",
+ "byteorder-lite",
+ "color_quant",
+ "exr",
+ "gif",
+ "image-webp",
+ "moxcms",
+ "num-traits",
+ "png",
+ "qoi",
+ "ravif",
+ "rayon",
+ "rgb",
+ "tiff",
+ "zune-core",
+ "zune-jpeg",
+]
+
+[[package]]
+name = "image-webp"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3"
+dependencies = [
+ "byteorder-lite",
+ "quick-error 2.0.1",
+]
+
+[[package]]
+name = "imgref"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7c5cedc30da3a610cac6b4ba17597bdf7152cf974e8aab3afb3d54455e371c8"
+
[[package]]
name = "indexmap"
version = "2.14.0"
@@ -1272,13 +1733,26 @@ version = "0.17.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
dependencies = [
- "console",
+ "console 0.15.11",
"number_prefix",
"portable-atomic",
"unicode-width",
"web-time",
]
+[[package]]
+name = "indicatif"
+version = "0.18.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb"
+dependencies = [
+ "console 0.16.3",
+ "portable-atomic",
+ "unicode-width",
+ "unit-prefix",
+ "web-time",
+]
+
[[package]]
name = "indoc"
version = "2.0.7"
@@ -1288,6 +1762,17 @@ dependencies = [
"rustversion",
]
+[[package]]
+name = "interpolate_name"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "ipnet"
version = "2.12.0"
@@ -1345,6 +1830,16 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+[[package]]
+name = "jobserver"
+version = "0.1.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
+dependencies = [
+ "getrandom 0.3.4",
+ "libc",
+]
+
[[package]]
name = "js-sys"
version = "0.3.95"
@@ -1369,12 +1864,28 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
+[[package]]
+name = "lebe"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8"
+
[[package]]
name = "libc"
version = "0.2.185"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
+[[package]]
+name = "libfuzzer-sys"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f12a681b7dd8ce12bff52488013ba614b869148d54dd79836ab85aafdd53f08d"
+dependencies = [
+ "arbitrary",
+ "cc",
+]
+
[[package]]
name = "libredox"
version = "0.1.16"
@@ -1396,18 +1907,39 @@ version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
+[[package]]
+name = "litrs"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
+
[[package]]
name = "log"
version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
+[[package]]
+name = "loop9"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062"
+dependencies = [
+ "imgref",
+]
+
[[package]]
name = "lru-slab"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
+[[package]]
+name = "lzma-rust2"
+version = "0.15.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1670343e58806300d87950e3401e820b519b9384281bbabfb15e3636689ffd69"
+
[[package]]
name = "macro_rules_attribute"
version = "0.2.2"
@@ -1439,6 +1971,26 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
+[[package]]
+name = "matrixmultiply"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08"
+dependencies = [
+ "autocfg",
+ "rawpointer",
+]
+
+[[package]]
+name = "maybe-rayon"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519"
+dependencies = [
+ "cfg-if",
+ "rayon",
+]
+
[[package]]
name = "md-5"
version = "0.10.6"
@@ -1519,6 +2071,63 @@ dependencies = [
"syn",
]
+[[package]]
+name = "moxcms"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
+dependencies = [
+ "num-traits",
+ "pxfm",
+]
+
+[[package]]
+name = "native-tls"
+version = "0.2.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
+dependencies = [
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
+[[package]]
+name = "ndarray"
+version = "0.17.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
+dependencies = [
+ "matrixmultiply",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "portable-atomic",
+ "portable-atomic-util",
+ "rawpointer",
+]
+
+[[package]]
+name = "new_debug_unreachable"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
+
+[[package]]
+name = "no_std_io2"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b51ed7824b6e07d354605f4abb3d9d300350701299da96642ee084f5ce631550"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "nom"
version = "7.1.3"
@@ -1529,6 +2138,21 @@ dependencies = [
"minimal-lexical",
]
+[[package]]
+name = "nom"
+version = "8.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "noop_proc_macro"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8"
+
[[package]]
name = "nu-ansi-term"
version = "0.50.3"
@@ -1538,6 +2162,62 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "num-bigint"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-complex"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-conv"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
+
+[[package]]
+name = "num-derive"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
+dependencies = [
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+]
+
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -1603,18 +2283,101 @@ version = "11.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
+[[package]]
+name = "openssl"
+version = "0.10.78"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.114"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
[[package]]
name = "option-ext"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
+[[package]]
+name = "ort"
+version = "2.0.0-rc.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133"
+dependencies = [
+ "ndarray",
+ "ort-sys",
+ "smallvec",
+ "tracing",
+ "ureq 3.3.0",
+]
+
+[[package]]
+name = "ort-sys"
+version = "2.0.0-rc.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90"
+dependencies = [
+ "hmac-sha256",
+ "lzma-rust2",
+ "ureq 3.3.0",
+]
+
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+[[package]]
+name = "pastey"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec"
+
+[[package]]
+name = "pem-rfc7468"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9"
+dependencies = [
+ "base64ct",
+]
+
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -1661,12 +2424,34 @@ dependencies = [
"plotters-backend",
]
+[[package]]
+name = "png"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
+dependencies = [
+ "bitflags",
+ "crc32fast",
+ "fdeflate",
+ "flate2",
+ "miniz_oxide",
+]
+
[[package]]
name = "portable-atomic"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618"
+dependencies = [
+ "portable-atomic",
+]
+
[[package]]
name = "potential_utf"
version = "0.1.5"
@@ -1676,6 +2461,12 @@ dependencies = [
"zerovec",
]
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
[[package]]
name = "ppv-lite86"
version = "0.2.21"
@@ -1704,6 +2495,25 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "profiling"
+version = "1.0.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3eb8486b569e12e2c32ad3e204dbaba5e4b5b216e9367044f25f1dba42341773"
+dependencies = [
+ "profiling-procmacros",
+]
+
+[[package]]
+name = "profiling-procmacros"
+version = "1.0.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52717f9a02b6965224f95ca2a81e2e0c5c43baacd28ca057577988930b6c3d5b"
+dependencies = [
+ "quote",
+ "syn",
+]
+
[[package]]
name = "proptest"
version = "1.11.0"
@@ -1723,6 +2533,12 @@ dependencies = [
"unarray",
]
+[[package]]
+name = "pxfm"
+version = "0.1.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f"
+
[[package]]
name = "pyo3"
version = "0.22.6"
@@ -1786,12 +2602,27 @@ dependencies = [
"syn",
]
+[[package]]
+name = "qoi"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001"
+dependencies = [
+ "bytemuck",
+]
+
[[package]]
name = "quick-error"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
+[[package]]
+name = "quick-error"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
+
[[package]]
name = "quinn"
version = "0.11.9"
@@ -1936,6 +2767,62 @@ dependencies = [
"rand_core 0.9.5",
]
+[[package]]
+name = "rav1e"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b"
+dependencies = [
+ "aligned-vec",
+ "arbitrary",
+ "arg_enum_proc_macro",
+ "arrayvec",
+ "av-scenechange",
+ "av1-grain",
+ "bitstream-io",
+ "built",
+ "cfg-if",
+ "interpolate_name",
+ "itertools 0.14.0",
+ "libc",
+ "libfuzzer-sys",
+ "log",
+ "maybe-rayon",
+ "new_debug_unreachable",
+ "noop_proc_macro",
+ "num-derive",
+ "num-traits",
+ "paste",
+ "profiling",
+ "rand 0.9.4",
+ "rand_chacha 0.9.0",
+ "simd_helpers",
+ "thiserror 2.0.18",
+ "v_frame",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "ravif"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e52310197d971b0f5be7fe6b57530dcd27beb35c1b013f29d66c1ad73fbbcc45"
+dependencies = [
+ "avif-serialize",
+ "imgref",
+ "loop9",
+ "quick-error 2.0.1",
+ "rav1e",
+ "rayon",
+ "rgb",
+]
+
+[[package]]
+name = "rawpointer"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
+
[[package]]
name = "rayon"
version = "1.12.0"
@@ -2015,6 +2902,7 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
dependencies = [
"base64 0.22.1",
"bytes",
+ "encoding_rs",
"futures-core",
"futures-util",
"h2",
@@ -2023,9 +2911,12 @@ dependencies = [
"http-body-util",
"hyper",
"hyper-rustls",
+ "hyper-tls",
"hyper-util",
"js-sys",
"log",
+ "mime",
+ "native-tls",
"percent-encoding",
"pin-project-lite",
"quinn",
@@ -2036,6 +2927,7 @@ dependencies = [
"serde_urlencoded",
"sync_wrapper",
"tokio",
+ "tokio-native-tls",
"tokio-rustls",
"tokio-util",
"tower",
@@ -2049,6 +2941,12 @@ dependencies = [
"webpki-roots 1.0.7",
]
+[[package]]
+name = "rgb"
+version = "0.8.53"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4"
+
[[package]]
name = "ring"
version = "0.17.14"
@@ -2137,7 +3035,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2"
dependencies = [
"fnv",
- "quick-error",
+ "quick-error 1.2.3",
"tempfile",
"wait-timeout",
]
@@ -2148,6 +3046,17 @@ version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
+[[package]]
+name = "safetensors"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "675656c1eabb620b921efea4f9199f97fc86e36dd6ffd1fbbe48d0f59a4987f5"
+dependencies = [
+ "hashbrown 0.16.1",
+ "serde",
+ "serde_json",
+]
+
[[package]]
name = "same-file"
version = "1.0.6"
@@ -2157,6 +3066,38 @@ dependencies = [
"winapi-util",
]
+[[package]]
+name = "schannel"
+version = "0.1.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "security-framework"
+version = "3.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
+dependencies = [
+ "bitflags",
+ "core-foundation 0.10.1",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
[[package]]
name = "semver"
version = "1.0.28"
@@ -2283,6 +3224,15 @@ version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
+[[package]]
+name = "simd_helpers"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6"
+dependencies = [
+ "quote",
+]
+
[[package]]
name = "slab"
version = "0.4.12"
@@ -2323,7 +3273,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
dependencies = [
"base64 0.13.1",
- "nom",
+ "nom 7.1.3",
"serde",
"unicode-segmentation",
]
@@ -2383,6 +3333,27 @@ dependencies = [
"syn",
]
+[[package]]
+name = "system-configuration"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
+dependencies = [
+ "bitflags",
+ "core-foundation 0.9.4",
+ "system-configuration-sys",
+]
+
+[[package]]
+name = "system-configuration-sys"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
[[package]]
name = "target-lexicon"
version = "0.12.16"
@@ -2451,6 +3422,20 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "tiff"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52"
+dependencies = [
+ "fax",
+ "flate2",
+ "half",
+ "quick-error 2.0.1",
+ "weezl",
+ "zune-jpeg",
+]
+
[[package]]
name = "tiktoken-rs"
version = "0.11.0"
@@ -2466,6 +3451,37 @@ dependencies = [
"rustc-hash 1.1.0",
]
+[[package]]
+name = "time"
+version = "0.3.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
+dependencies = [
+ "deranged",
+ "itoa",
+ "num-conv",
+ "powerfmt",
+ "serde_core",
+ "time-core",
+ "time-macros",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
+
+[[package]]
+name = "time-macros"
+version = "0.2.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
+dependencies = [
+ "num-conv",
+ "time-core",
+]
+
[[package]]
name = "tinystr"
version = "0.8.3"
@@ -2514,7 +3530,40 @@ dependencies = [
"derive_builder",
"esaxx-rs",
"getrandom 0.3.4",
- "indicatif",
+ "indicatif 0.17.11",
+ "itertools 0.14.0",
+ "log",
+ "macro_rules_attribute",
+ "monostate",
+ "onig",
+ "paste",
+ "rand 0.9.4",
+ "rayon",
+ "rayon-cond",
+ "regex",
+ "regex-syntax",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror 2.0.18",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
+[[package]]
+name = "tokenizers"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223"
+dependencies = [
+ "ahash",
+ "aho-corasick",
+ "compact_str",
+ "dary_heap",
+ "derive_builder",
+ "esaxx-rs",
+ "getrandom 0.3.4",
"itertools 0.14.0",
"log",
"macro_rules_attribute",
@@ -2562,6 +3611,16 @@ dependencies = [
"syn",
]
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
[[package]]
name = "tokio-rustls"
version = "0.26.4"
@@ -2818,6 +3877,12 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
+[[package]]
+name = "unit-prefix"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3"
+
[[package]]
name = "untrusted"
version = "0.9.0"
@@ -2843,6 +3908,42 @@ dependencies = [
"webpki-roots 0.26.11",
]
+[[package]]
+name = "ureq"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0"
+dependencies = [
+ "base64 0.22.1",
+ "cookie_store",
+ "der",
+ "flate2",
+ "log",
+ "native-tls",
+ "percent-encoding",
+ "rustls",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "socks",
+ "ureq-proto",
+ "utf8-zero",
+ "webpki-root-certs",
+ "webpki-roots 1.0.7",
+]
+
+[[package]]
+name = "ureq-proto"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c"
+dependencies = [
+ "base64 0.22.1",
+ "http",
+ "httparse",
+ "log",
+]
+
[[package]]
name = "url"
version = "2.5.8"
@@ -2861,6 +3962,12 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+[[package]]
+name = "utf8-zero"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
+
[[package]]
name = "utf8_iter"
version = "1.0.4"
@@ -2884,12 +3991,29 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "v_frame"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2"
+dependencies = [
+ "aligned-vec",
+ "num-traits",
+ "wasm-bindgen",
+]
+
[[package]]
name = "valuable"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
[[package]]
name = "version_check"
version = "0.9.5"
@@ -3070,6 +4194,15 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "webpki-root-certs"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
+dependencies = [
+ "rustls-pki-types",
+]
+
[[package]]
name = "webpki-roots"
version = "0.26.11"
@@ -3088,6 +4221,12 @@ dependencies = [
"rustls-pki-types",
]
+[[package]]
+name = "weezl"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
+
[[package]]
name = "winapi"
version = "0.3.9"
@@ -3125,6 +4264,35 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+[[package]]
+name = "windows-registry"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720"
+dependencies = [
+ "windows-link",
+ "windows-result",
+ "windows-strings",
+]
+
+[[package]]
+name = "windows-result"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
+dependencies = [
+ "windows-link",
+]
+
[[package]]
name = "windows-sys"
version = "0.52.0"
@@ -3413,6 +4581,12 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
+[[package]]
+name = "y4m"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448"
+
[[package]]
name = "yoke"
version = "0.8.2"
@@ -3521,3 +4695,27 @@ name = "zmij"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
+
+[[package]]
+name = "zune-core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9"
+
+[[package]]
+name = "zune-inflate"
+version = "0.2.54"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02"
+dependencies = [
+ "simd-adler32",
+]
+
+[[package]]
+name = "zune-jpeg"
+version = "0.5.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
+dependencies = [
+ "zune-core",
+]
diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.toml
index aa67c872f..6ea7c5026 100644
--- a/crates/headroom-core/Cargo.toml
+++ b/crates/headroom-core/Cargo.toml
@@ -41,6 +41,14 @@ regex = "1"
# outcome — but if a parity fixture flakes here, swap to
# `features = ["zlib"]` to link against system libz for byte-equal output.
flate2 = "1"
+# `fastembed` is the Rust port of the Python fastembed library. Used by
+# `relevance::EmbeddingScorer` for sentence embeddings in semantic relevance
+# scoring. Default features pull in `ort` (ONNX Runtime) with auto-download
+# of the runtime binary; the model file (BAAI/bge-small-en-v1.5, ~30 MB
+# int8-quantized ONNX) auto-downloads from HuggingFace Hub on first use.
+# Same library + same model = byte-equal embeddings between Python and Rust
+# (both call into ONNX Runtime over the identical ONNX file).
+fastembed = "5"
[dev-dependencies]
proptest = "1"
diff --git a/crates/headroom-core/src/relevance/embedding.rs b/crates/headroom-core/src/relevance/embedding.rs
index 91bc68957..bd3b93f14 100644
--- a/crates/headroom-core/src/relevance/embedding.rs
+++ b/crates/headroom-core/src/relevance/embedding.rs
@@ -1,115 +1,358 @@
-//! Embedding-based relevance scorer (sentence-transformers).
+//! Embedding-based relevance scorer using `fastembed-rs`.
//!
-//! # Status: STUB — real ONNX implementation lands in a follow-up commit.
+//! Uses BAAI/bge-small-en-v1.5 (33M params, 384 dims) by default —
+//! same model the Python side runs via the `fastembed` package, giving
+//! byte-equal embeddings on identical inputs. fastembed wraps ONNX
+//! Runtime under the hood, with the runtime binary auto-downloaded
+//! once at build time and the model weights auto-downloaded from
+//! Hugging Face Hub on first use (~30 MB int8-quantized ONNX).
//!
-//! Direct port of `headroom/relevance/embedding.py`. Python's
-//! `EmbeddingScorer.is_available()` returns `False` when
-//! `sentence-transformers` is not installed; this Rust stub mirrors
-//! that "not available" path exactly. `HybridScorer` already handles
-//! this case via its BM25-only fallback (with a small score boost),
-//! so the planning layer can call into hybrid right now and behave
-//! parity-equal with a Python deployment that has no ML deps.
+//! # Caching
//!
-//! When the real ONNX implementation lands:
-//! - Add `ort` (ONNX Runtime) and reuse the existing `tokenizers` dep.
-//! - Auto-download `sentence-transformers/all-MiniLM-L6-v2` via `hf-hub`.
-//! - Mean-pool the token embeddings, L2-normalize, cosine-similarity.
-//! - Flip `is_available()` to `true` and `score()`/`score_batch()` to
-//! the inference path.
+//! Loading a sentence-transformer model takes ~1-2 seconds (HF Hub
+//! call + ONNX session init). Construct the scorer once per process
+//! and reuse — `try_new` returns a `Result` because the first
+//! construction may need network access to fetch the model.
//!
-//! No public-API change required at the call site — `HybridScorer` can
-//! continue to reach for `EmbeddingScorer::new()` and check
-//! `is_available()` on it.
+//! When constructed, `is_available()` returns `true` and `HybridScorer`
+//! switches off the BM25-fallback path automatically. If construction
+//! fails (e.g. offline + model not cached), callers should fall back
+//! to `HybridScorer::default()` which uses the stub-fallback scorer.
+//!
+//! # Output stability vs Python
+//!
+//! Both languages call into the same ONNX file via ONNX Runtime (`ort`
+//! crate in Rust, `onnxruntime` package in Python's fastembed). Same
+//! kernels, same weights — embeddings agree to floating-point
+//! representation. Cosine similarity agrees to ~1e-6.
+
+use std::sync::Mutex;
+
+use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};
use super::base::{RelevanceScore, RelevanceScorer};
-/// Stub scorer. Always reports `is_available() == false`.
+/// fastembed-backed semantic relevance scorer.
///
-/// Construct one when you want to express "embedding scoring desired,
-/// fall back if unavailable". Production code goes through
-/// `HybridScorer` which handles the fallback transparently.
+/// Construct via `EmbeddingScorer::try_new()` to handle the model-load
+/// fallible step explicitly. `EmbeddingScorer::default()` is provided
+/// for backwards compatibility but `is_available()` returns `false`
+/// when the inner model failed to load (mimicking Python's
+/// "sentence-transformers not installed" branch).
pub struct EmbeddingScorer {
- /// Stored for future ONNX work. Currently unused since the stub
- /// can't actually load a model.
pub model_name: String,
+ /// `None` when model load failed — `is_available()` returns false
+ /// and `score`/`score_batch` return empty scores. This lets
+ /// `HybridScorer::default()` work even when the model can't be
+ /// loaded (e.g. offline, no model cache).
+ ///
+ /// Wrapped in a `Mutex` because `TextEmbedding::embed` requires
+ /// `&mut self` (the underlying ONNX session is single-threaded).
+ /// Concurrent callers serialize on the inner lock, which is fine
+ /// for the SmartCrusher hot path — embedding inference is the
+ /// dominant cost so contention is bounded by inference latency,
+ /// not lock latency.
+ model: Option<Mutex<TextEmbedding>>,
}
impl Default for EmbeddingScorer {
+ /// Returns an unloaded scorer (model = None, is_available = false).
+ ///
+ /// Mirrors Python's "sentence-transformers not installed" branch:
+ /// `HybridScorer::default()` constructs an EmbeddingScorer via
+ /// `default()`, finds it unavailable, and uses BM25-fallback.
+ ///
+ /// To get a real, model-backed scorer call `try_new()` explicitly
+ /// and pass it via `HybridScorer::with_scorers`. This separation
+ /// keeps `Default` cheap (no I/O) and predictable in tests —
+ /// otherwise model availability would depend on whether the user
+ /// has previously cached the weights.
fn default() -> Self {
EmbeddingScorer {
- model_name: "sentence-transformers/all-MiniLM-L6-v2".to_string(),
+ model_name: "BAAI/bge-small-en-v1.5".to_string(),
+ model: None,
}
}
}
impl EmbeddingScorer {
- pub fn new(model_name: impl Into<String>) -> Self {
- EmbeddingScorer {
- model_name: model_name.into(),
- }
+ /// Construct the scorer with the default model
+ /// (BAAI/bge-small-en-v1.5). May trigger a one-time HF Hub
+ /// download if the model isn't cached locally; subsequent calls
+ /// are fast.
+ ///
+ /// Returns an error from fastembed if model initialization fails
+ /// (network failure during download, missing ONNX runtime
+ /// binaries, etc.).
+ pub fn try_new() -> Result<Self, String> {
+ Self::try_new_with_model(EmbeddingModel::BGESmallENV15)
+ }
+
+ /// Construct with an explicit model. See `fastembed::EmbeddingModel`
+ /// for the catalog. The default `BGESmallENV15` is the best
+ /// quality/speed tradeoff for compression-relevance scoring on
+ /// short snippets.
+ pub fn try_new_with_model(model_kind: EmbeddingModel) -> Result<Self, String> {
+ let name = format!("{:?}", model_kind);
+ let model = TextEmbedding::try_new(InitOptions::new(model_kind))
+ .map_err(|e| format!("EmbeddingScorer model load failed: {}", e))?;
+ Ok(EmbeddingScorer {
+ model_name: name,
+ model: Some(Mutex::new(model)),
+ })
}
}
impl RelevanceScorer for EmbeddingScorer {
- fn score(&self, _item: &str, _context: &str) -> RelevanceScore {
- // Defensive: `HybridScorer` checks `is_available()` first so
- // this branch shouldn't be reached. Returning empty rather
- // than panicking keeps the trait safe to call directly.
- RelevanceScore::empty("Embedding: ONNX backend not yet implemented")
+ fn score(&self, item: &str, context: &str) -> RelevanceScore {
+ if item.is_empty() || context.is_empty() {
+ return RelevanceScore::empty("Embedding: empty input");
+ }
+ let Some(model) = &self.model else {
+ return RelevanceScore::empty("Embedding: model not available");
+ };
+ let mut guard = match model.lock() {
+ Ok(g) => g,
+ Err(_) => return RelevanceScore::empty("Embedding: lock poisoned"),
+ };
+ let embeddings = match guard.embed(vec![item.to_string(), context.to_string()], None) {
+ Ok(e) => e,
+ Err(e) => return RelevanceScore::empty(format!("Embedding: inference failed: {}", e)),
+ };
+ if embeddings.len() != 2 {
+ return RelevanceScore::empty("Embedding: unexpected embedding count");
+ }
+ let sim = cosine_similarity(&embeddings[0], &embeddings[1]);
+ RelevanceScore::new(
+ sim,
+ format!("Embedding: semantic similarity {:.2}", sim),
+ Vec::new(),
+ )
}
- fn score_batch(&self, items: &[&str], _context: &str) -> Vec<RelevanceScore> {
- items
+ fn score_batch(&self, items: &[&str], context: &str) -> Vec<RelevanceScore> {
+ if items.is_empty() {
+ return Vec::new();
+ }
+ if context.is_empty() {
+ return items
+ .iter()
+ .map(|_| RelevanceScore::empty("Embedding: empty context"))
+ .collect();
+ }
+ let Some(model) = &self.model else {
+ return items
+ .iter()
+ .map(|_| RelevanceScore::empty("Embedding: model not available"))
+ .collect();
+ };
+ let mut guard = match model.lock() {
+ Ok(g) => g,
+ Err(_) => {
+ return items
+ .iter()
+ .map(|_| RelevanceScore::empty("Embedding: lock poisoned"))
+ .collect();
+ }
+ };
+
+ // Encode items + context in one batch — saves model dispatch
+ // overhead. Mirrors Python fastembed batch encoding.
+ let mut all_texts: Vec<String> = items.iter().map(|s| s.to_string()).collect();
+ all_texts.push(context.to_string());
+ let embeddings = match guard.embed(all_texts, None) {
+ Ok(e) => e,
+ Err(e) => {
+ return items
+ .iter()
+ .map(|_| RelevanceScore::empty(format!("Embedding: inference failed: {}", e)))
+ .collect();
+ }
+ };
+ if embeddings.len() != items.len() + 1 {
+ return items
+ .iter()
+ .map(|_| RelevanceScore::empty("Embedding: unexpected embedding count"))
+ .collect();
+ }
+
+ let context_emb = embeddings.last().unwrap().clone();
+ embeddings
.iter()
- .map(|_| RelevanceScore::empty("Embedding: ONNX backend not yet implemented"))
+ .take(items.len())
+ .map(|emb| {
+ let sim = cosine_similarity(emb, &context_emb);
+ RelevanceScore::new(
+ sim,
+ format!("Embedding: {:.2}", sim),
+ Vec::new(),
+ )
+ })
.collect()
}
fn is_available(&self) -> bool {
- // Pinned to the Python "sentence-transformers not installed"
- // branch. Flips to `true` when the real ONNX impl lands.
- false
+ self.model.is_some()
}
}
+/// Cosine similarity for two vectors. Clamped to `[0, 1]` since we
+/// only care about positive similarity (mirrors Python `_cosine_similarity`).
+fn cosine_similarity(a: &[f32], b: &[f32]) -> f64 {
+ if a.is_empty() || b.is_empty() || a.len() != b.len() {
+ return 0.0;
+ }
+ let mut dot: f64 = 0.0;
+ let mut norm_a: f64 = 0.0;
+ let mut norm_b: f64 = 0.0;
+ for i in 0..a.len() {
+ let av = a[i] as f64;
+ let bv = b[i] as f64;
+ dot += av * bv;
+ norm_a += av * av;
+ norm_b += bv * bv;
+ }
+ if norm_a == 0.0 || norm_b == 0.0 {
+ return 0.0;
+ }
+ let sim = dot / (norm_a.sqrt() * norm_b.sqrt());
+ sim.clamp(0.0, 1.0)
+}
+
#[cfg(test)]
mod tests {
use super::*;
+ // The real-model tests are gated behind RUN_FASTEMBED_TESTS=1
+ // since they require network access on first run (~30 MB model
+ // download). Without the env var, only the offline-safe stub
+ // path is exercised.
+
+ fn fastembed_enabled() -> bool {
+ std::env::var("RUN_FASTEMBED_TESTS").is_ok()
+ }
+
+ /// Construct a stub scorer with `model = None` for offline-safe
+ /// tests of the unavailable-path behavior.
+ fn unavailable_scorer() -> EmbeddingScorer {
+ EmbeddingScorer {
+ model_name: "test".to_string(),
+ model: None,
+ }
+ }
+
#[test]
- fn stub_reports_unavailable() {
- assert!(!EmbeddingScorer::default().is_available());
+ fn cosine_similarity_orthogonal_vectors() {
+ let a = vec![1.0_f32, 0.0, 0.0, 0.0];
+ let b = vec![0.0_f32, 1.0, 0.0, 0.0];
+ assert_eq!(cosine_similarity(&a, &b), 0.0);
}
#[test]
- fn stub_score_returns_empty() {
- let s = EmbeddingScorer::default();
- let r = s.score("item text", "query");
- assert_eq!(r.score, 0.0);
+ fn cosine_similarity_identical_vectors() {
+ let v = vec![1.0_f32, 2.0, 3.0];
+ let sim = cosine_similarity(&v, &v);
+ assert!((sim - 1.0).abs() < 1e-9, "got {}", sim);
}
#[test]
- fn stub_score_batch_one_per_item() {
- let s = EmbeddingScorer::default();
- let items = ["a", "b", "c"];
- let scores = s.score_batch(&items, "ctx");
- assert_eq!(scores.len(), 3);
- for sc in scores {
+ fn cosine_similarity_opposite_clamped_to_zero() {
+ let a = vec![1.0_f32, 1.0];
+ let b = vec![-1.0_f32, -1.0];
+ // Raw cosine = -1.0; clamp to 0.0 since we only care about
+ // positive similarity for relevance scoring.
+ assert_eq!(cosine_similarity(&a, &b), 0.0);
+ }
+
+ #[test]
+ fn cosine_similarity_zero_vector_returns_zero() {
+ let zero = vec![0.0_f32; 4];
+ let v = vec![1.0_f32, 2.0, 3.0, 4.0];
+ assert_eq!(cosine_similarity(&zero, &v), 0.0);
+ assert_eq!(cosine_similarity(&v, &zero), 0.0);
+ }
+
+ #[test]
+ fn cosine_similarity_mismatched_dim_returns_zero() {
+ let a = vec![1.0_f32, 2.0];
+ let b = vec![1.0_f32, 2.0, 3.0];
+ assert_eq!(cosine_similarity(&a, &b), 0.0);
+ }
+
+ // ---------- offline-safe scorer behavior (no model needed) ----------
+
+ #[test]
+ fn unavailable_scorer_returns_empty_scores() {
+ // Construct a scorer with model=None to simulate the offline
+ // path. Default uses try_new which would download — bypass for
+ // unit tests.
+ let s = unavailable_scorer();
+ assert!(!s.is_available());
+
+ let r = s.score("item", "query");
+ assert_eq!(r.score, 0.0);
+
+ let batch = s.score_batch(&["a", "b", "c"], "query");
+ assert_eq!(batch.len(), 3);
+ for sc in batch {
assert_eq!(sc.score, 0.0);
}
}
#[test]
- fn stores_model_name() {
- let s = EmbeddingScorer::new("custom/model");
- assert_eq!(s.model_name, "custom/model");
+ fn unavailable_scorer_empty_inputs_short_circuit() {
+ let s = unavailable_scorer();
+ let r = s.score("", "query");
+ assert_eq!(r.score, 0.0);
+ assert!(r.reason.contains("empty"));
+ }
+
+ #[test]
+ fn batch_with_empty_items_returns_empty_vec() {
+ let s = unavailable_scorer();
+ let r = s.score_batch(&[], "anything");
+ assert!(r.is_empty());
+ }
+
+ // ---------- model-backed tests (gated on RUN_FASTEMBED_TESTS) ----------
+
+ #[test]
+ fn fastembed_loads_default_model() {
+ if !fastembed_enabled() {
+ return;
+ }
+ let s = EmbeddingScorer::try_new().expect("model loads");
+ assert!(s.is_available());
+ assert_eq!(s.model_name, "BGESmallENV15");
}
#[test]
- fn default_model_is_all_minilm() {
- assert_eq!(
- EmbeddingScorer::default().model_name,
- "sentence-transformers/all-MiniLM-L6-v2"
+ fn fastembed_semantic_match_outranks_unrelated() {
+ if !fastembed_enabled() {
+ return;
+ }
+ let s = EmbeddingScorer::try_new().expect("model loads");
+ let related = s.score("authentication failed for user", "login error");
+ let unrelated = s.score("the weather is nice today", "login error");
+ assert!(
+ related.score > unrelated.score,
+ "semantically-related text should score higher: related={}, unrelated={}",
+ related.score,
+ unrelated.score
);
}
+
+ #[test]
+ fn fastembed_batch_returns_one_score_per_item() {
+ if !fastembed_enabled() {
+ return;
+ }
+ let s = EmbeddingScorer::try_new().expect("model loads");
+ let items = ["foo", "bar", "baz"];
+ let scores = s.score_batch(&items, "query text");
+ assert_eq!(scores.len(), 3);
+ for sc in scores {
+ assert!((0.0..=1.0).contains(&sc.score));
+ }
+ }
}