Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(kompress): read the pinned snapshot from Python's one HF cache in the native proxy (#4438)
Kenneth Wongeef44f60cc56
diff --git a/crates/headroom-core/src/hf_cache.rs b/crates/headroom-core/src/hf_cache.rsnew file mode 100644index 000000000..83a9e230e--- /dev/null+++ b/crates/headroom-core/src/hf_cache.rs@@ -0,0 +1,712 @@+//! The HuggingFace hub cache, located and read the way Python's+//! `huggingface_hub` does, so the Rust loaders (the relevance embedder,+//! Kompress and the HuggingFace tokenizer) find the files the Python side+//! downloaded and pinned, and authenticate with the same token.+//!+//! All network-free: [`hub_cache_dir`] picks the one cache directory Python+//! would use, [`snapshot_path`] resolves a repo at a revision inside it,+//! [`hf_token`] finds the access token and [`hub_api`] builds a client from+//! them. A path variable that is set but empty counts as set, as in Python+//! (see [`hub_cache_dir`]). The pinned revisions themselves live+//! next to each loader and are kept equal to `_PINNED_REVISIONS` in+//! `headroom/onnx_runtime.py` by tests on the Python side.+//!+//! Not behind the `ml` feature: the tokenizer, which is not either, uses the+//! cache and token lookups.++use std::path::{Path, PathBuf};++use thiserror::Error;++/// True when `HEADROOM_HF_PIN` turns revision pinning off, so pinned repos+/// float on `main` (same values as Python's `onnx_runtime._resolve_revision`).+#[cfg_attr(not(feature = "ml"), allow(dead_code))] // the pinned loaders are `ml`-only+pub(crate) fn pin_disabled() -> bool {+ let pin = std::env::var("HEADROOM_HF_PIN").unwrap_or_default();+ matches!(+ pin.trim().to_ascii_lowercase().as_str(),+ "off" | "0" | "false" | "no"+ )+}++/// The revision to load for a repo pinned at `pinned`: the pin, or `main`+/// when `HEADROOM_HF_PIN` turns pinning off.+#[cfg_attr(not(feature = "ml"), allow(dead_code))]+pub(crate) fn pinned_or_main(pinned: &str) -> &str {+ if pin_disabled() {+ "main"+ } else {+ pinned+ }+}++/// The snapshot directory of `repo` (`"owner/name"`) at `revision` in the hub+/// cache `cache`, whether or not it exists. Never touches the network.+///+/// A branch name such as `main` resolves through `refs/<revision>`; a commit+/// SHA is used as the directory name directly. `hf_hub::Cache` resolves every+/// revision through `refs/`, which Python's `huggingface_hub` does not write+/// for a commit SHA, so a snapshot Python downloaded at a pin would be missed.+#[cfg_attr(not(feature = "ml"), allow(dead_code))]+pub(crate) fn snapshot_path(cache: &Path, repo: &str, revision: &str) -> PathBuf {+ let repo_dir = cache.join(format!("models--{}", repo.replace('/', "--")));+ let commit = std::fs::read_to_string(repo_dir.join("refs").join(revision))+ .map(|c| c.trim().to_string())+ .unwrap_or_else(|_| revision.to_string());+ repo_dir.join("snapshots").join(commit)+}++/// The HuggingFace hub cache directory, resolved the way Python's+/// `huggingface_hub` resolves `constants.HF_HUB_CACHE`, so both languages look+/// up and download into the same place: `HF_HUB_CACHE`, then+/// `HUGGINGFACE_HUB_CACHE` (its legacy name), then `HF_HOME/hub`, then+/// `XDG_CACHE_HOME/huggingface/hub`, then `~/.cache/huggingface/hub`.+///+/// It is exactly one directory. A snapshot in a lower-priority cache is not+/// used when a higher one is configured, which is also what Python does: with+/// `HF_HUB_CACHE` set, a model under `~/.cache` is neither found nor reused.+///+/// Both expansions happen as in `huggingface_hub.constants`:+/// `expandvars(expanduser(...))`, first on `HF_HOME` (or its+/// `XDG_CACHE_HOME` default) and then on the hub path itself. So+/// `HF_HUB_CACHE=${ROOT}/hub` names the same directory in both languages, and+/// a `~` that only appears once a variable is expanded stays literal, as it+/// does in Python. The home directory comes from `std::env::home_dir`, which+/// falls back to the user database when `HOME` is unset, as `expanduser`+/// does. A variable that is set but empty still counts, as with `os.getenv`:+/// `HF_HUB_CACHE=''` is the working directory itself, `HF_HOME=''` is `hub`+/// and `XDG_CACHE_HOME=''` is `huggingface/hub` under it, and none of them+/// falls through to a lower-priority cache. `tests/parity/fixtures/hf_hub_cache/`+/// holds the cases both languages are checked against.+pub(crate) fn hub_cache_dir() -> Option<PathBuf> {+ let hub = match env_path("HF_HUB_CACHE").or_else(|| env_path("HUGGINGFACE_HUB_CACHE")) {+ Some(dir) => dir,+ None => hf_home_dir()?.join("hub"),+ };+ Some(expand(hub))+}++/// `huggingface_hub.constants.HF_HOME`: `HF_HOME`, else+/// `XDG_CACHE_HOME/huggingface`, else `~/.cache/huggingface`, expanded as+/// Python does. The hub cache defaults to its `hub` folder and the login+/// token to its `token` file.+fn hf_home_dir() -> Option<PathBuf> {+ let dir = match env_path("HF_HOME") {+ Some(dir) => dir,+ None => env_path("XDG_CACHE_HOME")+ .or_else(|| std::env::home_dir().map(|home| home.join(".cache")))?+ .join("huggingface"),+ };+ Some(expand(dir))+}++/// The HuggingFace access token, found the way `huggingface_hub.get_token()`+/// finds it: `HF_TOKEN`, then its legacy name `HUGGING_FACE_HUB_TOKEN`, then+/// the file at `HF_TOKEN_PATH` (default `HF_HOME/token`, where+/// `hf auth login` writes it). Each value is cleaned like Python's+/// `_clean_token`: line breaks removed, outer whitespace trimmed, and an+/// empty result counts as no token.+///+/// `hf-hub`'s own lookup reads only a `token` file beside the hub cache, so+/// `HF_TOKEN`, an explicit `HF_TOKEN_PATH`, or a hub cache moved away from+/// `HF_HOME` would leave a gated download unauthenticated.+pub(crate) fn hf_token() -> Option<String> {+ let clean = |token: String| {+ let token = token.replace(['\r', '\n'], "");+ let token = token.trim();+ (!token.is_empty()).then(|| token.to_string())+ };+ // `HF_TOKEN or HUGGING_FACE_HUB_TOKEN`, then clean: a whitespace-only+ // `HF_TOKEN` is still the one picked, and cleans to no token, so the+ // legacy name is not consulted (the login file is).+ ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]+ .into_iter()+ .find_map(|name| std::env::var(name).ok().filter(|v| !v.is_empty()))+ .and_then(clean)+ .or_else(|| {+ let path = match env_path("HF_TOKEN_PATH") {+ Some(path) => expand(path),+ None => hf_home_dir()?.join("token"),+ };+ std::fs::read_to_string(path).ok().and_then(clean)+ })+}++/// The Hub endpoint: `HF_ENDPOINT` without trailing slashes (Python's+/// `constants.ENDPOINT` strips them, and `hf-hub` appends `/{repo}/…`, so a+/// kept slash would request `//repo`), else `https://huggingface.co`.+pub(crate) fn hf_endpoint() -> String {+ match std::env::var("HF_ENDPOINT") {+ Ok(endpoint) => endpoint.trim_end_matches('/').to_string(),+ Err(_) => "https://huggingface.co".to_string(),+ }+}++/// Why [`hub_api`] returned no client.+#[derive(Debug, Error)]+pub(crate) enum HubApiError {+ /// `HEADROOM_OFFLINE` is set; refused before the client existed.+ #[error(transparent)]+ Offline(#[from] crate::offline::OfflineEgressBlocked),+ /// `hf-hub` could not build the client.+ #[error(transparent)]+ Build(#[from] hf_hub::api::sync::ApiError),+}++/// An `hf-hub` client that caches in `cache` and talks to [`hf_endpoint`]+/// with [`hf_token`], as Python's `huggingface_hub` would, for fetching+/// `destination` (a repo id) for `purpose`.+///+/// This is the air-gap chokepoint for the tokenizer and embedder downloads:+/// under `HEADROOM_OFFLINE` it returns [`HubApiError::Offline`] before+/// building the client, because building it resolves the endpoint and the+/// `ureq` agent, and `hf-hub` downloads on a cache miss even with+/// `HF_HUB_OFFLINE` set. Guarding here rather than in each caller means no+/// caller of this function can skip it. (`Kompress::from_pretrained` still+/// builds its own `Api::new()` client behind its own guard; production loads+/// Kompress cache-only through `Kompress::from_cache`.)+///+/// The token is not sent over plain `http://` to a host other than loopback:+/// it would cross the network in clear text on every request, so the client+/// goes without it and says why. Python would send it; a gated repo behind+/// such a mirror then answers 401 rather than leak the credential.+pub(crate) fn hub_api(+ cache: &Path,+ purpose: &str,+ destination: &str,+) -> Result<hf_hub::api::sync::Api, HubApiError> {+ crate::offline::guard_egress(purpose, destination)?;+ let endpoint = hf_endpoint();+ let token = hf_token().filter(|_| {+ let exposed = sends_in_clear_text(&endpoint);+ if exposed {+ tracing::warn!(+ event = "hf_token_withheld",+ endpoint = %endpoint,+ "HF_ENDPOINT is plain http to a non-loopback host; not sending the \+ HuggingFace token to it. Use an https endpoint for gated repos."+ );+ }+ !exposed+ });+ hf_hub::api::sync::ApiBuilder::from_cache(hf_hub::Cache::new(cache.to_path_buf()))+ .with_progress(false)+ .with_endpoint(endpoint)+ .with_token(token)+ .build()+ .map_err(HubApiError::Build)+}++/// Whether a request to `endpoint` would carry its headers unencrypted off+/// this machine: `http://` to anything but a loopback host.+fn sends_in_clear_text(endpoint: &str) -> bool {+ let Some(rest) = endpoint+ .get(..7)+ .filter(|scheme| scheme.eq_ignore_ascii_case("http://"))+ .map(|_| &endpoint[7..])+ else {+ return false;+ };+ let authority = rest.split(['/', '?', '#']).next().unwrap_or("");+ let host_port = authority.rsplit('@').next().unwrap_or("");+ let host = match host_port.strip_prefix('[') {+ Some(v6) => v6.split(']').next().unwrap_or(""),+ None => host_port.split(':').next().unwrap_or(""),+ };+ let loopback = host.eq_ignore_ascii_case("localhost")+ || host+ .parse::<std::net::IpAddr>()+ .is_ok_and(|ip| ip.is_loopback());+ !loopback+}++/// A path-valued variable. An empty one is still set, as `os.getenv` reads+/// it, so it never falls through to the next variable.+fn env_path(name: &str) -> Option<PathBuf> {+ std::env::var_os(name).map(PathBuf::from)+}++/// `expandvars(expanduser(path))`, the order `huggingface_hub.constants` uses.+fn expand(path: PathBuf) -> PathBuf {+ expand_vars(&expand_user(path, std::env::home_dir().as_deref()))+}++/// Python's `os.path.expanduser` for a leading bare `~` component. `~user`+/// stays as written, which is also what `expanduser` does when it cannot+/// resolve the user.+fn expand_user(path: PathBuf, home: Option<&Path>) -> PathBuf {+ match (path.strip_prefix("~"), home) {+ (Ok(rest), Some(home)) => home.join(rest),+ _ => path,+ }+}++/// Python's `os.path.expandvars`: `posixpath` rules off Windows, `ntpath`+/// rules on it. A path that is not valid Unicode is returned unchanged.+fn expand_vars(path: &Path) -> PathBuf {+ let Some(text) = path.to_str() else {+ return path.to_path_buf();+ };+ #[cfg(windows)]+ let expanded = expand_vars_nt(text, env_lookup);+ #[cfg(not(windows))]+ let expanded = expand_vars_posix(text, env_lookup);+ PathBuf::from(expanded)+}++/// An environment value for `expand_vars`. Names the OS cannot hold (empty,+/// or containing `=` or NUL) are unset, which is how Python's `os.environ`+/// lookup fails for them too.+fn env_lookup(name: &str) -> Option<std::ffi::OsString> {+ if name.is_empty() || name.contains(['=', '\0']) {+ return None;+ }+ std::env::var_os(name)+}++/// `posixpath.expandvars`: `$name` (ASCII letters, digits, `_`) or+/// `${anything but a brace}`. An unset name, and a `$` that starts neither+/// form, stay as written. Expanded values are not scanned again.+#[cfg_attr(windows, allow(dead_code))] // tests run both rule sets everywhere+fn expand_vars_posix(+ text: &str,+ lookup: impl Fn(&str) -> Option<std::ffi::OsString>,+) -> std::ffi::OsString {+ let mut out = std::ffi::OsString::new();+ let mut rest = text;+ while let Some(at) = rest.find('$') {+ out.push(&rest[..at]);+ let after = &rest[at + 1..];+ let word = after+ .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_'))+ .unwrap_or(after.len());+ // (name, length of the reference after the `$`)+ let reference = if word > 0 {+ Some((&after[..word], word))+ } else {+ after+ .strip_prefix('{')+ .and_then(|braced| braced.find('}').map(|end| (&braced[..end], end + 2)))+ };+ match reference {+ Some((name, len)) => {+ match lookup(name) {+ Some(value) => out.push(value),+ None => out.push(&rest[at..at + 1 + len]),+ }+ rest = &after[len..];+ }+ None => {+ out.push("$");+ rest = after;+ }+ }+ }+ out.push(rest);+ out+}++/// `ntpath.expandvars`: `%name%`, `${name}` and `$name` (ASCII letters,+/// digits, `_`, `-`); `%%` and `$$` are a literal `%` and `$`; nothing inside+/// single quotes expands. An unset or unterminated reference stays as+/// written.+#[cfg_attr(not(windows), allow(dead_code))] // tests run both rule sets everywhere+fn expand_vars_nt(+ text: &str,+ lookup: impl Fn(&str) -> Option<std::ffi::OsString>,+) -> std::ffi::OsString {+ let mut out = std::ffi::OsString::new();+ let mut rest = text;+ while let Some(c) = rest.chars().next() {+ let after = &rest[c.len_utf8()..];+ match c {+ '\'' => match after.find('\'') {+ Some(end) => {+ out.push(&rest[..end + 2]);+ rest = &after[end + 1..];+ }+ None => {+ out.push(rest);+ rest = "";+ }+ },+ '%' if after.starts_with('%') => {+ out.push("%");+ rest = &after[1..];+ }+ '%' => match after.find('%') {+ Some(end) => {+ let name = &after[..end];+ match lookup(name) {+ Some(value) => out.push(value),+ None => out.push(&rest[..end + 2]),+ }+ rest = &after[end + 1..];+ }+ None => {+ out.push(rest);+ rest = "";+ }+ },+ '$' if after.starts_with('$') => {+ out.push("$");+ rest = &after[1..];+ }+ '$' if after.starts_with('{') => match after[1..].find('}') {+ Some(end) => {+ let name = &after[1..1 + end];+ match lookup(name) {+ Some(value) => out.push(value),+ None => out.push(&rest[..end + 3]),+ }+ rest = &after[end + 2..];+ }+ None => {+ out.push(rest);+ rest = "";+ }+ },+ '$' => {+ let end = after+ .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_' || c == '-'))+ .unwrap_or(after.len());+ let name = &after[..end];+ match lookup(name) {+ Some(value) => out.push(value),+ None => out.push(&rest[..end + 1]),+ }+ rest = &after[end..];+ }+ _ => {+ out.push(&rest[..c.len_utf8()]);+ rest = after;+ }+ }+ }+ out+}++#[cfg(test)]+pub(crate) mod tests {+ use super::*;++ /// Every variable `hub_cache_dir` reads.+ const CACHE_ENV: [&str; 6] = [+ "HF_HUB_CACHE",+ "HUGGINGFACE_HUB_CACHE",+ "HF_HOME",+ "XDG_CACHE_HOME",+ "HOME",+ "USERPROFILE",+ ];++ /// Saves [`CACHE_ENV`], clears it, and puts it back on drop, so a failing+ /// assertion cannot leak a fake cache into later tests. Take+ /// `env_lock()` first.+ pub(crate) struct CacheEnv(Vec<(String, Option<std::ffi::OsString>)>);++ impl CacheEnv {+ pub(crate) fn cleared() -> Self {+ Self::cleared_with(std::iter::empty())+ }++ /// [`Self::cleared`], plus `extra` variables (the ones a test's+ /// paths reference).+ pub(crate) fn cleared_with(extra: impl IntoIterator<Item = String>) -> Self {+ let names: Vec<String> = CACHE_ENV+ .iter()+ .map(|name| name.to_string())+ .chain(extra)+ .collect();+ let saved = names+ .into_iter()+ .map(|name| {+ let value = std::env::var_os(&name);+ std::env::remove_var(&name);+ (name, value)+ })+ .collect();+ CacheEnv(saved)+ }+ }++ impl Drop for CacheEnv {+ fn drop(&mut self) {+ for (name, value) in &self.0 {+ match value {+ Some(value) => std::env::set_var(name, value),+ None => std::env::remove_var(name),+ }+ }+ }+ }++ /// Same order as `huggingface_hub.constants.HF_HUB_CACHE`: each variable+ /// wins over every one after it, and only one directory comes back.+ #[test]+ fn hub_cache_dir_follows_huggingface_hub_precedence() {+ let _guard = crate::test_support::env_lock();+ let _env = CacheEnv::cleared();+ let t = tempfile::tempdir().unwrap();+ let p = |name: &str| t.path().join(name);+ std::env::set_var("HOME", p("home"));+ std::env::set_var("USERPROFILE", p("home"));+ std::env::set_var("XDG_CACHE_HOME", p("xdg"));+ std::env::set_var("HF_HOME", p("hf-home"));+ std::env::set_var("HUGGINGFACE_HUB_CACHE", p("legacy"));+ std::env::set_var("HF_HUB_CACHE", p("explicit"));++ assert_eq!(hub_cache_dir(), Some(p("explicit")));+ std::env::remove_var("HF_HUB_CACHE");+ assert_eq!(hub_cache_dir(), Some(p("legacy")));+ std::env::remove_var("HUGGINGFACE_HUB_CACHE");+ assert_eq!(hub_cache_dir(), Some(p("hf-home").join("hub")));+ std::env::remove_var("HF_HOME");+ assert_eq!(+ hub_cache_dir(),+ Some(p("xdg").join("huggingface").join("hub"))+ );+ std::env::remove_var("XDG_CACHE_HOME");+ assert_eq!(+ hub_cache_dir(),+ Some(p("home").join(".cache").join("huggingface").join("hub"))+ );++ // A leading `~` expands.+ std::env::set_var("HF_HOME", "~/models");+ assert_eq!(hub_cache_dir(), Some(p("home").join("models").join("hub")));++ // A set but empty variable is the working directory, as in Python,+ // and still wins over every variable after it.+ std::env::set_var("HF_HUB_CACHE", "");+ assert_eq!(hub_cache_dir(), Some(PathBuf::new()));+ std::env::remove_var("HF_HUB_CACHE");+ std::env::set_var("HF_HOME", "");+ std::env::set_var("XDG_CACHE_HOME", p("xdg"));+ assert_eq!(hub_cache_dir(), Some(PathBuf::from("hub")));+ }++ /// An empty `HF_TOKEN_PATH` names no readable file, so there is no token;+ /// it must not fall back to `HF_HOME/token`. Python's `get_token()` raises+ /// `IsADirectoryError` here (it reads `.`), so this case is Rust-only+ /// rather than in the shared fixture.+ #[test]+ fn an_empty_hf_token_path_does_not_fall_back_to_hf_home() {+ let _guard = crate::test_support::env_lock();+ let _env = CacheEnv::cleared_with(+ ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HF_TOKEN_PATH"].map(String::from),+ );+ let hf_home = tempfile::tempdir().unwrap();+ std::fs::write(hf_home.path().join("token"), "hf_from_home\n").unwrap();+ std::env::set_var("HF_HOME", hf_home.path());++ std::env::remove_var("HF_TOKEN_PATH");+ assert_eq!(hf_token().as_deref(), Some("hf_from_home"));+ std::env::set_var("HF_TOKEN_PATH", "");+ assert_eq!(hf_token(), None);+ }++ /// Cases checked against Python itself (`posixpath`, `ntpath` and+ /// `huggingface_hub.constants`) by `tests/test_fastembed_pinned_snapshot.py`.+ const HF_CACHE_FIXTURE: &str = include_str!(concat!(+ env!("CARGO_MANIFEST_DIR"),+ "/../../tests/parity/fixtures/hf_hub_cache/cases.json"+ ));++ /// Both rule sets on every platform, so the Windows port is exercised by+ /// the macOS and Linux runs too.+ #[test]+ fn expand_vars_matches_python_posixpath_and_ntpath() {+ let fixture: serde_json::Value = serde_json::from_str(HF_CACHE_FIXTURE).unwrap();+ let section = &fixture["expandvars"];+ let env: std::collections::HashMap<String, String> =+ serde_json::from_value(section["env"].clone()).unwrap();+ let lookup = |name: &str| env.get(name).map(std::ffi::OsString::from);+ for case in section["cases"].as_array().unwrap() {+ let input = case["input"].as_str().unwrap();+ let want = |key: &str| std::ffi::OsString::from(case[key].as_str().unwrap());+ assert_eq!(+ expand_vars_posix(input, lookup),+ want("posix"),+ "posix {input:?}"+ );+ assert_eq!(expand_vars_nt(input, lookup), want("nt"), "nt {input:?}");+ }+ }++ /// The review case among them: `HF_HUB_CACHE=${ROOT}/hub` is+ /// `<ROOT>/hub` in Python, so Rust must not look up or download into a+ /// literal `${ROOT}` directory instead.+ #[test]+ fn hub_cache_dir_matches_the_shared_fixture() {+ let fixture: serde_json::Value = serde_json::from_str(HF_CACHE_FIXTURE).unwrap();+ let section = &fixture["cache_dir"];+ let probe_vars: Vec<String> =+ serde_json::from_value(section["probe_vars"].clone()).unwrap();+ let _guard = crate::test_support::env_lock();+ let _env = CacheEnv::cleared_with(probe_vars.clone());+ let t = tempfile::tempdir().unwrap();+ let (root, home) = (t.path().join("root"), t.path().join("home"));+ let fill = |text: &str| {+ text.replace("{root}", root.to_str().unwrap())+ .replace("{home}", home.to_str().unwrap())+ };+ for case in section["cases"].as_array().unwrap() {+ for name in CACHE_ENV+ .iter()+ .copied()+ .chain(probe_vars.iter().map(String::as_str))+ {+ std::env::remove_var(name);+ }+ std::env::set_var("HOME", &home);+ std::env::set_var("USERPROFILE", &home);+ for (name, value) in case["env"].as_object().unwrap() {+ std::env::set_var(name, fill(value.as_str().unwrap()));+ }+ let expected = match &case["expected"] {+ serde_json::Value::Object(by_os) => {+ &by_os[if cfg!(windows) { "nt" } else { "posix" }]+ }+ expected => expected,+ };+ assert_eq!(+ hub_cache_dir(),+ Some(PathBuf::from(fill(expected.as_str().unwrap()))),+ "{}",+ case["name"]+ );+ }+ }++ /// With no `HOME` (a service started with a bare environment) the home+ /// directory still comes from the user database, as Python's+ /// `expanduser` does, instead of leaving the scorer with no cache.+ #[cfg(all(feature = "ml", unix))]+ #[test]+ fn hub_cache_dir_without_home_uses_the_user_database() {+ let _guard = crate::test_support::env_lock();+ let _env = CacheEnv::cleared();+ let dir = hub_cache_dir().expect("a cache directory without HOME");+ assert!(dir.is_absolute(), "{}", dir.display());+ assert!(+ dir.ends_with(Path::new(".cache").join("huggingface").join("hub")),+ "{}",+ dir.display()+ );+ }++ /// The fixture's `token` cases, which the Python half checks against+ /// `huggingface_hub.get_token()`.+ #[test]+ fn hf_token_matches_the_shared_fixture() {+ let fixture: serde_json::Value = serde_json::from_str(HF_CACHE_FIXTURE).unwrap();+ let probe_vars: Vec<String> =+ serde_json::from_value(fixture["cache_dir"]["probe_vars"].clone()).unwrap();+ let _guard = crate::test_support::env_lock();+ let _env = CacheEnv::cleared_with(probe_vars.clone());+ for case in fixture["token"]["cases"].as_array().unwrap() {+ let t = tempfile::tempdir().unwrap();+ let (root, home) = (t.path().join("root"), t.path().join("home"));+ let fill = |text: &str| {+ text.replace("{root}", root.to_str().unwrap())+ .replace("{home}", home.to_str().unwrap())+ };+ for name in CACHE_ENV+ .iter()+ .copied()+ .chain(probe_vars.iter().map(String::as_str))+ {+ std::env::remove_var(name);+ }+ std::env::set_var("HOME", &home);+ std::env::set_var("USERPROFILE", &home);+ for (name, value) in case["env"].as_object().unwrap() {+ std::env::set_var(name, fill(value.as_str().unwrap()));+ }+ for (path, content) in case["files"].as_object().unwrap() {+ let path = PathBuf::from(fill(path));+ std::fs::create_dir_all(path.parent().unwrap()).unwrap();+ std::fs::write(path, content.as_str().unwrap()).unwrap();+ }+ assert_eq!(+ hf_token().as_deref(),+ case["expected"].as_str(),+ "{}",+ case["name"]+ );+ }+ }++ #[test]+ fn hf_endpoint_drops_trailing_slashes_like_python() {+ let _guard = crate::test_support::env_lock();+ let saved = std::env::var_os("HF_ENDPOINT");+ std::env::remove_var("HF_ENDPOINT");+ assert_eq!(hf_endpoint(), "https://huggingface.co");+ std::env::set_var("HF_ENDPOINT", "https://mirror.example.com//");+ assert_eq!(hf_endpoint(), "https://mirror.example.com");+ match saved {+ Some(v) => std::env::set_var("HF_ENDPOINT", v),+ None => std::env::remove_var("HF_ENDPOINT"),+ }+ }++ /// The token is withheld only where it would cross the network unencrypted.+ #[test]+ fn plain_http_off_loopback_is_the_only_clear_text_endpoint() {+ for endpoint in [+ "http://mirror.example.com",+ "HTTP://mirror.example.com:8080/hub",+ "http://10.0.0.5",+ "http://[email protected]",+ "http://localhost.example.com",+ ] {+ assert!(sends_in_clear_text(endpoint), "{endpoint}");+ }+ for endpoint in [+ "https://huggingface.co",+ "https://mirror.example.com",+ "http://localhost:8000",+ "http://127.0.0.1:9",+ "http://127.8.0.1",+ "http://[::1]:8080/",+ ] {+ assert!(!sends_in_clear_text(endpoint), "{endpoint}");+ }+ }++ /// A commit SHA names the snapshot directly; a branch goes through its+ /// ref, as in Python's cache layout.+ #[test]+ fn snapshot_path_resolves_commits_directly_and_branches_through_refs() {+ let cache = tempfile::tempdir().unwrap();+ let repo = cache.path().join("models--owner--name");+ std::fs::create_dir_all(repo.join("refs")).unwrap();+ std::fs::write(repo.join("refs").join("main"), "abc123\n").unwrap();+ let snapshots = repo.join("snapshots");+ assert_eq!(+ snapshot_path(cache.path(), "owner/name", "def456"),+ snapshots.join("def456")+ );+ assert_eq!(+ snapshot_path(cache.path(), "owner/name", "main"),+ snapshots.join("abc123")+ );+ }++ #[test]+ fn pin_can_be_turned_off() {+ let _guard = crate::test_support::env_lock();+ let saved = std::env::var_os("HEADROOM_HF_PIN");+ for (value, disabled) in [("", false), ("on", false), ("off", true), (" No ", true)] {+ std::env::set_var("HEADROOM_HF_PIN", value);+ assert_eq!(pin_disabled(), disabled, "{value:?}");+ assert_eq!(pinned_or_main("sha"), if disabled { "main" } else { "sha" });+ }+ match saved {+ Some(v) => std::env::set_var("HEADROOM_HF_PIN", v),+ None => std::env::remove_var("HEADROOM_HF_PIN"),+ }+ }+}diff --git a/crates/headroom-core/src/lib.rs b/crates/headroom-core/src/lib.rsindex f209b4131..aceb50d86 100644--- a/crates/headroom-core/src/lib.rs+++ b/crates/headroom-core/src/lib.rs@@ -8,6 +8,7 @@ pub mod auth_mode;pub mod cache_control;pub mod ccr;pub mod compression_policy;+mod hf_cache;pub mod offline;#[cfg(feature = "ml")]mod onnx_cpu;diff --git a/crates/headroom-core/src/relevance/embedding.rs b/crates/headroom-core/src/relevance/embedding.rsindex 575e0ffc2..6c7db0811 100644--- a/crates/headroom-core/src/relevance/embedding.rs+++ b/crates/headroom-core/src/relevance/embedding.rs@@ -15,7 +15,7 @@//! construction may need network access to fetch the model. The files//! live in the HuggingFace hub cache directory Python's `huggingface_hub`//! would pick (`HF_HUB_CACHE`, `HF_HOME/hub`, `~/.cache/huggingface/hub`,-//! …; see `hf_hub_cache_dir`), shared with Python.+//! …; see `crate::hf_cache::hub_cache_dir`), shared with Python.//!//! When constructed, `is_available()` returns `true` and `HybridScorer`//! switches off the BM25-fallback path automatically. If construction@@ -95,230 +95,16 @@ const DEFAULT_MODEL_FILES: [&str; 5] = [/// pinning off (same values as Python's `onnx_runtime._resolve_revision`).#[cfg(feature = "ml")]fn default_model_revision() -> &'static str {- let pin = std::env::var("HEADROOM_HF_PIN").unwrap_or_default();- match pin.trim().to_ascii_lowercase().as_str() {- "off" | "0" | "false" | "no" => "main",- _ => DEFAULT_MODEL_REVISION,- }-}--/// The HuggingFace hub cache directory, resolved the way Python's-/// `huggingface_hub` resolves `constants.HF_HUB_CACHE`, so both languages look-/// up and download into the same place: `HF_HUB_CACHE`, then-/// `HUGGINGFACE_HUB_CACHE` (its legacy name), then `HF_HOME/hub`, then-/// `XDG_CACHE_HOME/huggingface/hub`, then `~/.cache/huggingface/hub`.-///-/// It is exactly one directory. A snapshot in a lower-priority cache is not-/// used when a higher one is configured, which is also what Python does: with-/// `HF_HUB_CACHE` set, a model under `~/.cache` is neither found nor reused.-///-/// Both expansions happen as in `huggingface_hub.constants`:-/// `expandvars(expanduser(...))`, first on `HF_HOME` (or its-/// `XDG_CACHE_HOME` default) and then on the hub path itself. So-/// `HF_HUB_CACHE=${ROOT}/hub` names the same directory in both languages, and-/// a `~` that only appears once a variable is expanded stays literal, as it-/// does in Python. The home directory comes from `std::env::home_dir`, which-/// falls back to the user database when `HOME` is unset, as `expanduser`-/// does. A variable that is set but empty still counts, as with `os.getenv`:-/// `HF_HUB_CACHE=''` is the working directory itself, `HF_HOME=''` is `hub`-/// and `XDG_CACHE_HOME=''` is `huggingface/hub` under it, and none of them-/// falls through to a lower-priority cache. `tests/parity/fixtures/hf_hub_cache/`-/// holds the cases both languages are checked against.-#[cfg(feature = "ml")]-fn hf_hub_cache_dir() -> Option<PathBuf> {- let var = |name: &str| std::env::var_os(name).map(PathBuf::from);- let home = std::env::home_dir();- let expand = |path: PathBuf| expand_vars(&expand_user(path, home.as_deref()));- let hub = match var("HF_HUB_CACHE").or_else(|| var("HUGGINGFACE_HUB_CACHE")) {- Some(dir) => dir,- None => {- let hf_home = match var("HF_HOME") {- Some(dir) => dir,- None => var("XDG_CACHE_HOME")- .or_else(|| home.as_ref().map(|home| home.join(".cache")))?- .join("huggingface"),- };- expand(hf_home).join("hub")- }- };- Some(expand(hub))-}--/// Python's `os.path.expanduser` for a leading bare `~` component. `~user`-/// stays as written, which is also what `expanduser` does when it cannot-/// resolve the user.-#[cfg(feature = "ml")]-fn expand_user(path: PathBuf, home: Option<&Path>) -> PathBuf {- match (path.strip_prefix("~"), home) {- (Ok(rest), Some(home)) => home.join(rest),- _ => path,- }-}--/// Python's `os.path.expandvars`: `posixpath` rules off Windows, `ntpath`-/// rules on it. A path that is not valid Unicode is returned unchanged.-#[cfg(feature = "ml")]-fn expand_vars(path: &Path) -> PathBuf {- let Some(text) = path.to_str() else {- return path.to_path_buf();- };- #[cfg(windows)]- let expanded = expand_vars_nt(text, env_lookup);- #[cfg(not(windows))]- let expanded = expand_vars_posix(text, env_lookup);- PathBuf::from(expanded)-}--/// An environment value for `expand_vars`. Names the OS cannot hold (empty,-/// or containing `=` or NUL) are unset, which is how Python's `os.environ`-/// lookup fails for them too.-#[cfg(feature = "ml")]-fn env_lookup(name: &str) -> Option<std::ffi::OsString> {- if name.is_empty() || name.contains(['=', '\0']) {- return None;- }- std::env::var_os(name)-}--/// `posixpath.expandvars`: `$name` (ASCII letters, digits, `_`) or-/// `${anything but a brace}`. An unset name, and a `$` that starts neither-/// form, stay as written. Expanded values are not scanned again.-#[cfg(feature = "ml")]-#[cfg_attr(windows, allow(dead_code))] // tests run both rule sets everywhere-fn expand_vars_posix(- text: &str,- lookup: impl Fn(&str) -> Option<std::ffi::OsString>,-) -> std::ffi::OsString {- let mut out = std::ffi::OsString::new();- let mut rest = text;- while let Some(at) = rest.find('$') {- out.push(&rest[..at]);- let after = &rest[at + 1..];- let word = after- .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_'))- .unwrap_or(after.len());- // (name, length of the reference after the `$`)- let reference = if word > 0 {- Some((&after[..word], word))- } else {- after- .strip_prefix('{')- .and_then(|braced| braced.find('}').map(|end| (&braced[..end], end + 2)))- };- match reference {- Some((name, len)) => {- match lookup(name) {- Some(value) => out.push(value),- None => out.push(&rest[at..at + 1 + len]),- }- rest = &after[len..];- }- None => {- out.push("$");- rest = after;- }- }- }- out.push(rest);- out-}--/// `ntpath.expandvars`: `%name%`, `${name}` and `$name` (ASCII letters,-/// digits, `_`, `-`); `%%` and `$$` are a literal `%` and `$`; nothing inside-/// single quotes expands. An unset or unterminated reference stays as-/// written.-#[cfg(feature = "ml")]-#[cfg_attr(not(windows), allow(dead_code))] // tests run both rule sets everywhere-fn expand_vars_nt(- text: &str,- lookup: impl Fn(&str) -> Option<std::ffi::OsString>,-) -> std::ffi::OsString {- let mut out = std::ffi::OsString::new();- let mut rest = text;- while let Some(c) = rest.chars().next() {- let after = &rest[c.len_utf8()..];- match c {- '\'' => match after.find('\'') {- Some(end) => {- out.push(&rest[..end + 2]);- rest = &after[end + 1..];- }- None => {- out.push(rest);- rest = "";- }- },- '%' if after.starts_with('%') => {- out.push("%");- rest = &after[1..];- }- '%' => match after.find('%') {- Some(end) => {- let name = &after[..end];- match lookup(name) {- Some(value) => out.push(value),- None => out.push(&rest[..end + 2]),- }- rest = &after[end + 1..];- }- None => {- out.push(rest);- rest = "";- }- },- '$' if after.starts_with('$') => {- out.push("$");- rest = &after[1..];- }- '$' if after.starts_with('{') => match after[1..].find('}') {- Some(end) => {- let name = &after[1..1 + end];- match lookup(name) {- Some(value) => out.push(value),- None => out.push(&rest[..end + 3]),- }- rest = &after[end + 2..];- }- None => {- out.push(rest);- rest = "";- }- },- '$' => {- let end = after- .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_' || c == '-'))- .unwrap_or(after.len());- let name = &after[..end];- match lookup(name) {- Some(value) => out.push(value),- None => out.push(&rest[..end + 1]),- }- rest = &after[end..];- }- _ => {- out.push(&rest[..c.len_utf8()]);- rest = after;- }- }- }- out+ crate::hf_cache::pinned_or_main(DEFAULT_MODEL_REVISION)}/// Find a snapshot of [`DEFAULT_MODEL_REPO`] at `revision` in the hub cache/// `cache` that holds every file in [`DEFAULT_MODEL_FILES`]. Never touches the-/// network.-///-/// Looks the snapshot up by directory rather than through `hf_hub::Cache`:-/// that resolves every revision through `refs/<revision>`, which Python's-/// `huggingface_hub` does not write for a commit SHA, so a snapshot Python-/// downloaded would be missed. A branch name still goes through its ref.+/// network; see [`crate::hf_cache::snapshot_path`] for how `revision`+/// resolves.#[cfg(feature = "ml")]fn cached_snapshot(cache: &Path, revision: &str) -> Option<PathBuf> {- let repo = cache.join(format!("models--{}", DEFAULT_MODEL_REPO.replace('/', "--")));- let commit = std::fs::read_to_string(repo.join("refs").join(revision))- .map(|c| c.trim().to_string())- .unwrap_or_else(|_| revision.to_string());- let snapshot = repo.join("snapshots").join(commit);+ let snapshot = crate::hf_cache::snapshot_path(cache, DEFAULT_MODEL_REPO, revision);// `is_file` follows the snapshot's symlinks, so blobs deleted from under// it count as a miss rather than a load failure.DEFAULT_MODEL_FILES@@ -328,32 +114,30 @@ fn cached_snapshot(cache: &Path, revision: &str) -> Option<PathBuf> {}/// The Hub handle for [`DEFAULT_MODEL_REPO`] at `revision`, caching under-/// `cache_root`. Honors `HF_ENDPOINT` like the Python side does.+/// `cache_root`, with Python's endpoint and token ([`crate::hf_cache::hub_api`]).#[cfg(feature = "ml")]fn default_model_repo(cache_root: &Path,revision: &str,) -> Result<hf_hub::api::sync::ApiRepo, String> {- // Air-gap chokepoint, ahead of the client: building it resolves the Hub- // endpoint and the `ureq` agent, and `hf-hub` downloads on a cache miss- // even with `HF_HUB_OFFLINE` set. Callers reach this only on a cache+ // `hub_api` is the air-gap chokepoint: under `HEADROOM_OFFLINE` it+ // refuses before building the client. Callers reach this only on a cache// miss, so a pre-seeded air-gapped host still loads.//// The refusal is soft by design: every caller degrades to the BM25// scorer on `Err`, so an air-gapped box loses embedding relevance instead// of failing a request. The message names the switch so the operator can// tell the causes apart in the log.- crate::offline::guard_egress("fastembed embedding model download", DEFAULT_MODEL_REPO)- .map_err(|e| format!("EmbeddingScorer model load refused: {e}"))?;- let mut builder =- hf_hub::api::sync::ApiBuilder::from_cache(hf_hub::Cache::new(cache_root.to_path_buf()))- .with_progress(false);- if let Ok(endpoint) = std::env::var("HF_ENDPOINT") {- builder = builder.with_endpoint(endpoint);- }- let api = builder- .build()- .map_err(|e| format!("EmbeddingScorer: HuggingFace client init failed: {e}"))?;+ use crate::hf_cache::HubApiError;+ let api = crate::hf_cache::hub_api(+ cache_root,+ "fastembed embedding model download",+ DEFAULT_MODEL_REPO,+ )+ .map_err(|e| match e {+ HubApiError::Offline(e) => format!("EmbeddingScorer model load refused: {e}"),+ HubApiError::Build(e) => format!("EmbeddingScorer: HuggingFace client init failed: {e}"),+ })?;Ok(api.repo(hf_hub::Repo::with_revision(DEFAULT_MODEL_REPO.to_string(),hf_hub::RepoType::Model,@@ -493,7 +277,7 @@ impl EmbeddingScorer {/// offline, network failure during download, missing ONNX runtime/// binaries, etc.).pub fn try_new() -> Result<Self, String> {- let cache = hf_hub_cache_dir().ok_or_else(|| {+ let cache = crate::hf_cache::hub_cache_dir().ok_or_else(|| {"EmbeddingScorer: no HuggingFace cache directory (set HF_HUB_CACHE or HF_HOME)".to_string()})?;@@ -729,6 +513,8 @@ fn cosine_similarity(a: &[f32], b: &[f32]) -> f64 {#[cfg(test)]mod tests {use super::*;+ #[cfg(feature = "ml")]+ use crate::hf_cache::tests::CacheEnv;// The real-model tests are gated behind RUN_FASTEMBED_TESTS=1// since they require network access on first run (~67 MB model@@ -843,209 +629,12 @@ mod tests {assert_eq!(cached_snapshot(cache.path(), DEFAULT_MODEL_REVISION), None);}- /// Every variable `hf_hub_cache_dir` reads.- #[cfg(feature = "ml")]- const CACHE_ENV: [&str; 6] = [- "HF_HUB_CACHE",- "HUGGINGFACE_HUB_CACHE",- "HF_HOME",- "XDG_CACHE_HOME",- "HOME",- "USERPROFILE",- ];-- /// Saves [`CACHE_ENV`], clears it, and puts it back on drop, so a failing- /// assertion cannot leak a fake cache into later tests. Take- /// `env_lock()` first.- #[cfg(feature = "ml")]- struct CacheEnv(Vec<(String, Option<std::ffi::OsString>)>);-- #[cfg(feature = "ml")]- impl CacheEnv {- fn cleared() -> Self {- Self::cleared_with(std::iter::empty())- }-- /// [`Self::cleared`], plus `extra` variables (the ones a test's- /// paths reference).- fn cleared_with(extra: impl IntoIterator<Item = String>) -> Self {- let names: Vec<String> = CACHE_ENV- .iter()- .map(|name| name.to_string())- .chain(extra)- .collect();- let saved = names- .into_iter()- .map(|name| {- let value = std::env::var_os(&name);- std::env::remove_var(&name);- (name, value)- })- .collect();- CacheEnv(saved)- }- }-- #[cfg(feature = "ml")]- impl Drop for CacheEnv {- fn drop(&mut self) {- for (name, value) in &self.0 {- match value {- Some(value) => std::env::set_var(name, value),- None => std::env::remove_var(name),- }- }- }- }-- /// Same order as `huggingface_hub.constants.HF_HUB_CACHE`: each variable- /// wins over every one after it, and only one directory comes back.- #[cfg(feature = "ml")]- #[test]- fn hf_hub_cache_dir_follows_huggingface_hub_precedence() {- let _guard = crate::test_support::env_lock();- let _env = CacheEnv::cleared();- let t = tempfile::tempdir().unwrap();- let p = |name: &str| t.path().join(name);- std::env::set_var("HOME", p("home"));- std::env::set_var("USERPROFILE", p("home"));- std::env::set_var("XDG_CACHE_HOME", p("xdg"));- std::env::set_var("HF_HOME", p("hf-home"));- std::env::set_var("HUGGINGFACE_HUB_CACHE", p("legacy"));- std::env::set_var("HF_HUB_CACHE", p("explicit"));-- assert_eq!(hf_hub_cache_dir(), Some(p("explicit")));- std::env::remove_var("HF_HUB_CACHE");- assert_eq!(hf_hub_cache_dir(), Some(p("legacy")));- std::env::remove_var("HUGGINGFACE_HUB_CACHE");- assert_eq!(hf_hub_cache_dir(), Some(p("hf-home").join("hub")));- std::env::remove_var("HF_HOME");- assert_eq!(- hf_hub_cache_dir(),- Some(p("xdg").join("huggingface").join("hub"))- );- std::env::remove_var("XDG_CACHE_HOME");- assert_eq!(- hf_hub_cache_dir(),- Some(p("home").join(".cache").join("huggingface").join("hub"))- );-- // A leading `~` expands.- std::env::set_var("HF_HOME", "~/models");- assert_eq!(- hf_hub_cache_dir(),- Some(p("home").join("models").join("hub"))- );-- // A set but empty variable is the working directory, as in Python,- // and still wins over every variable after it.- std::env::set_var("HF_HUB_CACHE", "");- assert_eq!(hf_hub_cache_dir(), Some(PathBuf::new()));- std::env::remove_var("HF_HUB_CACHE");- std::env::set_var("HF_HOME", "");- std::env::set_var("XDG_CACHE_HOME", p("xdg"));- assert_eq!(hf_hub_cache_dir(), Some(PathBuf::from("hub")));- }-- /// Cases checked against Python itself (`posixpath`, `ntpath` and- /// `huggingface_hub.constants`) by `tests/test_fastembed_pinned_snapshot.py`.- #[cfg(feature = "ml")]- const HF_CACHE_FIXTURE: &str = include_str!(concat!(- env!("CARGO_MANIFEST_DIR"),- "/../../tests/parity/fixtures/hf_hub_cache/cases.json"- ));-- /// Both rule sets on every platform, so the Windows port is exercised by- /// the macOS and Linux runs too.- #[cfg(feature = "ml")]- #[test]- fn expand_vars_matches_python_posixpath_and_ntpath() {- let fixture: serde_json::Value = serde_json::from_str(HF_CACHE_FIXTURE).unwrap();- let section = &fixture["expandvars"];- let env: std::collections::HashMap<String, String> =- serde_json::from_value(section["env"].clone()).unwrap();- let lookup = |name: &str| env.get(name).map(std::ffi::OsString::from);- for case in section["cases"].as_array().unwrap() {- let input = case["input"].as_str().unwrap();- let want = |key: &str| std::ffi::OsString::from(case[key].as_str().unwrap());- assert_eq!(- expand_vars_posix(input, lookup),- want("posix"),- "posix {input:?}"- );- assert_eq!(expand_vars_nt(input, lookup), want("nt"), "nt {input:?}");- }- }-- /// The review case among them: `HF_HUB_CACHE=${ROOT}/hub` is- /// `<ROOT>/hub` in Python, so Rust must not look up or download into a- /// literal `${ROOT}` directory instead.- #[cfg(feature = "ml")]- #[test]- fn hf_hub_cache_dir_matches_the_shared_fixture() {- let fixture: serde_json::Value = serde_json::from_str(HF_CACHE_FIXTURE).unwrap();- let section = &fixture["cache_dir"];- let probe_vars: Vec<String> =- serde_json::from_value(section["probe_vars"].clone()).unwrap();- let _guard = crate::test_support::env_lock();- let _env = CacheEnv::cleared_with(probe_vars.clone());- let t = tempfile::tempdir().unwrap();- let (root, home) = (t.path().join("root"), t.path().join("home"));- let fill = |text: &str| {- text.replace("{root}", root.to_str().unwrap())- .replace("{home}", home.to_str().unwrap())- };- for case in section["cases"].as_array().unwrap() {- for name in CACHE_ENV- .iter()- .copied()- .chain(probe_vars.iter().map(String::as_str))- {- std::env::remove_var(name);- }- std::env::set_var("HOME", &home);- std::env::set_var("USERPROFILE", &home);- for (name, value) in case["env"].as_object().unwrap() {- std::env::set_var(name, fill(value.as_str().unwrap()));- }- let expected = match &case["expected"] {- serde_json::Value::Object(by_os) => {- &by_os[if cfg!(windows) { "nt" } else { "posix" }]- }- expected => expected,- };- assert_eq!(- hf_hub_cache_dir(),- Some(PathBuf::from(fill(expected.as_str().unwrap()))),- "{}",- case["name"]- );- }- }-- /// With no `HOME` (a service started with a bare environment) the home- /// directory still comes from the user database, as Python's- /// `expanduser` does, instead of leaving the scorer with no cache.- #[cfg(all(feature = "ml", unix))]- #[test]- fn hf_hub_cache_dir_without_home_uses_the_user_database() {- let _guard = crate::test_support::env_lock();- let _env = CacheEnv::cleared();- let dir = hf_hub_cache_dir().expect("a cache directory without HOME");- assert!(dir.is_absolute(), "{}", dir.display());- assert!(- dir.ends_with(Path::new(".cache").join("huggingface").join("hub")),- "{}",- dir.display()- );- }-/// The case from review: an explicit, empty `HF_HUB_CACHE` next to a/// complete snapshot in the default cache under `HOME`. Python looks only/// in the explicit cache and downloads there, so Rust must not find the/// `HOME` copy, for the pinned revision or for floating `main`/// (`HEADROOM_HF_PIN=off`). `try_new` is `try_new_pinned` on- /// `hf_hub_cache_dir()`, which looks up and downloads only in the+ /// `hub_cache_dir()`, which looks up and downloads only in the/// directory it is given (see the two offline tests above), so checking/// the resolver and the lookup here covers it without flipping/// `HEADROOM_OFFLINE` for a long stretch of the parallel run.@@ -1068,14 +657,14 @@ mod tests {std::env::set_var("USERPROFILE", home.path());std::env::set_var("HF_HUB_CACHE", explicit.path());- let cache = hf_hub_cache_dir().expect("cache directory");+ let cache = crate::hf_cache::hub_cache_dir().expect("cache directory");assert_eq!(cache, explicit.path());assert_eq!(cached_snapshot(&cache, DEFAULT_MODEL_REVISION), None);assert_eq!(cached_snapshot(&cache, "main"), None);// Without the explicit cache, the same HOME snapshots are the cache.std::env::remove_var("HF_HUB_CACHE");- let cache = hf_hub_cache_dir().expect("cache directory");+ let cache = crate::hf_cache::hub_cache_dir().expect("cache directory");assert_eq!(cache, default_cache);assert_eq!(cached_snapshot(&cache, DEFAULT_MODEL_REVISION),diff --git a/crates/headroom-core/src/tokenizer/hf_impl.rs b/crates/headroom-core/src/tokenizer/hf_impl.rsindex f120c76ac..efb58c5d9 100644--- a/crates/headroom-core/src/tokenizer/hf_impl.rs+++ b/crates/headroom-core/src/tokenizer/hf_impl.rs@@ -12,11 +12,12 @@//! - [`HfTokenizer::from_bytes`] — for tokenizers embedded via `include_bytes!`.//! - [`HfTokenizer::from_file`] — for a local `tokenizer.json`.//! - [`HfTokenizer::from_pretrained`] — pulls `tokenizer.json` from the-//! HuggingFace Hub via `hf-hub`. First call downloads to-//! `~/.cache/huggingface/hub`, subsequent calls hit the cache. Blocking;+//! HuggingFace Hub via `hf-hub`. First call downloads into the hub cache+//! Python's `huggingface_hub` uses (`HF_HUB_CACHE`, `HF_HOME/hub`,+//! `~/.cache/huggingface/hub`, …), subsequent calls hit it. Blocking;//! call from `main()` or `tokio::task::spawn_blocking`. Gated repos-//! (Llama, Mistral) require an `HF_TOKEN` env var or a token in-//! `~/.cache/huggingface/token`.+//! (Llama, Mistral) need a token: `HF_TOKEN`, or the file+//! `hf auth login` writes (`HF_HOME/token`, or `HF_TOKEN_PATH`).//!//! # What's NOT here//! - **No tokenizer.json bundled in the binary.** Bundling Llama / Cohere@@ -112,9 +113,13 @@ impl HfTokenizer {/// e.g. `"CohereForAI/c4ai-command-r-v01"` or `"meta-llama/Meta-Llama-3-8B"`.////// Uses the `main` revision. Blocking — calls into `ureq` synchronously.- /// First successful call writes the file to `~/.cache/huggingface/hub`- /// (or `$HF_HOME` if set); subsequent calls in the same or later- /// processes hit the on-disk cache.+ /// The file lands in the hub cache directory Python's `huggingface_hub`+ /// would use ([`crate::hf_cache::hub_cache_dir`]: `HF_HUB_CACHE`, then+ /// `HUGGINGFACE_HUB_CACHE`, `HF_HOME/hub`, `XDG_CACHE_HOME/huggingface/hub`,+ /// `~/.cache/huggingface/hub`), so later calls, and the Python side, hit+ /// the same copy. The token and `HF_ENDPOINT` are read as Python reads+ /// them too ([`crate::hf_cache::hf_token`]); `hf-hub`'s own defaults+ /// ignore every variable but `HF_HOME`, and with `Api::new` even that.////// Errors:/// - [`HfTokenizerError::Offline`] when `HEADROOM_OFFLINE` is set. Refused@@ -129,11 +134,11 @@ impl HfTokenizer {// HEADROOM_OFFLINE=1 gets the same refusal whichever runtime reaches// for the Hub.//- // It has to sit here rather than around the `get` below: `Api::new`- // builds the `ureq` agent and resolves the endpoint, and hf-hub's own+ // `crate::hf_cache::hub_api` runs the guard before it builds the+ // client, rather than around the `get` below: building it creates the+ // `ureq` agent and resolves the endpoint, and hf-hub's own// HF_HUB_OFFLINE handling only covers the cache lookup — a cache miss- // still dials out. Guarding first means the air-gapped case never- // constructs a client at all.+ // still dials out. So the air-gapped case never constructs a client.//// The refusal is hard on purpose. Silently returning a cached or// estimating tokenizer would leave token counts subtly wrong with no@@ -145,11 +150,7 @@ impl HfTokenizer {// client exists" and "serve from the cache" cannot both hold here. An// air-gapped deployment with pre-seeded artifacts should point at them// with `from_file`, which is network-free by construction.- crate::offline::guard_egress("HuggingFace tokenizer download", repo)?;- let api = hf_hub::api::sync::Api::new().map_err(|e| HfTokenizerError::Hub {- repo: repo.to_string(),- source: Box::new(e),- })?;+ let api = hub_api(repo)?;let path = api.model(repo.to_string()).get("tokenizer.json")@@ -169,6 +170,31 @@ impl HfTokenizer {}}+/// The Hub client for [`HfTokenizer::from_pretrained`] fetching `repo`:+/// caching in Python's hub cache directory, with Python's token and endpoint.+/// Refuses with [`HfTokenizerError::Offline`] under `HEADROOM_OFFLINE`.+fn hub_api(repo: &str) -> Result<hf_hub::api::sync::Api, HfTokenizerError> {+ use crate::hf_cache::HubApiError;+ const PURPOSE: &str = "HuggingFace tokenizer download";++ let hub_error = |source: Box<dyn std::error::Error + Send + Sync>| HfTokenizerError::Hub {+ repo: repo.to_string(),+ source,+ };+ let Some(cache) = crate::hf_cache::hub_cache_dir() else {+ // Offline still wins: the caller treats `Offline` as final, and a+ // missing cache directory is no reason to report a Hub failure.+ crate::offline::guard_egress(PURPOSE, repo)?;+ return Err(hub_error(+ "no HuggingFace cache directory (set HF_HUB_CACHE or HF_HOME)".into(),+ ));+ };+ crate::hf_cache::hub_api(&cache, PURPOSE, repo).map_err(|e| match e {+ HubApiError::Offline(blocked) => HfTokenizerError::Offline(blocked),+ HubApiError::Build(source) => hub_error(Box::new(source)),+ })+}+impl Tokenizer for HfTokenizer {fn count_text(&self, text: &str) -> usize {if text.is_empty() {@@ -297,7 +323,7 @@ mod tests {/// this a real assertion rather than a tautology: without the guard the call/// reaches `hf-hub`, dials `huggingface.co`, gets a 401/404 and returns/// `HfTokenizerError::Hub`. Only the guard can produce `Offline`, and only- /// by returning before `Api::new` — so `Offline` *is* the "no socket was+ /// by returning before `hf_cache::hub_api` builds the client — so `Offline` *is* the "no socket was/// opened" assertion for this path. (A packet-level assertion is not/// available to a `cargo test` unit test the way `socket.socket` patching is/// in Python; this typed-error distinction is the equivalent signal.)@@ -349,6 +375,119 @@ mod tests {let _ = std::fs::remove_dir_all(&dir);}+ /// Lay out `tokenizer.json` for `repo` in hub cache `cache`, the way+ /// Python's `huggingface_hub` leaves a `main` download: a snapshot plus+ /// `refs/main` naming it.+ fn cache_main_tokenizer(cache: &std::path::Path, repo: &str) {+ let repo_dir = cache.join(format!("models--{}", repo.replace('/', "--")));+ let snapshot = repo_dir.join("snapshots").join("abc123");+ std::fs::create_dir_all(&snapshot).unwrap();+ std::fs::write(snapshot.join("tokenizer.json"), TINY_TOKENIZER_JSON).unwrap();+ std::fs::create_dir_all(repo_dir.join("refs")).unwrap();+ std::fs::write(repo_dir.join("refs").join("main"), "abc123").unwrap();+ }++ /// Variables `from_pretrained` reads, saved and restored around a test.+ fn hub_env() -> crate::hf_cache::tests::CacheEnv {+ crate::hf_cache::tests::CacheEnv::cleared_with(+ [+ "HF_TOKEN",+ "HUGGING_FACE_HUB_TOKEN",+ "HF_TOKEN_PATH",+ "HF_ENDPOINT",+ "HF_TEST_ROOT",+ ]+ .map(String::from),+ )+ }++ /// The tokenizer is read from (and would be downloaded into) the hub cache+ /// Python uses, here an `HF_HUB_CACHE` given through `${VAR}`. Before, it+ /// was always `~/.cache/huggingface/hub`, so this was a cache miss and a+ /// download. `HF_ENDPOINT` points at a closed loopback port, so a miss+ /// fails fast instead of reaching the internet.+ #[test]+ fn from_pretrained_reads_the_hub_cache_python_uses() {+ let _guard = crate::test_support::env_lock();+ let _env = hub_env();+ let t = tempfile::tempdir().unwrap();+ std::env::set_var("HOME", t.path().join("home"));+ std::env::set_var("USERPROFILE", t.path().join("home"));+ std::env::set_var("HF_TEST_ROOT", t.path());+ std::env::set_var("HF_HUB_CACHE", "${HF_TEST_ROOT}/hub");+ std::env::set_var("HF_ENDPOINT", "http://127.0.0.1:9");+ cache_main_tokenizer(&t.path().join("hub"), "acme/tiny");++ let tokenizer = HfTokenizer::from_pretrained("acme/tiny").expect("served from the cache");+ assert_eq!(tokenizer.count_text("hello world"), 2);+ }++ /// A cache miss asks the Hub at `HF_ENDPOINT` with the token from+ /// `HF_TOKEN`. `hf-hub`'s `Api::new` read neither, so a gated repo+ /// (Llama, Mistral) failed with 401 even with `HF_TOKEN` set. A loopback+ /// stand-in for the Hub records the request and answers 404.+ #[test]+ fn from_pretrained_sends_hf_token_to_hf_endpoint() {+ use std::io::{Read, Write};+ let _guard = crate::test_support::env_lock();+ let _env = hub_env();+ let t = tempfile::tempdir().unwrap();+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let port = listener.local_addr().unwrap().port();+ // Waits at most 10 s for the request, so a regression that never+ // reaches the endpoint fails instead of hanging the suite.+ listener.set_nonblocking(true).unwrap();+ let hub = std::thread::spawn(move || {+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);+ let mut conn = loop {+ match listener.accept() {+ Ok((conn, _)) => break conn,+ Err(_) if std::time::Instant::now() < deadline => {+ std::thread::sleep(std::time::Duration::from_millis(20))+ }+ Err(_) => return String::new(),+ }+ };+ conn.set_nonblocking(false).unwrap();+ let mut request = Vec::new();+ let mut buf = [0u8; 1024];+ while !request.windows(4).any(|w| w == b"\r\n\r\n") {+ let n = conn.read(&mut buf).unwrap();+ if n == 0 {+ break;+ }+ request.extend_from_slice(&buf[..n]);+ }+ conn.write_all(+ b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",+ )+ .unwrap();+ String::from_utf8_lossy(&request).to_ascii_lowercase()+ });+ std::env::set_var("HOME", t.path().join("home"));+ std::env::set_var("USERPROFILE", t.path().join("home"));+ std::env::set_var("HF_HUB_CACHE", t.path().join("hub"));+ // Trailing slash on purpose: Python strips it, and a kept one+ // would request `//acme/...`.+ std::env::set_var("HF_ENDPOINT", format!("http://127.0.0.1:{port}/"));+ std::env::set_var("HF_TOKEN", "hf_test_token");++ let result = HfTokenizer::from_pretrained("acme/gated");+ let request = hub.join().unwrap();+ assert!(+ matches!(result, Err(HfTokenizerError::Hub { .. })),+ "{result:?}"+ );+ assert!(+ request.contains(" /acme/gated/resolve/main/tokenizer.json "),+ "{request}"+ );+ assert!(+ request.contains("authorization: bearer hf_test_token"),+ "{request}"+ );+ }+/// Network-dependent: hits HuggingFace Hub. Run with/// `cargo test -p headroom-core -- --ignored from_pretrained_downloads_real_tokenizer`./// `gpt2` is a small public unauthenticated repo (~1.4 MB tokenizer.json).diff --git a/crates/headroom-core/src/transforms/kompress.rs b/crates/headroom-core/src/transforms/kompress.rsindex 06c51b075..17bb5aad8 100644--- a/crates/headroom-core/src/transforms/kompress.rs+++ b/crates/headroom-core/src/transforms/kompress.rs@@ -74,6 +74,12 @@ use tokenizers::{EncodeInput, InputSequence, Tokenizer};pub const DEFAULT_MODEL_ID: &str = "chopratejas/kompress-v2-base";/// HuggingFace repo holding the tokenizer Kompress reuses.pub const DEFAULT_TOKENIZER_REPO: &str = "answerdotai/ModernBERT-base";+/// Pinned commits of [`DEFAULT_MODEL_ID`] and [`DEFAULT_TOKENIZER_REPO`], the+/// snapshots the cache-only loader reads. Keep each equal to its entry in+/// `_PINNED_REVISIONS` (`headroom/onnx_runtime.py`), which is what the Python+/// side downloads; `tests/test_kompress_rust_pins.py` enforces it.+pub const DEFAULT_MODEL_REVISION: &str = "b1563631b35bfdcee37587ad530147497d820d4c";+pub const DEFAULT_TOKENIZER_REVISION: &str = "8949b909ec900327062f0ebf497f51aef5e6f0c8";/// Words per inference chunk. Coupled to the model's training window;/// kompress-v2-base was trained for 350.pub const DEFAULT_CHUNK_WORDS: usize = 350;@@ -261,8 +267,8 @@ impl Kompress {/// downloading on miss). Blocking — call off the hot path. Tries the/// [`ONNX_CANDIDATES`] in order.pub fn from_pretrained(config: KompressConfig) -> Result<Self, KompressError> {- // Air-gap chokepoint — the same one `hf_impl.rs` uses for the- // tokenizer, for the same reason. `HF_HUB_OFFLINE` only shortcuts the+ // Air-gap chokepoint — the same guard `hf_cache::hub_api` runs for+ // the tokenizer and embedder, for the same reason. `HF_HUB_OFFLINE` only shortcuts the// cache *lookup*; on a cache miss `hf-hub` still dials huggingface.co,// so the Kompress model download is a live egress path under// `HEADROOM_OFFLINE=1` and was not covered.@@ -333,20 +339,23 @@ impl Kompress {pub fn from_cache(config: KompressConfig) -> Result<Option<Self>, KompressError> {let Some(tok_path) = hf_cache_file(&config.tokenizer_repo, &["tokenizer.json"]) else {// Diagnostic: a `None` here is the #1 cause of a silent- // `kompress_ready=false`. Name the repo + the roots searched so- // operators don't have to guess between "not downloaded" and+ // `kompress_ready=false`. Name the repo, the cache and the revision+ // looked in so operators don't have to guess between "not+ // downloaded" and// "present but unreadable" (e.g. HF symlinks over `\\wsl$`, which- // native Windows can't follow — `path.exists()` returns false on+ // native Windows can't follow — `is_file()` returns false on// the unresolved symlink). Cache-only, so this is a defer, not an// error: the caller passes plain text through.tracing::warn!(event = "kompress_cache_miss",stage = "tokenizer",tokenizer_repo = %config.tokenizer_repo,- searched_roots = ?hf_hub_roots(),- "Kompress deferred: tokenizer.json not found in HF cache \- (not downloaded, or present but unreadable — e.g. HF symlinks \- over \\\\wsl$ which native Windows cannot follow)"+ revision = %cache_revision(&config.tokenizer_repo),+ hub_cache = ?crate::hf_cache::hub_cache_dir(),+ "Kompress deferred: tokenizer.json not found in the HF cache at the \+ pinned revision (not downloaded yet — the Python side fetches the \+ pin on its next Kompress load — or present but unreadable, e.g. HF \+ symlinks over \\\\wsl$ which native Windows cannot follow)");return Ok(None);};@@ -384,7 +393,8 @@ impl Kompress {model_id = %config.model_id,candidates = ?ONNX_CANDIDATES,any_onnx_found = found_onnx,- searched_roots = ?hf_hub_roots(),+ revision = %cache_revision(&config.model_id),+ hub_cache = ?crate::hf_cache::hub_cache_dir(),"Kompress deferred: no usable ONNX session \(no candidate file in cache, or every candidate failed to build)");@@ -603,59 +613,221 @@ fn build_session(path: &Path) -> Result<Session, Box<dyn std::error::Error + SenOk(session)}-/// Resolve `rel` (e.g. `["tokenizer.json"]` or `["onnx", "kompress-int8-wo.onnx"]`)-/// inside the local HuggingFace cache for `repo` (`"owner/name"`), searching-/// every snapshot under every candidate cache root. Returns `None` if not-/// present — never touches the network.-fn hf_cache_file(repo: &str, rel: &[&str]) -> Option<PathBuf> {- let repo_dir = format!("models--{}", repo.replace('/', "--"));- for hub in hf_hub_roots() {- let snapshots = hub.join(&repo_dir).join("snapshots");- let Ok(entries) = std::fs::read_dir(&snapshots) else {- continue;- };- for snap in entries.flatten() {- let mut cand = snap.path();- for part in rel {- cand = cand.join(part);- }- if cand.exists() {- return Some(cand);- }- }+/// The tokenizer and the ONNX artifact `onnx` (e.g.+/// `"onnx/kompress-int8-wo.onnx"`) that the cache-only loader reads for+/// `config`: the pinned snapshots in Python's hub cache, as+/// [`Kompress::from_cache`] finds them. `None` when either is missing. For+/// parity harnesses that load one specific artifact through+/// [`Kompress::from_files`], so they test the files production would load.+pub fn cached_artifacts(config: &KompressConfig, onnx: &str) -> Option<(PathBuf, PathBuf)> {+ let tokenizer = hf_cache_file(&config.tokenizer_repo, &["tokenizer.json"])?;+ let rel: Vec<&str> = onnx.split('/').collect();+ let model = hf_cache_file(&config.model_id, &rel)?;+ Some((tokenizer, model))+}++/// The revision of `repo` to load from the cache: its pin when it is one of+/// Kompress's default repos (see [`DEFAULT_MODEL_REVISION`]), otherwise+/// `main`, which is also what Python's `_resolve_revision` gives an unpinned+/// repo. `HEADROOM_HF_PIN=off` floats the pinned ones on `main` too.+fn cache_revision(repo: &str) -> &str {+ match repo {+ DEFAULT_MODEL_ID => crate::hf_cache::pinned_or_main(DEFAULT_MODEL_REVISION),+ DEFAULT_TOKENIZER_REPO => crate::hf_cache::pinned_or_main(DEFAULT_TOKENIZER_REVISION),+ _ => "main",}- None}-/// HuggingFace hub cache roots in resolution precedence. Cross-platform so-/// the cache-only loader works on Windows (native `headroom-proxy.exe`) as-/// well as Linux: `HF_HUB_CACHE` (the hub dir directly) → `HF_HOME/hub` →-/// `{HOME|USERPROFILE}/.cache/huggingface/hub`. `HOME` is the unix home; on-/// Windows the process sees `USERPROFILE` (and often no `HOME`), so both are-/// tried. Honoring `HF_HOME` also lets a Windows proxy point at a WSL cache.-fn hf_hub_roots() -> Vec<PathBuf> {- let mut roots = Vec::new();- let push_env = |roots: &mut Vec<PathBuf>, var: &str, suffix: &[&str]| {- if let Ok(v) = std::env::var(var) {- if !v.is_empty() {- let mut p = PathBuf::from(v);- for s in suffix {- p = p.join(s);- }- roots.push(p);- }- }- };- push_env(&mut roots, "HF_HUB_CACHE", &[]);- push_env(&mut roots, "HF_HOME", &["hub"]);- push_env(&mut roots, "HOME", &[".cache", "huggingface", "hub"]);- push_env(&mut roots, "USERPROFILE", &[".cache", "huggingface", "hub"]);- roots+/// Resolve `rel` (e.g. `["tokenizer.json"]` or `["onnx", "kompress-int8-wo.onnx"]`)+/// in `repo`'s snapshot at [`cache_revision`], inside the one hub cache+/// directory Python's `huggingface_hub` would use+/// ([`crate::hf_cache::hub_cache_dir`]). Returns `None` if not present —+/// never touches the network.+///+/// One snapshot, not the first file found in any: with several revisions in+/// the cache (every pin bump leaves the old one behind), which one a+/// directory scan returned depended on filesystem order, and the tokenizer+/// and the ONNX file could come from different snapshots. And one cache, not+/// every root that exists: Python does not fall back to `~/.cache` when+/// `HF_HUB_CACHE` names another directory, so neither does this. `is_file`+/// follows the cache's symlinks, so a blob missing under a snapshot (or a+/// symlink native Windows cannot follow over `\\wsl$`) is a miss.+fn hf_cache_file(repo: &str, rel: &[&str]) -> Option<PathBuf> {+ let cache = crate::hf_cache::hub_cache_dir()?;+ let mut path = crate::hf_cache::snapshot_path(&cache, repo, cache_revision(repo));+ for part in rel {+ path.push(part);+ }+ path.is_file().then_some(path)}#[cfg(test)]mod tests {use super::*;+ use crate::hf_cache::tests::CacheEnv;++ /// Lay out `rel` under `repo`'s snapshot `commit` in the hub cache `cache`.+ fn cached_file(cache: &Path, repo: &str, commit: &str, rel: &str) -> PathBuf {+ let path = cache+ .join(format!("models--{}", repo.replace('/', "--")))+ .join("snapshots")+ .join(commit)+ .join(rel);+ std::fs::create_dir_all(path.parent().unwrap()).unwrap();+ std::fs::write(&path, commit).unwrap();+ path+ }++ /// The cache after a pin bump: the pinned snapshot sits between an older+ /// one and a newer `main`, and both other snapshots hold the same files.+ fn cache_with_three_snapshots(cache: &Path) {+ for repo in [DEFAULT_MODEL_ID, DEFAULT_TOKENIZER_REPO] {+ let pin = cache_revision(repo).to_string();+ for commit in [+ "0000000000000000000000000000000000000000",+ &pin,+ "ffffffffffffffffffffffffffffffffffffffff",+ ] {+ cached_file(cache, repo, commit, "tokenizer.json");+ cached_file(cache, repo, commit, ONNX_CANDIDATES[1]);+ }+ let refs = cache+ .join(format!("models--{}", repo.replace('/', "--")))+ .join("refs");+ std::fs::create_dir_all(&refs).unwrap();+ std::fs::write(+ refs.join("main"),+ "ffffffffffffffffffffffffffffffffffffffff",+ )+ .unwrap();+ }+ }++ /// Python loads the pinned snapshot of both repos; so must the cache-only+ /// loader, whatever order the filesystem lists the snapshots in.+ #[test]+ fn the_cache_lookup_reads_the_pinned_snapshot() {+ let _guard = crate::test_support::env_lock();+ let _env = CacheEnv::cleared_with(["HEADROOM_HF_PIN".to_string()]);+ let cache = tempfile::tempdir().unwrap();+ std::env::set_var("HF_HUB_CACHE", cache.path());+ cache_with_three_snapshots(cache.path());++ let tokenizer = hf_cache_file(DEFAULT_TOKENIZER_REPO, &["tokenizer.json"]).unwrap();+ assert!(tokenizer.ends_with(Path::new(DEFAULT_TOKENIZER_REVISION).join("tokenizer.json")));+ let rel: Vec<&str> = ONNX_CANDIDATES[1].split('/').collect();+ let onnx = hf_cache_file(DEFAULT_MODEL_ID, &rel).unwrap();+ assert!(onnx.ends_with(Path::new(DEFAULT_MODEL_REVISION).join(ONNX_CANDIDATES[1])));++ // A candidate the pinned snapshot lacks is a miss, even when another+ // snapshot has it.+ cached_file(+ cache.path(),+ DEFAULT_MODEL_ID,+ "ffffffffffffffffffffffffffffffffffffffff",+ ONNX_CANDIDATES[2],+ );+ let rel: Vec<&str> = ONNX_CANDIDATES[2].split('/').collect();+ assert_eq!(hf_cache_file(DEFAULT_MODEL_ID, &rel), None);+ }++ /// `HEADROOM_HF_PIN=off` floats on `main` through `refs/main`, and a repo+ /// Kompress does not pin (a domain model) always does, as in Python.+ #[test]+ fn unpinned_lookups_follow_refs_main() {+ let _guard = crate::test_support::env_lock();+ let _env = CacheEnv::cleared_with(["HEADROOM_HF_PIN".to_string()]);+ let cache = tempfile::tempdir().unwrap();+ std::env::set_var("HF_HUB_CACHE", cache.path());+ cache_with_three_snapshots(cache.path());+ let newest = Path::new("ffffffffffffffffffffffffffffffffffffffff").join("tokenizer.json");++ std::env::set_var("HEADROOM_HF_PIN", "off");+ let tokenizer = hf_cache_file(DEFAULT_TOKENIZER_REPO, &["tokenizer.json"]).unwrap();+ assert!(tokenizer.ends_with(&newest));++ std::env::remove_var("HEADROOM_HF_PIN");+ let domain = "acme/kompress-legal";+ cached_file(+ cache.path(),+ domain,+ "ffffffffffffffffffffffffffffffffffffffff",+ "tokenizer.json",+ );+ let refs = cache+ .path()+ .join("models--acme--kompress-legal")+ .join("refs");+ std::fs::create_dir_all(&refs).unwrap();+ std::fs::write(+ refs.join("main"),+ "ffffffffffffffffffffffffffffffffffffffff",+ )+ .unwrap();+ assert!(hf_cache_file(domain, &["tokenizer.json"])+ .unwrap()+ .ends_with(&newest));+ }++ /// The Kompress lookup now names the directory Python would, including+ /// the cases the old root list missed: the legacy `HUGGINGFACE_HUB_CACHE`,+ /// `XDG_CACHE_HOME` and a `${VAR}` reference. And it reads only that one+ /// directory: a copy under `HOME` is not used when `HF_HUB_CACHE` points+ /// elsewhere.+ #[test]+ fn the_cache_lookup_uses_the_one_directory_python_uses() {+ let _guard = crate::test_support::env_lock();+ let _env = CacheEnv::cleared_with([+ "HEADROOM_HF_PIN".to_string(),+ "KOMPRESS_TEST_ROOT".to_string(),+ ]);+ let t = tempfile::tempdir().unwrap();+ let pin = cache_revision(DEFAULT_TOKENIZER_REPO).to_string();+ let lookup = || hf_cache_file(DEFAULT_TOKENIZER_REPO, &["tokenizer.json"]);+ std::env::set_var("HOME", t.path().join("home"));+ std::env::set_var("USERPROFILE", t.path().join("home"));++ let legacy = t.path().join("legacy");+ let want = cached_file(&legacy, DEFAULT_TOKENIZER_REPO, &pin, "tokenizer.json");+ std::env::set_var("HUGGINGFACE_HUB_CACHE", &legacy);+ assert_eq!(lookup(), Some(want));+ std::env::remove_var("HUGGINGFACE_HUB_CACHE");++ let xdg = t.path().join("xdg");+ let want = cached_file(+ &xdg.join("huggingface").join("hub"),+ DEFAULT_TOKENIZER_REPO,+ &pin,+ "tokenizer.json",+ );+ std::env::set_var("XDG_CACHE_HOME", &xdg);+ assert_eq!(lookup(), Some(want));+ std::env::remove_var("XDG_CACHE_HOME");++ let root = t.path().join("root");+ let want = cached_file(+ &root.join("hub"),+ DEFAULT_TOKENIZER_REPO,+ &pin,+ "tokenizer.json",+ );+ std::env::set_var("KOMPRESS_TEST_ROOT", &root);+ std::env::set_var("HF_HUB_CACHE", "${KOMPRESS_TEST_ROOT}/hub");+ assert_eq!(lookup(), Some(want));++ cached_file(+ &t.path()+ .join("home")+ .join(".cache")+ .join("huggingface")+ .join("hub"),+ DEFAULT_TOKENIZER_REPO,+ &pin,+ "tokenizer.json",+ );+ std::env::set_var("HF_HUB_CACHE", t.path().join("empty"));+ assert_eq!(lookup(), None);+ }/// `from_pretrained` must refuse before `Api::new()` when the operator/// air-gapped the box.diff --git a/crates/headroom-core/tests/kompress_parity.rs b/crates/headroom-core/tests/kompress_parity.rsindex 942fa8c11..f98bcb51c 100644--- a/crates/headroom-core/tests/kompress_parity.rs+++ b/crates/headroom-core/tests/kompress_parity.rs@@ -14,37 +14,20 @@use std::fs;use std::path::{Path, PathBuf};-use headroom_core::transforms::kompress::{Kompress, KompressConfig};+use headroom_core::transforms::kompress::{self, Kompress, KompressConfig};use serde_json::Value;-fn hf_cache_file(repo_dir: &str, rel: &[&str]) -> Option<PathBuf> {- let home = std::env::var("HOME").ok()?;- let snapshots = Path::new(&home)- .join(".cache/huggingface/hub")- .join(repo_dir)- .join("snapshots");- for snap in fs::read_dir(snapshots).ok()?.filter_map(|e| e.ok()) {- let mut cand = snap.path();- for part in rel {- cand = cand.join(part);- }- if cand.exists() {- return Some(cand);- }- }- None+/// The tokenizer + int8 ONNX artifact production would load: the pinned+/// snapshots in Python's hub cache. The fixtures were recorded against them.+fn cached_artifacts() -> Option<(PathBuf, PathBuf)> {+ kompress::cached_artifacts(&KompressConfig::default(), "onnx/kompress-int8-wo.onnx")}#[test]fn kompress_matches_python_fixtures_byte_for_byte() {- let tok = hf_cache_file("models--answerdotai--ModernBERT-base", &["tokenizer.json"]);- let onnx = hf_cache_file(- "models--chopratejas--kompress-v2-base",- &["onnx", "kompress-int8-wo.onnx"],- );- let (tok, onnx) = match (tok, onnx) {- (Some(t), Some(o)) => (t, o),- _ => {+ let (tok, onnx) = match cached_artifacts() {+ Some(paths) => paths,+ None => {eprintln!("SKIP: kompress model/tokenizer not in HF cache; \run `python scripts/record_kompress_trace.py` first"@@ -112,12 +95,7 @@ fn short_input_passes_through() {// Pure-logic check — no model needed. Fewer than MIN_WORDS words must// pass through unchanged regardless of model availability... but the// engine needs a model to construct. Guard on cache like the main test.- let tok = hf_cache_file("models--answerdotai--ModernBERT-base", &["tokenizer.json"]);- let onnx = hf_cache_file(- "models--chopratejas--kompress-v2-base",- &["onnx", "kompress-int8-wo.onnx"],- );- let (Some(tok), Some(onnx)) = (tok, onnx) else {+ let Some((tok, onnx)) = cached_artifacts() else {eprintln!("SKIP: model not cached");return;};diff --git a/crates/headroom-parity/src/lib.rs b/crates/headroom-parity/src/lib.rsindex adb5559cc..b78c61d30 100644--- a/crates/headroom-parity/src/lib.rs+++ b/crates/headroom-parity/src/lib.rs@@ -648,37 +648,16 @@ impl KompressComparator {Self::default()}- fn hf_cache_file(repo_dir: &str, rel: &[&str]) -> Option<PathBuf> {- let home = std::env::var("HOME").ok()?;- let snapshots = Path::new(&home)- .join(".cache/huggingface/hub")- .join(repo_dir)- .join("snapshots");- for snap in fs::read_dir(snapshots).ok()?.filter_map(|e| e.ok()) {- let mut cand = snap.path();- for part in rel {- cand = cand.join(part);- }- if cand.exists() {- return Some(cand);- }- }- None- }-fn model(&self) -> Option<&headroom_core::transforms::kompress::Kompress> {self.model.get_or_init(|| {- use headroom_core::transforms::kompress::{Kompress, KompressConfig};- let tok = Self::hf_cache_file(- "models--answerdotai--ModernBERT-base",- &["tokenizer.json"],- )?;- let onnx = Self::hf_cache_file(- "models--chopratejas--kompress-v2-base",- &["onnx", "kompress-int8-wo.onnx"],- )?;- Kompress::from_files(&tok, &onnx, KompressConfig::default()).ok()+ use headroom_core::transforms::kompress::{self, Kompress, KompressConfig};+ // The pinned snapshots in Python's hub cache, as production+ // loads them.+ let config = KompressConfig::default();+ let (tok, onnx) =+ kompress::cached_artifacts(&config, "onnx/kompress-int8-wo.onnx")?;+ Kompress::from_files(&tok, &onnx, config).ok()}).as_ref()}diff --git a/tests/parity/fixtures/hf_hub_cache/cases.json b/tests/parity/fixtures/hf_hub_cache/cases.jsonindex 0bb5c85a1..01273fd82 100644--- a/tests/parity/fixtures/hf_hub_cache/cases.json+++ b/tests/parity/fixtures/hf_hub_cache/cases.json@@ -1,5 +1,5 @@{- "description": "Cases both languages are checked against so Rust's hf_hub_cache_dir() names the directory huggingface_hub would. tests/test_fastembed_pinned_snapshot.py checks them against Python's posixpath/ntpath and huggingface_hub.constants; crates/headroom-core/src/relevance/embedding.rs checks its port against the same file.",+ "description": "Cases both languages are checked against so Rust's hf_hub_cache_dir() names the directory huggingface_hub would. tests/test_fastembed_pinned_snapshot.py checks them against Python's posixpath/ntpath and huggingface_hub.constants; crates/headroom-core/src/hf_cache.rs checks its port against the same file.","expandvars": {"comment": "posix = posixpath.expandvars, nt = ntpath.expandvars, with env set and unset absent (identical on CPython 3.10, 3.11, 3.13).","env": {@@ -130,7 +130,10 @@"comment": "{root} and {home} are temporary directories; HOME and USERPROFILE are set to {home}, every other cache variable is cleared first. expected may be {posix, nt}.","probe_vars": ["HEADROOM_HF_PROBE_ROOT",- "HEADROOM_HF_PROBE_UNSET"+ "HEADROOM_HF_PROBE_UNSET",+ "HF_TOKEN",+ "HF_TOKEN_PATH",+ "HUGGING_FACE_HUB_TOKEN"],"cases": [{@@ -275,5 +278,102 @@"expected": "{home}/.cache/huggingface/hub"}]+ },+ "token": {+ "comment": "huggingface_hub.get_token(): env, files written first ({root}/{home} as in cache_dir), expected token or null. HF_TOKEN, HUGGING_FACE_HUB_TOKEN and HF_TOKEN_PATH are cleared with the cache variables.",+ "cases": [+ {+ "name": "HF_TOKEN wins over the login file",+ "env": {+ "HF_TOKEN": "hf_env"+ },+ "files": {+ "{home}/.cache/huggingface/token": "hf_file"+ },+ "expected": "hf_env"+ },+ {+ "name": "the legacy HUGGING_FACE_HUB_TOKEN",+ "env": {+ "HUGGING_FACE_HUB_TOKEN": "hf_legacy"+ },+ "files": {},+ "expected": "hf_legacy"+ },+ {+ "name": "an empty HF_TOKEN falls through to the legacy name",+ "env": {+ "HF_TOKEN": "",+ "HUGGING_FACE_HUB_TOKEN": "hf_legacy"+ },+ "files": {},+ "expected": "hf_legacy"+ },+ {+ "name": "a whitespace-only HF_TOKEN is still the one picked, so the legacy name is skipped",+ "env": {+ "HF_TOKEN": " ",+ "HUGGING_FACE_HUB_TOKEN": "hf_legacy"+ },+ "files": {+ "{home}/.cache/huggingface/token": "hf_file"+ },+ "expected": "hf_file"+ },+ {+ "name": "the login file under the default HF_HOME, cleaned",+ "env": {},+ "files": {+ "{home}/.cache/huggingface/token": " hf_file\r\n"+ },+ "expected": "hf_file"+ },+ {+ "name": "the login file under HF_HOME",+ "env": {+ "HF_HOME": "{root}/hf"+ },+ "files": {+ "{root}/hf/token": "hf_home"+ },+ "expected": "hf_home"+ },+ {+ "name": "HF_TOKEN_PATH through ${VAR}",+ "env": {+ "HEADROOM_HF_PROBE_ROOT": "{root}",+ "HF_TOKEN_PATH": "${HEADROOM_HF_PROBE_ROOT}/tok"+ },+ "files": {+ "{root}/tok": "hf_path"+ },+ "expected": "hf_path"+ },+ {+ "name": "moving the hub cache does not move the token",+ "env": {+ "HF_HUB_CACHE": "{root}/elsewhere/hub"+ },+ "files": {+ "{home}/.cache/huggingface/token": "hf_file",+ "{root}/elsewhere/token": "hf_wrong"+ },+ "expected": "hf_file"+ },+ {+ "name": "a blank login file is no token",+ "env": {},+ "files": {+ "{home}/.cache/huggingface/token": " \n"+ },+ "expected": null+ },+ {+ "name": "nothing set",+ "env": {},+ "files": {},+ "expected": null+ }+ ]}}diff --git a/tests/test_fastembed_pinned_snapshot.py b/tests/test_fastembed_pinned_snapshot.pyindex 48ade3f2f..86b0dd13e 100644--- a/tests/test_fastembed_pinned_snapshot.py+++ b/tests/test_fastembed_pinned_snapshot.py@@ -228,11 +228,11 @@ def test_python_matches_parity_fixture():# ── the shared HuggingFace cache directory ──────────────────────────────────#-# Rust's ``hf_hub_cache_dir`` re-implements how ``huggingface_hub`` picks the+# Rust's ``hf_cache::hub_cache_dir`` re-implements how ``huggingface_hub`` picks the# hub cache, so both languages look up and download into one directory. These# check the shared fixture against Python itself; the Rust tests# ``expand_vars_matches_python_posixpath_and_ntpath`` and-# ``hf_hub_cache_dir_matches_the_shared_fixture`` check the port against the+# ``hub_cache_dir_matches_the_shared_fixture`` (``crates/headroom-core/src/hf_cache.rs``) check the port against the# same file._CACHE_VARS = ("HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE", "HF_HOME", "XDG_CACHE_HOME")@@ -288,3 +288,39 @@ def test_cache_dir_fixture_matches_huggingface_hub(case: dict, tmp_path: Path) -if isinstance(expected, dict):expected = expected["nt" if os.name == "nt" else "posix"]assert Path(result.stdout) == Path(fill(expected))++[email protected]("case", HF_CACHE_FIXTURE["token"]["cases"], ids=lambda case: case["name"])+def test_token_fixture_matches_huggingface_hub(case: dict, tmp_path: Path) -> None:+ """The token Rust's ``hf_cache::hf_token`` must find, from+ ``huggingface_hub.get_token()`` itself."""+ import subprocess++ pytest.importorskip("huggingface_hub")+ root, home = tmp_path / "root", tmp_path / "home"++ def fill(text: str) -> str:+ return text.replace("{root}", str(root)).replace("{home}", str(home))++ for path, content in case["files"].items():+ target = Path(fill(path))+ target.parent.mkdir(parents=True, exist_ok=True)+ target.write_text(content, encoding="utf-8")+ probe_vars = HF_CACHE_FIXTURE["cache_dir"]["probe_vars"]+ env = {k: v for k, v in os.environ.items() if k not in (*_CACHE_VARS, *probe_vars)}+ env["HOME"] = env["USERPROFILE"] = str(home)+ env.update({name: fill(value) for name, value in case["env"].items()})+ result = subprocess.run(+ [+ sys.executable,+ "-c",+ "import json, sys; from huggingface_hub import get_token; "+ "sys.stdout.write(json.dumps(get_token()))",+ ],+ capture_output=True,+ text=True,+ env=env,+ timeout=60,+ check=True,+ )+ assert json.loads(result.stdout) == case["expected"]diff --git a/tests/test_kompress_rust_pins.py b/tests/test_kompress_rust_pins.pynew file mode 100644index 000000000..11baf9a93--- /dev/null+++ b/tests/test_kompress_rust_pins.py@@ -0,0 +1,37 @@+"""The Rust Kompress loader reads the snapshots the Python side pinned.++The native proxy's Kompress is cache-only: it never downloads, it reads the+files Python's ``hf_hub_download_local_first`` put in the HuggingFace cache at+``_PINNED_REVISIONS``. ``crates/headroom-core/src/transforms/kompress.rs`` looks+up the same commits, so the two must not drift: a pin bump on one side only+would leave the Rust proxy on a snapshot Python no longer downloads.+"""++from __future__ import annotations++import re+from pathlib import Path++from headroom.onnx_runtime import _PINNED_REVISIONS++RUST_KOMPRESS = (+ Path(__file__).resolve().parents[1] / "crates/headroom-core/src/transforms/kompress.rs"+)+++def _rust_const(name: str) -> str:+ match = re.search(rf'pub const {name}: &str = "([^"]+)";', RUST_KOMPRESS.read_text())+ assert match, f"{name} not found in {RUST_KOMPRESS}"+ return match.group(1)+++def test_rust_kompress_model_pin_matches_python():+ repo = _rust_const("DEFAULT_MODEL_ID")+ assert repo in _PINNED_REVISIONS, f"{repo} has no Python pin"+ assert _rust_const("DEFAULT_MODEL_REVISION") == _PINNED_REVISIONS[repo]+++def test_rust_kompress_tokenizer_pin_matches_python():+ repo = _rust_const("DEFAULT_TOKENIZER_REPO")+ assert repo in _PINNED_REVISIONS, f"{repo} has no Python pin"+ assert _rust_const("DEFAULT_TOKENIZER_REVISION") == _PINNED_REVISIONS[repo]diff --git a/tests/test_offline_egress_chokepoint.py b/tests/test_offline_egress_chokepoint.pyindex 4d9dbef71..22bf0c9e3 100644--- a/tests/test_offline_egress_chokepoint.py+++ b/tests/test_offline_egress_chokepoint.py@@ -868,7 +868,8 @@ class TestHuggingFaceDownloadOffline:# ───────────────── path 3: the Rust HuggingFace fetch (parity) ──────────────_RUST_OFFLINE = REPO_ROOT / "crates" / "headroom-core" / "src" / "offline.rs"-_RUST_HF = REPO_ROOT / "crates" / "headroom-core" / "src" / "tokenizer" / "hf_impl.rs"+# Every Rust Hub download (tokenizer, embedder) builds its client in hf_cache::hub_api.+_RUST_HF = REPO_ROOT / "crates" / "headroom-core" / "src" / "hf_cache.rs"class TestRustOfflineParity:@@ -976,11 +977,14 @@ class TestRustOfflineParity:def test_hf_fetch_guards_before_it_builds_a_client(self) -> None:source = _RUST_HF.read_text(encoding="utf-8")- assert "guard_egress(" in source, "hf_impl.rs does not consult the offline guard"- guard_at = source.index("guard_egress(")- api_at = source.index("hf_hub::api::sync::Api::new()")+ # Within hub_api itself: a guard elsewhere in the file does not count.+ fn_at = source.index("pub(crate) fn hub_api(")+ body = source[fn_at : source.index("\n}\n", fn_at)]+ assert "guard_egress(" in body, "hf_cache::hub_api does not consult the offline guard"+ guard_at = body.index("guard_egress(")+ api_at = body.index("hf_hub::api::sync::ApiBuilder::from_cache(")assert guard_at < api_at, (- "the offline guard runs after Api::new(), which already resolves the "+ "the offline guard runs after ApiBuilder::build(), which already resolves the ""Hub endpoint and builds the ureq agent — guard before the client ""exists, not before the request")@@ -3225,7 +3229,7 @@ _RUST_ALLOWLIST: dict[str, tuple[int, str]] = {"cache-only: TextEmbedding::try_new_from_user_defined in load_snapshot ""builds the ONNX session from bytes already read off disk and has no ""Hub client at all. The files reach the disk through "- "default_model_repo, whose ApiBuilder is guarded, and only on a cache "+ "default_model_repo, whose hf_cache::hub_api is guarded, and only on a cache ""miss, so a pre-seeded air-gapped host still loads.",),"headroom-proxy/src/proxy.rs": (@@ -3329,7 +3333,7 @@ class TestRustEgressChokepointCoverage:def test_the_guarded_rust_paths_are_seen_as_guarded(self) -> None:sites = _rust_egress_sites()for relpath in (- "headroom-core/src/tokenizer/hf_impl.rs",+ "headroom-core/src/hf_cache.rs","headroom-core/src/transforms/kompress.rs","headroom-core/src/relevance/embedding.rs",):