Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(kompress): read the pinned snapshot from Python's one HF cache in the native proxy (#4438)

Kenneth Wongeef44f60cc56
diff --git a/crates/headroom-core/src/hf_cache.rs b/crates/headroom-core/src/hf_cache.rs
new file mode 100644
index 000000000..83a9e230e
--- /dev/null
+++ b/crates/headroom-core/src/hf_cache.rs
@@ -0,0 +1,712 @@
+//! The HuggingFace hub cache, located and read the way Python's
+//! `huggingface_hub` does, so the Rust loaders (the relevance embedder,
+//! Kompress and the HuggingFace tokenizer) find the files the Python side
+//! downloaded and pinned, and authenticate with the same token.
+//!
+//! All network-free: [`hub_cache_dir`] picks the one cache directory Python
+//! would use, [`snapshot_path`] resolves a repo at a revision inside it,
+//! [`hf_token`] finds the access token and [`hub_api`] builds a client from
+//! them. A path variable that is set but empty counts as set, as in Python
+//! (see [`hub_cache_dir`]). The pinned revisions themselves live
+//! next to each loader and are kept equal to `_PINNED_REVISIONS` in
+//! `headroom/onnx_runtime.py` by tests on the Python side.
+//!
+//! Not behind the `ml` feature: the tokenizer, which is not either, uses the
+//! cache and token lookups.
+
+use std::path::{Path, PathBuf};
+
+use thiserror::Error;
+
+/// True when `HEADROOM_HF_PIN` turns revision pinning off, so pinned repos
+/// float on `main` (same values as Python's `onnx_runtime._resolve_revision`).
+#[cfg_attr(not(feature = "ml"), allow(dead_code))] // the pinned loaders are `ml`-only
+pub(crate) fn pin_disabled() -> bool {
+ let pin = std::env::var("HEADROOM_HF_PIN").unwrap_or_default();
+ matches!(
+ pin.trim().to_ascii_lowercase().as_str(),
+ "off" | "0" | "false" | "no"
+ )
+}
+
+/// The revision to load for a repo pinned at `pinned`: the pin, or `main`
+/// when `HEADROOM_HF_PIN` turns pinning off.
+#[cfg_attr(not(feature = "ml"), allow(dead_code))]
+pub(crate) fn pinned_or_main(pinned: &str) -> &str {
+ if pin_disabled() {
+ "main"
+ } else {
+ pinned
+ }
+}
+
+/// The snapshot directory of `repo` (`"owner/name"`) at `revision` in the hub
+/// cache `cache`, whether or not it exists. Never touches the network.
+///
+/// A branch name such as `main` resolves through `refs/<revision>`; a commit
+/// SHA is used as the directory name directly. `hf_hub::Cache` resolves every
+/// revision through `refs/`, which Python's `huggingface_hub` does not write
+/// for a commit SHA, so a snapshot Python downloaded at a pin would be missed.
+#[cfg_attr(not(feature = "ml"), allow(dead_code))]
+pub(crate) fn snapshot_path(cache: &Path, repo: &str, revision: &str) -> PathBuf {
+ let repo_dir = cache.join(format!("models--{}", repo.replace('/', "--")));
+ let commit = std::fs::read_to_string(repo_dir.join("refs").join(revision))
+ .map(|c| c.trim().to_string())
+ .unwrap_or_else(|_| revision.to_string());
+ repo_dir.join("snapshots").join(commit)
+}
+
+/// The HuggingFace hub cache directory, resolved the way Python's
+/// `huggingface_hub` resolves `constants.HF_HUB_CACHE`, so both languages look
+/// up and download into the same place: `HF_HUB_CACHE`, then
+/// `HUGGINGFACE_HUB_CACHE` (its legacy name), then `HF_HOME/hub`, then
+/// `XDG_CACHE_HOME/huggingface/hub`, then `~/.cache/huggingface/hub`.
+///
+/// It is exactly one directory. A snapshot in a lower-priority cache is not
+/// used when a higher one is configured, which is also what Python does: with
+/// `HF_HUB_CACHE` set, a model under `~/.cache` is neither found nor reused.
+///
+/// Both expansions happen as in `huggingface_hub.constants`:
+/// `expandvars(expanduser(...))`, first on `HF_HOME` (or its
+/// `XDG_CACHE_HOME` default) and then on the hub path itself. So
+/// `HF_HUB_CACHE=${ROOT}/hub` names the same directory in both languages, and
+/// a `~` that only appears once a variable is expanded stays literal, as it
+/// does in Python. The home directory comes from `std::env::home_dir`, which
+/// falls back to the user database when `HOME` is unset, as `expanduser`
+/// does. A variable that is set but empty still counts, as with `os.getenv`:
+/// `HF_HUB_CACHE=''` is the working directory itself, `HF_HOME=''` is `hub`
+/// and `XDG_CACHE_HOME=''` is `huggingface/hub` under it, and none of them
+/// falls through to a lower-priority cache. `tests/parity/fixtures/hf_hub_cache/`
+/// holds the cases both languages are checked against.
+pub(crate) fn hub_cache_dir() -> Option<PathBuf> {
+ let hub = match env_path("HF_HUB_CACHE").or_else(|| env_path("HUGGINGFACE_HUB_CACHE")) {
+ Some(dir) => dir,
+ None => hf_home_dir()?.join("hub"),
+ };
+ Some(expand(hub))
+}
+
+/// `huggingface_hub.constants.HF_HOME`: `HF_HOME`, else
+/// `XDG_CACHE_HOME/huggingface`, else `~/.cache/huggingface`, expanded as
+/// Python does. The hub cache defaults to its `hub` folder and the login
+/// token to its `token` file.
+fn hf_home_dir() -> Option<PathBuf> {
+ let dir = match env_path("HF_HOME") {
+ Some(dir) => dir,
+ None => env_path("XDG_CACHE_HOME")
+ .or_else(|| std::env::home_dir().map(|home| home.join(".cache")))?
+ .join("huggingface"),
+ };
+ Some(expand(dir))
+}
+
+/// The HuggingFace access token, found the way `huggingface_hub.get_token()`
+/// finds it: `HF_TOKEN`, then its legacy name `HUGGING_FACE_HUB_TOKEN`, then
+/// the file at `HF_TOKEN_PATH` (default `HF_HOME/token`, where
+/// `hf auth login` writes it). Each value is cleaned like Python's
+/// `_clean_token`: line breaks removed, outer whitespace trimmed, and an
+/// empty result counts as no token.
+///
+/// `hf-hub`'s own lookup reads only a `token` file beside the hub cache, so
+/// `HF_TOKEN`, an explicit `HF_TOKEN_PATH`, or a hub cache moved away from
+/// `HF_HOME` would leave a gated download unauthenticated.
+pub(crate) fn hf_token() -> Option<String> {
+ let clean = |token: String| {
+ let token = token.replace(['\r', '\n'], "");
+ let token = token.trim();
+ (!token.is_empty()).then(|| token.to_string())
+ };
+ // `HF_TOKEN or HUGGING_FACE_HUB_TOKEN`, then clean: a whitespace-only
+ // `HF_TOKEN` is still the one picked, and cleans to no token, so the
+ // legacy name is not consulted (the login file is).
+ ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]
+ .into_iter()
+ .find_map(|name| std::env::var(name).ok().filter(|v| !v.is_empty()))
+ .and_then(clean)
+ .or_else(|| {
+ let path = match env_path("HF_TOKEN_PATH") {
+ Some(path) => expand(path),
+ None => hf_home_dir()?.join("token"),
+ };
+ std::fs::read_to_string(path).ok().and_then(clean)
+ })
+}
+
+/// The Hub endpoint: `HF_ENDPOINT` without trailing slashes (Python's
+/// `constants.ENDPOINT` strips them, and `hf-hub` appends `/{repo}/…`, so a
+/// kept slash would request `//repo`), else `https://huggingface.co`.
+pub(crate) fn hf_endpoint() -> String {
+ match std::env::var("HF_ENDPOINT") {
+ Ok(endpoint) => endpoint.trim_end_matches('/').to_string(),
+ Err(_) => "https://huggingface.co".to_string(),
+ }
+}
+
+/// Why [`hub_api`] returned no client.
+#[derive(Debug, Error)]
+pub(crate) enum HubApiError {
+ /// `HEADROOM_OFFLINE` is set; refused before the client existed.
+ #[error(transparent)]
+ Offline(#[from] crate::offline::OfflineEgressBlocked),
+ /// `hf-hub` could not build the client.
+ #[error(transparent)]
+ Build(#[from] hf_hub::api::sync::ApiError),
+}
+
+/// An `hf-hub` client that caches in `cache` and talks to [`hf_endpoint`]
+/// with [`hf_token`], as Python's `huggingface_hub` would, for fetching
+/// `destination` (a repo id) for `purpose`.
+///
+/// This is the air-gap chokepoint for the tokenizer and embedder downloads:
+/// under `HEADROOM_OFFLINE` it returns [`HubApiError::Offline`] before
+/// building the client, because building it resolves the endpoint and the
+/// `ureq` agent, and `hf-hub` downloads on a cache miss even with
+/// `HF_HUB_OFFLINE` set. Guarding here rather than in each caller means no
+/// caller of this function can skip it. (`Kompress::from_pretrained` still
+/// builds its own `Api::new()` client behind its own guard; production loads
+/// Kompress cache-only through `Kompress::from_cache`.)
+///
+/// The token is not sent over plain `http://` to a host other than loopback:
+/// it would cross the network in clear text on every request, so the client
+/// goes without it and says why. Python would send it; a gated repo behind
+/// such a mirror then answers 401 rather than leak the credential.
+pub(crate) fn hub_api(
+ cache: &Path,
+ purpose: &str,
+ destination: &str,
+) -> Result<hf_hub::api::sync::Api, HubApiError> {
+ crate::offline::guard_egress(purpose, destination)?;
+ let endpoint = hf_endpoint();
+ let token = hf_token().filter(|_| {
+ let exposed = sends_in_clear_text(&endpoint);
+ if exposed {
+ tracing::warn!(
+ event = "hf_token_withheld",
+ endpoint = %endpoint,
+ "HF_ENDPOINT is plain http to a non-loopback host; not sending the \
+ HuggingFace token to it. Use an https endpoint for gated repos."
+ );
+ }
+ !exposed
+ });
+ hf_hub::api::sync::ApiBuilder::from_cache(hf_hub::Cache::new(cache.to_path_buf()))
+ .with_progress(false)
+ .with_endpoint(endpoint)
+ .with_token(token)
+ .build()
+ .map_err(HubApiError::Build)
+}
+
+/// Whether a request to `endpoint` would carry its headers unencrypted off
+/// this machine: `http://` to anything but a loopback host.
+fn sends_in_clear_text(endpoint: &str) -> bool {
+ let Some(rest) = endpoint
+ .get(..7)
+ .filter(|scheme| scheme.eq_ignore_ascii_case("http://"))
+ .map(|_| &endpoint[7..])
+ else {
+ return false;
+ };
+ let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
+ let host_port = authority.rsplit('@').next().unwrap_or("");
+ let host = match host_port.strip_prefix('[') {
+ Some(v6) => v6.split(']').next().unwrap_or(""),
+ None => host_port.split(':').next().unwrap_or(""),
+ };
+ let loopback = host.eq_ignore_ascii_case("localhost")
+ || host
+ .parse::<std::net::IpAddr>()
+ .is_ok_and(|ip| ip.is_loopback());
+ !loopback
+}
+
+/// A path-valued variable. An empty one is still set, as `os.getenv` reads
+/// it, so it never falls through to the next variable.
+fn env_path(name: &str) -> Option<PathBuf> {
+ std::env::var_os(name).map(PathBuf::from)
+}
+
+/// `expandvars(expanduser(path))`, the order `huggingface_hub.constants` uses.
+fn expand(path: PathBuf) -> PathBuf {
+ expand_vars(&expand_user(path, std::env::home_dir().as_deref()))
+}
+
+/// Python's `os.path.expanduser` for a leading bare `~` component. `~user`
+/// stays as written, which is also what `expanduser` does when it cannot
+/// resolve the user.
+fn expand_user(path: PathBuf, home: Option<&Path>) -> PathBuf {
+ match (path.strip_prefix("~"), home) {
+ (Ok(rest), Some(home)) => home.join(rest),
+ _ => path,
+ }
+}
+
+/// Python's `os.path.expandvars`: `posixpath` rules off Windows, `ntpath`
+/// rules on it. A path that is not valid Unicode is returned unchanged.
+fn expand_vars(path: &Path) -> PathBuf {
+ let Some(text) = path.to_str() else {
+ return path.to_path_buf();
+ };
+ #[cfg(windows)]
+ let expanded = expand_vars_nt(text, env_lookup);
+ #[cfg(not(windows))]
+ let expanded = expand_vars_posix(text, env_lookup);
+ PathBuf::from(expanded)
+}
+
+/// An environment value for `expand_vars`. Names the OS cannot hold (empty,
+/// or containing `=` or NUL) are unset, which is how Python's `os.environ`
+/// lookup fails for them too.
+fn env_lookup(name: &str) -> Option<std::ffi::OsString> {
+ if name.is_empty() || name.contains(['=', '\0']) {
+ return None;
+ }
+ std::env::var_os(name)
+}
+
+/// `posixpath.expandvars`: `$name` (ASCII letters, digits, `_`) or
+/// `${anything but a brace}`. An unset name, and a `$` that starts neither
+/// form, stay as written. Expanded values are not scanned again.
+#[cfg_attr(windows, allow(dead_code))] // tests run both rule sets everywhere
+fn expand_vars_posix(
+ text: &str,
+ lookup: impl Fn(&str) -> Option<std::ffi::OsString>,
+) -> std::ffi::OsString {
+ let mut out = std::ffi::OsString::new();
+ let mut rest = text;
+ while let Some(at) = rest.find('$') {
+ out.push(&rest[..at]);
+ let after = &rest[at + 1..];
+ let word = after
+ .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_'))
+ .unwrap_or(after.len());
+ // (name, length of the reference after the `$`)
+ let reference = if word > 0 {
+ Some((&after[..word], word))
+ } else {
+ after
+ .strip_prefix('{')
+ .and_then(|braced| braced.find('}').map(|end| (&braced[..end], end + 2)))
+ };
+ match reference {
+ Some((name, len)) => {
+ match lookup(name) {
+ Some(value) => out.push(value),
+ None => out.push(&rest[at..at + 1 + len]),
+ }
+ rest = &after[len..];
+ }
+ None => {
+ out.push("$");
+ rest = after;
+ }
+ }
+ }
+ out.push(rest);
+ out
+}
+
+/// `ntpath.expandvars`: `%name%`, `${name}` and `$name` (ASCII letters,
+/// digits, `_`, `-`); `%%` and `$$` are a literal `%` and `$`; nothing inside
+/// single quotes expands. An unset or unterminated reference stays as
+/// written.
+#[cfg_attr(not(windows), allow(dead_code))] // tests run both rule sets everywhere
+fn expand_vars_nt(
+ text: &str,
+ lookup: impl Fn(&str) -> Option<std::ffi::OsString>,
+) -> std::ffi::OsString {
+ let mut out = std::ffi::OsString::new();
+ let mut rest = text;
+ while let Some(c) = rest.chars().next() {
+ let after = &rest[c.len_utf8()..];
+ match c {
+ '\'' => match after.find('\'') {
+ Some(end) => {
+ out.push(&rest[..end + 2]);
+ rest = &after[end + 1..];
+ }
+ None => {
+ out.push(rest);
+ rest = "";
+ }
+ },
+ '%' if after.starts_with('%') => {
+ out.push("%");
+ rest = &after[1..];
+ }
+ '%' => match after.find('%') {
+ Some(end) => {
+ let name = &after[..end];
+ match lookup(name) {
+ Some(value) => out.push(value),
+ None => out.push(&rest[..end + 2]),
+ }
+ rest = &after[end + 1..];
+ }
+ None => {
+ out.push(rest);
+ rest = "";
+ }
+ },
+ '$' if after.starts_with('$') => {
+ out.push("$");
+ rest = &after[1..];
+ }
+ '$' if after.starts_with('{') => match after[1..].find('}') {
+ Some(end) => {
+ let name = &after[1..1 + end];
+ match lookup(name) {
+ Some(value) => out.push(value),
+ None => out.push(&rest[..end + 3]),
+ }
+ rest = &after[end + 2..];
+ }
+ None => {
+ out.push(rest);
+ rest = "";
+ }
+ },
+ '$' => {
+ let end = after
+ .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_' || c == '-'))
+ .unwrap_or(after.len());
+ let name = &after[..end];
+ match lookup(name) {
+ Some(value) => out.push(value),
+ None => out.push(&rest[..end + 1]),
+ }
+ rest = &after[end..];
+ }
+ _ => {
+ out.push(&rest[..c.len_utf8()]);
+ rest = after;
+ }
+ }
+ }
+ out
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+ use super::*;
+
+ /// Every variable `hub_cache_dir` reads.
+ const CACHE_ENV: [&str; 6] = [
+ "HF_HUB_CACHE",
+ "HUGGINGFACE_HUB_CACHE",
+ "HF_HOME",
+ "XDG_CACHE_HOME",
+ "HOME",
+ "USERPROFILE",
+ ];
+
+ /// Saves [`CACHE_ENV`], clears it, and puts it back on drop, so a failing
+ /// assertion cannot leak a fake cache into later tests. Take
+ /// `env_lock()` first.
+ pub(crate) struct CacheEnv(Vec<(String, Option<std::ffi::OsString>)>);
+
+ impl CacheEnv {
+ pub(crate) fn cleared() -> Self {
+ Self::cleared_with(std::iter::empty())
+ }
+
+ /// [`Self::cleared`], plus `extra` variables (the ones a test's
+ /// paths reference).
+ pub(crate) fn cleared_with(extra: impl IntoIterator<Item = String>) -> Self {
+ let names: Vec<String> = CACHE_ENV
+ .iter()
+ .map(|name| name.to_string())
+ .chain(extra)
+ .collect();
+ let saved = names
+ .into_iter()
+ .map(|name| {
+ let value = std::env::var_os(&name);
+ std::env::remove_var(&name);
+ (name, value)
+ })
+ .collect();
+ CacheEnv(saved)
+ }
+ }
+
+ impl Drop for CacheEnv {
+ fn drop(&mut self) {
+ for (name, value) in &self.0 {
+ match value {
+ Some(value) => std::env::set_var(name, value),
+ None => std::env::remove_var(name),
+ }
+ }
+ }
+ }
+
+ /// Same order as `huggingface_hub.constants.HF_HUB_CACHE`: each variable
+ /// wins over every one after it, and only one directory comes back.
+ #[test]
+ fn hub_cache_dir_follows_huggingface_hub_precedence() {
+ let _guard = crate::test_support::env_lock();
+ let _env = CacheEnv::cleared();
+ let t = tempfile::tempdir().unwrap();
+ let p = |name: &str| t.path().join(name);
+ std::env::set_var("HOME", p("home"));
+ std::env::set_var("USERPROFILE", p("home"));
+ std::env::set_var("XDG_CACHE_HOME", p("xdg"));
+ std::env::set_var("HF_HOME", p("hf-home"));
+ std::env::set_var("HUGGINGFACE_HUB_CACHE", p("legacy"));
+ std::env::set_var("HF_HUB_CACHE", p("explicit"));
+
+ assert_eq!(hub_cache_dir(), Some(p("explicit")));
+ std::env::remove_var("HF_HUB_CACHE");
+ assert_eq!(hub_cache_dir(), Some(p("legacy")));
+ std::env::remove_var("HUGGINGFACE_HUB_CACHE");
+ assert_eq!(hub_cache_dir(), Some(p("hf-home").join("hub")));
+ std::env::remove_var("HF_HOME");
+ assert_eq!(
+ hub_cache_dir(),
+ Some(p("xdg").join("huggingface").join("hub"))
+ );
+ std::env::remove_var("XDG_CACHE_HOME");
+ assert_eq!(
+ hub_cache_dir(),
+ Some(p("home").join(".cache").join("huggingface").join("hub"))
+ );
+
+ // A leading `~` expands.
+ std::env::set_var("HF_HOME", "~/models");
+ assert_eq!(hub_cache_dir(), Some(p("home").join("models").join("hub")));
+
+ // A set but empty variable is the working directory, as in Python,
+ // and still wins over every variable after it.
+ std::env::set_var("HF_HUB_CACHE", "");
+ assert_eq!(hub_cache_dir(), Some(PathBuf::new()));
+ std::env::remove_var("HF_HUB_CACHE");
+ std::env::set_var("HF_HOME", "");
+ std::env::set_var("XDG_CACHE_HOME", p("xdg"));
+ assert_eq!(hub_cache_dir(), Some(PathBuf::from("hub")));
+ }
+
+ /// An empty `HF_TOKEN_PATH` names no readable file, so there is no token;
+ /// it must not fall back to `HF_HOME/token`. Python's `get_token()` raises
+ /// `IsADirectoryError` here (it reads `.`), so this case is Rust-only
+ /// rather than in the shared fixture.
+ #[test]
+ fn an_empty_hf_token_path_does_not_fall_back_to_hf_home() {
+ let _guard = crate::test_support::env_lock();
+ let _env = CacheEnv::cleared_with(
+ ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HF_TOKEN_PATH"].map(String::from),
+ );
+ let hf_home = tempfile::tempdir().unwrap();
+ std::fs::write(hf_home.path().join("token"), "hf_from_home\n").unwrap();
+ std::env::set_var("HF_HOME", hf_home.path());
+
+ std::env::remove_var("HF_TOKEN_PATH");
+ assert_eq!(hf_token().as_deref(), Some("hf_from_home"));
+ std::env::set_var("HF_TOKEN_PATH", "");
+ assert_eq!(hf_token(), None);
+ }
+
+ /// Cases checked against Python itself (`posixpath`, `ntpath` and
+ /// `huggingface_hub.constants`) by `tests/test_fastembed_pinned_snapshot.py`.
+ const HF_CACHE_FIXTURE: &str = include_str!(concat!(
+ env!("CARGO_MANIFEST_DIR"),
+ "/../../tests/parity/fixtures/hf_hub_cache/cases.json"
+ ));
+
+ /// Both rule sets on every platform, so the Windows port is exercised by
+ /// the macOS and Linux runs too.
+ #[test]
+ fn expand_vars_matches_python_posixpath_and_ntpath() {
+ let fixture: serde_json::Value = serde_json::from_str(HF_CACHE_FIXTURE).unwrap();
+ let section = &fixture["expandvars"];
+ let env: std::collections::HashMap<String, String> =
+ serde_json::from_value(section["env"].clone()).unwrap();
+ let lookup = |name: &str| env.get(name).map(std::ffi::OsString::from);
+ for case in section["cases"].as_array().unwrap() {
+ let input = case["input"].as_str().unwrap();
+ let want = |key: &str| std::ffi::OsString::from(case[key].as_str().unwrap());
+ assert_eq!(
+ expand_vars_posix(input, lookup),
+ want("posix"),
+ "posix {input:?}"
+ );
+ assert_eq!(expand_vars_nt(input, lookup), want("nt"), "nt {input:?}");
+ }
+ }
+
+ /// The review case among them: `HF_HUB_CACHE=${ROOT}/hub` is
+ /// `<ROOT>/hub` in Python, so Rust must not look up or download into a
+ /// literal `${ROOT}` directory instead.
+ #[test]
+ fn hub_cache_dir_matches_the_shared_fixture() {
+ let fixture: serde_json::Value = serde_json::from_str(HF_CACHE_FIXTURE).unwrap();
+ let section = &fixture["cache_dir"];
+ let probe_vars: Vec<String> =
+ serde_json::from_value(section["probe_vars"].clone()).unwrap();
+ let _guard = crate::test_support::env_lock();
+ let _env = CacheEnv::cleared_with(probe_vars.clone());
+ let t = tempfile::tempdir().unwrap();
+ let (root, home) = (t.path().join("root"), t.path().join("home"));
+ let fill = |text: &str| {
+ text.replace("{root}", root.to_str().unwrap())
+ .replace("{home}", home.to_str().unwrap())
+ };
+ for case in section["cases"].as_array().unwrap() {
+ for name in CACHE_ENV
+ .iter()
+ .copied()
+ .chain(probe_vars.iter().map(String::as_str))
+ {
+ std::env::remove_var(name);
+ }
+ std::env::set_var("HOME", &home);
+ std::env::set_var("USERPROFILE", &home);
+ for (name, value) in case["env"].as_object().unwrap() {
+ std::env::set_var(name, fill(value.as_str().unwrap()));
+ }
+ let expected = match &case["expected"] {
+ serde_json::Value::Object(by_os) => {
+ &by_os[if cfg!(windows) { "nt" } else { "posix" }]
+ }
+ expected => expected,
+ };
+ assert_eq!(
+ hub_cache_dir(),
+ Some(PathBuf::from(fill(expected.as_str().unwrap()))),
+ "{}",
+ case["name"]
+ );
+ }
+ }
+
+ /// With no `HOME` (a service started with a bare environment) the home
+ /// directory still comes from the user database, as Python's
+ /// `expanduser` does, instead of leaving the scorer with no cache.
+ #[cfg(all(feature = "ml", unix))]
+ #[test]
+ fn hub_cache_dir_without_home_uses_the_user_database() {
+ let _guard = crate::test_support::env_lock();
+ let _env = CacheEnv::cleared();
+ let dir = hub_cache_dir().expect("a cache directory without HOME");
+ assert!(dir.is_absolute(), "{}", dir.display());
+ assert!(
+ dir.ends_with(Path::new(".cache").join("huggingface").join("hub")),
+ "{}",
+ dir.display()
+ );
+ }
+
+ /// The fixture's `token` cases, which the Python half checks against
+ /// `huggingface_hub.get_token()`.
+ #[test]
+ fn hf_token_matches_the_shared_fixture() {
+ let fixture: serde_json::Value = serde_json::from_str(HF_CACHE_FIXTURE).unwrap();
+ let probe_vars: Vec<String> =
+ serde_json::from_value(fixture["cache_dir"]["probe_vars"].clone()).unwrap();
+ let _guard = crate::test_support::env_lock();
+ let _env = CacheEnv::cleared_with(probe_vars.clone());
+ for case in fixture["token"]["cases"].as_array().unwrap() {
+ let t = tempfile::tempdir().unwrap();
+ let (root, home) = (t.path().join("root"), t.path().join("home"));
+ let fill = |text: &str| {
+ text.replace("{root}", root.to_str().unwrap())
+ .replace("{home}", home.to_str().unwrap())
+ };
+ for name in CACHE_ENV
+ .iter()
+ .copied()
+ .chain(probe_vars.iter().map(String::as_str))
+ {
+ std::env::remove_var(name);
+ }
+ std::env::set_var("HOME", &home);
+ std::env::set_var("USERPROFILE", &home);
+ for (name, value) in case["env"].as_object().unwrap() {
+ std::env::set_var(name, fill(value.as_str().unwrap()));
+ }
+ for (path, content) in case["files"].as_object().unwrap() {
+ let path = PathBuf::from(fill(path));
+ std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+ std::fs::write(path, content.as_str().unwrap()).unwrap();
+ }
+ assert_eq!(
+ hf_token().as_deref(),
+ case["expected"].as_str(),
+ "{}",
+ case["name"]
+ );
+ }
+ }
+
+ #[test]
+ fn hf_endpoint_drops_trailing_slashes_like_python() {
+ let _guard = crate::test_support::env_lock();
+ let saved = std::env::var_os("HF_ENDPOINT");
+ std::env::remove_var("HF_ENDPOINT");
+ assert_eq!(hf_endpoint(), "https://huggingface.co");
+ std::env::set_var("HF_ENDPOINT", "https://mirror.example.com//");
+ assert_eq!(hf_endpoint(), "https://mirror.example.com");
+ match saved {
+ Some(v) => std::env::set_var("HF_ENDPOINT", v),
+ None => std::env::remove_var("HF_ENDPOINT"),
+ }
+ }
+
+ /// The token is withheld only where it would cross the network unencrypted.
+ #[test]
+ fn plain_http_off_loopback_is_the_only_clear_text_endpoint() {
+ for endpoint in [
+ "http://mirror.example.com",
+ "HTTP://mirror.example.com:8080/hub",
+ "http://10.0.0.5",
+ "http://[email protected]",
+ "http://localhost.example.com",
+ ] {
+ assert!(sends_in_clear_text(endpoint), "{endpoint}");
+ }
+ for endpoint in [
+ "https://huggingface.co",
+ "https://mirror.example.com",
+ "http://localhost:8000",
+ "http://127.0.0.1:9",
+ "http://127.8.0.1",
+ "http://[::1]:8080/",
+ ] {
+ assert!(!sends_in_clear_text(endpoint), "{endpoint}");
+ }
+ }
+
+ /// A commit SHA names the snapshot directly; a branch goes through its
+ /// ref, as in Python's cache layout.
+ #[test]
+ fn snapshot_path_resolves_commits_directly_and_branches_through_refs() {
+ let cache = tempfile::tempdir().unwrap();
+ let repo = cache.path().join("models--owner--name");
+ std::fs::create_dir_all(repo.join("refs")).unwrap();
+ std::fs::write(repo.join("refs").join("main"), "abc123\n").unwrap();
+ let snapshots = repo.join("snapshots");
+ assert_eq!(
+ snapshot_path(cache.path(), "owner/name", "def456"),
+ snapshots.join("def456")
+ );
+ assert_eq!(
+ snapshot_path(cache.path(), "owner/name", "main"),
+ snapshots.join("abc123")
+ );
+ }
+
+ #[test]
+ fn pin_can_be_turned_off() {
+ let _guard = crate::test_support::env_lock();
+ let saved = std::env::var_os("HEADROOM_HF_PIN");
+ for (value, disabled) in [("", false), ("on", false), ("off", true), (" No ", true)] {
+ std::env::set_var("HEADROOM_HF_PIN", value);
+ assert_eq!(pin_disabled(), disabled, "{value:?}");
+ assert_eq!(pinned_or_main("sha"), if disabled { "main" } else { "sha" });
+ }
+ match saved {
+ Some(v) => std::env::set_var("HEADROOM_HF_PIN", v),
+ None => std::env::remove_var("HEADROOM_HF_PIN"),
+ }
+ }
+}
diff --git a/crates/headroom-core/src/lib.rs b/crates/headroom-core/src/lib.rs
index f209b4131..aceb50d86 100644
--- a/crates/headroom-core/src/lib.rs
+++ b/crates/headroom-core/src/lib.rs
@@ -8,6 +8,7 @@ pub mod auth_mode;
pub mod cache_control;
pub mod ccr;
pub mod compression_policy;
+mod hf_cache;
pub mod offline;
#[cfg(feature = "ml")]
mod onnx_cpu;
diff --git a/crates/headroom-core/src/relevance/embedding.rs b/crates/headroom-core/src/relevance/embedding.rs
index 575e0ffc2..6c7db0811 100644
--- a/crates/headroom-core/src/relevance/embedding.rs
+++ b/crates/headroom-core/src/relevance/embedding.rs
@@ -15,7 +15,7 @@
//! construction may need network access to fetch the model. The files
//! live in the HuggingFace hub cache directory Python's `huggingface_hub`
//! would pick (`HF_HUB_CACHE`, `HF_HOME/hub`, `~/.cache/huggingface/hub`,
-//! …; see `hf_hub_cache_dir`), shared with Python.
+//! …; see `crate::hf_cache::hub_cache_dir`), shared with Python.
//!
//! When constructed, `is_available()` returns `true` and `HybridScorer`
//! switches off the BM25-fallback path automatically. If construction
@@ -95,230 +95,16 @@ const DEFAULT_MODEL_FILES: [&str; 5] = [
/// pinning off (same values as Python's `onnx_runtime._resolve_revision`).
#[cfg(feature = "ml")]
fn default_model_revision() -> &'static str {
- let pin = std::env::var("HEADROOM_HF_PIN").unwrap_or_default();
- match pin.trim().to_ascii_lowercase().as_str() {
- "off" | "0" | "false" | "no" => "main",
- _ => DEFAULT_MODEL_REVISION,
- }
-}
-
-/// The HuggingFace hub cache directory, resolved the way Python's
-/// `huggingface_hub` resolves `constants.HF_HUB_CACHE`, so both languages look
-/// up and download into the same place: `HF_HUB_CACHE`, then
-/// `HUGGINGFACE_HUB_CACHE` (its legacy name), then `HF_HOME/hub`, then
-/// `XDG_CACHE_HOME/huggingface/hub`, then `~/.cache/huggingface/hub`.
-///
-/// It is exactly one directory. A snapshot in a lower-priority cache is not
-/// used when a higher one is configured, which is also what Python does: with
-/// `HF_HUB_CACHE` set, a model under `~/.cache` is neither found nor reused.
-///
-/// Both expansions happen as in `huggingface_hub.constants`:
-/// `expandvars(expanduser(...))`, first on `HF_HOME` (or its
-/// `XDG_CACHE_HOME` default) and then on the hub path itself. So
-/// `HF_HUB_CACHE=${ROOT}/hub` names the same directory in both languages, and
-/// a `~` that only appears once a variable is expanded stays literal, as it
-/// does in Python. The home directory comes from `std::env::home_dir`, which
-/// falls back to the user database when `HOME` is unset, as `expanduser`
-/// does. A variable that is set but empty still counts, as with `os.getenv`:
-/// `HF_HUB_CACHE=''` is the working directory itself, `HF_HOME=''` is `hub`
-/// and `XDG_CACHE_HOME=''` is `huggingface/hub` under it, and none of them
-/// falls through to a lower-priority cache. `tests/parity/fixtures/hf_hub_cache/`
-/// holds the cases both languages are checked against.
-#[cfg(feature = "ml")]
-fn hf_hub_cache_dir() -> Option<PathBuf> {
- let var = |name: &str| std::env::var_os(name).map(PathBuf::from);
- let home = std::env::home_dir();
- let expand = |path: PathBuf| expand_vars(&expand_user(path, home.as_deref()));
- let hub = match var("HF_HUB_CACHE").or_else(|| var("HUGGINGFACE_HUB_CACHE")) {
- Some(dir) => dir,
- None => {
- let hf_home = match var("HF_HOME") {
- Some(dir) => dir,
- None => var("XDG_CACHE_HOME")
- .or_else(|| home.as_ref().map(|home| home.join(".cache")))?
- .join("huggingface"),
- };
- expand(hf_home).join("hub")
- }
- };
- Some(expand(hub))
-}
-
-/// Python's `os.path.expanduser` for a leading bare `~` component. `~user`
-/// stays as written, which is also what `expanduser` does when it cannot
-/// resolve the user.
-#[cfg(feature = "ml")]
-fn expand_user(path: PathBuf, home: Option<&Path>) -> PathBuf {
- match (path.strip_prefix("~"), home) {
- (Ok(rest), Some(home)) => home.join(rest),
- _ => path,
- }
-}
-
-/// Python's `os.path.expandvars`: `posixpath` rules off Windows, `ntpath`
-/// rules on it. A path that is not valid Unicode is returned unchanged.
-#[cfg(feature = "ml")]
-fn expand_vars(path: &Path) -> PathBuf {
- let Some(text) = path.to_str() else {
- return path.to_path_buf();
- };
- #[cfg(windows)]
- let expanded = expand_vars_nt(text, env_lookup);
- #[cfg(not(windows))]
- let expanded = expand_vars_posix(text, env_lookup);
- PathBuf::from(expanded)
-}
-
-/// An environment value for `expand_vars`. Names the OS cannot hold (empty,
-/// or containing `=` or NUL) are unset, which is how Python's `os.environ`
-/// lookup fails for them too.
-#[cfg(feature = "ml")]
-fn env_lookup(name: &str) -> Option<std::ffi::OsString> {
- if name.is_empty() || name.contains(['=', '\0']) {
- return None;
- }
- std::env::var_os(name)
-}
-
-/// `posixpath.expandvars`: `$name` (ASCII letters, digits, `_`) or
-/// `${anything but a brace}`. An unset name, and a `$` that starts neither
-/// form, stay as written. Expanded values are not scanned again.
-#[cfg(feature = "ml")]
-#[cfg_attr(windows, allow(dead_code))] // tests run both rule sets everywhere
-fn expand_vars_posix(
- text: &str,
- lookup: impl Fn(&str) -> Option<std::ffi::OsString>,
-) -> std::ffi::OsString {
- let mut out = std::ffi::OsString::new();
- let mut rest = text;
- while let Some(at) = rest.find('$') {
- out.push(&rest[..at]);
- let after = &rest[at + 1..];
- let word = after
- .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_'))
- .unwrap_or(after.len());
- // (name, length of the reference after the `$`)
- let reference = if word > 0 {
- Some((&after[..word], word))
- } else {
- after
- .strip_prefix('{')
- .and_then(|braced| braced.find('}').map(|end| (&braced[..end], end + 2)))
- };
- match reference {
- Some((name, len)) => {
- match lookup(name) {
- Some(value) => out.push(value),
- None => out.push(&rest[at..at + 1 + len]),
- }
- rest = &after[len..];
- }
- None => {
- out.push("$");
- rest = after;
- }
- }
- }
- out.push(rest);
- out
-}
-
-/// `ntpath.expandvars`: `%name%`, `${name}` and `$name` (ASCII letters,
-/// digits, `_`, `-`); `%%` and `$$` are a literal `%` and `$`; nothing inside
-/// single quotes expands. An unset or unterminated reference stays as
-/// written.
-#[cfg(feature = "ml")]
-#[cfg_attr(not(windows), allow(dead_code))] // tests run both rule sets everywhere
-fn expand_vars_nt(
- text: &str,
- lookup: impl Fn(&str) -> Option<std::ffi::OsString>,
-) -> std::ffi::OsString {
- let mut out = std::ffi::OsString::new();
- let mut rest = text;
- while let Some(c) = rest.chars().next() {
- let after = &rest[c.len_utf8()..];
- match c {
- '\'' => match after.find('\'') {
- Some(end) => {
- out.push(&rest[..end + 2]);
- rest = &after[end + 1..];
- }
- None => {
- out.push(rest);
- rest = "";
- }
- },
- '%' if after.starts_with('%') => {
- out.push("%");
- rest = &after[1..];
- }
- '%' => match after.find('%') {
- Some(end) => {
- let name = &after[..end];
- match lookup(name) {
- Some(value) => out.push(value),
- None => out.push(&rest[..end + 2]),
- }
- rest = &after[end + 1..];
- }
- None => {
- out.push(rest);
- rest = "";
- }
- },
- '$' if after.starts_with('$') => {
- out.push("$");
- rest = &after[1..];
- }
- '$' if after.starts_with('{') => match after[1..].find('}') {
- Some(end) => {
- let name = &after[1..1 + end];
- match lookup(name) {
- Some(value) => out.push(value),
- None => out.push(&rest[..end + 3]),
- }
- rest = &after[end + 2..];
- }
- None => {
- out.push(rest);
- rest = "";
- }
- },
- '$' => {
- let end = after
- .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_' || c == '-'))
- .unwrap_or(after.len());
- let name = &after[..end];
- match lookup(name) {
- Some(value) => out.push(value),
- None => out.push(&rest[..end + 1]),
- }
- rest = &after[end..];
- }
- _ => {
- out.push(&rest[..c.len_utf8()]);
- rest = after;
- }
- }
- }
- out
+ crate::hf_cache::pinned_or_main(DEFAULT_MODEL_REVISION)
}
/// Find a snapshot of [`DEFAULT_MODEL_REPO`] at `revision` in the hub cache
/// `cache` that holds every file in [`DEFAULT_MODEL_FILES`]. Never touches the
-/// network.
-///
-/// Looks the snapshot up by directory rather than through `hf_hub::Cache`:
-/// that resolves every revision through `refs/<revision>`, which Python's
-/// `huggingface_hub` does not write for a commit SHA, so a snapshot Python
-/// downloaded would be missed. A branch name still goes through its ref.
+/// network; see [`crate::hf_cache::snapshot_path`] for how `revision`
+/// resolves.
#[cfg(feature = "ml")]
fn cached_snapshot(cache: &Path, revision: &str) -> Option<PathBuf> {
- let repo = cache.join(format!("models--{}", DEFAULT_MODEL_REPO.replace('/', "--")));
- let commit = std::fs::read_to_string(repo.join("refs").join(revision))
- .map(|c| c.trim().to_string())
- .unwrap_or_else(|_| revision.to_string());
- let snapshot = repo.join("snapshots").join(commit);
+ let snapshot = crate::hf_cache::snapshot_path(cache, DEFAULT_MODEL_REPO, revision);
// `is_file` follows the snapshot's symlinks, so blobs deleted from under
// it count as a miss rather than a load failure.
DEFAULT_MODEL_FILES
@@ -328,32 +114,30 @@ fn cached_snapshot(cache: &Path, revision: &str) -> Option<PathBuf> {
}
/// The Hub handle for [`DEFAULT_MODEL_REPO`] at `revision`, caching under
-/// `cache_root`. Honors `HF_ENDPOINT` like the Python side does.
+/// `cache_root`, with Python's endpoint and token ([`crate::hf_cache::hub_api`]).
#[cfg(feature = "ml")]
fn default_model_repo(
cache_root: &Path,
revision: &str,
) -> Result<hf_hub::api::sync::ApiRepo, String> {
- // Air-gap chokepoint, ahead of the client: building it resolves the Hub
- // endpoint and the `ureq` agent, and `hf-hub` downloads on a cache miss
- // even with `HF_HUB_OFFLINE` set. Callers reach this only on a cache
+ // `hub_api` is the air-gap chokepoint: under `HEADROOM_OFFLINE` it
+ // refuses before building the client. Callers reach this only on a cache
// miss, so a pre-seeded air-gapped host still loads.
//
// The refusal is soft by design: every caller degrades to the BM25
// scorer on `Err`, so an air-gapped box loses embedding relevance instead
// of failing a request. The message names the switch so the operator can
// tell the causes apart in the log.
- crate::offline::guard_egress("fastembed embedding model download", DEFAULT_MODEL_REPO)
- .map_err(|e| format!("EmbeddingScorer model load refused: {e}"))?;
- let mut builder =
- hf_hub::api::sync::ApiBuilder::from_cache(hf_hub::Cache::new(cache_root.to_path_buf()))
- .with_progress(false);
- if let Ok(endpoint) = std::env::var("HF_ENDPOINT") {
- builder = builder.with_endpoint(endpoint);
- }
- let api = builder
- .build()
- .map_err(|e| format!("EmbeddingScorer: HuggingFace client init failed: {e}"))?;
+ use crate::hf_cache::HubApiError;
+ let api = crate::hf_cache::hub_api(
+ cache_root,
+ "fastembed embedding model download",
+ DEFAULT_MODEL_REPO,
+ )
+ .map_err(|e| match e {
+ HubApiError::Offline(e) => format!("EmbeddingScorer model load refused: {e}"),
+ HubApiError::Build(e) => format!("EmbeddingScorer: HuggingFace client init failed: {e}"),
+ })?;
Ok(api.repo(hf_hub::Repo::with_revision(
DEFAULT_MODEL_REPO.to_string(),
hf_hub::RepoType::Model,
@@ -493,7 +277,7 @@ impl EmbeddingScorer {
/// offline, network failure during download, missing ONNX runtime
/// binaries, etc.).
pub fn try_new() -> Result<Self, String> {
- let cache = hf_hub_cache_dir().ok_or_else(|| {
+ let cache = crate::hf_cache::hub_cache_dir().ok_or_else(|| {
"EmbeddingScorer: no HuggingFace cache directory (set HF_HUB_CACHE or HF_HOME)"
.to_string()
})?;
@@ -729,6 +513,8 @@ fn cosine_similarity(a: &[f32], b: &[f32]) -> f64 {
#[cfg(test)]
mod tests {
use super::*;
+ #[cfg(feature = "ml")]
+ use crate::hf_cache::tests::CacheEnv;
// The real-model tests are gated behind RUN_FASTEMBED_TESTS=1
// since they require network access on first run (~67 MB model
@@ -843,209 +629,12 @@ mod tests {
assert_eq!(cached_snapshot(cache.path(), DEFAULT_MODEL_REVISION), None);
}
- /// Every variable `hf_hub_cache_dir` reads.
- #[cfg(feature = "ml")]
- const CACHE_ENV: [&str; 6] = [
- "HF_HUB_CACHE",
- "HUGGINGFACE_HUB_CACHE",
- "HF_HOME",
- "XDG_CACHE_HOME",
- "HOME",
- "USERPROFILE",
- ];
-
- /// Saves [`CACHE_ENV`], clears it, and puts it back on drop, so a failing
- /// assertion cannot leak a fake cache into later tests. Take
- /// `env_lock()` first.
- #[cfg(feature = "ml")]
- struct CacheEnv(Vec<(String, Option<std::ffi::OsString>)>);
-
- #[cfg(feature = "ml")]
- impl CacheEnv {
- fn cleared() -> Self {
- Self::cleared_with(std::iter::empty())
- }
-
- /// [`Self::cleared`], plus `extra` variables (the ones a test's
- /// paths reference).
- fn cleared_with(extra: impl IntoIterator<Item = String>) -> Self {
- let names: Vec<String> = CACHE_ENV
- .iter()
- .map(|name| name.to_string())
- .chain(extra)
- .collect();
- let saved = names
- .into_iter()
- .map(|name| {
- let value = std::env::var_os(&name);
- std::env::remove_var(&name);
- (name, value)
- })
- .collect();
- CacheEnv(saved)
- }
- }
-
- #[cfg(feature = "ml")]
- impl Drop for CacheEnv {
- fn drop(&mut self) {
- for (name, value) in &self.0 {
- match value {
- Some(value) => std::env::set_var(name, value),
- None => std::env::remove_var(name),
- }
- }
- }
- }
-
- /// Same order as `huggingface_hub.constants.HF_HUB_CACHE`: each variable
- /// wins over every one after it, and only one directory comes back.
- #[cfg(feature = "ml")]
- #[test]
- fn hf_hub_cache_dir_follows_huggingface_hub_precedence() {
- let _guard = crate::test_support::env_lock();
- let _env = CacheEnv::cleared();
- let t = tempfile::tempdir().unwrap();
- let p = |name: &str| t.path().join(name);
- std::env::set_var("HOME", p("home"));
- std::env::set_var("USERPROFILE", p("home"));
- std::env::set_var("XDG_CACHE_HOME", p("xdg"));
- std::env::set_var("HF_HOME", p("hf-home"));
- std::env::set_var("HUGGINGFACE_HUB_CACHE", p("legacy"));
- std::env::set_var("HF_HUB_CACHE", p("explicit"));
-
- assert_eq!(hf_hub_cache_dir(), Some(p("explicit")));
- std::env::remove_var("HF_HUB_CACHE");
- assert_eq!(hf_hub_cache_dir(), Some(p("legacy")));
- std::env::remove_var("HUGGINGFACE_HUB_CACHE");
- assert_eq!(hf_hub_cache_dir(), Some(p("hf-home").join("hub")));
- std::env::remove_var("HF_HOME");
- assert_eq!(
- hf_hub_cache_dir(),
- Some(p("xdg").join("huggingface").join("hub"))
- );
- std::env::remove_var("XDG_CACHE_HOME");
- assert_eq!(
- hf_hub_cache_dir(),
- Some(p("home").join(".cache").join("huggingface").join("hub"))
- );
-
- // A leading `~` expands.
- std::env::set_var("HF_HOME", "~/models");
- assert_eq!(
- hf_hub_cache_dir(),
- Some(p("home").join("models").join("hub"))
- );
-
- // A set but empty variable is the working directory, as in Python,
- // and still wins over every variable after it.
- std::env::set_var("HF_HUB_CACHE", "");
- assert_eq!(hf_hub_cache_dir(), Some(PathBuf::new()));
- std::env::remove_var("HF_HUB_CACHE");
- std::env::set_var("HF_HOME", "");
- std::env::set_var("XDG_CACHE_HOME", p("xdg"));
- assert_eq!(hf_hub_cache_dir(), Some(PathBuf::from("hub")));
- }
-
- /// Cases checked against Python itself (`posixpath`, `ntpath` and
- /// `huggingface_hub.constants`) by `tests/test_fastembed_pinned_snapshot.py`.
- #[cfg(feature = "ml")]
- const HF_CACHE_FIXTURE: &str = include_str!(concat!(
- env!("CARGO_MANIFEST_DIR"),
- "/../../tests/parity/fixtures/hf_hub_cache/cases.json"
- ));
-
- /// Both rule sets on every platform, so the Windows port is exercised by
- /// the macOS and Linux runs too.
- #[cfg(feature = "ml")]
- #[test]
- fn expand_vars_matches_python_posixpath_and_ntpath() {
- let fixture: serde_json::Value = serde_json::from_str(HF_CACHE_FIXTURE).unwrap();
- let section = &fixture["expandvars"];
- let env: std::collections::HashMap<String, String> =
- serde_json::from_value(section["env"].clone()).unwrap();
- let lookup = |name: &str| env.get(name).map(std::ffi::OsString::from);
- for case in section["cases"].as_array().unwrap() {
- let input = case["input"].as_str().unwrap();
- let want = |key: &str| std::ffi::OsString::from(case[key].as_str().unwrap());
- assert_eq!(
- expand_vars_posix(input, lookup),
- want("posix"),
- "posix {input:?}"
- );
- assert_eq!(expand_vars_nt(input, lookup), want("nt"), "nt {input:?}");
- }
- }
-
- /// The review case among them: `HF_HUB_CACHE=${ROOT}/hub` is
- /// `<ROOT>/hub` in Python, so Rust must not look up or download into a
- /// literal `${ROOT}` directory instead.
- #[cfg(feature = "ml")]
- #[test]
- fn hf_hub_cache_dir_matches_the_shared_fixture() {
- let fixture: serde_json::Value = serde_json::from_str(HF_CACHE_FIXTURE).unwrap();
- let section = &fixture["cache_dir"];
- let probe_vars: Vec<String> =
- serde_json::from_value(section["probe_vars"].clone()).unwrap();
- let _guard = crate::test_support::env_lock();
- let _env = CacheEnv::cleared_with(probe_vars.clone());
- let t = tempfile::tempdir().unwrap();
- let (root, home) = (t.path().join("root"), t.path().join("home"));
- let fill = |text: &str| {
- text.replace("{root}", root.to_str().unwrap())
- .replace("{home}", home.to_str().unwrap())
- };
- for case in section["cases"].as_array().unwrap() {
- for name in CACHE_ENV
- .iter()
- .copied()
- .chain(probe_vars.iter().map(String::as_str))
- {
- std::env::remove_var(name);
- }
- std::env::set_var("HOME", &home);
- std::env::set_var("USERPROFILE", &home);
- for (name, value) in case["env"].as_object().unwrap() {
- std::env::set_var(name, fill(value.as_str().unwrap()));
- }
- let expected = match &case["expected"] {
- serde_json::Value::Object(by_os) => {
- &by_os[if cfg!(windows) { "nt" } else { "posix" }]
- }
- expected => expected,
- };
- assert_eq!(
- hf_hub_cache_dir(),
- Some(PathBuf::from(fill(expected.as_str().unwrap()))),
- "{}",
- case["name"]
- );
- }
- }
-
- /// With no `HOME` (a service started with a bare environment) the home
- /// directory still comes from the user database, as Python's
- /// `expanduser` does, instead of leaving the scorer with no cache.
- #[cfg(all(feature = "ml", unix))]
- #[test]
- fn hf_hub_cache_dir_without_home_uses_the_user_database() {
- let _guard = crate::test_support::env_lock();
- let _env = CacheEnv::cleared();
- let dir = hf_hub_cache_dir().expect("a cache directory without HOME");
- assert!(dir.is_absolute(), "{}", dir.display());
- assert!(
- dir.ends_with(Path::new(".cache").join("huggingface").join("hub")),
- "{}",
- dir.display()
- );
- }
-
/// The case from review: an explicit, empty `HF_HUB_CACHE` next to a
/// complete snapshot in the default cache under `HOME`. Python looks only
/// in the explicit cache and downloads there, so Rust must not find the
/// `HOME` copy, for the pinned revision or for floating `main`
/// (`HEADROOM_HF_PIN=off`). `try_new` is `try_new_pinned` on
- /// `hf_hub_cache_dir()`, which looks up and downloads only in the
+ /// `hub_cache_dir()`, which looks up and downloads only in the
/// directory it is given (see the two offline tests above), so checking
/// the resolver and the lookup here covers it without flipping
/// `HEADROOM_OFFLINE` for a long stretch of the parallel run.
@@ -1068,14 +657,14 @@ mod tests {
std::env::set_var("USERPROFILE", home.path());
std::env::set_var("HF_HUB_CACHE", explicit.path());
- let cache = hf_hub_cache_dir().expect("cache directory");
+ let cache = crate::hf_cache::hub_cache_dir().expect("cache directory");
assert_eq!(cache, explicit.path());
assert_eq!(cached_snapshot(&cache, DEFAULT_MODEL_REVISION), None);
assert_eq!(cached_snapshot(&cache, "main"), None);
// Without the explicit cache, the same HOME snapshots are the cache.
std::env::remove_var("HF_HUB_CACHE");
- let cache = hf_hub_cache_dir().expect("cache directory");
+ let cache = crate::hf_cache::hub_cache_dir().expect("cache directory");
assert_eq!(cache, default_cache);
assert_eq!(
cached_snapshot(&cache, DEFAULT_MODEL_REVISION),
diff --git a/crates/headroom-core/src/tokenizer/hf_impl.rs b/crates/headroom-core/src/tokenizer/hf_impl.rs
index f120c76ac..efb58c5d9 100644
--- a/crates/headroom-core/src/tokenizer/hf_impl.rs
+++ b/crates/headroom-core/src/tokenizer/hf_impl.rs
@@ -12,11 +12,12 @@
//! - [`HfTokenizer::from_bytes`] — for tokenizers embedded via `include_bytes!`.
//! - [`HfTokenizer::from_file`] — for a local `tokenizer.json`.
//! - [`HfTokenizer::from_pretrained`] — pulls `tokenizer.json` from the
-//! HuggingFace Hub via `hf-hub`. First call downloads to
-//! `~/.cache/huggingface/hub`, subsequent calls hit the cache. Blocking;
+//! HuggingFace Hub via `hf-hub`. First call downloads into the hub cache
+//! Python's `huggingface_hub` uses (`HF_HUB_CACHE`, `HF_HOME/hub`,
+//! `~/.cache/huggingface/hub`, …), subsequent calls hit it. Blocking;
//! call from `main()` or `tokio::task::spawn_blocking`. Gated repos
-//! (Llama, Mistral) require an `HF_TOKEN` env var or a token in
-//! `~/.cache/huggingface/token`.
+//! (Llama, Mistral) need a token: `HF_TOKEN`, or the file
+//! `hf auth login` writes (`HF_HOME/token`, or `HF_TOKEN_PATH`).
//!
//! # What's NOT here
//! - **No tokenizer.json bundled in the binary.** Bundling Llama / Cohere
@@ -112,9 +113,13 @@ impl HfTokenizer {
/// e.g. `"CohereForAI/c4ai-command-r-v01"` or `"meta-llama/Meta-Llama-3-8B"`.
///
/// Uses the `main` revision. Blocking — calls into `ureq` synchronously.
- /// First successful call writes the file to `~/.cache/huggingface/hub`
- /// (or `$HF_HOME` if set); subsequent calls in the same or later
- /// processes hit the on-disk cache.
+ /// The file lands in the hub cache directory Python's `huggingface_hub`
+ /// would use ([`crate::hf_cache::hub_cache_dir`]: `HF_HUB_CACHE`, then
+ /// `HUGGINGFACE_HUB_CACHE`, `HF_HOME/hub`, `XDG_CACHE_HOME/huggingface/hub`,
+ /// `~/.cache/huggingface/hub`), so later calls, and the Python side, hit
+ /// the same copy. The token and `HF_ENDPOINT` are read as Python reads
+ /// them too ([`crate::hf_cache::hf_token`]); `hf-hub`'s own defaults
+ /// ignore every variable but `HF_HOME`, and with `Api::new` even that.
///
/// Errors:
/// - [`HfTokenizerError::Offline`] when `HEADROOM_OFFLINE` is set. Refused
@@ -129,11 +134,11 @@ impl HfTokenizer {
// HEADROOM_OFFLINE=1 gets the same refusal whichever runtime reaches
// for the Hub.
//
- // It has to sit here rather than around the `get` below: `Api::new`
- // builds the `ureq` agent and resolves the endpoint, and hf-hub's own
+ // `crate::hf_cache::hub_api` runs the guard before it builds the
+ // client, rather than around the `get` below: building it creates the
+ // `ureq` agent and resolves the endpoint, and hf-hub's own
// HF_HUB_OFFLINE handling only covers the cache lookup — a cache miss
- // still dials out. Guarding first means the air-gapped case never
- // constructs a client at all.
+ // still dials out. So the air-gapped case never constructs a client.
//
// The refusal is hard on purpose. Silently returning a cached or
// estimating tokenizer would leave token counts subtly wrong with no
@@ -145,11 +150,7 @@ impl HfTokenizer {
// client exists" and "serve from the cache" cannot both hold here. An
// air-gapped deployment with pre-seeded artifacts should point at them
// with `from_file`, which is network-free by construction.
- crate::offline::guard_egress("HuggingFace tokenizer download", repo)?;
- let api = hf_hub::api::sync::Api::new().map_err(|e| HfTokenizerError::Hub {
- repo: repo.to_string(),
- source: Box::new(e),
- })?;
+ let api = hub_api(repo)?;
let path = api
.model(repo.to_string())
.get("tokenizer.json")
@@ -169,6 +170,31 @@ impl HfTokenizer {
}
}
+/// The Hub client for [`HfTokenizer::from_pretrained`] fetching `repo`:
+/// caching in Python's hub cache directory, with Python's token and endpoint.
+/// Refuses with [`HfTokenizerError::Offline`] under `HEADROOM_OFFLINE`.
+fn hub_api(repo: &str) -> Result<hf_hub::api::sync::Api, HfTokenizerError> {
+ use crate::hf_cache::HubApiError;
+ const PURPOSE: &str = "HuggingFace tokenizer download";
+
+ let hub_error = |source: Box<dyn std::error::Error + Send + Sync>| HfTokenizerError::Hub {
+ repo: repo.to_string(),
+ source,
+ };
+ let Some(cache) = crate::hf_cache::hub_cache_dir() else {
+ // Offline still wins: the caller treats `Offline` as final, and a
+ // missing cache directory is no reason to report a Hub failure.
+ crate::offline::guard_egress(PURPOSE, repo)?;
+ return Err(hub_error(
+ "no HuggingFace cache directory (set HF_HUB_CACHE or HF_HOME)".into(),
+ ));
+ };
+ crate::hf_cache::hub_api(&cache, PURPOSE, repo).map_err(|e| match e {
+ HubApiError::Offline(blocked) => HfTokenizerError::Offline(blocked),
+ HubApiError::Build(source) => hub_error(Box::new(source)),
+ })
+}
+
impl Tokenizer for HfTokenizer {
fn count_text(&self, text: &str) -> usize {
if text.is_empty() {
@@ -297,7 +323,7 @@ mod tests {
/// this a real assertion rather than a tautology: without the guard the call
/// reaches `hf-hub`, dials `huggingface.co`, gets a 401/404 and returns
/// `HfTokenizerError::Hub`. Only the guard can produce `Offline`, and only
- /// by returning before `Api::new` — so `Offline` *is* the "no socket was
+ /// by returning before `hf_cache::hub_api` builds the client — so `Offline` *is* the "no socket was
/// opened" assertion for this path. (A packet-level assertion is not
/// available to a `cargo test` unit test the way `socket.socket` patching is
/// in Python; this typed-error distinction is the equivalent signal.)
@@ -349,6 +375,119 @@ mod tests {
let _ = std::fs::remove_dir_all(&dir);
}
+ /// Lay out `tokenizer.json` for `repo` in hub cache `cache`, the way
+ /// Python's `huggingface_hub` leaves a `main` download: a snapshot plus
+ /// `refs/main` naming it.
+ fn cache_main_tokenizer(cache: &std::path::Path, repo: &str) {
+ let repo_dir = cache.join(format!("models--{}", repo.replace('/', "--")));
+ let snapshot = repo_dir.join("snapshots").join("abc123");
+ std::fs::create_dir_all(&snapshot).unwrap();
+ std::fs::write(snapshot.join("tokenizer.json"), TINY_TOKENIZER_JSON).unwrap();
+ std::fs::create_dir_all(repo_dir.join("refs")).unwrap();
+ std::fs::write(repo_dir.join("refs").join("main"), "abc123").unwrap();
+ }
+
+ /// Variables `from_pretrained` reads, saved and restored around a test.
+ fn hub_env() -> crate::hf_cache::tests::CacheEnv {
+ crate::hf_cache::tests::CacheEnv::cleared_with(
+ [
+ "HF_TOKEN",
+ "HUGGING_FACE_HUB_TOKEN",
+ "HF_TOKEN_PATH",
+ "HF_ENDPOINT",
+ "HF_TEST_ROOT",
+ ]
+ .map(String::from),
+ )
+ }
+
+ /// The tokenizer is read from (and would be downloaded into) the hub cache
+ /// Python uses, here an `HF_HUB_CACHE` given through `${VAR}`. Before, it
+ /// was always `~/.cache/huggingface/hub`, so this was a cache miss and a
+ /// download. `HF_ENDPOINT` points at a closed loopback port, so a miss
+ /// fails fast instead of reaching the internet.
+ #[test]
+ fn from_pretrained_reads_the_hub_cache_python_uses() {
+ let _guard = crate::test_support::env_lock();
+ let _env = hub_env();
+ let t = tempfile::tempdir().unwrap();
+ std::env::set_var("HOME", t.path().join("home"));
+ std::env::set_var("USERPROFILE", t.path().join("home"));
+ std::env::set_var("HF_TEST_ROOT", t.path());
+ std::env::set_var("HF_HUB_CACHE", "${HF_TEST_ROOT}/hub");
+ std::env::set_var("HF_ENDPOINT", "http://127.0.0.1:9");
+ cache_main_tokenizer(&t.path().join("hub"), "acme/tiny");
+
+ let tokenizer = HfTokenizer::from_pretrained("acme/tiny").expect("served from the cache");
+ assert_eq!(tokenizer.count_text("hello world"), 2);
+ }
+
+ /// A cache miss asks the Hub at `HF_ENDPOINT` with the token from
+ /// `HF_TOKEN`. `hf-hub`'s `Api::new` read neither, so a gated repo
+ /// (Llama, Mistral) failed with 401 even with `HF_TOKEN` set. A loopback
+ /// stand-in for the Hub records the request and answers 404.
+ #[test]
+ fn from_pretrained_sends_hf_token_to_hf_endpoint() {
+ use std::io::{Read, Write};
+ let _guard = crate::test_support::env_lock();
+ let _env = hub_env();
+ let t = tempfile::tempdir().unwrap();
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let port = listener.local_addr().unwrap().port();
+ // Waits at most 10 s for the request, so a regression that never
+ // reaches the endpoint fails instead of hanging the suite.
+ listener.set_nonblocking(true).unwrap();
+ let hub = std::thread::spawn(move || {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
+ let mut conn = loop {
+ match listener.accept() {
+ Ok((conn, _)) => break conn,
+ Err(_) if std::time::Instant::now() < deadline => {
+ std::thread::sleep(std::time::Duration::from_millis(20))
+ }
+ Err(_) => return String::new(),
+ }
+ };
+ conn.set_nonblocking(false).unwrap();
+ let mut request = Vec::new();
+ let mut buf = [0u8; 1024];
+ while !request.windows(4).any(|w| w == b"\r\n\r\n") {
+ let n = conn.read(&mut buf).unwrap();
+ if n == 0 {
+ break;
+ }
+ request.extend_from_slice(&buf[..n]);
+ }
+ conn.write_all(
+ b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
+ )
+ .unwrap();
+ String::from_utf8_lossy(&request).to_ascii_lowercase()
+ });
+ std::env::set_var("HOME", t.path().join("home"));
+ std::env::set_var("USERPROFILE", t.path().join("home"));
+ std::env::set_var("HF_HUB_CACHE", t.path().join("hub"));
+ // Trailing slash on purpose: Python strips it, and a kept one
+ // would request `//acme/...`.
+ std::env::set_var("HF_ENDPOINT", format!("http://127.0.0.1:{port}/"));
+ std::env::set_var("HF_TOKEN", "hf_test_token");
+
+ let result = HfTokenizer::from_pretrained("acme/gated");
+ let request = hub.join().unwrap();
+ assert!(
+ matches!(result, Err(HfTokenizerError::Hub { .. })),
+ "{result:?}"
+ );
+ assert!(
+ request.contains(" /acme/gated/resolve/main/tokenizer.json "),
+ "{request}"
+ );
+ assert!(
+ request.contains("authorization: bearer hf_test_token"),
+ "{request}"
+ );
+ }
+
/// Network-dependent: hits HuggingFace Hub. Run with
/// `cargo test -p headroom-core -- --ignored from_pretrained_downloads_real_tokenizer`.
/// `gpt2` is a small public unauthenticated repo (~1.4 MB tokenizer.json).
diff --git a/crates/headroom-core/src/transforms/kompress.rs b/crates/headroom-core/src/transforms/kompress.rs
index 06c51b075..17bb5aad8 100644
--- a/crates/headroom-core/src/transforms/kompress.rs
+++ b/crates/headroom-core/src/transforms/kompress.rs
@@ -74,6 +74,12 @@ use tokenizers::{EncodeInput, InputSequence, Tokenizer};
pub const DEFAULT_MODEL_ID: &str = "chopratejas/kompress-v2-base";
/// HuggingFace repo holding the tokenizer Kompress reuses.
pub const DEFAULT_TOKENIZER_REPO: &str = "answerdotai/ModernBERT-base";
+/// Pinned commits of [`DEFAULT_MODEL_ID`] and [`DEFAULT_TOKENIZER_REPO`], the
+/// snapshots the cache-only loader reads. Keep each equal to its entry in
+/// `_PINNED_REVISIONS` (`headroom/onnx_runtime.py`), which is what the Python
+/// side downloads; `tests/test_kompress_rust_pins.py` enforces it.
+pub const DEFAULT_MODEL_REVISION: &str = "b1563631b35bfdcee37587ad530147497d820d4c";
+pub const DEFAULT_TOKENIZER_REVISION: &str = "8949b909ec900327062f0ebf497f51aef5e6f0c8";
/// Words per inference chunk. Coupled to the model's training window;
/// kompress-v2-base was trained for 350.
pub const DEFAULT_CHUNK_WORDS: usize = 350;
@@ -261,8 +267,8 @@ impl Kompress {
/// downloading on miss). Blocking — call off the hot path. Tries the
/// [`ONNX_CANDIDATES`] in order.
pub fn from_pretrained(config: KompressConfig) -> Result<Self, KompressError> {
- // Air-gap chokepoint — the same one `hf_impl.rs` uses for the
- // tokenizer, for the same reason. `HF_HUB_OFFLINE` only shortcuts the
+ // Air-gap chokepoint — the same guard `hf_cache::hub_api` runs for
+ // the tokenizer and embedder, for the same reason. `HF_HUB_OFFLINE` only shortcuts the
// cache *lookup*; on a cache miss `hf-hub` still dials huggingface.co,
// so the Kompress model download is a live egress path under
// `HEADROOM_OFFLINE=1` and was not covered.
@@ -333,20 +339,23 @@ impl Kompress {
pub fn from_cache(config: KompressConfig) -> Result<Option<Self>, KompressError> {
let Some(tok_path) = hf_cache_file(&config.tokenizer_repo, &["tokenizer.json"]) else {
// Diagnostic: a `None` here is the #1 cause of a silent
- // `kompress_ready=false`. Name the repo + the roots searched so
- // operators don't have to guess between "not downloaded" and
+ // `kompress_ready=false`. Name the repo, the cache and the revision
+ // looked in so operators don't have to guess between "not
+ // downloaded" and
// "present but unreadable" (e.g. HF symlinks over `\\wsl$`, which
- // native Windows can't follow — `path.exists()` returns false on
+ // native Windows can't follow — `is_file()` returns false on
// the unresolved symlink). Cache-only, so this is a defer, not an
// error: the caller passes plain text through.
tracing::warn!(
event = "kompress_cache_miss",
stage = "tokenizer",
tokenizer_repo = %config.tokenizer_repo,
- searched_roots = ?hf_hub_roots(),
- "Kompress deferred: tokenizer.json not found in HF cache \
- (not downloaded, or present but unreadable — e.g. HF symlinks \
- over \\\\wsl$ which native Windows cannot follow)"
+ revision = %cache_revision(&config.tokenizer_repo),
+ hub_cache = ?crate::hf_cache::hub_cache_dir(),
+ "Kompress deferred: tokenizer.json not found in the HF cache at the \
+ pinned revision (not downloaded yet — the Python side fetches the \
+ pin on its next Kompress load — or present but unreadable, e.g. HF \
+ symlinks over \\\\wsl$ which native Windows cannot follow)"
);
return Ok(None);
};
@@ -384,7 +393,8 @@ impl Kompress {
model_id = %config.model_id,
candidates = ?ONNX_CANDIDATES,
any_onnx_found = found_onnx,
- searched_roots = ?hf_hub_roots(),
+ revision = %cache_revision(&config.model_id),
+ hub_cache = ?crate::hf_cache::hub_cache_dir(),
"Kompress deferred: no usable ONNX session \
(no candidate file in cache, or every candidate failed to build)"
);
@@ -603,59 +613,221 @@ fn build_session(path: &Path) -> Result<Session, Box<dyn std::error::Error + Sen
Ok(session)
}
-/// Resolve `rel` (e.g. `["tokenizer.json"]` or `["onnx", "kompress-int8-wo.onnx"]`)
-/// inside the local HuggingFace cache for `repo` (`"owner/name"`), searching
-/// every snapshot under every candidate cache root. Returns `None` if not
-/// present — never touches the network.
-fn hf_cache_file(repo: &str, rel: &[&str]) -> Option<PathBuf> {
- let repo_dir = format!("models--{}", repo.replace('/', "--"));
- for hub in hf_hub_roots() {
- let snapshots = hub.join(&repo_dir).join("snapshots");
- let Ok(entries) = std::fs::read_dir(&snapshots) else {
- continue;
- };
- for snap in entries.flatten() {
- let mut cand = snap.path();
- for part in rel {
- cand = cand.join(part);
- }
- if cand.exists() {
- return Some(cand);
- }
- }
+/// The tokenizer and the ONNX artifact `onnx` (e.g.
+/// `"onnx/kompress-int8-wo.onnx"`) that the cache-only loader reads for
+/// `config`: the pinned snapshots in Python's hub cache, as
+/// [`Kompress::from_cache`] finds them. `None` when either is missing. For
+/// parity harnesses that load one specific artifact through
+/// [`Kompress::from_files`], so they test the files production would load.
+pub fn cached_artifacts(config: &KompressConfig, onnx: &str) -> Option<(PathBuf, PathBuf)> {
+ let tokenizer = hf_cache_file(&config.tokenizer_repo, &["tokenizer.json"])?;
+ let rel: Vec<&str> = onnx.split('/').collect();
+ let model = hf_cache_file(&config.model_id, &rel)?;
+ Some((tokenizer, model))
+}
+
+/// The revision of `repo` to load from the cache: its pin when it is one of
+/// Kompress's default repos (see [`DEFAULT_MODEL_REVISION`]), otherwise
+/// `main`, which is also what Python's `_resolve_revision` gives an unpinned
+/// repo. `HEADROOM_HF_PIN=off` floats the pinned ones on `main` too.
+fn cache_revision(repo: &str) -> &str {
+ match repo {
+ DEFAULT_MODEL_ID => crate::hf_cache::pinned_or_main(DEFAULT_MODEL_REVISION),
+ DEFAULT_TOKENIZER_REPO => crate::hf_cache::pinned_or_main(DEFAULT_TOKENIZER_REVISION),
+ _ => "main",
}
- None
}
-/// HuggingFace hub cache roots in resolution precedence. Cross-platform so
-/// the cache-only loader works on Windows (native `headroom-proxy.exe`) as
-/// well as Linux: `HF_HUB_CACHE` (the hub dir directly) → `HF_HOME/hub` →
-/// `{HOME|USERPROFILE}/.cache/huggingface/hub`. `HOME` is the unix home; on
-/// Windows the process sees `USERPROFILE` (and often no `HOME`), so both are
-/// tried. Honoring `HF_HOME` also lets a Windows proxy point at a WSL cache.
-fn hf_hub_roots() -> Vec<PathBuf> {
- let mut roots = Vec::new();
- let push_env = |roots: &mut Vec<PathBuf>, var: &str, suffix: &[&str]| {
- if let Ok(v) = std::env::var(var) {
- if !v.is_empty() {
- let mut p = PathBuf::from(v);
- for s in suffix {
- p = p.join(s);
- }
- roots.push(p);
- }
- }
- };
- push_env(&mut roots, "HF_HUB_CACHE", &[]);
- push_env(&mut roots, "HF_HOME", &["hub"]);
- push_env(&mut roots, "HOME", &[".cache", "huggingface", "hub"]);
- push_env(&mut roots, "USERPROFILE", &[".cache", "huggingface", "hub"]);
- roots
+/// Resolve `rel` (e.g. `["tokenizer.json"]` or `["onnx", "kompress-int8-wo.onnx"]`)
+/// in `repo`'s snapshot at [`cache_revision`], inside the one hub cache
+/// directory Python's `huggingface_hub` would use
+/// ([`crate::hf_cache::hub_cache_dir`]). Returns `None` if not present —
+/// never touches the network.
+///
+/// One snapshot, not the first file found in any: with several revisions in
+/// the cache (every pin bump leaves the old one behind), which one a
+/// directory scan returned depended on filesystem order, and the tokenizer
+/// and the ONNX file could come from different snapshots. And one cache, not
+/// every root that exists: Python does not fall back to `~/.cache` when
+/// `HF_HUB_CACHE` names another directory, so neither does this. `is_file`
+/// follows the cache's symlinks, so a blob missing under a snapshot (or a
+/// symlink native Windows cannot follow over `\\wsl$`) is a miss.
+fn hf_cache_file(repo: &str, rel: &[&str]) -> Option<PathBuf> {
+ let cache = crate::hf_cache::hub_cache_dir()?;
+ let mut path = crate::hf_cache::snapshot_path(&cache, repo, cache_revision(repo));
+ for part in rel {
+ path.push(part);
+ }
+ path.is_file().then_some(path)
}
#[cfg(test)]
mod tests {
use super::*;
+ use crate::hf_cache::tests::CacheEnv;
+
+ /// Lay out `rel` under `repo`'s snapshot `commit` in the hub cache `cache`.
+ fn cached_file(cache: &Path, repo: &str, commit: &str, rel: &str) -> PathBuf {
+ let path = cache
+ .join(format!("models--{}", repo.replace('/', "--")))
+ .join("snapshots")
+ .join(commit)
+ .join(rel);
+ std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+ std::fs::write(&path, commit).unwrap();
+ path
+ }
+
+ /// The cache after a pin bump: the pinned snapshot sits between an older
+ /// one and a newer `main`, and both other snapshots hold the same files.
+ fn cache_with_three_snapshots(cache: &Path) {
+ for repo in [DEFAULT_MODEL_ID, DEFAULT_TOKENIZER_REPO] {
+ let pin = cache_revision(repo).to_string();
+ for commit in [
+ "0000000000000000000000000000000000000000",
+ &pin,
+ "ffffffffffffffffffffffffffffffffffffffff",
+ ] {
+ cached_file(cache, repo, commit, "tokenizer.json");
+ cached_file(cache, repo, commit, ONNX_CANDIDATES[1]);
+ }
+ let refs = cache
+ .join(format!("models--{}", repo.replace('/', "--")))
+ .join("refs");
+ std::fs::create_dir_all(&refs).unwrap();
+ std::fs::write(
+ refs.join("main"),
+ "ffffffffffffffffffffffffffffffffffffffff",
+ )
+ .unwrap();
+ }
+ }
+
+ /// Python loads the pinned snapshot of both repos; so must the cache-only
+ /// loader, whatever order the filesystem lists the snapshots in.
+ #[test]
+ fn the_cache_lookup_reads_the_pinned_snapshot() {
+ let _guard = crate::test_support::env_lock();
+ let _env = CacheEnv::cleared_with(["HEADROOM_HF_PIN".to_string()]);
+ let cache = tempfile::tempdir().unwrap();
+ std::env::set_var("HF_HUB_CACHE", cache.path());
+ cache_with_three_snapshots(cache.path());
+
+ let tokenizer = hf_cache_file(DEFAULT_TOKENIZER_REPO, &["tokenizer.json"]).unwrap();
+ assert!(tokenizer.ends_with(Path::new(DEFAULT_TOKENIZER_REVISION).join("tokenizer.json")));
+ let rel: Vec<&str> = ONNX_CANDIDATES[1].split('/').collect();
+ let onnx = hf_cache_file(DEFAULT_MODEL_ID, &rel).unwrap();
+ assert!(onnx.ends_with(Path::new(DEFAULT_MODEL_REVISION).join(ONNX_CANDIDATES[1])));
+
+ // A candidate the pinned snapshot lacks is a miss, even when another
+ // snapshot has it.
+ cached_file(
+ cache.path(),
+ DEFAULT_MODEL_ID,
+ "ffffffffffffffffffffffffffffffffffffffff",
+ ONNX_CANDIDATES[2],
+ );
+ let rel: Vec<&str> = ONNX_CANDIDATES[2].split('/').collect();
+ assert_eq!(hf_cache_file(DEFAULT_MODEL_ID, &rel), None);
+ }
+
+ /// `HEADROOM_HF_PIN=off` floats on `main` through `refs/main`, and a repo
+ /// Kompress does not pin (a domain model) always does, as in Python.
+ #[test]
+ fn unpinned_lookups_follow_refs_main() {
+ let _guard = crate::test_support::env_lock();
+ let _env = CacheEnv::cleared_with(["HEADROOM_HF_PIN".to_string()]);
+ let cache = tempfile::tempdir().unwrap();
+ std::env::set_var("HF_HUB_CACHE", cache.path());
+ cache_with_three_snapshots(cache.path());
+ let newest = Path::new("ffffffffffffffffffffffffffffffffffffffff").join("tokenizer.json");
+
+ std::env::set_var("HEADROOM_HF_PIN", "off");
+ let tokenizer = hf_cache_file(DEFAULT_TOKENIZER_REPO, &["tokenizer.json"]).unwrap();
+ assert!(tokenizer.ends_with(&newest));
+
+ std::env::remove_var("HEADROOM_HF_PIN");
+ let domain = "acme/kompress-legal";
+ cached_file(
+ cache.path(),
+ domain,
+ "ffffffffffffffffffffffffffffffffffffffff",
+ "tokenizer.json",
+ );
+ let refs = cache
+ .path()
+ .join("models--acme--kompress-legal")
+ .join("refs");
+ std::fs::create_dir_all(&refs).unwrap();
+ std::fs::write(
+ refs.join("main"),
+ "ffffffffffffffffffffffffffffffffffffffff",
+ )
+ .unwrap();
+ assert!(hf_cache_file(domain, &["tokenizer.json"])
+ .unwrap()
+ .ends_with(&newest));
+ }
+
+ /// The Kompress lookup now names the directory Python would, including
+ /// the cases the old root list missed: the legacy `HUGGINGFACE_HUB_CACHE`,
+ /// `XDG_CACHE_HOME` and a `${VAR}` reference. And it reads only that one
+ /// directory: a copy under `HOME` is not used when `HF_HUB_CACHE` points
+ /// elsewhere.
+ #[test]
+ fn the_cache_lookup_uses_the_one_directory_python_uses() {
+ let _guard = crate::test_support::env_lock();
+ let _env = CacheEnv::cleared_with([
+ "HEADROOM_HF_PIN".to_string(),
+ "KOMPRESS_TEST_ROOT".to_string(),
+ ]);
+ let t = tempfile::tempdir().unwrap();
+ let pin = cache_revision(DEFAULT_TOKENIZER_REPO).to_string();
+ let lookup = || hf_cache_file(DEFAULT_TOKENIZER_REPO, &["tokenizer.json"]);
+ std::env::set_var("HOME", t.path().join("home"));
+ std::env::set_var("USERPROFILE", t.path().join("home"));
+
+ let legacy = t.path().join("legacy");
+ let want = cached_file(&legacy, DEFAULT_TOKENIZER_REPO, &pin, "tokenizer.json");
+ std::env::set_var("HUGGINGFACE_HUB_CACHE", &legacy);
+ assert_eq!(lookup(), Some(want));
+ std::env::remove_var("HUGGINGFACE_HUB_CACHE");
+
+ let xdg = t.path().join("xdg");
+ let want = cached_file(
+ &xdg.join("huggingface").join("hub"),
+ DEFAULT_TOKENIZER_REPO,
+ &pin,
+ "tokenizer.json",
+ );
+ std::env::set_var("XDG_CACHE_HOME", &xdg);
+ assert_eq!(lookup(), Some(want));
+ std::env::remove_var("XDG_CACHE_HOME");
+
+ let root = t.path().join("root");
+ let want = cached_file(
+ &root.join("hub"),
+ DEFAULT_TOKENIZER_REPO,
+ &pin,
+ "tokenizer.json",
+ );
+ std::env::set_var("KOMPRESS_TEST_ROOT", &root);
+ std::env::set_var("HF_HUB_CACHE", "${KOMPRESS_TEST_ROOT}/hub");
+ assert_eq!(lookup(), Some(want));
+
+ cached_file(
+ &t.path()
+ .join("home")
+ .join(".cache")
+ .join("huggingface")
+ .join("hub"),
+ DEFAULT_TOKENIZER_REPO,
+ &pin,
+ "tokenizer.json",
+ );
+ std::env::set_var("HF_HUB_CACHE", t.path().join("empty"));
+ assert_eq!(lookup(), None);
+ }
/// `from_pretrained` must refuse before `Api::new()` when the operator
/// air-gapped the box.
diff --git a/crates/headroom-core/tests/kompress_parity.rs b/crates/headroom-core/tests/kompress_parity.rs
index 942fa8c11..f98bcb51c 100644
--- a/crates/headroom-core/tests/kompress_parity.rs
+++ b/crates/headroom-core/tests/kompress_parity.rs
@@ -14,37 +14,20 @@
use std::fs;
use std::path::{Path, PathBuf};
-use headroom_core::transforms::kompress::{Kompress, KompressConfig};
+use headroom_core::transforms::kompress::{self, Kompress, KompressConfig};
use serde_json::Value;
-fn hf_cache_file(repo_dir: &str, rel: &[&str]) -> Option<PathBuf> {
- let home = std::env::var("HOME").ok()?;
- let snapshots = Path::new(&home)
- .join(".cache/huggingface/hub")
- .join(repo_dir)
- .join("snapshots");
- for snap in fs::read_dir(snapshots).ok()?.filter_map(|e| e.ok()) {
- let mut cand = snap.path();
- for part in rel {
- cand = cand.join(part);
- }
- if cand.exists() {
- return Some(cand);
- }
- }
- None
+/// The tokenizer + int8 ONNX artifact production would load: the pinned
+/// snapshots in Python's hub cache. The fixtures were recorded against them.
+fn cached_artifacts() -> Option<(PathBuf, PathBuf)> {
+ kompress::cached_artifacts(&KompressConfig::default(), "onnx/kompress-int8-wo.onnx")
}
#[test]
fn kompress_matches_python_fixtures_byte_for_byte() {
- let tok = hf_cache_file("models--answerdotai--ModernBERT-base", &["tokenizer.json"]);
- let onnx = hf_cache_file(
- "models--chopratejas--kompress-v2-base",
- &["onnx", "kompress-int8-wo.onnx"],
- );
- let (tok, onnx) = match (tok, onnx) {
- (Some(t), Some(o)) => (t, o),
- _ => {
+ let (tok, onnx) = match cached_artifacts() {
+ Some(paths) => paths,
+ None => {
eprintln!(
"SKIP: kompress model/tokenizer not in HF cache; \
run `python scripts/record_kompress_trace.py` first"
@@ -112,12 +95,7 @@ fn short_input_passes_through() {
// Pure-logic check — no model needed. Fewer than MIN_WORDS words must
// pass through unchanged regardless of model availability... but the
// engine needs a model to construct. Guard on cache like the main test.
- let tok = hf_cache_file("models--answerdotai--ModernBERT-base", &["tokenizer.json"]);
- let onnx = hf_cache_file(
- "models--chopratejas--kompress-v2-base",
- &["onnx", "kompress-int8-wo.onnx"],
- );
- let (Some(tok), Some(onnx)) = (tok, onnx) else {
+ let Some((tok, onnx)) = cached_artifacts() else {
eprintln!("SKIP: model not cached");
return;
};
diff --git a/crates/headroom-parity/src/lib.rs b/crates/headroom-parity/src/lib.rs
index adb5559cc..b78c61d30 100644
--- a/crates/headroom-parity/src/lib.rs
+++ b/crates/headroom-parity/src/lib.rs
@@ -648,37 +648,16 @@ impl KompressComparator {
Self::default()
}
- fn hf_cache_file(repo_dir: &str, rel: &[&str]) -> Option<PathBuf> {
- let home = std::env::var("HOME").ok()?;
- let snapshots = Path::new(&home)
- .join(".cache/huggingface/hub")
- .join(repo_dir)
- .join("snapshots");
- for snap in fs::read_dir(snapshots).ok()?.filter_map(|e| e.ok()) {
- let mut cand = snap.path();
- for part in rel {
- cand = cand.join(part);
- }
- if cand.exists() {
- return Some(cand);
- }
- }
- None
- }
-
fn model(&self) -> Option<&headroom_core::transforms::kompress::Kompress> {
self.model
.get_or_init(|| {
- use headroom_core::transforms::kompress::{Kompress, KompressConfig};
- let tok = Self::hf_cache_file(
- "models--answerdotai--ModernBERT-base",
- &["tokenizer.json"],
- )?;
- let onnx = Self::hf_cache_file(
- "models--chopratejas--kompress-v2-base",
- &["onnx", "kompress-int8-wo.onnx"],
- )?;
- Kompress::from_files(&tok, &onnx, KompressConfig::default()).ok()
+ use headroom_core::transforms::kompress::{self, Kompress, KompressConfig};
+ // The pinned snapshots in Python's hub cache, as production
+ // loads them.
+ let config = KompressConfig::default();
+ let (tok, onnx) =
+ kompress::cached_artifacts(&config, "onnx/kompress-int8-wo.onnx")?;
+ Kompress::from_files(&tok, &onnx, config).ok()
})
.as_ref()
}
diff --git a/tests/parity/fixtures/hf_hub_cache/cases.json b/tests/parity/fixtures/hf_hub_cache/cases.json
index 0bb5c85a1..01273fd82 100644
--- a/tests/parity/fixtures/hf_hub_cache/cases.json
+++ b/tests/parity/fixtures/hf_hub_cache/cases.json
@@ -1,5 +1,5 @@
{
- "description": "Cases both languages are checked against so Rust's hf_hub_cache_dir() names the directory huggingface_hub would. tests/test_fastembed_pinned_snapshot.py checks them against Python's posixpath/ntpath and huggingface_hub.constants; crates/headroom-core/src/relevance/embedding.rs checks its port against the same file.",
+ "description": "Cases both languages are checked against so Rust's hf_hub_cache_dir() names the directory huggingface_hub would. tests/test_fastembed_pinned_snapshot.py checks them against Python's posixpath/ntpath and huggingface_hub.constants; crates/headroom-core/src/hf_cache.rs checks its port against the same file.",
"expandvars": {
"comment": "posix = posixpath.expandvars, nt = ntpath.expandvars, with env set and unset absent (identical on CPython 3.10, 3.11, 3.13).",
"env": {
@@ -130,7 +130,10 @@
"comment": "{root} and {home} are temporary directories; HOME and USERPROFILE are set to {home}, every other cache variable is cleared first. expected may be {posix, nt}.",
"probe_vars": [
"HEADROOM_HF_PROBE_ROOT",
- "HEADROOM_HF_PROBE_UNSET"
+ "HEADROOM_HF_PROBE_UNSET",
+ "HF_TOKEN",
+ "HF_TOKEN_PATH",
+ "HUGGING_FACE_HUB_TOKEN"
],
"cases": [
{
@@ -275,5 +278,102 @@
"expected": "{home}/.cache/huggingface/hub"
}
]
+ },
+ "token": {
+ "comment": "huggingface_hub.get_token(): env, files written first ({root}/{home} as in cache_dir), expected token or null. HF_TOKEN, HUGGING_FACE_HUB_TOKEN and HF_TOKEN_PATH are cleared with the cache variables.",
+ "cases": [
+ {
+ "name": "HF_TOKEN wins over the login file",
+ "env": {
+ "HF_TOKEN": "hf_env"
+ },
+ "files": {
+ "{home}/.cache/huggingface/token": "hf_file"
+ },
+ "expected": "hf_env"
+ },
+ {
+ "name": "the legacy HUGGING_FACE_HUB_TOKEN",
+ "env": {
+ "HUGGING_FACE_HUB_TOKEN": "hf_legacy"
+ },
+ "files": {},
+ "expected": "hf_legacy"
+ },
+ {
+ "name": "an empty HF_TOKEN falls through to the legacy name",
+ "env": {
+ "HF_TOKEN": "",
+ "HUGGING_FACE_HUB_TOKEN": "hf_legacy"
+ },
+ "files": {},
+ "expected": "hf_legacy"
+ },
+ {
+ "name": "a whitespace-only HF_TOKEN is still the one picked, so the legacy name is skipped",
+ "env": {
+ "HF_TOKEN": " ",
+ "HUGGING_FACE_HUB_TOKEN": "hf_legacy"
+ },
+ "files": {
+ "{home}/.cache/huggingface/token": "hf_file"
+ },
+ "expected": "hf_file"
+ },
+ {
+ "name": "the login file under the default HF_HOME, cleaned",
+ "env": {},
+ "files": {
+ "{home}/.cache/huggingface/token": " hf_file\r\n"
+ },
+ "expected": "hf_file"
+ },
+ {
+ "name": "the login file under HF_HOME",
+ "env": {
+ "HF_HOME": "{root}/hf"
+ },
+ "files": {
+ "{root}/hf/token": "hf_home"
+ },
+ "expected": "hf_home"
+ },
+ {
+ "name": "HF_TOKEN_PATH through ${VAR}",
+ "env": {
+ "HEADROOM_HF_PROBE_ROOT": "{root}",
+ "HF_TOKEN_PATH": "${HEADROOM_HF_PROBE_ROOT}/tok"
+ },
+ "files": {
+ "{root}/tok": "hf_path"
+ },
+ "expected": "hf_path"
+ },
+ {
+ "name": "moving the hub cache does not move the token",
+ "env": {
+ "HF_HUB_CACHE": "{root}/elsewhere/hub"
+ },
+ "files": {
+ "{home}/.cache/huggingface/token": "hf_file",
+ "{root}/elsewhere/token": "hf_wrong"
+ },
+ "expected": "hf_file"
+ },
+ {
+ "name": "a blank login file is no token",
+ "env": {},
+ "files": {
+ "{home}/.cache/huggingface/token": " \n"
+ },
+ "expected": null
+ },
+ {
+ "name": "nothing set",
+ "env": {},
+ "files": {},
+ "expected": null
+ }
+ ]
}
}
diff --git a/tests/test_fastembed_pinned_snapshot.py b/tests/test_fastembed_pinned_snapshot.py
index 48ade3f2f..86b0dd13e 100644
--- a/tests/test_fastembed_pinned_snapshot.py
+++ b/tests/test_fastembed_pinned_snapshot.py
@@ -228,11 +228,11 @@ def test_python_matches_parity_fixture():
# ── the shared HuggingFace cache directory ──────────────────────────────────
#
-# Rust's ``hf_hub_cache_dir`` re-implements how ``huggingface_hub`` picks the
+# Rust's ``hf_cache::hub_cache_dir`` re-implements how ``huggingface_hub`` picks the
# hub cache, so both languages look up and download into one directory. These
# check the shared fixture against Python itself; the Rust tests
# ``expand_vars_matches_python_posixpath_and_ntpath`` and
-# ``hf_hub_cache_dir_matches_the_shared_fixture`` check the port against the
+# ``hub_cache_dir_matches_the_shared_fixture`` (``crates/headroom-core/src/hf_cache.rs``) check the port against the
# same file.
_CACHE_VARS = ("HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE", "HF_HOME", "XDG_CACHE_HOME")
@@ -288,3 +288,39 @@ def test_cache_dir_fixture_matches_huggingface_hub(case: dict, tmp_path: Path) -
if isinstance(expected, dict):
expected = expected["nt" if os.name == "nt" else "posix"]
assert Path(result.stdout) == Path(fill(expected))
+
+
[email protected]("case", HF_CACHE_FIXTURE["token"]["cases"], ids=lambda case: case["name"])
+def test_token_fixture_matches_huggingface_hub(case: dict, tmp_path: Path) -> None:
+ """The token Rust's ``hf_cache::hf_token`` must find, from
+ ``huggingface_hub.get_token()`` itself."""
+ import subprocess
+
+ pytest.importorskip("huggingface_hub")
+ root, home = tmp_path / "root", tmp_path / "home"
+
+ def fill(text: str) -> str:
+ return text.replace("{root}", str(root)).replace("{home}", str(home))
+
+ for path, content in case["files"].items():
+ target = Path(fill(path))
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text(content, encoding="utf-8")
+ probe_vars = HF_CACHE_FIXTURE["cache_dir"]["probe_vars"]
+ env = {k: v for k, v in os.environ.items() if k not in (*_CACHE_VARS, *probe_vars)}
+ env["HOME"] = env["USERPROFILE"] = str(home)
+ env.update({name: fill(value) for name, value in case["env"].items()})
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ "import json, sys; from huggingface_hub import get_token; "
+ "sys.stdout.write(json.dumps(get_token()))",
+ ],
+ capture_output=True,
+ text=True,
+ env=env,
+ timeout=60,
+ check=True,
+ )
+ assert json.loads(result.stdout) == case["expected"]
diff --git a/tests/test_kompress_rust_pins.py b/tests/test_kompress_rust_pins.py
new file mode 100644
index 000000000..11baf9a93
--- /dev/null
+++ b/tests/test_kompress_rust_pins.py
@@ -0,0 +1,37 @@
+"""The Rust Kompress loader reads the snapshots the Python side pinned.
+
+The native proxy's Kompress is cache-only: it never downloads, it reads the
+files Python's ``hf_hub_download_local_first`` put in the HuggingFace cache at
+``_PINNED_REVISIONS``. ``crates/headroom-core/src/transforms/kompress.rs`` looks
+up the same commits, so the two must not drift: a pin bump on one side only
+would leave the Rust proxy on a snapshot Python no longer downloads.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from headroom.onnx_runtime import _PINNED_REVISIONS
+
+RUST_KOMPRESS = (
+ Path(__file__).resolve().parents[1] / "crates/headroom-core/src/transforms/kompress.rs"
+)
+
+
+def _rust_const(name: str) -> str:
+ match = re.search(rf'pub const {name}: &str = "([^"]+)";', RUST_KOMPRESS.read_text())
+ assert match, f"{name} not found in {RUST_KOMPRESS}"
+ return match.group(1)
+
+
+def test_rust_kompress_model_pin_matches_python():
+ repo = _rust_const("DEFAULT_MODEL_ID")
+ assert repo in _PINNED_REVISIONS, f"{repo} has no Python pin"
+ assert _rust_const("DEFAULT_MODEL_REVISION") == _PINNED_REVISIONS[repo]
+
+
+def test_rust_kompress_tokenizer_pin_matches_python():
+ repo = _rust_const("DEFAULT_TOKENIZER_REPO")
+ assert repo in _PINNED_REVISIONS, f"{repo} has no Python pin"
+ assert _rust_const("DEFAULT_TOKENIZER_REVISION") == _PINNED_REVISIONS[repo]
diff --git a/tests/test_offline_egress_chokepoint.py b/tests/test_offline_egress_chokepoint.py
index 4d9dbef71..22bf0c9e3 100644
--- a/tests/test_offline_egress_chokepoint.py
+++ b/tests/test_offline_egress_chokepoint.py
@@ -868,7 +868,8 @@ class TestHuggingFaceDownloadOffline:
# ───────────────── path 3: the Rust HuggingFace fetch (parity) ──────────────
_RUST_OFFLINE = REPO_ROOT / "crates" / "headroom-core" / "src" / "offline.rs"
-_RUST_HF = REPO_ROOT / "crates" / "headroom-core" / "src" / "tokenizer" / "hf_impl.rs"
+# Every Rust Hub download (tokenizer, embedder) builds its client in hf_cache::hub_api.
+_RUST_HF = REPO_ROOT / "crates" / "headroom-core" / "src" / "hf_cache.rs"
class TestRustOfflineParity:
@@ -976,11 +977,14 @@ class TestRustOfflineParity:
def test_hf_fetch_guards_before_it_builds_a_client(self) -> None:
source = _RUST_HF.read_text(encoding="utf-8")
- assert "guard_egress(" in source, "hf_impl.rs does not consult the offline guard"
- guard_at = source.index("guard_egress(")
- api_at = source.index("hf_hub::api::sync::Api::new()")
+ # Within hub_api itself: a guard elsewhere in the file does not count.
+ fn_at = source.index("pub(crate) fn hub_api(")
+ body = source[fn_at : source.index("\n}\n", fn_at)]
+ assert "guard_egress(" in body, "hf_cache::hub_api does not consult the offline guard"
+ guard_at = body.index("guard_egress(")
+ api_at = body.index("hf_hub::api::sync::ApiBuilder::from_cache(")
assert guard_at < api_at, (
- "the offline guard runs after Api::new(), which already resolves the "
+ "the offline guard runs after ApiBuilder::build(), which already resolves the "
"Hub endpoint and builds the ureq agent — guard before the client "
"exists, not before the request"
)
@@ -3225,7 +3229,7 @@ _RUST_ALLOWLIST: dict[str, tuple[int, str]] = {
"cache-only: TextEmbedding::try_new_from_user_defined in load_snapshot "
"builds the ONNX session from bytes already read off disk and has no "
"Hub client at all. The files reach the disk through "
- "default_model_repo, whose ApiBuilder is guarded, and only on a cache "
+ "default_model_repo, whose hf_cache::hub_api is guarded, and only on a cache "
"miss, so a pre-seeded air-gapped host still loads.",
),
"headroom-proxy/src/proxy.rs": (
@@ -3329,7 +3333,7 @@ class TestRustEgressChokepointCoverage:
def test_the_guarded_rust_paths_are_seen_as_guarded(self) -> None:
sites = _rust_egress_sites()
for relpath in (
- "headroom-core/src/tokenizer/hf_impl.rs",
+ "headroom-core/src/hf_cache.rs",
"headroom-core/src/transforms/kompress.rs",
"headroom-core/src/relevance/embedding.rs",
):