Skip to content

Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.

fix(embeddings): hash the whole model file, and serialise the env tests

Nicolas Geysse72195c35c562
diff --git a/docs/embeddings.md b/docs/embeddings.md
index 7ff6bd36..c9be3a18 100644
--- a/docs/embeddings.md
+++ b/docs/embeddings.md
@@ -85,7 +85,7 @@ composite hash of `(provider, model, revision)`:
| provider | `model` | `revision` |
|---|---|---|
-| `local` | model file name | file identity — size, mtime, sha256 of the head |
+| `local` | model file name | sha256 of the **whole file** |
| `openai`-compatible | resolved model name, plus `api_base` and `dimensions` | none — the API exposes no revision |
The fingerprint describes the **effective** configuration, so an environment
@@ -94,6 +94,12 @@ local provider it hashes the file *contents*: replacing the `.onnx` in place
leaves the path and filename identical, and that is the most likely way a local
model gets swapped.
+The whole file, not a head prefix — two fine-tunes of one architecture have
+identical sizes and identical ONNX graph protos, and `cp -p` preserves mtime, so
+`(size, mtime, head)` cannot tell them apart. Cost, measured on the 470 MB
+default model: **1.4 s**, once per process, and only when the local provider is
+in use — against a model load that already reads and optimises the same file.
+
On a mismatch, the affected vectors and index caches are discarded and rebuilt,
with a `warn` naming what happened. What to expect:
diff --git a/src/cache.rs b/src/cache.rs
index 614e0584..baa49ac0 100644
--- a/src/cache.rs
+++ b/src/cache.rs
@@ -75,33 +75,31 @@ pub(crate) fn sha256_hex(input: &[u8]) -> String {
s
}
-fn sha256(input: &[u8]) -> [u8; 32] {
- const K: [u32; 64] = [
- 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
- 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
- 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
- 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
- 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
- 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
- 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
- 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
- ];
- let mut h: [u32; 8] = [
- 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
- 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19,
- ];
-
- // Pre-processing: padding
- let bit_len = (input.len() as u64).wrapping_mul(8);
- let mut msg = Vec::with_capacity(input.len() + 64);
- msg.extend_from_slice(input);
- msg.push(0x80);
- while msg.len() % 64 != 56 {
- msg.push(0);
- }
- msg.extend_from_slice(&bit_len.to_be_bytes());
-
- for chunk in msg.chunks(64) {
+/// SHA-256 round constants (FIPS 180-4 §4.2.2), shared by the one-shot and
+/// streaming paths below.
+#[rustfmt::skip]
+const SHA256_K: [u32; 64] = [
+ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+ 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+ 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+ 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+ 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+ 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+ 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+ 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
+];
+
+/// Initial hash values (FIPS 180-4 §5.3.3).
+#[rustfmt::skip]
+const SHA256_H0: [u32; 8] = [
+ 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
+ 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19,
+];
+
+/// Compress one 64-byte block into the running state. Extracted from
+/// [`sha256`] unchanged so the streaming variant below shares exactly the same
+/// arithmetic rather than a second copy of it.
+fn sha256_compress(h: &mut [u32; 8], chunk: &[u8]) {
let mut w = [0u32; 64];
for (i, word) in chunk.chunks(4).enumerate() {
w[i] = u32::from_be_bytes([word[0], word[1], word[2], word[3]]);
@@ -121,7 +119,7 @@ fn sha256(input: &[u8]) -> [u8; 32] {
for i in 0..64 {
let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25);
let ch = (e & f) ^ ((!e) & g);
- let temp1 = hh.wrapping_add(s1).wrapping_add(ch).wrapping_add(K[i]).wrapping_add(w[i]);
+ let temp1 = hh.wrapping_add(s1).wrapping_add(ch).wrapping_add(SHA256_K[i]).wrapping_add(w[i]);
let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22);
let maj = (a & b) ^ (a & c) ^ (b & c);
let temp2 = s0.wrapping_add(maj);
@@ -144,6 +142,23 @@ fn sha256(input: &[u8]) -> [u8; 32] {
h[5] = h[5].wrapping_add(f);
h[6] = h[6].wrapping_add(g);
h[7] = h[7].wrapping_add(hh);
+}
+
+fn sha256(input: &[u8]) -> [u8; 32] {
+ let mut h = SHA256_H0;
+
+ // Pre-processing: padding
+ let bit_len = (input.len() as u64).wrapping_mul(8);
+ let mut msg = Vec::with_capacity(input.len() + 64);
+ msg.extend_from_slice(input);
+ msg.push(0x80);
+ while msg.len() % 64 != 56 {
+ msg.push(0);
+ }
+ msg.extend_from_slice(&bit_len.to_be_bytes());
+
+ for chunk in msg.chunks(64) {
+ sha256_compress(&mut h, chunk);
}
let mut out = [0u8; 32];
@@ -153,6 +168,78 @@ fn sha256(input: &[u8]) -> [u8; 32] {
out
}
+/// Streaming SHA-256 over a reader, in 1 MiB chunks.
+///
+/// [`sha256`] copies its whole input into a padded buffer, so hashing a 470 MB
+/// ONNX model through it would allocate roughly twice that. This variant holds
+/// 1 MiB whatever the file size.
+fn sha256_reader<R: std::io::Read>(r: &mut R) -> std::io::Result<[u8; 32]> {
+ let mut h = SHA256_H0;
+ let mut buf = vec![0u8; 1 << 20];
+ let mut carry: Vec<u8> = Vec::with_capacity(64);
+ let mut total: u64 = 0;
+
+ loop {
+ let n = r.read(&mut buf)?;
+ if n == 0 {
+ break;
+ }
+ total += n as u64;
+ let mut data: &[u8] = &buf[..n];
+
+ // Finish a partial block left over from the previous read.
+ if !carry.is_empty() {
+ let take = (64 - carry.len()).min(data.len());
+ carry.extend_from_slice(&data[..take]);
+ data = &data[take..];
+ if carry.len() == 64 {
+ sha256_compress(&mut h, &carry);
+ carry.clear();
+ }
+ }
+
+ let full = data.len() / 64 * 64;
+ for chunk in data[..full].chunks(64) {
+ sha256_compress(&mut h, chunk);
+ }
+ carry.extend_from_slice(&data[full..]);
+ }
+
+ // Final padding: 0x80, zeros to 56 mod 64, then the length in bits.
+ let bit_len = total.wrapping_mul(8);
+ let mut tail = carry;
+ tail.push(0x80);
+ while tail.len() % 64 != 56 {
+ tail.push(0);
+ }
+ tail.extend_from_slice(&bit_len.to_be_bytes());
+ for chunk in tail.chunks(64) {
+ sha256_compress(&mut h, chunk);
+ }
+
+ let mut out = [0u8; 32];
+ for (i, word) in h.iter().enumerate() {
+ out[i * 4..(i + 1) * 4].copy_from_slice(&word.to_be_bytes());
+ }
+ Ok(out)
+}
+
+/// Streaming SHA-256 of a whole file, as lowercase hex.
+///
+/// Used by `embed_fingerprint` to identify a local model by its contents.
+/// Unlike [`SourceFingerprint`], which hashes only a head prefix because a
+/// missed change there costs a re-parse, a missed change here would mean
+/// serving vectors from one model against queries from another.
+pub(crate) fn sha256_file_hex(path: &Path) -> Result<String> {
+ let mut file = fs::File::open(path).with_context(|| format!("open({})", path.display()))?;
+ let digest = sha256_reader(&mut file).with_context(|| format!("read({})", path.display()))?;
+ let mut s = String::with_capacity(64);
+ for b in digest.iter() {
+ s.push_str(&format!("{:02x}", b));
+ }
+ Ok(s)
+}
+
/// One row from the `ontology_cache` table.
#[derive(Debug, Clone)]
pub struct CacheEntry {
@@ -388,3 +475,78 @@ mod tests {
assert_eq!(derive_name(""), "default");
}
}
+
+#[cfg(test)]
+mod sha256_stream_tests {
+ use super::*;
+
+ /// The streaming variant must agree with the one-shot on every length that
+ /// exercises a padding or buffering edge. Hand-rolled block padding is
+ /// exactly the kind of code that is right for 1 MiB and wrong for 64 bytes.
+ #[test]
+ fn streaming_agrees_with_one_shot_on_every_boundary() {
+ const MIB: usize = 1 << 20;
+ for len in [
+ 0,
+ 1,
+ 55, // last length that fits padding in the same block
+ 56, // first length that forces an extra block
+ 63,
+ 64, // exactly one block
+ 65,
+ 64 * 3,
+ MIB - 1,
+ MIB, // exactly the read buffer
+ MIB + 1, // forces a carry across reads
+ MIB + 63,
+ MIB + 64,
+ 2 * MIB + 17,
+ ] {
+ // Not all zeros: a constant-byte input would hide an ordering bug.
+ let data: Vec<u8> = (0..len).map(|i| (i % 251) as u8).collect();
+ let one_shot = sha256(&data);
+ let streamed = sha256_reader(&mut data.as_slice()).unwrap();
+ assert_eq!(
+ one_shot, streamed,
+ "streaming and one-shot disagree at len = {len}"
+ );
+ }
+ }
+
+ /// A reader that hands back small, awkward chunks — the real file case is
+ /// not the only one, and `read` is allowed to return short.
+ #[test]
+ fn a_dribbling_reader_produces_the_same_digest() {
+ struct Dribble<'a>(&'a [u8], usize);
+ impl std::io::Read for Dribble<'_> {
+ fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+ let n = self.1.min(self.0.len()).min(buf.len());
+ buf[..n].copy_from_slice(&self.0[..n]);
+ self.0 = &self.0[n..];
+ Ok(n)
+ }
+ }
+ let data: Vec<u8> = (0..5000).map(|i| (i % 251) as u8).collect();
+ let expected = sha256(&data);
+ for chunk in [1usize, 7, 63, 64, 65, 1000] {
+ let mut r = Dribble(&data, chunk);
+ assert_eq!(
+ sha256_reader(&mut r).unwrap(),
+ expected,
+ "disagreement when the reader returns {chunk} bytes at a time"
+ );
+ }
+ }
+
+ #[test]
+ fn hashing_a_file_matches_hashing_its_bytes() {
+ let dir = std::env::temp_dir().join(format!("oo-sha-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let p = dir.join("blob.bin");
+ let data: Vec<u8> = (0..200_000).map(|i| (i % 251) as u8).collect();
+ std::fs::write(&p, &data).unwrap();
+ let from_file = sha256_file_hex(&p).unwrap();
+ std::fs::remove_dir_all(&dir).ok();
+ assert_eq!(from_file, sha256_hex(&data));
+ }
+}
diff --git a/src/embed_fingerprint.rs b/src/embed_fingerprint.rs
index e11e8ef1..031d83c4 100644
--- a/src/embed_fingerprint.rs
+++ b/src/embed_fingerprint.rs
@@ -37,8 +37,19 @@
//!
//! Hashing the model file *path* would not do: the default download URL and the
//! on-disk filename are both stable across a model change made by replacing the
-//! file in place. The file identity is computed exactly the way `ontology_cache`
-//! fingerprints its sources ([`crate::cache::SourceFingerprint`]).
+//! file in place.
+//!
+//! The revision is a **sha256 of the whole file**, not of a head prefix.
+//! `SourceFingerprint` (mtime + size + sha256 of the first 64 KiB) is what
+//! `ontology_cache` uses, and it is right there because a missed change costs a
+//! re-parse. Here a missed change costs vectors from one model served against
+//! queries from another — the very thing this module exists to prevent — and
+//! the miss is reachable: two fine-tunes of one architecture have byte-identical
+//! sizes and graph protos, and `cp -p` preserves mtime. Measured cost of doing
+//! it properly: **1.4 s for the 470 MB default model** (330 MB/s, release, the
+//! crate's own sha256), against a `TextEmbedder::load` that already reads and
+//! optimises that same file. It is paid once per process, and only by the local
+//! provider.
//!
//! Two inputs are included beyond the agreed triple, for the openai-compatible
//! arm only. Both are one line each and either can be dropped without touching
@@ -103,13 +114,8 @@ pub fn describe(cfg: &EmbeddingsConfig) -> String {
.unwrap_or_else(|| UNAVAILABLE.to_string());
let revision = path
.as_ref()
- .and_then(|p| crate::cache::SourceFingerprint::from_path(p).ok())
- .map(|fp| {
- format!(
- "size={},mtime={},sha={}",
- fp.size, fp.mtime_secs, fp.sha_prefix
- )
- })
+ .and_then(|p| crate::cache::sha256_file_hex(p).ok())
+ .map(|sha| format!("sha256={sha}"))
.unwrap_or_else(|| UNAVAILABLE.to_string());
format!("provider=local\nmodel={name}\nrevision={revision}")
}
@@ -147,9 +153,25 @@ mod tests {
}
}
+ /// `cargo test` runs the tests of one binary on several threads, and the
+ /// environment is process-wide: without this, one test clearing
+ /// `OPEN_ONTOLOGIES_EMBEDDINGS_MODEL` between another's two `fingerprint`
+ /// calls makes that other test fail for no reason of its own. The RAII
+ /// guard below restores state but grants no mutual exclusion, which is a
+ /// different thing.
+ static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
+
/// Guard the env-var overrides the resolvers read, so a developer's shell
- /// cannot make these pass or fail for reasons unrelated to the code.
- struct EnvGuard(Vec<(&'static str, Option<String>)>);
+ /// cannot make these pass or fail for reasons unrelated to the code, and
+ /// hold [`ENV_LOCK`] for the duration so no sibling test observes the
+ /// mutation.
+ struct EnvGuard {
+ saved: Vec<(&'static str, Option<String>)>,
+ /// Held for its `Drop`, never read: it is what serialises these tests
+ /// against each other. Named rather than `_0` so that intent survives.
+ #[allow(dead_code)]
+ lock: std::sync::MutexGuard<'static, ()>,
+ }
impl EnvGuard {
fn clear() -> Self {
let keys = [
@@ -157,6 +179,12 @@ mod tests {
"OPEN_ONTOLOGIES_EMBEDDINGS_API_BASE",
"OPEN_ONTOLOGIES_EMBEDDINGS_MODEL",
];
+ // Take the lock BEFORE reading or mutating anything: saving the
+ // old values while another test is mid-mutation would restore its
+ // scratch state as if it were ours.
+ // Poisoning only means an earlier test panicked while holding it;
+ // the environment is still ours to restore, so take it either way.
+ let lock = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let saved = keys
.iter()
.map(|k| (*k, std::env::var(k).ok()))
@@ -164,12 +192,12 @@ mod tests {
for (k, _) in &saved {
unsafe { std::env::remove_var(k) };
}
- Self(saved)
+ Self { saved, lock }
}
}
impl Drop for EnvGuard {
fn drop(&mut self) {
- for (k, v) in &self.0 {
+ for (k, v) in &self.saved {
match v {
Some(val) => unsafe { std::env::set_var(k, val) },
None => unsafe { std::env::remove_var(k) },
@@ -267,6 +295,51 @@ mod tests {
assert_ne!(before, after, "a model replaced in place went undetected");
}
+ #[test]
+ fn a_same_size_swap_with_preserved_mtime_is_still_detected() {
+ // The case a head-prefix fingerprint misses, and it is not contrived:
+ // two fine-tunes of one architecture have byte-identical sizes AND
+ // byte-identical ONNX graph protos, `cp -p` preserves mtime, and the
+ // weights that differ can sit past the first 64 KiB. Under
+ // (mtime, size, sha256-of-head) the two are indistinguishable.
+ let _g = EnvGuard::clear();
+ let dir = std::env::temp_dir().join(format!("oo-fp3-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let model = dir.join("model.onnx");
+
+ // 96 KiB: a shared 64 KiB head, then weights.
+ const HEAD: usize = 64 * 1024;
+ let mut a = vec![0xABu8; HEAD];
+ a.extend(std::iter::repeat_n(0x01u8, 32 * 1024));
+ let mut b = vec![0xABu8; HEAD];
+ b.extend(std::iter::repeat_n(0x02u8, 32 * 1024));
+ assert_eq!(a.len(), b.len(), "the scenario requires identical sizes");
+ assert_eq!(a[..HEAD], b[..HEAD], "the scenario requires an identical head");
+
+ let cfg = local_cfg(Some(model.to_str().unwrap()));
+
+ std::fs::write(&model, &a).unwrap();
+ let mtime = std::fs::metadata(&model).unwrap().modified().unwrap();
+ let before = fingerprint(&cfg);
+
+ // `cp -p`: same bytes count, same modification time.
+ std::fs::write(&model, &b).unwrap();
+ std::fs::File::open(&model)
+ .unwrap()
+ .set_modified(mtime)
+ .unwrap();
+
+ let after = fingerprint(&cfg);
+ std::fs::remove_dir_all(&dir).ok();
+
+ assert_ne!(
+ before, after,
+ "a same-size, same-mtime model swap differing only past the head \
+ went undetected — this is the corruption path the module exists \
+ to close"
+ );
+ }
+
#[test]
fn a_missing_model_file_does_not_collide_with_a_real_one() {
let _g = EnvGuard::clear();