Skip to content

Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.

fix(cache): cache datasets as N-Quads so named graphs survive a reload (#113)

Nicolasa08ca93b679a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 00ad44a3..50bbbacb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,34 @@ All notable changes to Open Ontologies are documented here.
## [Unreleased]
### Fixed
+- **The compile cache no longer flattens named graphs** (issue #112). It was
+ written as N-Triples, a format that cannot carry a graph name, so the cached
+ artefact was not equivalent to the source it stood for: a dataset went in and
+ a flattened graph came out. The first load parsed the source and answered
+ correctly; every load after it read the cache back and returned a store with
+ no named graphs at all. Measured on an unchanged TriG file, twice through
+ `onto_load`: `origin: "source"` gave two named graphs, `origin: "cache"` gave
+ none, and `onto_temporal_snapshot` reported `{"ok": true, "in_scope": []}` —
+ a clean, confident, empty answer.
+
+ The freshness key `(source_path, mtime, size, sha256)` was never at fault and
+ is unchanged; the cache was correctly judged fresh, and what it held was
+ wrong. A second path needed no second load at all: an idle-evicted ontology
+ reloads through `ensure_loaded`, from that same flattened file, so a
+ long-running `serve-http --idle-ttl-secs` lost its named graphs by being
+ idle.
+
+ Cache files are now N-Quads (`.nq`) — line-based and just as fast to parse,
+ which was the reason N-Triples was chosen, but able to name a graph. The
+ extension doubles as the format marker: an entry pointing at a `.nt` file was
+ written before this fix, holds a flattened dataset, and is recompiled instead
+ of read. That costs one re-parse per ontology, once, and heals warm caches
+ rather than leaving them quietly wrong. Anything whose meaning lives in the
+ graph name is affected — the bi-temporal tools above all, which read
+ `GRAPH ?g` and saw an empty store. Three tests in `tests/registry_test.rs`,
+ each loading TWICE on purpose: a test that loads once passes on the broken
+ code, which is why this went unnoticed.
+
- **Named graphs are preserved when exporting to TriG and N-Quads.**
`GraphStore::serialize` rendered every format with `serialize_triple`, which
flattens a quad from a named graph into the default graph. Import and the
diff --git a/src/cache.rs b/src/cache.rs
index c0b165c0..24e98dfa 100644
--- a/src/cache.rs
+++ b/src/cache.rs
@@ -1,9 +1,16 @@
//! Compile cache for ontology files.
//!
-//! A loaded ontology is serialized as N-Triples (the simplest, fastest-to-parse
-//! RDF format) and written to `<cache_dir>/<sha>.nt`. Metadata is stored in
-//! the `ontology_cache` SQLite table so we can quickly answer "is the cache
-//! still valid for this source file?" without re-parsing.
+//! A loaded ontology is serialized as N-Quads (line-based and just as fast to
+//! parse as N-Triples, but able to name a graph) and written to
+//! `<cache_dir>/<sha>.nq`. Metadata is stored in the `ontology_cache` SQLite
+//! table so we can quickly answer "is the cache still valid for this source
+//! file?" without re-parsing.
+//!
+//! The format is load-bearing, not a detail. N-Triples cannot carry a graph
+//! name, so caching in it made the cached artefact non-equivalent to the
+//! source: a dataset went in and a flattened graph came out, silently, on
+//! every load after the first. Anything whose meaning lives in the graph name
+//! — the bi-temporal layer above all — lost it there.
//!
//! Validity key: `(source_path, mtime_secs, size, sha256(prefix))`.
//! For most workflows the (mtime, size) pair is sufficient and avoids hashing
@@ -18,6 +25,13 @@ use rusqlite::params;
use crate::state::StateDb;
+/// Extension of a cache file, and the format it holds.
+///
+/// N-Quads rather than N-Triples: line-based and just as fast to parse, but it
+/// can name a graph. The extension doubles as the format marker — a `.nt` file
+/// left by an older build holds a flattened dataset and must not be read back.
+pub const CACHE_EXT: &str = "nq";
+
// Number of head-bytes hashed for the cache fingerprint — overridable via
// `[cache] hash_prefix_bytes` in config.toml. See `crate::runtime`.
@@ -317,6 +331,11 @@ impl CacheManager {
}
/// Build a cache file path for a given source key.
+ ///
+ /// The extension is part of the cache contract: a file that does not end
+ /// in [`CACHE_EXT`] was written by a version that could not represent
+ /// named graphs, and is treated as stale rather than read back (see
+ /// `OntologyRegistry::load_file`).
pub fn cache_path_for(&self, name: &str, sha: &str) -> PathBuf {
// Use both the name and the sha so renames don't collide and we can
// garbage-collect stale entries by `name`.
@@ -324,7 +343,8 @@ impl CacheManager {
.chars()
.map(|c| if c.is_ascii_alphanumeric() || c == '-' || c == '_' { c } else { '_' })
.collect::<String>();
- self.cache_dir.join(format!("{}.{}.nt", safe, &sha[..sha.len().min(16)]))
+ self.cache_dir
+ .join(format!("{}.{}.{CACHE_EXT}", safe, &sha[..sha.len().min(16)]))
}
/// Atomically write `content` to `path` (writes to `<path>.tmp` then renames).
diff --git a/src/graph.rs b/src/graph.rs
index 84fc689c..f4488ebf 100644
--- a/src/graph.rs
+++ b/src/graph.rs
@@ -419,9 +419,23 @@ impl GraphStore {
}
pub fn load_ntriples(&self, content: &str) -> anyhow::Result<usize> {
+ self.load_lines(content, RdfFormat::NTriples)
+ }
+
+ /// Load N-Quads, keeping every graph name.
+ ///
+ /// The line-based sibling of [`load_ntriples`](Self::load_ntriples), for
+ /// the paths that round-trip a whole dataset rather than one graph — the
+ /// compile cache above all, where N-Triples silently flattened everything
+ /// it was asked to hold.
+ pub fn load_nquads(&self, content: &str) -> anyhow::Result<usize> {
+ self.load_lines(content, RdfFormat::NQuads)
+ }
+
+ fn load_lines(&self, content: &str, format: RdfFormat) -> anyhow::Result<usize> {
let store = self.store.lock().unwrap();
let reader = Cursor::new(content.as_bytes());
- let parser = RdfParser::from_format(RdfFormat::NTriples).for_reader(reader);
+ let parser = RdfParser::from_format(format).for_reader(reader);
let mut count = 0;
for quad in parser {
store.insert(&quad?)?;
diff --git a/src/registry.rs b/src/registry.rs
index 921ffe19..fe7bfe50 100644
--- a/src/registry.rs
+++ b/src/registry.rs
@@ -113,6 +113,14 @@ impl OntologyRegistry {
&& e.source_size == fp.size
&& e.source_sha == fp.sha_prefix
&& Path::new(&e.cache_path).exists()
+ // A cache written before the format carried graph
+ // names holds a flattened dataset. The source has not
+ // changed, so every other check passes and it would be
+ // read back as authoritative — recompile instead. One
+ // extra parse per ontology, once, and warm caches heal
+ // themselves instead of staying quietly wrong.
+ && Path::new(&e.cache_path).extension().and_then(|x| x.to_str())
+ == Some(crate::cache::CACHE_EXT)
})
.unwrap_or(false);
@@ -121,9 +129,9 @@ impl OntologyRegistry {
let (triple_count, origin, cache_path) = if cache_is_fresh {
let entry = existing.unwrap();
- let nt = std::fs::read_to_string(&entry.cache_path)
+ let quads = std::fs::read_to_string(&entry.cache_path)
.with_context(|| format!("read cache {}", entry.cache_path))?;
- let count = self.graph.load_ntriples(&nt)?;
+ let count = self.graph.load_nquads(&quads)?;
self.cache.touch(&name)?;
(count, "cache", PathBuf::from(entry.cache_path))
} else {
@@ -133,8 +141,11 @@ impl OntologyRegistry {
.with_context(|| format!("parse source {}", path))?;
let cache_path = if self.config.enabled {
let cp = self.cache.cache_path_for(&name, &fp.sha_prefix);
- let nt = self.graph.serialize("ntriples")?;
- CacheManager::atomic_write(&cp, &nt)?;
+ // N-Quads, not N-Triples: the cache has to be able to give
+ // back what it was given, and a named graph does not survive
+ // the trip through a triple format.
+ let quads = self.graph.serialize("nquads")?;
+ CacheManager::atomic_write(&cp, &quads)?;
self.cache.upsert(&name, path, &fp, &cp, count)?;
cp
} else {
@@ -209,8 +220,8 @@ impl OntologyRegistry {
self.graph.clear()?;
let count = self.graph.load_file(&source_path)?;
let new_cache = self.cache.cache_path_for(&name, &cur.sha_prefix);
- let nt = self.graph.serialize("ntriples")?;
- CacheManager::atomic_write(&new_cache, &nt)?;
+ let quads = self.graph.serialize("nquads")?;
+ CacheManager::atomic_write(&new_cache, &quads)?;
self.cache.upsert(&name, &source_path, &cur, &new_cache, count)?;
let mut active_guard = self.active.lock().unwrap();
@@ -227,9 +238,9 @@ impl OntologyRegistry {
// Reload from N-Triples cache; fall back to source if cache file
// is missing for some reason.
if cache_path.exists() {
- let nt = std::fs::read_to_string(&cache_path)?;
+ let quads = std::fs::read_to_string(&cache_path)?;
self.graph.clear()?;
- self.graph.load_ntriples(&nt)?;
+ self.graph.load_nquads(&quads)?;
} else if Path::new(&source_path).exists() {
self.graph.clear()?;
self.graph.load_file(&source_path)?;
@@ -364,10 +375,10 @@ impl OntologyRegistry {
.with_context(|| format!("parse source {}", entry.source_path))?;
let fp = SourceFingerprint::from_path(path)?;
let cache_path = self.cache.cache_path_for(name, &fp.sha_prefix);
- let nt = isolated_graph.serialize("ntriples")?;
- CacheManager::atomic_write(&cache_path, &nt)?;
+ let quads = isolated_graph.serialize("nquads")?;
+ CacheManager::atomic_write(&cache_path, &quads)?;
// If the sha-prefix changed, the new cache_path differs from the old
- // one. Remove the old file to avoid leaking stale .nt files.
+ // one. Remove the old file to avoid leaking stale cache files.
if entry.cache_path != cache_path.to_string_lossy() {
let _ = std::fs::remove_file(&entry.cache_path);
}
diff --git a/tests/registry_test.rs b/tests/registry_test.rs
index f4ea599d..819ddef5 100644
--- a/tests/registry_test.rs
+++ b/tests/registry_test.rs
@@ -56,6 +56,7 @@ struct Harness {
_tmp: tempfile::TempDir,
pub source_path: PathBuf,
pub cache_dir: PathBuf,
+ pub db_path: PathBuf,
pub registry: Arc<OntologyRegistry>,
pub graph: Arc<GraphStore>,
}
@@ -82,6 +83,7 @@ fn setup(idle_ttl_secs: u64) -> Harness {
_tmp: tmp,
source_path,
cache_dir,
+ db_path,
registry,
graph,
}
@@ -107,7 +109,10 @@ fn first_load_writes_cache_file() {
std::path::Path::new(&res.cache_path).exists(),
"cache file should exist on disk"
);
- // Cache directory should contain at least one .nt file.
+ // The cache directory should hold a file in the cache format. Asserted
+ // through the constant rather than a literal: the extension IS the format
+ // marker (a file in any other one is treated as stale), so the two must
+ // not be able to drift apart.
let entries: Vec<_> = std::fs::read_dir(&h.cache_dir)
.unwrap()
.filter_map(|e| e.ok())
@@ -115,11 +120,15 @@ fn first_load_writes_cache_file() {
e.path()
.extension()
.and_then(|s| s.to_str())
- .map(|s| s == "nt")
+ .map(|s| s == open_ontologies::cache::CACHE_EXT)
.unwrap_or(false)
})
.collect();
- assert!(!entries.is_empty(), "expected an .nt file in cache dir");
+ assert!(
+ !entries.is_empty(),
+ "expected a .{} file in cache dir",
+ open_ontologies::cache::CACHE_EXT
+ );
}
#[test]
@@ -429,3 +438,136 @@ fn auto_refresh_after_eviction_uses_new_source() {
let v2 = h.graph.triple_count();
assert!(v2 > v1, "expected refreshed (larger) ontology after change");
}
+
+// ────────────────────────────────────────────────────────────────────────────
+// Named graphs survive the cache
+//
+// The cache is only useful if what comes out equals what went in. It was
+// written as N-Triples, which cannot carry a graph name, so a dataset went in
+// and a flattened graph came out — on every load after the first, and on every
+// reload after an idle eviction. The source never changed, so the freshness
+// key was right and nothing anywhere reported a loss.
+//
+// Every test below loads TWICE on purpose. A test that loads once passes on
+// the broken code, which is why this went unnoticed.
+// ────────────────────────────────────────────────────────────────────────────
+
+/// Two named graphs plus a description of them in the default graph — the
+/// shape the bi-temporal tools read (`src/temporal.rs`).
+const SAMPLE_TRIG: &str = r#"
+@prefix ex: <http://example.org/test#> .
+@prefix t: <https://open-ontologies.org/temporal#> .
+
+ex:g_v1 { ex:Doc ex:status ex:Draft . }
+ex:g_v2 { ex:Doc ex:status ex:Published . }
+
+{
+ ex:g_v1 t:validFrom "2024-01-01" ; t:validTo "2026-05-01" .
+ ex:g_v2 t:validFrom "2026-05-01" .
+}
+"#;
+
+fn setup_trig(idle_ttl_secs: u64) -> Harness {
+ let h = setup(idle_ttl_secs);
+ let trig_path = h._tmp.path().join("sample.trig");
+ std::fs::write(&trig_path, SAMPLE_TRIG).unwrap();
+ Harness {
+ source_path: trig_path,
+ ..h
+ }
+}
+
+/// The graph names the store currently holds, as reported by SPARQL rather
+/// than by the loader that just claimed to have loaded them.
+fn named_graphs(h: &Harness) -> Vec<String> {
+ let raw = h
+ .graph
+ .sparql_select("SELECT DISTINCT ?g WHERE { GRAPH ?g { ?s ?p ?o } } ORDER BY ?g")
+ .unwrap();
+ let parsed: serde_json::Value = serde_json::from_str(&raw).unwrap();
+ parsed["results"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .filter_map(|row| row["g"].as_str().map(str::to_string))
+ .collect()
+}
+
+#[test]
+fn a_second_load_from_cache_keeps_the_named_graphs() {
+ let h = setup_trig(0);
+ let path = h.source_path.to_str().unwrap();
+
+ let first = h.registry.load_file(path, LoadOptions::default()).unwrap();
+ assert_eq!(first.origin, "source");
+ let from_source = named_graphs(&h);
+ assert_eq!(from_source.len(), 2, "fixture has two named graphs: {from_source:?}");
+
+ // Same file, untouched: the cache is legitimately fresh and IS used. That
+ // is the point — the bug was never in the freshness key.
+ let second = h.registry.load_file(path, LoadOptions::default()).unwrap();
+ assert_eq!(second.origin, "cache", "second load should hit the cache");
+ assert_eq!(
+ named_graphs(&h),
+ from_source,
+ "a cache round trip must return the dataset it was given, graph names included"
+ );
+ assert_eq!(second.triple_count, first.triple_count);
+}
+
+#[test]
+fn a_reload_after_eviction_keeps_the_named_graphs() {
+ // The path that needs no second load and no user action at all: the entry
+ // goes idle, the evictor drops it, and the next read reloads from cache.
+ let h = setup_trig(1);
+ let path = h.source_path.to_str().unwrap();
+ h.registry.load_file(path, LoadOptions::default()).unwrap();
+ let before = named_graphs(&h);
+
+ sleep(Duration::from_millis(1100));
+ assert!(h.registry.evictor_tick().unwrap(), "entry should be evicted");
+ h.registry.ensure_loaded().unwrap();
+
+ assert_eq!(
+ named_graphs(&h),
+ before,
+ "an eviction is a memory event, not a data change"
+ );
+}
+
+#[test]
+fn a_cache_file_left_by_an_older_build_is_recompiled_not_read() {
+ // A `.nt` cache holds a flattened dataset. The source has not changed, so
+ // every other freshness check passes: without the format check it would be
+ // read back as authoritative and the graph names would be gone.
+ let h = setup_trig(0);
+ let path = h.source_path.to_str().unwrap();
+ let first = h.registry.load_file(path, LoadOptions::default()).unwrap();
+ let expected = named_graphs(&h);
+
+ // Rewrite history: flatten the cache to N-Triples under the old name, and
+ // point the metadata at it, exactly as an older build would have left it.
+ let legacy = std::path::Path::new(&first.cache_path).with_extension("nt");
+ let flattened = h.graph.serialize("ntriples").unwrap();
+ std::fs::write(&legacy, &flattened).unwrap();
+ let fp = SourceFingerprint::from_path(std::path::Path::new(path)).unwrap();
+ let cm = CacheManager::new(
+ h.cache_dir.clone(),
+ StateDb::open(&h.db_path).unwrap(),
+ )
+ .unwrap();
+ cm.upsert("sample", path, &fp, &legacy, first.triple_count)
+ .unwrap();
+
+ let second = h.registry.load_file(path, LoadOptions::default()).unwrap();
+ assert_eq!(
+ second.origin, "source",
+ "a legacy cache file must be recompiled, not trusted"
+ );
+ assert_eq!(named_graphs(&h), expected);
+ assert!(
+ second.cache_path.ends_with(".nq"),
+ "the rewritten cache carries the format that can hold graph names: {}",
+ second.cache_path
+ );
+}