Code graph and memory for your coding agents, in one Rust program. For Claude Code, Codex, Cursor and any MCP client.
Rust agent-memoryai-agentsclaude-codecode-analysiscode-graphcode-intelligencecode-searchcodexcoding-agentscontext-engineeringcursordeveloper-toolsgraphragknowledge-graphlocal-firstmcpmcp-servermodel-context-protocolrusttree-sitter
// Adapted from Graphify 0.9.95 (92741d16), Apache-2.0; see NOTICE.md.
// Upstream: graphify/cache.py, graphify/extract.py
//! Port of graphify/cache.py: owned per-run cache state with explicit flush.
//! The native release has its own AST version; semantic entries retain their
//! original content/prompt namespaces so compatible extraction is reusable.
pub use crate::cache_paths::*;
use crate::{Result, paths, python_compat, python_json};
use serde_json::{Map, Value, json};
use sha2::{Digest, Sha256};
use std::{
collections::{BTreeSet, HashSet},
fs,
io::Write,
path::{Path, PathBuf},
time::{SystemTime, UNIX_EPOCH},
};
/// Bump when AST cache-key semantics change independently of the package
/// version; 10 carries the extraction changes of Graphify 0.9.92-0.9.95
/// (upstream schemas 17-25: pytest fixture markers, Python module-level class
/// salting, TS `this.field` from any method, Solidity arity and inherited-call
/// records, Go and Rust case-distinct ids, Kotlin `typealias`, C# `delegate`,
/// Swift `macro`, Rust `use` of sibling `mod`s, Dart `typedef`, Metal/CUDA
/// qualifiers, `extern "C"` headers, Obsidian vault-relative wikilinks,
/// CommonJS `require` bindings, SQL brackets, routine bodies, `GO` and
/// recovered tables, structural YAML, Objective-C `NS_CLOSED_ENUM`, Python,
/// Go, C++ and Rust type aliases, C# overloads and package.json `declares`);
/// 9 carries the per-file record fields of Graphify 0.9.89-0.9.91
/// (upstream schemas 13-16: Go and PHP receiver type tables and member
/// receivers, `_callable`/`_callable_class` markers on Go and Objective-C
/// declarations, Objective-C NS_ENUM/NS_OPTIONS, generic type parameters that
/// no longer mint stubs, C# `_`/case-distinct ids and receiver markers, TS
/// class field tables, Go generic receivers and Rust `use` names); 8 carries
/// the per-file record fields of Graphify 0.9.78-0.9.88
/// (upstream schemas 6-12: Python typed receivers, `self.X` bindings, opaque
/// bases, `super(args)` and property reads, C# `type_path`, receiver chains and
/// return tables, Java receiver chains and `_java_ret`, Rust `use` groups,
/// quoted-include provenance) and records the target files an entry's edges
/// resolved to; 7 invalidated records whose source paths leaked a walked
/// directory alias when the extraction root was canonicalized (6 added
/// receiver shadows).
pub const AST_CACHE_SCHEMA: i64 = 10;
/// Where an AST entry lists the `target_file` stamps that named an existing
/// file when it was written; a hit whose target is gone is a miss (#4185).
pub const CACHED_TARGETS_KEY: &str = "_cached_target_files";
/// `_salted_key`: the on-disk key of an entry whose validity depends on more
/// than the file's bytes (a file the project remaps to another language,
/// #2961); without a salt, the content hash itself.
pub fn salted_key(hash: &str, salt: Option<&str>) -> String {
match salt.filter(|s| !s.is_empty()) {
None => hash.to_string(),
Some(salt) => format!("{:x}", Sha256::digest(format!("{hash}:{salt}").as_bytes())),
}
}
/// `_grammar_fingerprint` over `(name, version)` pairs: the first 12 hex
/// digits of the SHA-256 of the sorted `name==version` list, `"none"` for an
/// empty set.
pub fn grammar_fingerprint_of(parts: impl IntoIterator<Item = (String, String)>) -> String {
let mut parts = parts
.into_iter()
.map(|(name, version)| format!("{}=={version}", name.to_lowercase()))
.collect::<Vec<_>>();
if parts.is_empty() {
return "none".into();
}
parts.sort();
format!("{:x}", Sha256::digest(parts.join(";").as_bytes()))[..12].into()
}
/// The tree-sitter runtime and grammar crates the engine pins, from its own
/// `Cargo.lock` (vendored `graphicle-tree-sitter-*` crates included).
/// Upstream fingerprints the installed `tree-sitter-*` distributions; the
/// native links its grammars statically, so the versions it is built with are
/// its grammar set (a grammar pack must match the pinned grammar exactly, see
/// `grammar_packs`). A workspace that embeds the engine resolves its own lock
/// (the `graphicle` program links tree-sitter 0.27), which this list does not
/// see.
pub fn compiled_grammars() -> Vec<(String, String)> {
const LOCK: &str = include_str!("../Cargo.lock");
let grammar = |name: &str| {
let name = name.strip_prefix("graphicle-").unwrap_or(name);
name == "tree-sitter" || name.starts_with("tree-sitter-") || name.starts_with("tree_sitter")
};
let mut out = vec![];
let mut name = None;
for line in LOCK.lines().map(str::trim) {
if line == "[[package]]" {
name = None;
} else if let Some(value) = line.strip_prefix("name = ") {
name = Some(value.trim_matches('"'));
} else if let Some(value) = line.strip_prefix("version = ")
&& let Some(package) = name.take()
&& grammar(package)
{
out.push((package.to_string(), value.trim_matches('"').to_string()));
}
}
out
}
/// The grammar set this build's AST entries depend on (#4236), memoised.
pub fn grammar_fingerprint() -> &'static str {
static FINGERPRINT: std::sync::OnceLock<String> = std::sync::OnceLock::new();
FINGERPRINT.get_or_init(|| grammar_fingerprint_of(compiled_grammars()))
}
/// `_target_file_present`: `target` (relative ones under `root`) is a file.
pub fn target_file_present(target: &str, root: &Path) -> bool {
let candidate = Path::new(target);
if candidate.is_absolute() {
candidate.is_file()
} else {
root.join(candidate).is_file()
}
}
/// `_present_target_files`: the edges' `target_file` stamps that name an
/// existing file, sorted and distinct.
pub fn present_target_files(result: &Value, root: &Path) -> Vec<String> {
let mut present = BTreeSet::new();
for edge in ar(&result["edges"]) {
if let Some(target) = edge.get("target_file").and_then(Value::as_str)
&& !target.is_empty()
&& !present.contains(target)
&& target_file_present(target, root)
{
present.insert(target.to_string());
}
}
present.into_iter().collect()
}
pub const MTIME_GRANULARITY_NS: i64 = 2_000_000_000;
pub fn clock_ns() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos().min(i64::MAX as u128) as i64)
.unwrap_or(0)
}
pub fn body_content(bytes: &[u8]) -> Vec<u8> {
static FENCE: std::sync::LazyLock<regex::Regex> =
std::sync::LazyLock::new(|| regex::Regex::new(r"(?m)^---[ \t]*\r?$").unwrap());
let text = String::from_utf8_lossy(bytes);
let re = &*FENCE;
let Some(open) = re.find(&text).filter(|m| m.start() == 0) else {
return bytes.into();
};
let Some(close) = re.find_at(&text, open.end()) else {
return bytes.into();
};
text[close.start() + 3..].as_bytes().into()
}
pub fn prompt_fingerprint(text: &str) -> String {
let normalized = text
.replace("\r\n", "\n")
.replace('\r', "\n")
.split('\n')
.map(|l| l.trim_end_matches(|c: char| c.is_whitespace() || matches!(c, '\x1c'..='\x1f')))
.collect::<Vec<_>>()
.join("\n");
format!(
"{:x}",
Sha256::digest(
normalized
.trim_matches(|c: char| c.is_whitespace() || matches!(c, '\x1c'..='\x1f'))
.as_bytes()
)
)[..12]
.into()
}
pub fn mtime_granularity(raw: Option<&str>) -> i64 {
let parsed = raw.unwrap_or("").trim().parse::<f64>();
match parsed {
Ok(n) if n >= 0.0 && n.is_finite() => (n * 1_000_000.0) as i64,
_ => MTIME_GRANULARITY_NS,
}
}
#[derive(Clone, Copy, serde::Serialize, serde::Deserialize)]
pub struct Stat {
pub size: u64,
pub mtime_ns: i64,
}
impl Stat {
pub fn read(path: &Path) -> std::io::Result<Self> {
let m = fs::metadata(path)?;
#[cfg(unix)]
let mtime_ns = {
use std::os::unix::fs::MetadataExt;
m.mtime()
.saturating_mul(1_000_000_000)
.saturating_add(m.mtime_nsec())
};
#[cfg(not(unix))]
let mtime_ns = match m.modified()?.duration_since(UNIX_EPOCH) {
Ok(d) => d.as_nanos() as i64,
Err(e) => -(e.duration().as_nanos() as i64),
};
Ok(Self {
size: m.len(),
mtime_ns,
})
}
}
pub fn stat_fresh(entry: &Value, st: Stat, granularity: i64) -> bool {
entry.is_object()
&& entry["size"] == json!(st.size)
&& entry["mtime_ns"] == json!(st.mtime_ns)
&& entry["indexed_at_ns"]
.as_i64()
.is_some_and(|at| st.mtime_ns as i128 + granularity as i128 <= at as i128)
}
/// A content hash taken like [`ExtractionCache::file_hash`] without touching
/// the cache, so workers can hash in parallel; [`ExtractionCache::apply_hash`]
/// records it.
pub struct HashProbe {
result: Result<String>,
update: Option<StatUpdate>,
read_source: bool,
}
impl HashProbe {
fn failed(error: crate::Error) -> Self {
Self {
result: Err(error),
update: None,
read_source: false,
}
}
pub fn hash(&self) -> Option<&str> {
self.result.as_deref().ok()
}
}
/// The stat-index entry a freshly computed hash refreshes.
struct StatUpdate {
key: String,
salt: String,
stat: Stat,
at: i64,
}
/// Resolutions shared by the probes of one batch: the root and each source
/// directory resolve once instead of once per file (every component of a
/// resolved path costs an `lstat`). Resolving a directory and then the file
/// name from it walks the same components as resolving the whole path.
#[derive(Default)]
pub struct HashMemo {
shared: bool,
granularity: Option<i64>,
root: std::sync::OnceLock<PathBuf>,
dirs: std::sync::RwLock<std::collections::HashMap<PathBuf, PathBuf>>,
}
impl HashMemo {
pub fn shared(granularity: i64) -> Self {
Self {
shared: true,
granularity: Some(granularity),
..Self::default()
}
}
fn granularity(&self, cache: &ExtractionCache) -> i64 {
self.granularity.unwrap_or_else(|| cache.granularity())
}
fn resolve_root(&self, root: &Path) -> Result<PathBuf> {
if !self.shared {
return resolve(root);
}
if let Some(r) = self.root.get() {
return Ok(r.clone());
}
let r = resolve(root)?;
Ok(self.root.get_or_init(|| r).clone())
}
fn resolve_file(&self, p: &Path) -> Result<PathBuf> {
let (Some(parent), Some(name)) = (p.parent(), p.file_name()) else {
return resolve(p);
};
if !self.shared || !p.is_absolute() {
return resolve(p);
}
let known = self
.dirs
.read()
.unwrap_or_else(|e| e.into_inner())
.get(parent)
.cloned();
let dir = match known {
Some(dir) => dir,
None => {
let dir = resolve(parent)?;
self.dirs
.write()
.unwrap_or_else(|e| e.into_inner())
.insert(parent.to_path_buf(), dir.clone());
dir
}
};
crate::fsutil::resolve_missing_from(&dir, Path::new(name)).map_err(crate::Error::from)
}
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
#[serde(tag = "kind", content = "value", rename_all = "snake_case")]
pub enum Prompt {
Text(String),
File(PathBuf),
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
#[serde(default)]
pub struct CacheOptions {
pub root: PathBuf,
pub kind: String,
pub cache_root: Option<PathBuf>,
pub prompt: Option<Prompt>,
pub prompt_file: Option<PathBuf>,
pub allow_legacy: bool,
pub allow_partial: bool,
/// Extra key material (`load_cached`/`save_cached` `salt`): the entry key
/// becomes [`salted_key`] of the content hash and this.
pub salt: Option<String>,
}
impl Default for CacheOptions {
fn default() -> Self {
Self {
root: PathBuf::from("."),
kind: "ast".into(),
cache_root: None,
prompt: None,
prompt_file: None,
allow_legacy: true,
allow_partial: false,
salt: None,
}
}
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
#[serde(default)]
pub struct ExtractionCache {
pub output: String,
pub extractor_version: String,
pub schema: i64,
pub cleaned_ast_dirs: HashSet<String>,
pub legacy_semantic_hits: usize,
pub corrupt_cache_entries: usize,
pub prompt_fp_cache: Map<String, Value>,
pub stat_index: Map<String, Value>,
pub stat_index_root: Option<PathBuf>,
pub stat_index_anchor: Option<PathBuf>,
pub stat_index_dirty: bool,
/// The on-disk index file of the bound root, captured at bind time so a
/// root switch flushes the outgoing root to its own file (#3989).
pub stat_index_path: Option<PathBuf>,
pub warnings: Vec<String>,
pub source_reads: usize,
pub computed_word_counts: usize,
/// A supplied clock/granularity is useful for embedded deterministic callers.
pub now_ns: Option<i64>,
pub granularity_ns: Option<i64>,
/// The grammar fingerprint of the AST directory; `None` is this build's
/// [`grammar_fingerprint`] (a comparison may supply the reference's).
pub grammar_fingerprint: Option<String>,
}
impl Default for ExtractionCache {
fn default() -> Self {
Self {
output: paths::OUT_DIR.to_string(),
extractor_version: format!(
"{}-graphicle.{}",
crate::UPSTREAM_VERSION,
env!("CARGO_PKG_VERSION")
),
schema: AST_CACHE_SCHEMA,
cleaned_ast_dirs: HashSet::new(),
legacy_semantic_hits: 0,
corrupt_cache_entries: 0,
prompt_fp_cache: Map::new(),
stat_index: Map::new(),
stat_index_root: None,
stat_index_anchor: None,
stat_index_dirty: false,
stat_index_path: None,
warnings: vec![],
source_reads: 0,
computed_word_counts: 0,
now_ns: None,
granularity_ns: None,
grammar_fingerprint: None,
}
}
}
impl ExtractionCache {
/// `v{version}-s{schema}-g{grammars}`: the versioned AST directory name.
pub fn ast_version(&self) -> String {
format!(
"v{}-s{}-g{}",
self.extractor_version,
self.schema,
self.grammar_fingerprint
.as_deref()
.unwrap_or_else(|| grammar_fingerprint())
)
}
pub fn now(&self) -> i64 {
self.now_ns.unwrap_or_else(clock_ns)
}
pub fn granularity(&self) -> i64 {
self.granularity_ns.unwrap_or_else(|| {
mtime_granularity(
std::env::var("GRAPHICLE_MTIME_GRANULARITY_MS")
.ok()
.as_deref(),
)
})
}
pub fn base(&self, root: &Path) -> Result<PathBuf> {
let out = Path::new(&self.output);
Ok(if out.is_absolute() {
out.into()
} else {
resolve(root)?.join(out)
})
}
pub fn stat_file(&self, root: &Path) -> Result<PathBuf> {
Ok(self.base(root)?.join("cache/stat-index.json"))
}
pub fn cleanup_stale(&mut self, base: &Path, current: &Path) {
if !self.cleaned_ast_dirs.insert(ps(current)) || !base.is_dir() {
return;
}
let Ok(children) = fs::read_dir(base) else {
return;
};
for child in children.flatten() {
let p = child.path();
if p == current {
continue;
}
if p.is_dir() && child.file_name().to_string_lossy().starts_with('v') {
if !fs::symlink_metadata(&p).is_ok_and(|m| m.file_type().is_symlink()) {
let _ = fs::remove_dir_all(p);
}
} else if p.extension().is_some_and(|s| s == "json") {
let _ = fs::remove_file(p);
}
}
}
pub fn cache_dir(&mut self, root: &Path, kind: &str, fp: Option<&str>) -> Result<PathBuf> {
let mut d = self.base(root)?.join("cache").join(kind);
if kind == "ast" {
let base = d;
d = base.join(self.ast_version());
self.cleanup_stale(&base, &d);
} else if let Some(fp) = fp.filter(|s| !s.is_empty()) {
d = d.join(format!("p{fp}"));
}
fs::create_dir_all(&d).map_err(crate::Error::from)?;
Ok(d)
}
/// Where the parser-derived facts of files cached for plain AST `options`
/// are kept (`cache/syntax/v…-s…/<entry hash>.json`, keyed like the AST
/// entries and versioned with them), so a run that reads a record from
/// the cache does not parse the file for its cross-file passes either.
/// Stale versions are cleared once; the directory is created on write.
pub fn syntax_dir(&mut self, options: &CacheOptions) -> Option<PathBuf> {
if !Self::plain_ast(options) {
return None;
}
let location = options.cache_root.as_deref().unwrap_or(&options.root);
let base = self.base(location).ok()?.join("cache").join("syntax");
let d = base.join(self.ast_version());
self.cleanup_stale(&base, &d);
Some(d)
}
pub fn resolve_prompt(
&mut self,
prompt: Option<&Prompt>,
file: Option<&Path>,
) -> Option<String> {
let prompt = file
.map(|p| Prompt::File(p.into()))
.or_else(|| prompt.cloned())?;
let memo_key = file.and_then(|p| {
Stat::read(p)
.ok()
.map(|st| json!([ps(p), st.size, st.mtime_ns]).to_string())
});
if let Some(key) = &memo_key
&& let Some(v) = self.prompt_fp_cache.get(key).and_then(Value::as_str)
{
return Some(v.into());
}
let content = match &prompt {
Prompt::Text(s) => Ok(s.clone()),
Prompt::File(p) => fs::read(p).map(|b| String::from_utf8_lossy(&b).into_owned()),
};
match content {
Ok(s) => {
let fp = prompt_fingerprint(&s);
if let Some(k) = memo_key {
self.prompt_fp_cache.insert(k, json!(fp));
}
Some(fp)
}
Err(e) => {
let p = match prompt {
Prompt::Text(s) => s,
Prompt::File(p) => ps(&p),
};
self.warnings.push(format!("could not read extraction prompt {} ({}); semantic cache entries cannot be attributed to a prompt version and fall back to the unversioned layout, so this run may replay entries from an older extraction prompt (#1939).",python_compat::repr(&json!(p)),crate::io_compat::read_error(&e,Path::new(&p))));
None
}
}
}
pub fn ensure_index(&mut self, root: &Path, cache_root: Option<&Path>) -> Result<()> {
let new_root = resolve(cache_root.unwrap_or(root))?;
if let Some(current) = self.stat_index_root.clone() {
// `Path ==` in pathlib: on Windows `file_hash` binds the
// normcased root, which names the same folder.
if crate::fsutil::path_key(¤t) == crate::fsutil::path_key(&new_root) {
return Ok(());
}
// A later call naming a different cache root rebinds (#3989): the
// outgoing root's pending entries go to its own file first, so no
// root's stat-index.json ever holds another project's paths.
self.warnings.push(format!(
"stat index switched from cache root {} to {} within one process; each root's stat-index.json now only reflects its own files, but library callers crossing project roots in one process should still use a fresh process per root to avoid losing the fastpath (#3989).",
python_compat::repr(&json!(ps(¤t))),
python_compat::repr(&json!(ps(&new_root)))
));
self.flush();
}
let anchor = resolve(root)?;
let p = self.stat_file(&new_root)?;
self.stat_index_root = Some(new_root);
self.stat_index_anchor = Some(anchor.clone());
self.stat_index_path = Some(p.clone());
self.stat_index.clear();
if let Ok(s) = fs::read_to_string(&p)
&& let Ok(Value::Object(raw)) = serde_json::from_str(&s)
{
for (k, v) in raw {
if Path::new(&k).is_absolute() {
self.stat_index.entry(k).or_insert(v);
} else {
self.stat_index.insert(stat_absolute(&k, &anchor), v);
}
}
}
self.stat_index_dirty = false;
Ok(())
}
pub fn flush(&mut self) {
if !self.stat_index_dirty {
return;
}
let Some(root) = self.stat_index_root.as_ref() else {
return;
};
if !root.is_dir() {
self.stat_index_dirty = false;
return;
}
let Some(path) = self
.stat_index_path
.clone()
.or_else(|| self.stat_file(root).ok())
else {
self.stat_index_dirty = false;
return;
};
let mut on_disk = Map::new();
for (k, v) in &self.stat_index {
if Path::new(k).exists() {
let key = self
.stat_index_anchor
.as_ref()
.map(|a| stat_relative(k, a))
.unwrap_or(k.clone());
on_disk.insert(key, v.clone());
}
}
let result = (|| -> Result<()> {
fs::create_dir_all(path.parent().unwrap()).map_err(crate::Error::from)?;
let mut data = serde_json::to_vec(&on_disk).map_err(crate::Error::from)?; // The disk layout uses ASCII escaping like json.dumps.
if data.iter().any(|b| *b >= 128) {
data = compact_ascii(&Value::Object(on_disk))?;
}
atomic_write(&path, &data, "stat-index.")
})();
let _ = result;
self.stat_index_dirty = false;
}
pub fn stat_entry(&mut self, key: &str, st: Stat, at: i64) -> &mut Value {
let e = self
.stat_index
.entry(key.to_string())
.or_insert(Value::Null);
if !e.is_object() || e["size"] != json!(st.size) || e["mtime_ns"] != json!(st.mtime_ns) {
*e = json!({"size":st.size,"mtime_ns":st.mtime_ns});
}
e["indexed_at_ns"] = json!(at);
e
}
pub fn file_hash(
&mut self,
path: &Path,
root: &Path,
cache_root: Option<&Path>,
) -> Result<String> {
let p = normalize(path);
let root = normalize(root);
if !p.is_file() {
return Err(format!("file_hash requires a file, got: {}", ps(&p)).into());
}
self.ensure_index(&root, cache_root)?;
let probe = self.probe_hash(&p, &root, &HashMemo::default());
self.apply_hash(probe)
}
/// [`Self::file_hash`] for many files: the stats, reads and SHA-256 run on
/// up to `workers` threads, then the stat-index updates are applied in
/// `paths` order, so the index holds what hashing the files one by one
/// would leave (up to the observation timestamps).
pub fn file_hashes(
&mut self,
paths: &[&Path],
root: &Path,
cache_root: Option<&Path>,
workers: usize,
) -> Vec<Result<String>> {
let mut out = Vec::with_capacity(paths.len());
// One by one until a file has loaded the index, so the first
// `ensure_index` sees exactly what sequential hashing would.
let mut next = 0;
while next < paths.len() && self.stat_index_root.is_none() {
out.push(self.file_hash(paths[next], root, cache_root));
next += 1;
}
let root = normalize(root);
let memo = HashMemo::shared(self.granularity());
let this = &*self;
let probes = crate::par::map(&paths[next..], workers, |path| {
let p = normalize(path);
if !p.is_file() {
return HashProbe::failed(
format!("file_hash requires a file, got: {}", ps(&p)).into(),
);
}
this.probe_hash(&p, &root, &memo)
});
out.extend(probes.into_iter().map(|probe| self.apply_hash(probe)));
out
}
/// The read-only part of [`Self::file_hash`] (after its file check and
/// index load): the digest, plus the stat-index change it implies, which
/// [`Self::apply_hash`] records.
pub fn probe_hash(&self, p: &Path, root: &Path, memo: &HashMemo) -> HashProbe {
let probe = || -> Result<HashProbe> {
let resolved = memo.resolve_file(p)?;
let abs_key = ps(&resolved);
let resolved_root = memo.resolve_root(root)?;
let salt = if let Ok(rel) = resolved.strip_prefix(&resolved_root) {
let walked = absolute(p)?;
let walked_root = absolute(root)?;
let rel = if let Ok(r) = walked.strip_prefix(&walked_root) {
r.into()
} else {
walked
.ancestors()
.skip(1)
.find(|parent| resolve(parent).is_ok_and(|p| p == resolved_root))
.and_then(|parent| walked.strip_prefix(parent).ok())
.unwrap_or(rel)
.to_path_buf()
};
ps(&rel)
.replace(std::path::MAIN_SEPARATOR, "/")
.to_lowercase()
} else {
abs_key
.replace(std::path::MAIN_SEPARATOR, "/")
.to_lowercase()
};
let st = Stat::read(p).ok();
if let Some(st) = st
&& let Some(entry) = self.stat_index.get(&abs_key)
&& stat_fresh(entry, st, memo.granularity(self))
&& let Some(hash) = entry
.get("hashes")
.and_then(|v| v.get(&salt))
.and_then(Value::as_str)
{
return Ok(HashProbe {
result: Ok(hash.into()),
update: None,
read_source: false,
});
}
let at = self.now();
let raw = fs::read(p).map_err(|e| crate::io_compat::read_error(&e, p))?;
let content = if p
.extension()
.is_some_and(|s| s.to_string_lossy().to_lowercase() == "md")
{
body_content(&raw)
} else {
raw
};
let mut h = Sha256::new();
h.update(content);
h.update([0]);
h.update(salt.as_bytes());
let digest = format!("{:x}", h.finalize());
Ok(HashProbe {
result: Ok(digest),
update: st.map(|stat| StatUpdate {
key: abs_key,
salt,
stat,
at,
}),
read_source: true,
})
};
probe().unwrap_or_else(HashProbe::failed)
}
/// Record a [`HashProbe`] in the stat index and counters; yields its digest.
pub fn apply_hash(&mut self, probe: HashProbe) -> Result<String> {
if probe.read_source {
self.source_reads += 1;
}
let digest = probe.result?;
if let Some(u) = probe.update {
let entry = self.stat_entry(&u.key, u.stat, u.at);
if !entry["hashes"].is_object() {
entry["hashes"] = json!({});
}
entry["hashes"][u.salt] = json!(digest);
entry.as_object_mut().unwrap().shift_remove("hash");
self.stat_index_dirty = true;
}
Ok(digest)
}
pub fn cached_word_count(
&mut self,
path: &Path,
root: &Path,
compute: impl FnOnce(&Path) -> Result<Value>,
cache_root: Option<&Path>,
) -> Result<Value> {
let lookup = self.word_count_lookup(path, root, cache_root)?;
match lookup {
WordCountLookup::Hit { value } => Ok(value),
WordCountLookup::Missing { ticket } => {
let n = compute(path)?;
self.complete_word_count(ticket, n)
}
}
}
pub fn word_count_lookup(
&mut self,
path: &Path,
root: &Path,
cache_root: Option<&Path>,
) -> Result<WordCountLookup> {
let p = normalize(path);
let root = normalize(root);
self.ensure_index(&root, cache_root)?;
let key = ps(&resolve(&p)?);
let st = Stat::read(&p).ok();
if let Some(st) = st
&& let Some(e) = self.stat_index.get(&key)
&& stat_fresh(e, st, self.granularity())
&& let Some(n) = e.get("word_count")
{
return Ok(WordCountLookup::Hit { value: n.clone() });
}
Ok(WordCountLookup::Missing {
ticket: WordCountTicket {
key,
st,
observed_at_ns: self.now(),
},
})
}
pub fn complete_word_count(&mut self, ticket: WordCountTicket, n: Value) -> Result<Value> {
self.computed_word_counts += 1;
if let Some(st) = ticket.st {
self.stat_entry(&ticket.key, st, ticket.observed_at_ns)["word_count"] = n.clone();
self.stat_index_dirty = true;
}
Ok(n)
}
/// [`Self::cached_word_count`] for many files: lookups and counting run on
/// up to `workers` threads, then the index updates are applied in `paths`
/// order, stopping at (and returning) the first failed lookup, as a
/// sequential loop would.
pub fn cached_word_counts(
&mut self,
paths: &[&Path],
root: &Path,
cache_root: Option<&Path>,
workers: usize,
compute: impl Fn(&Path) -> Value + Sync,
) -> Result<Vec<Value>> {
if paths.is_empty() {
return Ok(vec![]);
}
self.ensure_index(&normalize(root), cache_root)?;
let memo = HashMemo::shared(self.granularity());
let this = &*self;
let looked = crate::par::map(paths, workers, |path| -> Result<WordCount> {
let p = normalize(path);
let key = ps(&memo.resolve_file(&p)?);
let st = Stat::read(&p).ok();
if let Some(st) = st
&& let Some(e) = this.stat_index.get(&key)
&& stat_fresh(e, st, memo.granularity(this))
&& let Some(n) = e.get("word_count")
{
return Ok(WordCount::Hit(n.clone()));
}
let ticket = WordCountTicket {
key,
st,
observed_at_ns: this.now(),
};
Ok(WordCount::Counted(ticket, compute(path)))
});
let mut out = Vec::with_capacity(paths.len());
for looked in looked {
out.push(match looked? {
WordCount::Hit(n) => n,
WordCount::Counted(ticket, n) => self.complete_word_count(ticket, n)?,
});
}
Ok(out)
}
pub fn load(&mut self, path: &Path, options: &CacheOptions) -> Result<Option<Value>> {
let Some(location) = self.locate(path, options)? else {
return Ok(None);
};
let parsed = Self::parse_located(&location, path, options)?;
Ok(self.record_located(&location, parsed))
}
/// Where the cache entry for `path` lives, if one exists. This is the
/// part of [`Self::load`] that touches cache state (content hash index,
/// prompt fingerprint, directory layout); reading the entry itself is the
/// pure [`Self::parse_located`], so many entries can be parsed in parallel
/// after locating them one by one.
pub fn locate(
&mut self,
path: &Path,
options: &CacheOptions,
) -> Result<Option<CacheEntryLocation>> {
let Ok(hash) = self.file_hash(path, &options.root, options.cache_root.as_deref()) else {
return Ok(None);
};
let hash = salted_key(&hash, options.salt.as_deref());
let fp = self.resolve_prompt(options.prompt.as_ref(), options.prompt_file.as_deref());
let location = options.cache_root.as_deref().unwrap_or(&options.root);
let mut entry = self
.cache_dir(location, &options.kind, fp.as_deref())?
.join(format!("{hash}.json"));
let mut legacy = false;
if fp.is_some() && !entry.exists() && options.allow_legacy {
let p = self
.cache_dir(location, &options.kind, None)?
.join(format!("{hash}.json"));
if p.exists() {
entry = p;
legacy = true;
}
}
if !entry.exists() {
return Ok(None);
}
Ok(Some(CacheEntryLocation { entry, legacy }))
}
/// Read and validate a located entry without touching cache state.
pub fn parse_located(
location: &CacheEntryLocation,
path: &Path,
options: &CacheOptions,
) -> Result<LocatedEntry> {
let data = match fs::read_to_string(&location.entry) {
Ok(d) => d,
Err(e) if e.kind() == std::io::ErrorKind::InvalidData => return Err(e.into()),
Err(_) => return Ok(LocatedEntry::Miss),
};
let mut result: Value = match serde_json::from_str(&data) {
Ok(d) => d,
Err(_) => return Ok(LocatedEntry::Corrupt),
};
drop(data);
if !options.allow_partial && result.is_object() {
if truth(&result["partial"]) {
return Ok(LocatedEntry::Miss);
}
if options.kind.starts_with("semantic")
&& !truth(&result["nodes"])
&& !truth(&result["hyperedges"])
{
return Ok(LocatedEntry::Miss);
}
}
if options.kind.starts_with("semantic")
&& result.is_object()
&& !semantic_matches(&result, path, &options.root)?
{
return Ok(LocatedEntry::Miss);
}
if result.is_object() {
// A source under a walked directory alias can carry ROOT_MARKER
// too. Restore it before treating ordinary relative paths as such.
restore_entry_paths(&mut result, path, &options.root);
// A recorded target file that is gone makes the replayed edges
// name a path the run cannot remap: re-extract instead (#4185).
let targets = result
.as_object_mut()
.and_then(|m| m.shift_remove(CACHED_TARGETS_KEY));
if let Some(Value::Array(targets)) = targets
&& !targets
.iter()
.filter_map(Value::as_str)
.all(|t| target_file_present(t, &options.root))
{
return Ok(LocatedEntry::Miss);
}
}
Ok(LocatedEntry::Hit(result))
}
/// Account a parsed entry in this cache's counters and yield the hit.
pub fn record_located(
&mut self,
location: &CacheEntryLocation,
parsed: LocatedEntry,
) -> Option<Value> {
match parsed {
LocatedEntry::Hit(value) => {
if location.legacy {
self.legacy_semantic_hits += 1;
}
Some(value)
}
LocatedEntry::Corrupt => {
self.corrupt_cache_entries += 1;
None
}
LocatedEntry::Miss => None,
}
}
/// Remove only the current content's entries in this prompt/mode namespace.
/// Rejected extraction results must not become permanent cache hits on retry.
pub fn invalidate(&mut self, path: &Path, options: &CacheOptions) -> Result<()> {
let hash = self.file_hash(path, &options.root, options.cache_root.as_deref())?;
let fp = self.resolve_prompt(options.prompt.as_ref(), options.prompt_file.as_deref());
let location = options.cache_root.as_deref().unwrap_or(&options.root);
let mut directories = vec![self.cache_dir(location, &options.kind, fp.as_deref())?];
if fp.is_some() && options.allow_legacy {
directories.push(self.cache_dir(location, &options.kind, None)?);
}
for directory in directories {
match fs::remove_file(directory.join(format!("{hash}.json"))) {
Ok(()) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(error.into()),
}
}
Ok(())
}
pub fn save(&mut self, path: &Path, result: &Value, options: &CacheOptions) -> Result<()> {
if !path.is_file() {
return Ok(());
}
let bytes = Self::encode_entry(path, result, options)?;
self.save_encoded(path, &bytes, options, None)
}
/// The on-disk bytes [`Self::save`] writes for `result`: the per-file
/// record with sources and ids made relative to the root. Pure, so
/// extraction workers can encode their own records.
pub fn encode_entry(path: &Path, result: &Value, options: &CacheOptions) -> Result<Vec<u8>> {
let mut on_disk = result.clone();
if on_disk.is_object() {
if options.kind == "ast" {
let targets = present_target_files(&on_disk, &options.root);
if !targets.is_empty() {
on_disk[CACHED_TARGETS_KEY] = json!(targets);
}
}
relativize_sources(&mut on_disk, &options.root);
relativize_ids(&mut on_disk, path, &options.root);
}
let mut bytes = vec![];
python_json::write(&mut bytes, &on_disk, None, true).map_err(crate::Error::from)?;
Ok(bytes)
}
/// Write bytes from [`Self::encode_entry`] as the entry for `path`.
pub fn save_encoded(
&mut self,
path: &Path,
bytes: &[u8],
options: &CacheOptions,
syntax: Option<&[u8]>,
) -> Result<()> {
if !path.is_file() {
return Ok(());
}
let hash = salted_key(
&self.file_hash(path, &options.root, options.cache_root.as_deref())?,
options.salt.as_deref(),
);
let fp = self.resolve_prompt(options.prompt.as_ref(), options.prompt_file.as_deref());
let dir = self.cache_dir(
options.cache_root.as_deref().unwrap_or(&options.root),
&options.kind,
fp.as_deref(),
)?;
let entry = dir.join(format!("{hash}.json"));
atomic_write(&entry, bytes, &format!("{hash}."))?;
if let Some(syntax) = syntax
&& let Some(dir) = self.syntax_dir(options)
{
write_syntax(&dir, &hash, syntax);
}
Ok(())
}
/// Whether AST entries for `options` can be located and saved without the
/// prompt/legacy handling: no prompt, so the entry directory is the one
/// versioned AST directory.
fn plain_ast(options: &CacheOptions) -> bool {
options.kind == "ast" && options.prompt.is_none() && options.prompt_file.is_none()
}
/// The AST entry directory [`Self::save_encoded`] would write to, once
/// this cache has loaded its stat index and run [`Self::cache_dir`] for
/// it (which clears stale versions once per cache). Until then, `None`:
/// saving must go through [`Self::save_encoded`].
pub fn shared_ast_dir(&self, options: &CacheOptions) -> Option<PathBuf> {
if !Self::plain_ast(options) || self.stat_index_root.is_none() {
return None;
}
let location = options.cache_root.as_deref().unwrap_or(&options.root);
let dir = self
.base(location)
.ok()?
.join("cache")
.join("ast")
.join(self.ast_version());
self.cleaned_ast_dirs.contains(&ps(&dir)).then_some(dir)
}
/// [`Self::save_encoded`] from a worker thread, for a `dir` from
/// [`Self::shared_ast_dir`]: the same file check, hash, directory and
/// bytes. The stat-index change the hash implies comes back as a probe
/// for [`Self::apply_hash`] (applied even when the write fails, as the
/// sequential save records the hash before writing).
pub fn save_encoded_shared(
&self,
path: &Path,
bytes: &[u8],
options: &CacheOptions,
dir: &Path,
memo: &HashMemo,
syntax: Option<(&Path, &[u8])>,
) -> (Option<HashProbe>, Result<()>) {
if !path.is_file() {
return (None, Ok(()));
}
let p = normalize(path);
if !p.is_file() {
return (
None,
Err(format!("file_hash requires a file, got: {}", ps(&p)).into()),
);
}
let mut probe = self.probe_hash(&p, &normalize(&options.root), memo);
let hash = match std::mem::replace(&mut probe.result, Ok(String::new())) {
Ok(hash) => {
probe.result = Ok(hash.clone());
salted_key(&hash, options.salt.as_deref())
}
Err(e) => return (None, Err(e)),
};
let written = fs::create_dir_all(dir)
.map_err(crate::Error::from)
.and_then(|()| {
atomic_write(
&dir.join(format!("{hash}.json")),
bytes,
&format!("{hash}."),
)
});
if written.is_ok()
&& let Some((syntax_dir, syntax)) = syntax
{
write_syntax(syntax_dir, &hash, syntax);
}
(Some(probe), written)
}
/// [`Self::load`] for many AST entries with `post` applied to each hit:
/// the hashing and entry reads run on up to `workers` threads, and the
/// stat-index updates and counters are applied in `paths` order, as a
/// sequential loop would. `Ok(None)` is a miss.
pub fn load_many(
&mut self,
paths: &[&Path],
options: &CacheOptions,
workers: usize,
post: impl Fn(&Path, &Path, &mut Value) + Sync,
) -> Vec<Result<Option<Value>>> {
self.load_many_salted(paths, options, workers, |_| options.salt.clone(), post)
}
/// [`Self::load_many`] with each path's own key `salt` (`cache_salt`),
/// computed on the calling thread.
pub fn load_many_salted(
&mut self,
paths: &[&Path],
options: &CacheOptions,
workers: usize,
salt: impl Fn(&Path) -> Option<String>,
post: impl Fn(&Path, &Path, &mut Value) + Sync,
) -> Vec<Result<Option<Value>>> {
let salts = paths.iter().map(|p| salt(p)).collect::<Vec<_>>();
if !Self::plain_ast(options) {
// Prompted or semantic entries: locate one by one (prompt
// fingerprints, legacy layout), then parse in parallel.
let located = paths
.iter()
.zip(&salts)
.map(|(p, salt)| {
let options = CacheOptions {
salt: salt.clone(),
..options.clone()
};
self.locate(p, &options)
})
.collect::<Vec<_>>();
let parsed = crate::par::map(
&located.iter().zip(paths).collect::<Vec<_>>(),
workers,
|(location, path)| match location {
Ok(Some(location)) => Some(Self::parse_located(location, path, options).map(
|mut entry| {
if let LocatedEntry::Hit(value) = &mut entry {
post(path, &location.entry, value);
}
entry
},
)),
_ => None,
},
);
return located
.into_iter()
.zip(parsed)
.map(|(location, parsed)| match (location, parsed) {
(Ok(Some(location)), Some(parsed)) => {
parsed.map(|entry| self.record_located(&location, entry))
}
(Ok(_), _) => Ok(None),
(Err(e), _) => Err(e),
})
.collect();
}
let hashes = self.file_hashes(paths, &options.root, options.cache_root.as_deref(), workers);
let location = options.cache_root.as_deref().unwrap_or(&options.root);
// `locate` resolves the entry directory after each successful hash;
// it is the same directory every time, created (and stale versions
// cleared) by the first call.
let dir = hashes
.iter()
.any(Result::is_ok)
.then(|| self.cache_dir(location, &options.kind, None));
let entries = hashes
.into_iter()
.zip(&salts)
.map(|(hash, salt)| match (hash, &dir) {
(Err(_), _) | (_, None) => Ok(None),
(Ok(hash), Some(Ok(dir))) => Ok(Some(
dir.join(format!("{}.json", salted_key(&hash, salt.as_deref()))),
)),
(Ok(_), Some(Err(e))) => Err(crate::Error::Internal(e.to_string())),
})
.collect::<Vec<_>>();
let parsed = crate::par::map(
&entries.iter().zip(paths).collect::<Vec<_>>(),
workers,
|(entry, path)| match entry {
Ok(Some(entry)) if entry.exists() => {
let location = CacheEntryLocation {
entry: entry.clone(),
legacy: false,
};
Some(
Self::parse_located(&location, path, options).map(|mut entry| {
if let LocatedEntry::Hit(value) = &mut entry {
post(path, &location.entry, value);
}
(location, entry)
}),
)
}
_ => None,
},
);
entries
.into_iter()
.zip(parsed)
.map(|(entry, parsed)| {
entry?;
match parsed {
Some(Ok((location, entry))) => Ok(self.record_located(&location, entry)),
Some(Err(e)) => Err(e),
None => Ok(None),
}
})
.collect()
}
fn cache_entries(&self, root: &Path, kinds: &[&str], flat: bool) -> Result<Vec<PathBuf>> {
let base = self.base(root)?.join("cache");
let mut entries = vec![];
if flat {
json_files(&base, false, &mut entries)
}
for kind in kinds {
json_files(&base.join(kind), true, &mut entries);
}
Ok(entries)
}
pub fn cached_files(&self, root: &Path) -> Result<BTreeSet<String>> {
Ok(self
.cache_entries(root, &["ast", "semantic", "semantic-deep"], true)?
.iter()
.filter_map(|p| p.file_stem().map(|s| s.to_string_lossy().into_owned()))
.collect())
}
pub fn clear(&self, root: &Path) -> Result<()> {
for f in self.cache_entries(root, &["ast", "semantic", "semantic-deep"], true)? {
fs::remove_file(f).map_err(crate::Error::from)?;
}
Ok(())
}
pub fn prune(&self, root: &Path, live: &HashSet<String>) -> Result<usize> {
let mut pruned = 0;
for p in self.cache_entries(root, &["semantic", "semantic-deep"], false)? {
let stem = p.file_stem().unwrap_or_default().to_string_lossy();
if !live.contains(stem.as_ref()) && fs::remove_file(p).is_ok() {
pruned += 1
}
}
Ok(pruned)
}
pub fn check_semantic(&mut self, files: &[String], options: &SemanticOptions) -> Result<Value> {
let opt = options.cache_options();
let mut nodes = vec![];
let mut edges = vec![];
let mut hyper = vec![];
let mut uncached = vec![];
let before = self.legacy_semantic_hits;
let corrupt = self.corrupt_cache_entries;
for f in files {
let p = Path::new(f);
let p = if p.is_absolute() {
p.into()
} else {
options.root.join(p)
};
if let Some(result) = self.load(&p, &opt)? {
nodes.extend_from_slice(ar(&result["nodes"]));
edges.extend_from_slice(ar(&result["edges"]));
hyper.extend_from_slice(ar(&result["hyperedges"]));
} else {
uncached.push(f.clone())
}
}
let n = self.legacy_semantic_hits - before;
if n > 0 {
self.warnings.push(format!("{n} semantic cache entr{} predate extraction-prompt fingerprinting and were written by an unknown prompt version; they were replayed as-is, so this graph may mix extraction vintages. Re-run with --force (or GRAPHICLE_FORCE=1) to re-extract them with the current prompt (#1939).",if n==1{"y"}else{"ies"}));
}
let n = self.corrupt_cache_entries - corrupt;
if n > 0 {
self.warnings.push(format!("{n} semantic cache entr{} could not be parsed as JSON and were treated as misses, so those files were re-extracted. A corrupt entry stays on disk and fails again every run; run with --force (or GRAPHICLE_FORCE=1) to rewrite them, or clear the cache to stop paying for the re-extraction (#2405).",if n==1{"y"}else{"ies"}));
}
Ok(json!([nodes, edges, hyper, uncached]))
}
pub fn save_semantic(
&mut self,
nodes: &[Value],
edges: &[Value],
hyper: &[Value],
options: &SemanticOptions,
) -> Result<usize> {
let matcher = SourceMatcher::new(&options.root)?;
let mut groups = Map::<String, Value>::new();
for (bucket, items) in [("nodes", nodes), ("edges", edges), ("hyperedges", hyper)] {
for item in items {
let Some(src) = item
.get("source_file")
.filter(|s| truth(s))
.and_then(Value::as_str)
else {
continue;
};
let norm = matcher.normalized(src);
let mut item = item.clone();
if norm != src {
item["source_file"] = json!(norm);
}
groups.entry(norm).or_insert_with(empty_group)[bucket]
.as_array_mut()
.unwrap()
.push(item);
}
}
let allowed = options
.allowed_source_files
.as_ref()
.map(|p| {
p.iter()
.map(|p| matcher.identity(p))
.collect::<Result<BTreeSet<_>>>()
})
.transpose()?;
let partial = options
.partial_source_files
.as_ref()
.map(|p| {
p.iter()
.map(|p| matcher.identity(p))
.collect::<Result<BTreeSet<_>>>()
})
.transpose()?;
let recover = |f: &str| -> Result<(PathBuf, PathBuf)> {
let mut path = matcher.identity(f)?;
let mut resolved = resolve(&path).or_else(|_| absolute(&path))?;
if !resolved.is_file()
&& let Some(allowed) = &allowed
{
let matches = allowed
.iter()
.filter(|p| p.file_name() == path.file_name())
.collect::<Vec<_>>();
if matches.len() == 1 {
path = matches[0].clone();
resolved = path.clone();
}
}
let normalized = normalize(&path);
Ok((normalized, resolved))
};
if let Some(partial) = &partial {
let present = groups
.keys()
.map(|k| matcher.identity(k))
.collect::<Result<BTreeSet<_>>>()?;
for path in partial {
if !present.contains(path) {
groups.entry(ps(path)).or_insert_with(empty_group);
}
}
}
let skipped = |f: &str| -> Result<bool> {
let (path, resolved) = recover(f)?;
Ok(!resolved.is_file()
|| allowed
.as_ref()
.is_some_and(|allowed| !allowed.contains(&path)))
};
if allowed.is_some() {
let mut dropped = BTreeSet::new();
let mut written = BTreeSet::new();
for (f, result) in &groups {
let target = if skipped(f)? {
&mut dropped
} else {
&mut written
};
for n in ar(&result["nodes"]) {
if !n["id"].is_null()
&& let Some(k) = python_compat::scalar_key(&n["id"])
{
target.insert(k);
}
}
}
dropped.retain(|id| !written.contains(id));
if !dropped.is_empty() {
for (f, result) in &mut groups {
if !skipped(f)? {
prune_dangling(result, &dropped);
}
}
}
}
let mut saved = 0;
let mut not_file = 0;
let group_count = groups.len();
for (f, mut result) in groups {
let (path, p) = recover(&f)?;
if !p.is_file() {
not_file += 1;
continue;
}
if path != matcher.identity(&f)? {
let corrected = matcher.normalized(&ps(&path));
for bucket in ["nodes", "edges", "hyperedges"] {
for item in result[bucket].as_array_mut().unwrap() {
item["source_file"] = json!(corrected);
}
}
}
if allowed
.as_ref()
.is_some_and(|allowed| !allowed.contains(&path))
{
self.warnings.push(format!("semantic cache skipped out-of-scope source_file {}; the file was not dispatched for extraction",python_compat::repr(&json!(f))));
continue;
}
let mut prev_partial = false;
let mut opt = options.cache_options();
if options.merge_existing {
opt.allow_legacy = false;
opt.allow_partial = true;
if let Some(prev) = self.load(&path, &opt)?.filter(truth) {
prev_partial = truth(&prev["partial"]);
for bucket in ["nodes", "edges", "hyperedges"] {
let mut items = ar(&prev[bucket]).to_vec();
items.extend_from_slice(ar(&result[bucket]));
result[bucket] = json!(items);
}
}
}
let partial = partial.as_ref().is_some_and(|p| p.contains(&path))
|| group_partial(&result)
|| prev_partial;
if partial {
result["partial"] = json!(true)
}
if !partial && !truth(&result["nodes"]) && !truth(&result["hyperedges"]) {
continue;
}
self.save(&path, &result, &opt)?;
saved += 1;
}
if not_file > 0 && not_file == group_count {
self.warnings.push(format!("save_semantic_cache: all {not_file} source_file group(s) were skipped because their paths do not resolve to real files. This usually means ``root`` is anchored to the wrong directory (e.g. the --out directory instead of the corpus root). Pass the corpus directory as ``root`` and the output directory as ``cache_root`` (#1991)."));
}
Ok(saved)
}
}
/// On-disk position of one cache entry, from [`ExtractionCache::locate`].
#[derive(Clone, Debug)]
pub struct CacheEntryLocation {
pub entry: PathBuf,
/// Found under the unversioned prompt layout of an older extraction.
pub legacy: bool,
}
/// Outcome of parsing a located entry, from [`ExtractionCache::parse_located`].
#[derive(Debug)]
pub enum LocatedEntry {
Hit(Value),
Corrupt,
Miss,
}
/// Own this guard for an embedded run; dropping it flushes the stat index.
/// The standalone application flushes its process-wide instance at shutdown.
#[derive(Default)]
pub struct CacheSession(pub ExtractionCache);
impl std::ops::Deref for CacheSession {
type Target = ExtractionCache;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl std::ops::DerefMut for CacheSession {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl Drop for CacheSession {
fn drop(&mut self) {
self.0.flush()
}
}
#[derive(serde::Serialize, serde::Deserialize)]
pub struct WordCountTicket {
pub key: String,
pub st: Option<Stat>,
pub observed_at_ns: i64,
}
#[derive(serde::Serialize, serde::Deserialize)]
#[serde(tag = "status", rename_all = "snake_case")]
pub enum WordCountLookup {
Hit { value: Value },
Missing { ticket: WordCountTicket },
}
/// One file's outcome in [`ExtractionCache::cached_word_counts`].
enum WordCount {
Hit(Value),
Counted(WordCountTicket, Value),
}
fn empty_group() -> Value {
json!({"nodes":[],"edges":[],"hyperedges":[]})
}
pub fn group_partial(v: &Value) -> bool {
["nodes", "edges", "hyperedges"].iter().any(|k| {
ar(&v[*k])
.iter()
.any(|v| v.is_object() && truth(&v["_partial"]))
})
}
#[derive(Default, Clone, serde::Serialize, serde::Deserialize)]
#[serde(default)]
pub struct SemanticOptions {
pub root: PathBuf,
pub cache_root: Option<PathBuf>,
pub mode: Option<String>,
pub prompt: Option<Prompt>,
pub prompt_file: Option<PathBuf>,
pub merge_existing: bool,
pub allowed_source_files: Option<Vec<String>>,
pub partial_source_files: Option<Vec<String>>,
}
impl SemanticOptions {
pub fn cache_options(&self) -> CacheOptions {
CacheOptions {
root: self.root.clone(),
kind: self
.mode
.as_ref()
.map(|s| format!("semantic-{s}"))
.unwrap_or("semantic".into()),
cache_root: self.cache_root.clone(),
prompt: self.prompt.clone(),
prompt_file: self.prompt_file.clone(),
..Default::default()
}
}
}
fn compact_ascii(v: &Value) -> Result<Vec<u8>> {
let mut full = vec![];
python_json::write(&mut full, v, None, true).map_err(crate::Error::from)?;
let mut in_string = false;
let mut escape = false;
let mut out = vec![];
for b in full {
if b == b'"' && !escape {
in_string = !in_string;
}
if in_string || b != b' ' {
out.push(b);
}
escape = b == b'\\' && !escape;
}
Ok(out)
}
/// Write a file's syntax facts beside its AST entry (see
/// [`ExtractionCache::syntax_dir`]). They are an optimisation: a failed
/// write only means the next run parses the file.
fn write_syntax(dir: &Path, hash: &str, bytes: &[u8]) {
let _ = fs::create_dir_all(dir)
.map_err(crate::Error::from)
.and_then(|()| {
atomic_write(
&dir.join(format!("{hash}.json")),
bytes,
&format!("{hash}."),
)
});
}
fn atomic_write(path: &Path, bytes: &[u8], prefix: &str) -> Result<()> {
let parent = path.parent().ok_or("cache entry has no parent")?;
let mut f = tempfile::Builder::new()
.prefix(prefix)
.suffix(".tmp")
.tempfile_in(parent)
.map_err(crate::Error::from)?;
f.write_all(bytes).map_err(crate::Error::from)?;
let temp = f.into_temp_path();
paths::os_replace_with_fallback(&temp, path).map_err(crate::Error::from)?;
Ok(())
}
fn json_files(dir: &Path, recursive: bool, files: &mut Vec<PathBuf>) {
let Ok(entries) = fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let p = entry.path();
if p.extension().is_some_and(|s| s == "json") {
files.push(p.clone())
}
if recursive
&& p.is_dir()
&& !fs::symlink_metadata(&p).is_ok_and(|s| s.file_type().is_symlink())
{
json_files(&p, true, files)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn embedded_session_flushes_when_its_owner_leaves_scope() {
let root = tempfile::tempdir().unwrap();
let file = root.path().join("a.txt");
fs::write(&file, "text").unwrap();
let index = root.path().join("cache-output/cache/stat-index.json");
{
let mut session = CacheSession::default();
session.output = "cache-output".into();
session.file_hash(&file, root.path(), None).unwrap();
assert!(!index.exists());
}
let stored: Value = serde_json::from_slice(&fs::read(index).unwrap()).unwrap();
assert!(stored.get("a.txt").is_some());
}
/// A tree with plain files, a Markdown front-matter file, a symlinked
/// directory, a duplicate path, a missing path and a directory.
fn corpus() -> (tempfile::TempDir, Vec<PathBuf>) {
let root = tempfile::tempdir().unwrap();
let src = root.path().join("src");
fs::create_dir_all(src.join("deep/er")).unwrap();
let mut paths = vec![];
for i in 0..40 {
let p = src
.join(if i % 3 == 0 { "deep/er" } else { "." })
.join(format!("f{i}.py"));
fs::write(&p, format!("x = {i}\n")).unwrap();
paths.push(p);
}
fs::write(
src.join("notes.md"),
"---\ntitle: t\n---\nbody words here\n",
)
.unwrap();
paths.push(src.join("notes.md"));
#[cfg(unix)]
{
std::os::unix::fs::symlink(src.join("deep"), root.path().join("linked")).unwrap();
paths.push(root.path().join("linked/er/f0.py"));
}
paths.push(paths[1].clone());
paths.push(src.join("missing.py"));
paths.push(src.join("deep"));
(root, paths)
}
/// The stat index as written, without the observation clock.
fn index_without_clock(cache: &ExtractionCache) -> Value {
let mut index = Value::Object(cache.stat_index.clone());
for entry in index.as_object_mut().unwrap().values_mut() {
entry.as_object_mut().unwrap().shift_remove("indexed_at_ns");
}
index
}
#[test]
fn batched_hashes_match_hashing_one_by_one() {
let (root, paths) = corpus();
let refs = paths.iter().map(PathBuf::as_path).collect::<Vec<_>>();
// Old enough that the second pass is served from the index.
let now = clock_ns() + 10 * MTIME_GRANULARITY_NS;
for pass in 0..2 {
let mut serial = ExtractionCache {
now_ns: Some(now),
..Default::default()
};
let mut batched = ExtractionCache {
now_ns: Some(now),
..Default::default()
};
if pass == 1 {
for cache in [&mut serial, &mut batched] {
cache.ensure_index(root.path(), None).unwrap();
for p in &refs {
let _ = cache.file_hash(p, root.path(), None);
}
}
}
let one = refs
.iter()
.map(|p| {
serial
.file_hash(p, root.path(), None)
.map_err(|e| e.to_string())
})
.collect::<Vec<_>>();
let many = batched
.file_hashes(&refs, root.path(), None, 8)
.into_iter()
.map(|r| r.map_err(|e| e.to_string()))
.collect::<Vec<_>>();
assert_eq!(one, many, "pass {pass}");
assert!(one.iter().filter(|r| r.is_err()).count() == 2);
assert_eq!(index_without_clock(&serial), index_without_clock(&batched));
assert_eq!(
serial.stat_index.keys().collect::<Vec<_>>(),
batched.stat_index.keys().collect::<Vec<_>>()
);
// One by one, the repeated path is served by the entry its first
// occurrence just wrote; a batch hashes both occurrences.
assert_eq!(
serial.source_reads + usize::from(pass == 0),
batched.source_reads,
"pass {pass}"
);
}
}
#[test]
fn batched_word_counts_match_counting_one_by_one() {
let (root, paths) = corpus();
let files = paths
.iter()
.filter(|p| p.is_file())
.map(PathBuf::as_path)
.collect::<Vec<_>>();
let count = |p: &Path| json!(fs::read(p).map(|b| b.len()).unwrap_or(0));
let mut serial = ExtractionCache {
now_ns: Some(clock_ns()),
..Default::default()
};
let one = files
.iter()
.map(|p| {
serial
.cached_word_count(p, root.path(), |p| Ok(count(p)), None)
.unwrap()
})
.collect::<Vec<_>>();
let mut batched = ExtractionCache {
now_ns: serial.now_ns,
..Default::default()
};
let many = batched
.cached_word_counts(&files, root.path(), None, 8, count)
.unwrap();
assert_eq!(one, many);
assert_eq!(index_without_clock(&serial), index_without_clock(&batched));
assert_eq!(serial.computed_word_counts, batched.computed_word_counts);
}
#[test]
fn shared_saves_write_the_entry_the_sequential_save_writes() {
let (root, paths) = corpus();
let options = CacheOptions {
root: root.path().into(),
cache_root: Some(root.path().into()),
..Default::default()
};
let record = json!({"nodes":[{"id":"a","source_file":"src/f1.py"}],"edges":[]});
let path = &paths[1];
let bytes = ExtractionCache::encode_entry(path, &record, &options).unwrap();
let mut sequential = ExtractionCache {
output: "seq-out".into(),
..Default::default()
};
sequential
.save_encoded(path, &bytes, &options, None)
.unwrap();
let mut shared = ExtractionCache {
output: "shared-out".into(),
..Default::default()
};
assert!(shared.shared_ast_dir(&options).is_none());
// Locating prepares the directory the shared save needs.
assert!(shared.locate(path, &options).unwrap().is_none());
let dir = shared.shared_ast_dir(&options).unwrap();
let memo = HashMemo::shared(shared.granularity());
let (probe, written) =
shared.save_encoded_shared(path, &bytes, &options, &dir, &memo, None);
written.unwrap();
shared.apply_hash(probe.unwrap()).unwrap();
let entries = |out: &str| {
let mut found = vec![];
json_files(&root.path().join(out).join("cache"), true, &mut found);
found
.into_iter()
.filter(|p| p.file_name().unwrap() != "stat-index.json")
.map(|p| {
(
p.strip_prefix(root.path().join(out)).unwrap().to_path_buf(),
fs::read(&p).unwrap(),
)
})
.collect::<Vec<_>>()
};
assert_eq!(entries("seq-out"), entries("shared-out"));
assert_eq!(entries("seq-out").len(), 1);
assert_eq!(
index_without_clock(&sequential),
index_without_clock(&shared)
);
}
#[test]
fn a_later_root_rebinds_the_stat_index_to_its_own_file() {
// Graphify 985a017 (#3989): the index bound to the first root forever.
let a = tempfile::tempdir().unwrap();
let b = tempfile::tempdir().unwrap();
fs::write(a.path().join("a.py"), "x = 1\n").unwrap();
fs::write(b.path().join("b.py"), "y = 2\n").unwrap();
let now = clock_ns() + 10 * MTIME_GRANULARITY_NS;
let mut cache = ExtractionCache {
now_ns: Some(now),
..Default::default()
};
cache
.file_hash(&a.path().join("a.py"), a.path(), None)
.unwrap();
cache
.file_hash(&b.path().join("b.py"), b.path(), None)
.unwrap();
cache.flush();
let keys = |root: &Path| {
let file = cache.stat_file(&resolve(root).unwrap()).unwrap();
let raw: Map<String, Value> =
serde_json::from_str(&fs::read_to_string(file).unwrap()).unwrap();
raw.keys().cloned().collect::<Vec<_>>()
};
assert_eq!(keys(a.path()), ["a.py"]);
assert_eq!(keys(b.path()), ["b.py"]);
assert!(cache.warnings.iter().any(|w| w.contains("#3989")));
// Same root again: no rebind and no further warning.
let warned = cache.warnings.len();
cache.ensure_index(b.path(), None).unwrap();
assert_eq!(cache.warnings.len(), warned);
}
}