Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
//! Persistent commit index ("Fast Path 1" stat cache).
//!
//! After each successful commit, `GitService::commit` saves a snapshot of
//! `(path -> size, mtime_ns, oid)` for every file that was part of the
//! resulting tree. The next commit can then skip the read+SHA-1+write path
//! for any file whose `(size, mtime_ns)` match the cached entry — saving
//! the expensive blob materialization that produces the same OID we already
//! have.
//!
//! Correctness guard: every saved index records its `parent_oid` (the commit
//! the index reflects). On load, if `parent_oid != prev_head` (concurrent
//! commit, branch switch, first run) the cache is silently discarded and
//! commit proceeds via the slow path. Cache misses are *always* a soft
//! failure — we never produce an incorrect commit because of a stale or
//! corrupt index.
//!
//! Racy-clean guard: `(size, mtime_ns)` alone cannot distinguish "unchanged"
//! from "changed to the same size within the same filesystem clock tick as
//! the last commit". To close that window (the classic git "racy clean"
//! problem) the store records *when the index was written* in
//! [`CommitIndex::saved_at_ns`] — populated at `load()` time from the index
//! file's own mtime (local) or object `last_modified` (S3), i.e. the same
//! clock and granularity as the working-tree file mtimes it is compared
//! against. A cache entry is only trusted when its `mtime_ns` is strictly
//! older than `saved_at_ns`; an entry whose mtime is in the same tick as (or
//! newer than) the index write is "racy" and forced down the slow path. This
//! costs at most one extra read per file, and only for files that shared a
//! clock tick with a commit — steady-state Fast Path 1 is unaffected.
//!
//! The wire format is JSON for debuggability. OIDs are stored as 40-char hex.
//! Any deserialization or backend error is mapped to `Ok(None)` on `load()`,
//! and `save()` failures are logged by the caller (the commit itself has
//! already succeeded by the time we save).
//!
//! Per-(account, branch). The branch component is `validate_ref_name`-checked
//! before any path is constructed, so attempts at path traversal via crafted
//! branch names are rejected.
use std::collections::HashMap;
use std::time::SystemTime;
use async_trait::async_trait;
use gix_hash::ObjectId;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::git::types::IndexEntry;
/// Convert a [`SystemTime`] to nanoseconds since the Unix epoch, matching the
/// `i128` representation used for `IndexEntry::mtime_ns`. Returns `None` for
/// pre-epoch times (which the racy-clean guard treats conservatively). Shared
/// by the `IndexStore` backends to stamp [`CommitIndex::saved_at_ns`] from the
/// index file's own modification time on load.
pub fn systemtime_to_ns(t: SystemTime) -> Option<i128> {
t.duration_since(SystemTime::UNIX_EPOCH)
.ok()
.map(|d| d.as_nanos() as i128)
}
/// Snapshot of the working tree's `(size, mtime_ns, oid)` after the commit
/// identified by `parent_oid`.
///
/// `entries` is keyed by account-relative path (same form as `CommitRequest::paths`).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CommitIndex {
/// Commit OID this snapshot reflects. Used to invalidate the cache when
/// the branch head has moved (concurrent commit, branch switch, etc.).
pub parent_oid: ObjectId,
/// Account-relative path → cached `(size, mtime_ns, oid)`.
pub entries: HashMap<String, IndexEntry>,
/// Wall-clock time (ns since epoch) the index file was written, as
/// observed from the backend's own metadata at `load()` time. `None` for
/// freshly-built (not-yet-persisted) indexes and when the backend cannot
/// report a modification time. Used for the racy-clean guard: a cache
/// entry whose `mtime_ns >= saved_at_ns` may have been modified within the
/// same clock tick as the last commit and is forced down the slow path.
/// Not part of the on-disk wire format — it is derived from the file's own
/// timestamp on load.
pub saved_at_ns: Option<i128>,
}
/// Error type returned by [`IndexStore`] operations. All error variants are
/// soft failures from the caller's perspective — `load()` callers map them to
/// `Ok(None)` and `save()` callers log and continue.
#[derive(Debug, Error)]
pub enum IndexStoreError {
/// Underlying I/O error from the backend (filesystem, network, etc.).
#[error("i/o error: {0}")]
Io(#[from] std::io::Error),
/// On-disk format could not be parsed (corruption, version skew, bad oid).
#[error("decode error: {0}")]
Decode(String),
/// Branch component failed `validate_ref_name` (path-traversal guard).
#[error("invalid branch name: {0}")]
InvalidBranch(String),
/// Non-I/O backend failure (e.g. S3 SDK error).
#[error("backend error: {0}")]
Backend(String),
}
/// Per-(account, branch) commit-index storage.
///
/// `load` returns `Ok(None)` for any "no usable index here" outcome — missing
/// file, decode failure, version mismatch, etc. The caller treats every miss
/// as "skip Fast Path 1, fall back to the full read/hash path".
///
/// `save` is fire-and-forget from the caller's perspective: the commit has
/// already succeeded; the worst-case cost of a save failure is one extra
/// slow-path commit next time.
#[async_trait]
pub trait IndexStore: Send + Sync + 'static {
/// Load the latest persisted index for `(account, branch)`. Returns
/// `Ok(None)` if no index has been written yet, or if the persisted bytes
/// fail to decode (treated as a soft miss so commit falls back to the
/// slow path).
async fn load(
&self,
account: &str,
branch: &str,
) -> Result<Option<CommitIndex>, IndexStoreError>;
/// Persist `index` for `(account, branch)`, replacing any prior snapshot.
/// Last-write-wins semantics — there is no CAS because the index is a
/// soft-state cache; correctness is enforced at load time via the
/// `parent_oid` check.
async fn save(
&self,
account: &str,
branch: &str,
index: &CommitIndex,
) -> Result<(), IndexStoreError>;
}
// ---- Wire format ---------------------------------------------------------
/// Bumped only when a backwards-incompatible change to the layout ships.
/// Older readers see an unknown version and treat the file as absent.
const INDEX_FORMAT_VERSION: u32 = 1;
#[derive(Debug, Serialize, Deserialize)]
struct WireIndex {
version: u32,
parent_oid: String,
entries: Vec<WireEntry>,
}
#[derive(Debug, Serialize, Deserialize)]
struct WireEntry {
path: String,
size: u64,
mtime_ns: i128,
oid: String,
}
/// Serialize a `CommitIndex` into the on-disk JSON wire format.
pub fn encode_index(index: &CommitIndex) -> Result<Vec<u8>, IndexStoreError> {
let mut entries: Vec<WireEntry> = index
.entries
.iter()
.map(|(path, e)| WireEntry {
path: path.clone(),
size: e.size,
mtime_ns: e.mtime_ns,
oid: e.oid.to_hex().to_string(),
})
.collect();
// Sorted output → deterministic byte content for tests / hashing.
entries.sort_by(|a, b| a.path.cmp(&b.path));
let wire = WireIndex {
version: INDEX_FORMAT_VERSION,
parent_oid: index.parent_oid.to_hex().to_string(),
entries,
};
serde_json::to_vec(&wire).map_err(|e| IndexStoreError::Decode(e.to_string()))
}
/// Decode the on-disk JSON wire format. An unknown `version` field returns
/// `Ok(None)`; malformed JSON or invalid OIDs return `Err(Decode)` (which
/// the trait `load()` callers also map to `None`).
pub fn decode_index(bytes: &[u8]) -> Result<Option<CommitIndex>, IndexStoreError> {
let wire: WireIndex = match serde_json::from_slice(bytes) {
Ok(w) => w,
Err(e) => return Err(IndexStoreError::Decode(e.to_string())),
};
if wire.version != INDEX_FORMAT_VERSION {
// Forward-compat: unknown version → silently treat as missing.
return Ok(None);
}
let parent_oid = ObjectId::from_hex(wire.parent_oid.as_bytes())
.map_err(|e| IndexStoreError::Decode(format!("parent_oid: {e}")))?;
let mut entries = HashMap::with_capacity(wire.entries.len());
for w in wire.entries {
let oid = ObjectId::from_hex(w.oid.as_bytes())
.map_err(|e| IndexStoreError::Decode(format!("entry oid {}: {e}", w.path)))?;
entries.insert(
w.path,
IndexEntry {
size: w.size,
mtime_ns: w.mtime_ns,
oid,
},
);
}
Ok(Some(CommitIndex {
parent_oid,
entries,
// The wire format carries no timestamp; callers that need the
// racy-clean guard populate `saved_at_ns` from the backing file's own
// mtime after a successful decode.
saved_at_ns: None,
}))
}
#[cfg(test)]
mod tests {
use super::*;
fn oid_from_byte(b: u8) -> ObjectId {
let mut bytes = [0u8; 20];
bytes.fill(b);
ObjectId::from_bytes_or_panic(&bytes)
}
#[test]
fn round_trip_preserves_entries() {
let mut entries = HashMap::new();
entries.insert(
"resources/a.md".into(),
IndexEntry {
size: 42,
mtime_ns: 1_700_000_000_000_000_000,
oid: oid_from_byte(0xAA),
},
);
entries.insert(
"agent/b.py".into(),
IndexEntry {
size: 7,
mtime_ns: -1,
oid: oid_from_byte(0xBB),
},
);
let idx = CommitIndex {
parent_oid: oid_from_byte(0xCC),
entries,
saved_at_ns: None,
};
let bytes = encode_index(&idx).unwrap();
let decoded = decode_index(&bytes).unwrap().unwrap();
assert_eq!(decoded, idx);
}
#[test]
fn unknown_version_is_silent_miss() {
let bogus = serde_json::json!({
"version": 9999,
"parent_oid": format!("{:040}", 0),
"entries": []
});
let bytes = serde_json::to_vec(&bogus).unwrap();
assert!(decode_index(&bytes).unwrap().is_none());
}
#[test]
fn malformed_json_errors() {
assert!(decode_index(b"not-json").is_err());
}
#[test]
fn invalid_oid_errors() {
let bogus = serde_json::json!({
"version": INDEX_FORMAT_VERSION,
"parent_oid": "zzzz",
"entries": []
});
let bytes = serde_json::to_vec(&bogus).unwrap();
assert!(decode_index(&bytes).is_err());
}
}