Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
//! VFS enumeration with snapshot pruning rules.
//!
//! Walks an account's VFS subtree at `/local/{account}` and returns
//! account-relative paths of files that must be included in a Git snapshot.
//!
//! Pruning rules:
//!
//! | Skip if account-relative path matches | Reason |
//! |---|---|
//! | First path segment ∈ {`_system`, `tasks`, `temp`, `queue`, `upload`} | Internal scopes + runtime locks |
//! | Any segment equals `.path.ovlock` OR starts with `.path.ovlock` | Runtime lock |
//! | File extension is `.faiss` or `.index`, OR path contains an `embedding_cache/` segment | Vector index — derived data |
//! | File name ends with `.abstract.md` or `.overview.md` | L0/L1 summaries — derived data |
use crate::core::internal_names::{is_hidden_runtime_lock_name, PATH_LOCK_FILE};
use std::sync::Arc;
use crate::core::filesystem::FileSystem;
use crate::git::error::GitError;
/// First-segment internal scope names that are pruned.
///
/// Combines `INTERNAL_SCOPES` (system/tasks/temp/queue/upload) and
/// `VikingFS._INTERNAL_NAMES` from the design.
const INTERNAL_FIRST_SEGMENTS: &[&str] = &["_system", "tasks", "temp", "queue", "upload"];
/// Returns true if this account-relative path must be excluded from commits.
///
/// `rel` is the path relative to the account root (no leading "/", no
/// "/local/{account}/" prefix). Examples of valid input:
/// - "resources/a.md"
/// - "_system/lock"
/// - ".path.ovlock"
pub fn prune_path(rel: &str) -> bool {
if rel.is_empty() {
return true;
}
let segments: Vec<&str> = rel.split('/').filter(|s| !s.is_empty()).collect();
if segments.is_empty() {
return true;
}
// Rule 1: first path segment in INTERNAL_FIRST_SEGMENTS
if INTERNAL_FIRST_SEGMENTS.contains(&segments[0]) {
return true;
}
// Rule 2: any segment is a runtime path-lock file.
if segments
.iter()
.any(|segment| segment.starts_with(PATH_LOCK_FILE) || is_hidden_runtime_lock_name(segment))
{
return true;
}
// Rule 3a: any intermediate segment named exactly "embedding_cache"
// (i.e. path contains an `embedding_cache/` segment).
for seg in &segments[..segments.len().saturating_sub(1)] {
if *seg == "embedding_cache" {
return true;
}
}
// Rule 3b: derived indexes and summaries.
if let Some(last) = segments.last() {
if last.ends_with(".faiss")
|| last.ends_with(".index")
|| last.ends_with(".abstract.md")
|| last.ends_with(".overview.md")
{
return true;
}
}
false
}
/// Enumerate every versionable file in an account's VFS subtree.
///
/// Returns paths relative to the account root (no leading "/", no
/// "/local/{account}/" prefix). Directories are filtered out; only file
/// entries that survive pruning are returned.
pub async fn collect_all(
vfs: &Arc<dyn FileSystem>,
account: &str,
) -> Result<Vec<String>, GitError> {
collect_under(vfs, account, "").await
}
/// Enumerate every versionable file under an account-relative subdirectory.
///
/// `sub_path` is account-relative (no leading "/", no trailing "/"). An empty
/// `sub_path` enumerates the entire account tree (equivalent to
/// [`collect_all`]). Returned paths are account-relative and include the
/// `sub_path` prefix. Directories are filtered out; only file entries that
/// survive [`prune_path`] (applied to the account-relative form) are returned.
pub async fn collect_under(
vfs: &Arc<dyn FileSystem>,
account: &str,
sub_path: &str,
) -> Result<Vec<String>, GitError> {
let root = if sub_path.is_empty() {
format!("/local/{}", account)
} else {
format!("/local/{}/{}", account, sub_path)
};
let account_prefix = format!("/local/{}/", account);
let entries = vfs
.tree_directory(&root, true, None, None, None, None, None, false)
.await?;
let mut survivors = Vec::new();
for entry in entries {
if entry.info.is_dir {
continue;
}
// Strip "/local/{account}/" so pruning sees account-relative paths.
// If the entry doesn't carry that prefix, skip it (defensive — should
// not happen for a well-formed VFS).
let rel = match entry.path.strip_prefix(&account_prefix) {
Some(r) => r,
None => continue,
};
if prune_path(rel) {
continue;
}
survivors.push(rel.to_string());
}
Ok(survivors)
}
#[cfg(test)]
mod tests {
use super::*;
use async_trait::async_trait;
use std::collections::HashMap;
use std::sync::Arc;
use crate::core::errors::Result;
use crate::core::filesystem::FileSystem;
use crate::core::types::{FileInfo, TreeEntry, WriteFlag};
/// In-test mock that directly overrides `tree_directory` to return a
/// precomputed list of (path, is_dir). All other trait methods are
/// unimplemented because `collect_all` only calls `tree_directory`.
struct MockFS {
/// Map: root path -> list of (absolute path, is_dir)
entries_by_root: HashMap<String, Vec<(String, bool)>>,
}
impl MockFS {
fn new(root: &str, entries: Vec<(&str, bool)>) -> Self {
let mut map = HashMap::new();
map.insert(
root.to_string(),
entries
.into_iter()
.map(|(p, d)| (p.to_string(), d))
.collect(),
);
Self {
entries_by_root: map,
}
}
}
#[async_trait]
impl FileSystem for MockFS {
async fn create(&self, _path: &str) -> Result<()> {
unimplemented!()
}
async fn mkdir(&self, _path: &str, _mode: u32) -> Result<()> {
unimplemented!()
}
async fn remove(&self, _path: &str) -> Result<()> {
unimplemented!()
}
async fn remove_all(&self, _path: &str) -> Result<()> {
unimplemented!()
}
async fn read(&self, _path: &str, _offset: u64, _size: u64) -> Result<Vec<u8>> {
unimplemented!()
}
async fn write(
&self,
_path: &str,
_data: &[u8],
_offset: u64,
_flags: WriteFlag,
) -> Result<u64> {
unimplemented!()
}
async fn read_dir(
&self,
_path: &str,
_offset: Option<usize>,
_limit: Option<usize>,
_sort_by: Option<crate::core::ListSortBy>,
_sort_order: Option<crate::core::SortOrder>,
) -> Result<Vec<FileInfo>> {
unimplemented!()
}
async fn stat(&self, _path: &str) -> Result<FileInfo> {
unimplemented!()
}
async fn rename(&self, _old_path: &str, _new_path: &str) -> Result<()> {
unimplemented!()
}
async fn chmod(&self, _path: &str, _mode: u32) -> Result<()> {
unimplemented!()
}
/// Override tree_directory to return precomputed entries, bypassing
/// the default read_dir/stat-based recursion.
async fn tree_directory(
&self,
path: &str,
_show_hidden: bool,
_node_limit: Option<usize>,
_level_limit: Option<usize>,
_offset: Option<usize>,
_sort_by: Option<crate::core::ListSortBy>,
_sort_order: Option<crate::core::SortOrder>,
_directories_only: bool,
) -> Result<Vec<TreeEntry>> {
let raw = self.entries_by_root.get(path).cloned().unwrap_or_default();
let prefix = if path == "/" {
"/".to_string()
} else {
format!("{}/", path)
};
let mut out = Vec::new();
for (full_path, is_dir) in raw {
let rel_path = full_path
.strip_prefix(&prefix)
.unwrap_or(&full_path)
.to_string();
let name = full_path
.rsplit('/')
.next()
.unwrap_or(&full_path)
.to_string();
let info = if is_dir {
FileInfo::new_dir(name, 0o755)
} else {
FileInfo::new_file(name, 0, 0o644)
};
out.push(TreeEntry {
path: full_path,
rel_path,
info,
extra: HashMap::new(),
});
}
Ok(out)
}
}
#[tokio::test]
async fn test_collect_all_prunes_internal_scopes_and_vector_indexes() {
let mock = MockFS::new(
"/local/acct",
vec![
("/local/acct/resources/a.md", false),
("/local/acct/agent/b.py", false),
("/local/acct/_system/lock", false),
("/local/acct/tasks/job.json", false),
("/local/acct/temp/upload.bin", false),
("/local/acct/queue/x", false),
("/local/acct/upload/y", false),
("/local/acct/resources/x.faiss", false),
("/local/acct/resources/x.index", false),
("/local/acct/resources/embedding_cache/v.bin", false),
(
"/local/acct/temp/.encrypt_stage/823e7fa1698ab91f00481e4960d38764c6d77fa9230a86a0cca178bf87becc93.encrypt",
false,
),
],
);
let fs: Arc<dyn FileSystem> = Arc::new(mock);
let mut got = collect_all(&fs, "acct").await.unwrap();
got.sort();
let mut expected = vec!["agent/b.py".to_string(), "resources/a.md".to_string()];
expected.sort();
assert_eq!(got, expected);
}
#[tokio::test]
async fn test_collect_all_prunes_derived_l0_l1_files() {
let mock = MockFS::new(
"/local/acct",
vec![
("/local/acct/resources/x.md", false),
("/local/acct/resources/.abstract.md", false),
("/local/acct/resources/.overview.md", false),
("/local/acct/resources/x.md.abstract.md", false),
("/local/acct/resources/x.md.overview.md", false),
],
);
let fs: Arc<dyn FileSystem> = Arc::new(mock);
assert_eq!(collect_all(&fs, "acct").await.unwrap(), vec!["resources/x.md"]);
}
#[tokio::test]
async fn test_collect_all_returns_account_relative_paths() {
let mock = MockFS::new("/local/acct", vec![("/local/acct/resources/a.md", false)]);
let fs: Arc<dyn FileSystem> = Arc::new(mock);
let got = collect_all(&fs, "acct").await.unwrap();
assert_eq!(got, vec!["resources/a.md".to_string()]);
// Defensive: ensure no absolute leakage.
for p in &got {
assert!(!p.starts_with('/'), "path should not be absolute: {}", p);
assert!(
!p.contains("/local/acct"),
"path should not contain account prefix: {}",
p
);
}
}
#[tokio::test]
async fn test_collect_under_returns_paths_below_subdir_with_prefix() {
let mock = MockFS::new(
"/local/acct/resources",
vec![
("/local/acct/resources/a.md", false),
("/local/acct/resources/sub/b.md", false),
("/local/acct/resources/x.faiss", false),
],
);
let fs: Arc<dyn FileSystem> = Arc::new(mock);
let mut got = collect_under(&fs, "acct", "resources").await.unwrap();
got.sort();
let mut expected = vec![
"resources/a.md".to_string(),
"resources/sub/b.md".to_string(),
];
expected.sort();
assert_eq!(got, expected);
}
#[tokio::test]
async fn test_collect_under_empty_sub_path_equivalent_to_collect_all() {
let mock = MockFS::new(
"/local/acct",
vec![
("/local/acct/resources/a.md", false),
("/local/acct/_system/lock", false),
],
);
let fs: Arc<dyn FileSystem> = Arc::new(mock);
let got_under = collect_under(&fs, "acct", "").await.unwrap();
let got_all = collect_all(&fs, "acct").await.unwrap();
assert_eq!(got_under, got_all);
assert_eq!(got_under, vec!["resources/a.md".to_string()]);
}
#[tokio::test]
async fn test_collect_under_applies_pruning_on_account_relative_form() {
// A `_system` sub_path is itself a pruned prefix — every file under it
// is pruned because the first account-relative segment is "_system".
let mock = MockFS::new(
"/local/acct/_system",
vec![("/local/acct/_system/lock", false)],
);
let fs: Arc<dyn FileSystem> = Arc::new(mock);
let got = collect_under(&fs, "acct", "_system").await.unwrap();
assert!(got.is_empty(), "everything under _system must be pruned");
}
#[test]
fn test_prune_path_table() {
// Pruned
assert!(prune_path("_system/lock"));
assert!(prune_path("tasks/job.json"));
assert!(prune_path("temp/x"));
assert!(prune_path("queue/x"));
assert!(prune_path("upload/x"));
assert!(prune_path("resources/.path.ovlock"));
assert!(prune_path(".path.ovlock"));
assert!(prune_path(
"resources/.exact.ovlock.file.txt.0123456789abcdef"
));
assert!(prune_path("resources/x.faiss"));
assert!(prune_path("resources/x.index"));
assert!(prune_path("resources/embedding_cache/v.bin"));
assert!(prune_path("agent/embedding_cache/something"));
assert!(prune_path("resources/.abstract.md"));
assert!(prune_path("resources/.overview.md"));
assert!(prune_path("resources/x.md.abstract.md"));
assert!(prune_path("resources/x.md.overview.md"));
// Survivors
assert!(!prune_path("resources/a.md"));
assert!(!prune_path("agent/skills/b.py"));
// "_systemfoo" is NOT "_system", must survive.
assert!(!prune_path("_systemfoo/x"));
// Per "any segment starting with .path.ovlock", ".path.ovlocking"
// is pruned even though it isn't exactly ".path.ovlock".
assert!(prune_path("resources/.path.ovlocking"));
}
}