Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
perf(ragfs): skip redirect metadata without redirects (#4683)
Qin Haojiee273459c6478
diff --git a/crates/ragfs/src/core/multibackend_wrapper.rs b/crates/ragfs/src/core/multibackend_wrapper.rsindex db2f7e1ef..4c2804859 100644--- a/crates/ragfs/src/core/multibackend_wrapper.rs+++ b/crates/ragfs/src/core/multibackend_wrapper.rs@@ -14,6 +14,7 @@ use std::sync::Arc;use std::time::Duration;use async_trait::async_trait;+use futures::stream::{self, StreamExt};use regex::Regex;use serde_json::Value;use tokio::sync::{Mutex, Notify};@@ -21,11 +22,10 @@ use tokio::sync::{Mutex, Notify};use super::context::{FsContext, FS_CTX};use super::encryption_wrapper::EncryptionWrappedFS;use super::errors::{Error, Result};-use super::filesystem::{normalize_prefix_path, relative_match_file, FileSystem};+use super::filesystem::{normalize_prefix_path, relative_depth, relative_match_file, FileSystem};use super::types::{- BackendRole, BackendSyncState, FileInfo, GlobEntry, GlobPage, GrepResult,- OperationItemConfig, RedirectEntry, RedirectPolicy, SyncLogEntry, SyncOp, SyncType, TreeEntry,- WriteFlag,+ BackendRole, BackendSyncState, FileInfo, GlobEntry, GlobPage, GrepResult, OperationItemConfig,+ RedirectEntry, RedirectPolicy, SyncLogEntry, SyncOp, SyncType, TreeEntry, WriteFlag,};use crate::core::glob::{compare_rel_paths, decode_offset_token, encode_offset_token, PreparedGlob,@@ -53,6 +53,8 @@ const DEFAULT_MAX_RETRIES_PER_ROUND: usize = 3;const DEFAULT_QUARANTINE_AFTER_FAILURES: u32 = 9;/// Default wait timeout when shutting down background tasks.const DEFAULT_SHUTDOWN_WAIT: Duration = Duration::from_secs(5);+/// Maximum concurrent metadata reads when merging redirect entries into tree output.+const DEFAULT_TREE_REDIRECT_META_CONCURRENCY: usize = 32;#[derive(Debug)]struct SyncFanoutTaskResult {@@ -717,6 +719,10 @@ impl Inner {path: &str,ctx: &FsContext,) -> Result<Option<Vec<String>>> {+ if self.redirects.is_empty() {+ return Ok(None);+ }+let dir = parent_dir(path);let name = file_name(path);let redirect_meta = self.meta_store.get_redirect_meta(&dir, ctx).await?;@@ -1391,6 +1397,10 @@ impl FileSystem for MultiWriteWrappedFS {// Filter multi-write internal names.entries.retain(|e| !MULTIWRITE_INTERNAL_NAMES.contains(&e.name.as_str()));+ if inner.redirects.is_empty() {+ return Ok(entries);+ }+// Merge redirect entries so users can see redirected files in listings.let ctx =current_required_ctx().or_else(|_| inner.meta_store.ctx_resolver().resolve(path))?;@@ -1547,6 +1557,10 @@ impl FileSystem for MultiWriteWrappedFS {});result.count = result.matches.len();+ if inner.redirects.is_empty() {+ return Ok(result);+ }+// For redirect files, also grep in target backends.let ctx = current_required_ctx().or_else(|_| inner.meta_store.ctx_resolver().resolve(&path_owned))?;@@ -1737,25 +1751,60 @@ impl FileSystem for MultiWriteWrappedFS {!MULTIWRITE_INTERNAL_NAMES.contains(&name)});+ if self.inner.redirects.is_empty() {+ return Ok(entries);+ }++ if node_limit.is_some_and(|limit| entries.len() >= limit) {+ return Ok(entries);+ }+let ctx = current_required_ctx().or_else(|_| self.inner.meta_store.ctx_resolver().resolve(&base))?;let mut seen_paths: HashSet<String> = entries.iter().map(|e| e.path.clone()).collect();- let mut dir_paths = vec![base.clone()];+ let mut dir_paths = Vec::new();+ let mut queued_dirs: HashSet<String> = HashSet::new();+ if !matches!(level_limit, Some(0)) {+ queued_dirs.insert(base.clone());+ dir_paths.push(base.clone());+ }for entry in &entries {if entry.info.is_dir {let dir = normalize_prefix_path(&entry.path);- if !dir_paths.iter().any(|p| p == &dir) {+ let may_emit_redirect_children = match level_limit {+ Some(limit) => {+ let rel = relative_match_file(&base, &dir);+ relative_depth(&rel) < limit+ }+ None => true,+ };+ if may_emit_redirect_children && queued_dirs.insert(dir.clone()) {dir_paths.push(dir);}}}- for dir in dir_paths {- let redirect_meta = match self.inner.meta_store.get_redirect_meta(&dir, &ctx).await {- Ok(meta) => meta,- Err(_) => continue,+ let inner = Arc::clone(&self.inner);+ let mut redirect_results = stream::iter(dir_paths)+ .map(|dir| {+ let inner = Arc::clone(&inner);+ let ctx = ctx.clone();+ async move {+ let redirect_meta = inner.meta_store.get_redirect_meta(&dir, &ctx).await.ok();+ (dir, redirect_meta)+ }+ })+ .buffered(DEFAULT_TREE_REDIRECT_META_CONCURRENCY);++ 'redirect_dirs: while let Some((dir, redirect_meta)) = redirect_results.next().await {+ let Some(redirect_meta) = redirect_meta else {+ continue;};for (name, redirect_entry) in redirect_meta.entries {+ if node_limit.is_some_and(|limit| entries.len() >= limit) {+ break 'redirect_dirs;+ }+let virtual_path = if dir == "/" {format!("/{}", name)} else {@@ -1773,6 +1822,9 @@ impl FileSystem for MultiWriteWrappedFS {.trim_start_matches('/').to_string()};+ if level_limit.is_some_and(|limit| relative_depth(&rel_path) > limit) {+ continue;+ }let mut extra = HashMap::new();extra.insert("redirect".to_string(), Value::Bool(true));entries.push(TreeEntry {diff --git a/crates/ragfs/src/core/multibackend_wrapper/routing.rs b/crates/ragfs/src/core/multibackend_wrapper/routing.rsindex 8de9babf2..1dff3f491 100644--- a/crates/ragfs/src/core/multibackend_wrapper/routing.rs+++ b/crates/ragfs/src/core/multibackend_wrapper/routing.rs@@ -80,6 +80,11 @@ impl Inner {return Some(self.primary().backend.clone());}+ if self.redirects.is_empty() {+ self.record_read_route(ReadRouteSource::Miss);+ return None;+ }+let dir = parent_dir(&normalized);let name = file_name(&normalized).to_string();let ctx = current_required_ctx()diff --git a/crates/ragfs/src/core/multibackend_wrapper/tests.rs b/crates/ragfs/src/core/multibackend_wrapper/tests.rsindex 357c56fcf..d0ec7bced 100644--- a/crates/ragfs/src/core/multibackend_wrapper/tests.rs+++ b/crates/ragfs/src/core/multibackend_wrapper/tests.rs@@ -91,6 +91,93 @@ async fn test_read_dir_redirect_entries_use_target_stat() {}).await.unwrap();++ let no_redirect_fs = test_multiwrite_fs(Vec::new());+ let no_redirect_ctx = test_ctx();++ FS_CTX+ .scope(no_redirect_ctx.clone(), async {+ no_redirect_fs+ .ensure_parent_dirs("/local/acct/docs/real.txt", 0o755)+ .await?;+ no_redirect_fs+ .write(+ "/local/acct/docs/real.txt",+ b"real body",+ 0,+ WriteFlag::Create,+ )+ .await?;+ no_redirect_fs+ .inner+ .meta_store+ .update_dir_meta(+ "/local/acct/docs",+ &no_redirect_ctx,+ |redirect, _sync_log| {+ redirect.entries.insert(+ "ghost.pdf".to_string(),+ RedirectEntry {+ targets: vec!["backup1".to_string()],+ },+ );+ Ok(())+ },+ )+ .await?;+ let backup = no_redirect_fs+ .inner+ .backup_by_name("backup1")+ .expect("backup exists")+ .backend+ .clone();+ backup+ .ensure_parent_dirs("/local/acct/docs/ghost.pdf", 0o755)+ .await?;+ backup+ .write(+ "/local/acct/docs/ghost.pdf",+ b"GHOST_MARKER\n",+ 0,+ WriteFlag::Create,+ )+ .await?;++ let entries = no_redirect_fs.read_dir("/local/acct/docs").await?;+ let names: Vec<&str> = entries.iter().map(|entry| entry.name.as_str()).collect();+ assert_eq!(names, vec!["real.txt"]);++ let tree = no_redirect_fs+ .tree_directory("/local/acct", false, None, Some(3))+ .await?;+ let paths: Vec<&str> = tree.iter().map(|entry| entry.path.as_str()).collect();+ assert!(!paths.contains(&"/local/acct/docs/ghost.pdf"));++ let grep = no_redirect_fs+ .grep(+ "/local/acct/docs",+ "GHOST_MARKER",+ false,+ false,+ None,+ None,+ Some(1),+ )+ .await?;+ assert_eq!(grep.count, 0);++ let err = no_redirect_fs+ .rename("/local/acct/docs/ghost.pdf", "/local/acct/docs/renamed.pdf")+ .await+ .expect_err("stale redirect metadata is ignored when redirects are disabled");+ assert!(matches!(err, Error::NotFound(_)));+ assert!(backup.exists("/local/acct/docs/ghost.pdf").await);+ assert!(!backup.exists("/local/acct/docs/renamed.pdf").await);++ Ok::<(), Error>(())+ })+ .await+ .unwrap();}#[tokio::test]@@ -113,6 +200,36 @@ async fn test_recursive_grep_finds_nested_redirected_files() {).await?;+ let shallow = fs+ .tree_directory("/local/acct/resources", false, None, Some(1))+ .await?;+ let shallow_paths: Vec<&str> =+ shallow.iter().map(|entry| entry.path.as_str()).collect();+ assert_eq!(shallow_paths, vec!["/local/acct/resources/doc-1"]);++ let root_only = fs+ .tree_directory("/local/acct/resources", false, None, Some(0))+ .await?;+ assert!(root_only.is_empty());++ let deep = fs+ .tree_directory("/local/acct/resources", false, None, Some(2))+ .await?;+ let deep_paths: Vec<&str> = deep.iter().map(|entry| entry.path.as_str()).collect();+ assert_eq!(+ deep_paths,+ vec![+ "/local/acct/resources/doc-1",+ "/local/acct/resources/doc-1/page.md",+ ]+ );++ let capped = fs+ .tree_directory("/local/acct/resources", false, Some(1), Some(2))+ .await?;+ let capped_paths: Vec<&str> = capped.iter().map(|entry| entry.path.as_str()).collect();+ assert_eq!(capped_paths, vec!["/local/acct/resources/doc-1"]);+let result = fs.grep("/local/acct/resources",diff --git a/crates/ragfs/src/multibackend/meta.rs b/crates/ragfs/src/multibackend/meta.rsindex 457722cd8..777a3cdcf 100644--- a/crates/ragfs/src/multibackend/meta.rs+++ b/crates/ragfs/src/multibackend/meta.rs@@ -345,16 +345,12 @@ impl MetaStateStore {/// Read redirect metadata for a directory (public, used by read_dir to merge redirect entries).pub async fn get_redirect_meta(&self, dir: &str, ctx: &FsContext) -> Result<RedirectMeta> {- self.read_dir_meta_pair(dir, ctx)- .await- .map(|(redirect, _)| redirect)+ self.read_redirect_meta(dir, ctx).await}/// Read sync log metadata for a directory (public, used by retry_loop).pub async fn get_sync_log_meta(&self, dir: &str, ctx: &FsContext) -> Result<SyncLogMeta> {- self.read_dir_meta_pair(dir, ctx)- .await- .map(|(_, sync_log)| sync_log)+ self.read_sync_log_meta(dir, ctx).await}/// Get a reference to the context resolver.