Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

perf(ragfs): skip redirect metadata without redirects (#4683)

Qin Haojiee273459c6478
diff --git a/crates/ragfs/src/core/multibackend_wrapper.rs b/crates/ragfs/src/core/multibackend_wrapper.rs
index db2f7e1ef..4c2804859 100644
--- a/crates/ragfs/src/core/multibackend_wrapper.rs
+++ b/crates/ragfs/src/core/multibackend_wrapper.rs
@@ -14,6 +14,7 @@ use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
+use futures::stream::{self, StreamExt};
use regex::Regex;
use serde_json::Value;
use tokio::sync::{Mutex, Notify};
@@ -21,11 +22,10 @@ use tokio::sync::{Mutex, Notify};
use super::context::{FsContext, FS_CTX};
use super::encryption_wrapper::EncryptionWrappedFS;
use super::errors::{Error, Result};
-use super::filesystem::{normalize_prefix_path, relative_match_file, FileSystem};
+use super::filesystem::{normalize_prefix_path, relative_depth, relative_match_file, FileSystem};
use super::types::{
- BackendRole, BackendSyncState, FileInfo, GlobEntry, GlobPage, GrepResult,
- OperationItemConfig, RedirectEntry, RedirectPolicy, SyncLogEntry, SyncOp, SyncType, TreeEntry,
- WriteFlag,
+ BackendRole, BackendSyncState, FileInfo, GlobEntry, GlobPage, GrepResult, OperationItemConfig,
+ RedirectEntry, RedirectPolicy, SyncLogEntry, SyncOp, SyncType, TreeEntry, WriteFlag,
};
use crate::core::glob::{
compare_rel_paths, decode_offset_token, encode_offset_token, PreparedGlob,
@@ -53,6 +53,8 @@ const DEFAULT_MAX_RETRIES_PER_ROUND: usize = 3;
const DEFAULT_QUARANTINE_AFTER_FAILURES: u32 = 9;
/// Default wait timeout when shutting down background tasks.
const DEFAULT_SHUTDOWN_WAIT: Duration = Duration::from_secs(5);
+/// Maximum concurrent metadata reads when merging redirect entries into tree output.
+const DEFAULT_TREE_REDIRECT_META_CONCURRENCY: usize = 32;
#[derive(Debug)]
struct SyncFanoutTaskResult {
@@ -717,6 +719,10 @@ impl Inner {
path: &str,
ctx: &FsContext,
) -> Result<Option<Vec<String>>> {
+ if self.redirects.is_empty() {
+ return Ok(None);
+ }
+
let dir = parent_dir(path);
let name = file_name(path);
let redirect_meta = self.meta_store.get_redirect_meta(&dir, ctx).await?;
@@ -1391,6 +1397,10 @@ impl FileSystem for MultiWriteWrappedFS {
// Filter multi-write internal names.
entries.retain(|e| !MULTIWRITE_INTERNAL_NAMES.contains(&e.name.as_str()));
+ if inner.redirects.is_empty() {
+ return Ok(entries);
+ }
+
// Merge redirect entries so users can see redirected files in listings.
let ctx =
current_required_ctx().or_else(|_| inner.meta_store.ctx_resolver().resolve(path))?;
@@ -1547,6 +1557,10 @@ impl FileSystem for MultiWriteWrappedFS {
});
result.count = result.matches.len();
+ if inner.redirects.is_empty() {
+ return Ok(result);
+ }
+
// For redirect files, also grep in target backends.
let ctx = current_required_ctx()
.or_else(|_| inner.meta_store.ctx_resolver().resolve(&path_owned))?;
@@ -1737,25 +1751,60 @@ impl FileSystem for MultiWriteWrappedFS {
!MULTIWRITE_INTERNAL_NAMES.contains(&name)
});
+ if self.inner.redirects.is_empty() {
+ return Ok(entries);
+ }
+
+ if node_limit.is_some_and(|limit| entries.len() >= limit) {
+ return Ok(entries);
+ }
+
let ctx = current_required_ctx()
.or_else(|_| self.inner.meta_store.ctx_resolver().resolve(&base))?;
let mut seen_paths: HashSet<String> = entries.iter().map(|e| e.path.clone()).collect();
- let mut dir_paths = vec![base.clone()];
+ let mut dir_paths = Vec::new();
+ let mut queued_dirs: HashSet<String> = HashSet::new();
+ if !matches!(level_limit, Some(0)) {
+ queued_dirs.insert(base.clone());
+ dir_paths.push(base.clone());
+ }
for entry in &entries {
if entry.info.is_dir {
let dir = normalize_prefix_path(&entry.path);
- if !dir_paths.iter().any(|p| p == &dir) {
+ let may_emit_redirect_children = match level_limit {
+ Some(limit) => {
+ let rel = relative_match_file(&base, &dir);
+ relative_depth(&rel) < limit
+ }
+ None => true,
+ };
+ if may_emit_redirect_children && queued_dirs.insert(dir.clone()) {
dir_paths.push(dir);
}
}
}
- for dir in dir_paths {
- let redirect_meta = match self.inner.meta_store.get_redirect_meta(&dir, &ctx).await {
- Ok(meta) => meta,
- Err(_) => continue,
+ let inner = Arc::clone(&self.inner);
+ let mut redirect_results = stream::iter(dir_paths)
+ .map(|dir| {
+ let inner = Arc::clone(&inner);
+ let ctx = ctx.clone();
+ async move {
+ let redirect_meta = inner.meta_store.get_redirect_meta(&dir, &ctx).await.ok();
+ (dir, redirect_meta)
+ }
+ })
+ .buffered(DEFAULT_TREE_REDIRECT_META_CONCURRENCY);
+
+ 'redirect_dirs: while let Some((dir, redirect_meta)) = redirect_results.next().await {
+ let Some(redirect_meta) = redirect_meta else {
+ continue;
};
for (name, redirect_entry) in redirect_meta.entries {
+ if node_limit.is_some_and(|limit| entries.len() >= limit) {
+ break 'redirect_dirs;
+ }
+
let virtual_path = if dir == "/" {
format!("/{}", name)
} else {
@@ -1773,6 +1822,9 @@ impl FileSystem for MultiWriteWrappedFS {
.trim_start_matches('/')
.to_string()
};
+ if level_limit.is_some_and(|limit| relative_depth(&rel_path) > limit) {
+ continue;
+ }
let mut extra = HashMap::new();
extra.insert("redirect".to_string(), Value::Bool(true));
entries.push(TreeEntry {
diff --git a/crates/ragfs/src/core/multibackend_wrapper/routing.rs b/crates/ragfs/src/core/multibackend_wrapper/routing.rs
index 8de9babf2..1dff3f491 100644
--- a/crates/ragfs/src/core/multibackend_wrapper/routing.rs
+++ b/crates/ragfs/src/core/multibackend_wrapper/routing.rs
@@ -80,6 +80,11 @@ impl Inner {
return Some(self.primary().backend.clone());
}
+ if self.redirects.is_empty() {
+ self.record_read_route(ReadRouteSource::Miss);
+ return None;
+ }
+
let dir = parent_dir(&normalized);
let name = file_name(&normalized).to_string();
let ctx = current_required_ctx()
diff --git a/crates/ragfs/src/core/multibackend_wrapper/tests.rs b/crates/ragfs/src/core/multibackend_wrapper/tests.rs
index 357c56fcf..d0ec7bced 100644
--- a/crates/ragfs/src/core/multibackend_wrapper/tests.rs
+++ b/crates/ragfs/src/core/multibackend_wrapper/tests.rs
@@ -91,6 +91,93 @@ async fn test_read_dir_redirect_entries_use_target_stat() {
})
.await
.unwrap();
+
+ let no_redirect_fs = test_multiwrite_fs(Vec::new());
+ let no_redirect_ctx = test_ctx();
+
+ FS_CTX
+ .scope(no_redirect_ctx.clone(), async {
+ no_redirect_fs
+ .ensure_parent_dirs("/local/acct/docs/real.txt", 0o755)
+ .await?;
+ no_redirect_fs
+ .write(
+ "/local/acct/docs/real.txt",
+ b"real body",
+ 0,
+ WriteFlag::Create,
+ )
+ .await?;
+ no_redirect_fs
+ .inner
+ .meta_store
+ .update_dir_meta(
+ "/local/acct/docs",
+ &no_redirect_ctx,
+ |redirect, _sync_log| {
+ redirect.entries.insert(
+ "ghost.pdf".to_string(),
+ RedirectEntry {
+ targets: vec!["backup1".to_string()],
+ },
+ );
+ Ok(())
+ },
+ )
+ .await?;
+ let backup = no_redirect_fs
+ .inner
+ .backup_by_name("backup1")
+ .expect("backup exists")
+ .backend
+ .clone();
+ backup
+ .ensure_parent_dirs("/local/acct/docs/ghost.pdf", 0o755)
+ .await?;
+ backup
+ .write(
+ "/local/acct/docs/ghost.pdf",
+ b"GHOST_MARKER\n",
+ 0,
+ WriteFlag::Create,
+ )
+ .await?;
+
+ let entries = no_redirect_fs.read_dir("/local/acct/docs").await?;
+ let names: Vec<&str> = entries.iter().map(|entry| entry.name.as_str()).collect();
+ assert_eq!(names, vec!["real.txt"]);
+
+ let tree = no_redirect_fs
+ .tree_directory("/local/acct", false, None, Some(3))
+ .await?;
+ let paths: Vec<&str> = tree.iter().map(|entry| entry.path.as_str()).collect();
+ assert!(!paths.contains(&"/local/acct/docs/ghost.pdf"));
+
+ let grep = no_redirect_fs
+ .grep(
+ "/local/acct/docs",
+ "GHOST_MARKER",
+ false,
+ false,
+ None,
+ None,
+ Some(1),
+ )
+ .await?;
+ assert_eq!(grep.count, 0);
+
+ let err = no_redirect_fs
+ .rename("/local/acct/docs/ghost.pdf", "/local/acct/docs/renamed.pdf")
+ .await
+ .expect_err("stale redirect metadata is ignored when redirects are disabled");
+ assert!(matches!(err, Error::NotFound(_)));
+ assert!(backup.exists("/local/acct/docs/ghost.pdf").await);
+ assert!(!backup.exists("/local/acct/docs/renamed.pdf").await);
+
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
}
#[tokio::test]
@@ -113,6 +200,36 @@ async fn test_recursive_grep_finds_nested_redirected_files() {
)
.await?;
+ let shallow = fs
+ .tree_directory("/local/acct/resources", false, None, Some(1))
+ .await?;
+ let shallow_paths: Vec<&str> =
+ shallow.iter().map(|entry| entry.path.as_str()).collect();
+ assert_eq!(shallow_paths, vec!["/local/acct/resources/doc-1"]);
+
+ let root_only = fs
+ .tree_directory("/local/acct/resources", false, None, Some(0))
+ .await?;
+ assert!(root_only.is_empty());
+
+ let deep = fs
+ .tree_directory("/local/acct/resources", false, None, Some(2))
+ .await?;
+ let deep_paths: Vec<&str> = deep.iter().map(|entry| entry.path.as_str()).collect();
+ assert_eq!(
+ deep_paths,
+ vec![
+ "/local/acct/resources/doc-1",
+ "/local/acct/resources/doc-1/page.md",
+ ]
+ );
+
+ let capped = fs
+ .tree_directory("/local/acct/resources", false, Some(1), Some(2))
+ .await?;
+ let capped_paths: Vec<&str> = capped.iter().map(|entry| entry.path.as_str()).collect();
+ assert_eq!(capped_paths, vec!["/local/acct/resources/doc-1"]);
+
let result = fs
.grep(
"/local/acct/resources",
diff --git a/crates/ragfs/src/multibackend/meta.rs b/crates/ragfs/src/multibackend/meta.rs
index 457722cd8..777a3cdcf 100644
--- a/crates/ragfs/src/multibackend/meta.rs
+++ b/crates/ragfs/src/multibackend/meta.rs
@@ -345,16 +345,12 @@ impl MetaStateStore {
/// Read redirect metadata for a directory (public, used by read_dir to merge redirect entries).
pub async fn get_redirect_meta(&self, dir: &str, ctx: &FsContext) -> Result<RedirectMeta> {
- self.read_dir_meta_pair(dir, ctx)
- .await
- .map(|(redirect, _)| redirect)
+ self.read_redirect_meta(dir, ctx).await
}
/// Read sync log metadata for a directory (public, used by retry_loop).
pub async fn get_sync_log_meta(&self, dir: &str, ctx: &FsContext) -> Result<SyncLogMeta> {
- self.read_dir_meta_pair(dir, ctx)
- .await
- .map(|(_, sync_log)| sync_log)
+ self.read_sync_log_meta(dir, ctx).await
}
/// Get a reference to the context resolver.