Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(ragfs): gate existing data reconciliation (#6617)
baojun-zhangdf87b21857c9
diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rsindex a452e4e10..c39c75cc6 100644--- a/crates/ragfs/src/core/mountable.rs+++ b/crates/ragfs/src/core/mountable.rs@@ -1254,6 +1254,7 @@ mod tests {service_name: "mw-test".to_string(),initial_partitions: 16,checkpoint_interval_secs: 86_400,+ reconcile_existing_data: false,provider: "filesystem".to_string(),items: vec![BackendItemConfig {name: "backup1".to_string(),diff --git a/crates/ragfs/src/core/multibackend_wrapper.rs b/crates/ragfs/src/core/multibackend_wrapper.rsindex 8d5553a37..6e0addebf 100644--- a/crates/ragfs/src/core/multibackend_wrapper.rs+++ b/crates/ragfs/src/core/multibackend_wrapper.rs@@ -67,6 +67,7 @@ pub struct MultiWriteWrappedFSBuilder {metadata_provider: Option<Arc<dyn MultiWriteProvider>>,initial_partitions: u32,checkpoint_interval: Duration,+ reconcile_existing_data: bool,ctx_resolver: Arc<dyn FsContextResolver>,}@@ -110,6 +111,12 @@ impl MultiWriteWrappedFSBuilder {self}+ /// Set whether startup should reconcile existing primary data into V2 metadata.+ pub fn reconcile_existing_data(mut self, reconcile_existing_data: bool) -> Self {+ self.reconcile_existing_data = reconcile_existing_data;+ self+ }+/// Configure the resolver for mount-relative foreground paths.pub fn ctx_resolver(mut self, ctx_resolver: Arc<dyn FsContextResolver>) -> Self {self.ctx_resolver = ctx_resolver;@@ -210,6 +217,7 @@ impl MultiWriteWrappedFSBuilder {metadata_provider,self.initial_partitions,backup_names,+ self.reconcile_existing_data,);fs.inner.runtime.set_import_worker(async move {let result = importer.run().await;@@ -234,6 +242,7 @@ impl MultiWriteWrappedFS {metadata_provider: None,initial_partitions: 16,checkpoint_interval: Duration::from_secs(86_400),+ reconcile_existing_data: false,ctx_resolver: Arc::new(DefaultFsContextResolver),}}diff --git a/crates/ragfs/src/core/types.rs b/crates/ragfs/src/core/types.rsindex 9056b334c..819189d69 100644--- a/crates/ragfs/src/core/types.rs+++ b/crates/ragfs/src/core/types.rs@@ -568,6 +568,9 @@ pub struct BackendsConfig {/// Checkpoint interval in seconds.#[serde(default = "default_checkpoint_interval_secs")]pub checkpoint_interval_secs: u64,+ /// Whether to scan existing primary data and reconcile it into V2 metadata.+ #[serde(default)]+ pub reconcile_existing_data: bool,/// Metadata storage provider.#[serde(default = "default_backups_provider")]pub provider: String,diff --git a/crates/ragfs/src/git/service.rs b/crates/ragfs/src/git/service.rsindex 60eab5527..15ae4ff87 100644--- a/crates/ragfs/src/git/service.rs+++ b/crates/ragfs/src/git/service.rs@@ -6702,6 +6702,7 @@ mod fast_path1_tests {service_name: "mw-test".to_string(),initial_partitions: 16,checkpoint_interval_secs: 86_400,+ reconcile_existing_data: false,provider: "filesystem".to_string(),items: vec![BackendItemConfig {name: "backup1".to_string(),diff --git a/crates/ragfs/src/multibackend/factory.rs b/crates/ragfs/src/multibackend/factory.rsindex 5bf37e7f1..1588642d6 100644--- a/crates/ragfs/src/multibackend/factory.rs+++ b/crates/ragfs/src/multibackend/factory.rs@@ -188,6 +188,7 @@ pub(crate) async fn build_inactive_multi_write_fs(.with_backups(backup_entries).initial_partitions(bc.initial_partitions).checkpoint_interval(Duration::from_secs(bc.checkpoint_interval_secs))+ .reconcile_existing_data(bc.reconcile_existing_data).ctx_resolver(Arc::new(RelativePathFsContextResolver)).build_inactive().awaitdiff --git a/crates/ragfs/src/multibackend/migration/mod.rs b/crates/ragfs/src/multibackend/migration/mod.rsindex 1789f4600..9aff6ee8d 100644--- a/crates/ragfs/src/multibackend/migration/mod.rs+++ b/crates/ragfs/src/multibackend/migration/mod.rs@@ -59,6 +59,7 @@ pub struct FullDataImporter {provider: Arc<dyn MultiWriteProvider>,initial_partitions: u32,backup_names: Vec<String>,+ reconcile_existing_data: bool,}impl FullDataImporter {@@ -69,6 +70,7 @@ impl FullDataImporter {provider: Arc<dyn MultiWriteProvider>,initial_partitions: u32,backup_names: Vec<String>,+ reconcile_existing_data: bool,) -> Self {Self {primary,@@ -76,6 +78,7 @@ impl FullDataImporter {provider,initial_partitions,backup_names,+ reconcile_existing_data,}}@@ -94,7 +97,8 @@ impl FullDataImporter {.into_iter().collect::<Vec<_>>();accounts.sort();- self.import_scope(SYSTEM_DIR).await?;+ self.import_scope(SYSTEM_DIR, self.reconcile_existing_data)+ .await?;for account in &accounts {if !read_active_accounts(self.primary.as_ref()).await?@@ -102,7 +106,8 @@ impl FullDataImporter {{continue;}- self.import_scope(account).await?;+ self.import_scope(account, self.reconcile_existing_data)+ .await?;}self.cleanup_v1_global_metadata().await?;@@ -129,7 +134,7 @@ impl FullDataImporter {}/// Initialize one V2 scope and import its current file states.- async fn import_scope(&self, account: &str) -> Result<()> {+ async fn import_scope(&self, account: &str, reconcile_existing_data: bool) -> Result<()> {let partitions = self.store.initialize_account(account, self.initial_partitions, &self.backup_names)@@ -141,6 +146,13 @@ impl FullDataImporter {&partitions,).await?;+ if !reconcile_existing_data {+ tracing::info!(+ account = %account,+ "multi-write existing data reconciliation skipped"+ );+ return Ok(());+ }let context = Arc::new(FsContextInner::new(account).with_auto_pathlock_disabled());FS_CTX.scope(context, self.import_account(account)).await}diff --git a/docs/en/concepts/14-multi-write-storage.md b/docs/en/concepts/14-multi-write-storage.mdindex 356cab111..1bd5ef13e 100644--- a/docs/en/concepts/14-multi-write-storage.md+++ b/docs/en/concepts/14-multi-write-storage.md@@ -27,12 +27,17 @@ created. Existing accounts keep their persisted partition count and routes whenthe configuration changes. Partition expansion and shrinking are not supportedin this release.-## First Enable Import+## First Enable-When `.multiwrite.json` is absent, startup creates it as `V2/migrating`,-starts foreground V2 event submission, and imports the current primary files-in the background. Successful import changes the status to `stable`. Extra-old files on backups are not removed.+When `.multiwrite.json` is absent, startup creates it as `V2/migrating` and+starts foreground V2 event submission. If `reconcile_existing_data=true`,+startup scans current primary files and records them in V2 metadata. After+initialization and optional reconciliation succeed, the status changes to+`stable`. Extra old files on backups are not removed.++With the default `reconcile_existing_data=false`, pre-existing primary files are+not recorded as V2 write events. Use it only when backups are already aligned or+historical backfill is intentionally skipped.Old multi-write protocols and old migration fields are rejected. They are notconverted or interpreted.diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex 42289d67b..b03a93e6a 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -1378,6 +1378,7 @@ top-level backend remains the only direct read source."backups": {"initial_partitions": 16,"checkpoint_interval_secs": 86400,+ "reconcile_existing_data": false,"provider": "filesystem","items": [{@@ -1400,6 +1401,7 @@ Common `backups` fields:|-----------|------|-------------|---------|| `initial_partitions` | int | Initial metadata partitions; changes do not affect existing accounts | `16` || `checkpoint_interval_secs` | int | Checkpoint discovery interval; minimum `60` | `86400` |+| `reconcile_existing_data` | bool | Scan primary files that already exist when multi-write is first enabled and record them in V2 metadata. Keep `false` only when backups are already aligned or historical backfill is intentionally skipped. | `false` || `provider` | str | Metadata provider: `filesystem` or top-level CacheRuntime-backed `cache` | `"filesystem"` || `items` | array | Backups with `name`, `backend`, `params`, and optional `encryption` | `[]` |diff --git a/docs/en/guides/13-multi-write-storage.md b/docs/en/guides/13-multi-write-storage.mdindex 4a30c18ab..24e159478 100644--- a/docs/en/guides/13-multi-write-storage.md+++ b/docs/en/guides/13-multi-write-storage.md@@ -14,6 +14,7 @@ backups asynchronously. The primary is the only direct read source."backups": {"initial_partitions": 16,"checkpoint_interval_secs": 86400,+ "reconcile_existing_data": false,"provider": "filesystem","items": [{@@ -30,12 +31,13 @@ backups asynchronously. The primary is the only direct read source.}```-The three top-level V2 settings are:+The top-level V2 settings are:| Field | Default | Meaning || --- | --- | --- || `initial_partitions` | `16` | Metadata partitions created for an account || `checkpoint_interval_secs` | `86400` | Checkpoint discovery interval; minimum `60` |+| `reconcile_existing_data` | `false` | Scan existing primary files and record them in V2 metadata || `provider` | `"filesystem"` | Metadata segment provider: `filesystem` or `cache` |Each `items[]` entry needs `name`, `backend`, and backend-specific `params`.@@ -71,8 +73,13 @@ supported in this release.## First Enable And RollbackWhen `.multiwrite.json` is absent, the mount starts foreground V2 event-submission and imports current primary files in the background. Extra old-files on backups are not removed.+submission and initializes V2 metadata. If `reconcile_existing_data=true`,+startup also scans current primary files and records them in V2 metadata. Extra+old files on backups are not removed.++With the default `reconcile_existing_data=false`, files that already existed on+the primary are not recorded as V2 write events. Use this only when backups are+already aligned with the primary or historical backfill is intentionally skipped.Old multi-write protocols and old migration fields are rejected. Take acomplete backup before enabling multi-write. Restore that backup beforediff --git a/docs/zh/concepts/14-multi-write-storage.md b/docs/zh/concepts/14-multi-write-storage.mdindex d7d0dbdc3..a72525a88 100644--- a/docs/zh/concepts/14-multi-write-storage.md+++ b/docs/zh/concepts/14-multi-write-storage.md@@ -23,11 +23,16 @@ V2 多写以一个 primary backend 作为业务真相,并把文件复制到配已有 account 始终使用持久化的 partition 数量和 routes,修改配置不会改变现有布局。本期不支持扩容或缩容。-## 首次启用导入+## 首次启用-`.multiwrite.json` 不存在时,启动流程创建 `V2/migrating` 状态,先启用前台-V2 事件提交,再在后台导入 primary 当前文件。导入成功后切换为 `stable`。-导入不会删除 backup 中多余的旧文件。+`.multiwrite.json` 不存在时,启动流程创建 `V2/migrating` 状态,并启用前台+V2 事件提交。若 `reconcile_existing_data=true`,启动流程会扫描 primary 当前文件+并写入 V2 metadata。初始化和可选对账成功后,状态切换为 `stable`。该流程+不会删除 backup 中多余的旧文件。++默认 `reconcile_existing_data=false` 时,primary 上启用前已存在的文件不会作为+V2 write event 记录。只有在 backup 已与 primary 对齐,或明确跳过历史回填时+才保持默认值。旧多写协议和旧迁移字段直接拒绝,不执行转换或解释。diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex 8f424723d..46929682d 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -1364,6 +1364,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系"backups": {"initial_partitions": 16,"checkpoint_interval_secs": 86400,+ "reconcile_existing_data": false,"provider": "filesystem","items": [{@@ -1386,6 +1387,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系|------|------|------|--------|| `initial_partitions` | int | account 首次初始化的 metadata partition 数;修改不影响已有 account | `16` || `checkpoint_interval_secs` | int | Checkpoint 扫描间隔,最小值为 `60` | `86400` |+| `reconcile_existing_data` | bool | 首次启用多写时,扫描 primary 已存在文件并写入 V2 metadata。仅在 backup 已对齐或明确跳过历史回填时保持 `false`。 | `false` || `provider` | str | Metadata provider:`filesystem` 或复用顶层 CacheRuntime 的 `cache` | `"filesystem"` || `items` | array | backup 列表,包含 `name`、`backend`、`params` 和可选 `encryption` | `[]` |diff --git a/docs/zh/guides/13-multi-write-storage.md b/docs/zh/guides/13-multi-write-storage.mdindex 3211082db..82d1abc4d 100644--- a/docs/zh/guides/13-multi-write-storage.md+++ b/docs/zh/guides/13-multi-write-storage.md@@ -14,6 +14,7 @@ V2 多写在 primary 成功后返回,metadata 和 backup 在后台异步更新"backups": {"initial_partitions": 16,"checkpoint_interval_secs": 86400,+ "reconcile_existing_data": false,"provider": "filesystem","items": [{@@ -30,12 +31,13 @@ V2 多写在 primary 成功后返回,metadata 和 backup 在后台异步更新}```-V2 的三个顶层配置是:+V2 的顶层配置是:| 字段 | 默认值 | 含义 || --- | --- | --- || `initial_partitions` | `16` | 为 account 创建的 metadata partition 数量 || `checkpoint_interval_secs` | `86400` | Checkpoint 扫描间隔,最小值为 `60` |+| `reconcile_existing_data` | `false` | 扫描 primary 已存在文件并写入 V2 metadata || `provider` | `"filesystem"` | Metadata segment provider:`filesystem` 或 `cache` |每个 `items[]` 需要 `name`、`backend` 和 backend 专用 `params`。@@ -66,8 +68,13 @@ redirects 字段,输出 warning 后会在调用 Rust 前删除,字段不生## 首次启用与回滚-`.multiwrite.json` 不存在时,mount 先启用前台 V2 事件提交,再在后台导入-primary 当前文件。导入不会删除 backup 中多余的旧文件。+`.multiwrite.json` 不存在时,mount 先启用前台 V2 事件提交,并初始化 V2+metadata。若 `reconcile_existing_data=true`,启动流程还会扫描 primary 当前文件+并写入 V2 metadata。该流程不会删除 backup 中多余的旧文件。++默认 `reconcile_existing_data=false` 时,primary 上启用前已存在的文件不会作为+V2 write event 记录。只有在 backup 已与 primary 对齐,或明确跳过历史回填时+才保持默认值。旧多写协议和旧迁移字段直接拒绝。启用多写前必须创建完整备份;回滚二进制前必须先恢复该备份。diff --git a/examples/ov.conf.example b/examples/ov.conf.exampleindex ff4766f6d..9ee9dabd6 100644--- a/examples/ov.conf.example+++ b/examples/ov.conf.example@@ -82,6 +82,7 @@"_comment": "Rename this key to backups to enable V2 multi-write.","initial_partitions": 16,"checkpoint_interval_secs": 86400,+ "reconcile_existing_data": false,"provider": "cache","items": [{"name": "local-backup",