Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(ragfs): gate existing data reconciliation (#6617)

baojun-zhangdf87b21857c9
diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rs
index a452e4e10..c39c75cc6 100644
--- a/crates/ragfs/src/core/mountable.rs
+++ b/crates/ragfs/src/core/mountable.rs
@@ -1254,6 +1254,7 @@ mod tests {
service_name: "mw-test".to_string(),
initial_partitions: 16,
checkpoint_interval_secs: 86_400,
+ reconcile_existing_data: false,
provider: "filesystem".to_string(),
items: vec![BackendItemConfig {
name: "backup1".to_string(),
diff --git a/crates/ragfs/src/core/multibackend_wrapper.rs b/crates/ragfs/src/core/multibackend_wrapper.rs
index 8d5553a37..6e0addebf 100644
--- a/crates/ragfs/src/core/multibackend_wrapper.rs
+++ b/crates/ragfs/src/core/multibackend_wrapper.rs
@@ -67,6 +67,7 @@ pub struct MultiWriteWrappedFSBuilder {
metadata_provider: Option<Arc<dyn MultiWriteProvider>>,
initial_partitions: u32,
checkpoint_interval: Duration,
+ reconcile_existing_data: bool,
ctx_resolver: Arc<dyn FsContextResolver>,
}
@@ -110,6 +111,12 @@ impl MultiWriteWrappedFSBuilder {
self
}
+ /// Set whether startup should reconcile existing primary data into V2 metadata.
+ pub fn reconcile_existing_data(mut self, reconcile_existing_data: bool) -> Self {
+ self.reconcile_existing_data = reconcile_existing_data;
+ self
+ }
+
/// Configure the resolver for mount-relative foreground paths.
pub fn ctx_resolver(mut self, ctx_resolver: Arc<dyn FsContextResolver>) -> Self {
self.ctx_resolver = ctx_resolver;
@@ -210,6 +217,7 @@ impl MultiWriteWrappedFSBuilder {
metadata_provider,
self.initial_partitions,
backup_names,
+ self.reconcile_existing_data,
);
fs.inner.runtime.set_import_worker(async move {
let result = importer.run().await;
@@ -234,6 +242,7 @@ impl MultiWriteWrappedFS {
metadata_provider: None,
initial_partitions: 16,
checkpoint_interval: Duration::from_secs(86_400),
+ reconcile_existing_data: false,
ctx_resolver: Arc::new(DefaultFsContextResolver),
}
}
diff --git a/crates/ragfs/src/core/types.rs b/crates/ragfs/src/core/types.rs
index 9056b334c..819189d69 100644
--- a/crates/ragfs/src/core/types.rs
+++ b/crates/ragfs/src/core/types.rs
@@ -568,6 +568,9 @@ pub struct BackendsConfig {
/// Checkpoint interval in seconds.
#[serde(default = "default_checkpoint_interval_secs")]
pub checkpoint_interval_secs: u64,
+ /// Whether to scan existing primary data and reconcile it into V2 metadata.
+ #[serde(default)]
+ pub reconcile_existing_data: bool,
/// Metadata storage provider.
#[serde(default = "default_backups_provider")]
pub provider: String,
diff --git a/crates/ragfs/src/git/service.rs b/crates/ragfs/src/git/service.rs
index 60eab5527..15ae4ff87 100644
--- a/crates/ragfs/src/git/service.rs
+++ b/crates/ragfs/src/git/service.rs
@@ -6702,6 +6702,7 @@ mod fast_path1_tests {
service_name: "mw-test".to_string(),
initial_partitions: 16,
checkpoint_interval_secs: 86_400,
+ reconcile_existing_data: false,
provider: "filesystem".to_string(),
items: vec![BackendItemConfig {
name: "backup1".to_string(),
diff --git a/crates/ragfs/src/multibackend/factory.rs b/crates/ragfs/src/multibackend/factory.rs
index 5bf37e7f1..1588642d6 100644
--- a/crates/ragfs/src/multibackend/factory.rs
+++ b/crates/ragfs/src/multibackend/factory.rs
@@ -188,6 +188,7 @@ pub(crate) async fn build_inactive_multi_write_fs(
.with_backups(backup_entries)
.initial_partitions(bc.initial_partitions)
.checkpoint_interval(Duration::from_secs(bc.checkpoint_interval_secs))
+ .reconcile_existing_data(bc.reconcile_existing_data)
.ctx_resolver(Arc::new(RelativePathFsContextResolver))
.build_inactive()
.await
diff --git a/crates/ragfs/src/multibackend/migration/mod.rs b/crates/ragfs/src/multibackend/migration/mod.rs
index 1789f4600..9aff6ee8d 100644
--- a/crates/ragfs/src/multibackend/migration/mod.rs
+++ b/crates/ragfs/src/multibackend/migration/mod.rs
@@ -59,6 +59,7 @@ pub struct FullDataImporter {
provider: Arc<dyn MultiWriteProvider>,
initial_partitions: u32,
backup_names: Vec<String>,
+ reconcile_existing_data: bool,
}
impl FullDataImporter {
@@ -69,6 +70,7 @@ impl FullDataImporter {
provider: Arc<dyn MultiWriteProvider>,
initial_partitions: u32,
backup_names: Vec<String>,
+ reconcile_existing_data: bool,
) -> Self {
Self {
primary,
@@ -76,6 +78,7 @@ impl FullDataImporter {
provider,
initial_partitions,
backup_names,
+ reconcile_existing_data,
}
}
@@ -94,7 +97,8 @@ impl FullDataImporter {
.into_iter()
.collect::<Vec<_>>();
accounts.sort();
- self.import_scope(SYSTEM_DIR).await?;
+ self.import_scope(SYSTEM_DIR, self.reconcile_existing_data)
+ .await?;
for account in &accounts {
if !read_active_accounts(self.primary.as_ref())
.await?
@@ -102,7 +106,8 @@ impl FullDataImporter {
{
continue;
}
- self.import_scope(account).await?;
+ self.import_scope(account, self.reconcile_existing_data)
+ .await?;
}
self.cleanup_v1_global_metadata().await?;
@@ -129,7 +134,7 @@ impl FullDataImporter {
}
/// Initialize one V2 scope and import its current file states.
- async fn import_scope(&self, account: &str) -> Result<()> {
+ async fn import_scope(&self, account: &str, reconcile_existing_data: bool) -> Result<()> {
let partitions = self
.store
.initialize_account(account, self.initial_partitions, &self.backup_names)
@@ -141,6 +146,13 @@ impl FullDataImporter {
&partitions,
)
.await?;
+ if !reconcile_existing_data {
+ tracing::info!(
+ account = %account,
+ "multi-write existing data reconciliation skipped"
+ );
+ return Ok(());
+ }
let context = Arc::new(FsContextInner::new(account).with_auto_pathlock_disabled());
FS_CTX.scope(context, self.import_account(account)).await
}
diff --git a/docs/en/concepts/14-multi-write-storage.md b/docs/en/concepts/14-multi-write-storage.md
index 356cab111..1bd5ef13e 100644
--- a/docs/en/concepts/14-multi-write-storage.md
+++ b/docs/en/concepts/14-multi-write-storage.md
@@ -27,12 +27,17 @@ created. Existing accounts keep their persisted partition count and routes when
the configuration changes. Partition expansion and shrinking are not supported
in this release.
-## First Enable Import
+## First Enable
-When `.multiwrite.json` is absent, startup creates it as `V2/migrating`,
-starts foreground V2 event submission, and imports the current primary files
-in the background. Successful import changes the status to `stable`. Extra
-old files on backups are not removed.
+When `.multiwrite.json` is absent, startup creates it as `V2/migrating` and
+starts foreground V2 event submission. If `reconcile_existing_data=true`,
+startup scans current primary files and records them in V2 metadata. After
+initialization and optional reconciliation succeed, the status changes to
+`stable`. Extra old files on backups are not removed.
+
+With the default `reconcile_existing_data=false`, pre-existing primary files are
+not recorded as V2 write events. Use it only when backups are already aligned or
+historical backfill is intentionally skipped.
Old multi-write protocols and old migration fields are rejected. They are not
converted or interpreted.
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md
index 42289d67b..b03a93e6a 100644
--- a/docs/en/guides/01-configuration.md
+++ b/docs/en/guides/01-configuration.md
@@ -1378,6 +1378,7 @@ top-level backend remains the only direct read source.
"backups": {
"initial_partitions": 16,
"checkpoint_interval_secs": 86400,
+ "reconcile_existing_data": false,
"provider": "filesystem",
"items": [
{
@@ -1400,6 +1401,7 @@ Common `backups` fields:
|-----------|------|-------------|---------|
| `initial_partitions` | int | Initial metadata partitions; changes do not affect existing accounts | `16` |
| `checkpoint_interval_secs` | int | Checkpoint discovery interval; minimum `60` | `86400` |
+| `reconcile_existing_data` | bool | Scan primary files that already exist when multi-write is first enabled and record them in V2 metadata. Keep `false` only when backups are already aligned or historical backfill is intentionally skipped. | `false` |
| `provider` | str | Metadata provider: `filesystem` or top-level CacheRuntime-backed `cache` | `"filesystem"` |
| `items` | array | Backups with `name`, `backend`, `params`, and optional `encryption` | `[]` |
diff --git a/docs/en/guides/13-multi-write-storage.md b/docs/en/guides/13-multi-write-storage.md
index 4a30c18ab..24e159478 100644
--- a/docs/en/guides/13-multi-write-storage.md
+++ b/docs/en/guides/13-multi-write-storage.md
@@ -14,6 +14,7 @@ backups asynchronously. The primary is the only direct read source.
"backups": {
"initial_partitions": 16,
"checkpoint_interval_secs": 86400,
+ "reconcile_existing_data": false,
"provider": "filesystem",
"items": [
{
@@ -30,12 +31,13 @@ backups asynchronously. The primary is the only direct read source.
}
```
-The three top-level V2 settings are:
+The top-level V2 settings are:
| Field | Default | Meaning |
| --- | --- | --- |
| `initial_partitions` | `16` | Metadata partitions created for an account |
| `checkpoint_interval_secs` | `86400` | Checkpoint discovery interval; minimum `60` |
+| `reconcile_existing_data` | `false` | Scan existing primary files and record them in V2 metadata |
| `provider` | `"filesystem"` | Metadata segment provider: `filesystem` or `cache` |
Each `items[]` entry needs `name`, `backend`, and backend-specific `params`.
@@ -71,8 +73,13 @@ supported in this release.
## First Enable And Rollback
When `.multiwrite.json` is absent, the mount starts foreground V2 event
-submission and imports current primary files in the background. Extra old
-files on backups are not removed.
+submission and initializes V2 metadata. If `reconcile_existing_data=true`,
+startup also scans current primary files and records them in V2 metadata. Extra
+old files on backups are not removed.
+
+With the default `reconcile_existing_data=false`, files that already existed on
+the primary are not recorded as V2 write events. Use this only when backups are
+already aligned with the primary or historical backfill is intentionally skipped.
Old multi-write protocols and old migration fields are rejected. Take a
complete backup before enabling multi-write. Restore that backup before
diff --git a/docs/zh/concepts/14-multi-write-storage.md b/docs/zh/concepts/14-multi-write-storage.md
index d7d0dbdc3..a72525a88 100644
--- a/docs/zh/concepts/14-multi-write-storage.md
+++ b/docs/zh/concepts/14-multi-write-storage.md
@@ -23,11 +23,16 @@ V2 多写以一个 primary backend 作为业务真相,并把文件复制到配
已有 account 始终使用持久化的 partition 数量和 routes,修改配置不会改变
现有布局。本期不支持扩容或缩容。
-## 首次启用导入
+## 首次启用
-`.multiwrite.json` 不存在时,启动流程创建 `V2/migrating` 状态,先启用前台
-V2 事件提交,再在后台导入 primary 当前文件。导入成功后切换为 `stable`。
-导入不会删除 backup 中多余的旧文件。
+`.multiwrite.json` 不存在时,启动流程创建 `V2/migrating` 状态,并启用前台
+V2 事件提交。若 `reconcile_existing_data=true`,启动流程会扫描 primary 当前文件
+并写入 V2 metadata。初始化和可选对账成功后,状态切换为 `stable`。该流程
+不会删除 backup 中多余的旧文件。
+
+默认 `reconcile_existing_data=false` 时,primary 上启用前已存在的文件不会作为
+V2 write event 记录。只有在 backup 已与 primary 对齐,或明确跳过历史回填时
+才保持默认值。
旧多写协议和旧迁移字段直接拒绝,不执行转换或解释。
diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md
index 8f424723d..46929682d 100644
--- a/docs/zh/guides/01-configuration.md
+++ b/docs/zh/guides/01-configuration.md
@@ -1364,6 +1364,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
"backups": {
"initial_partitions": 16,
"checkpoint_interval_secs": 86400,
+ "reconcile_existing_data": false,
"provider": "filesystem",
"items": [
{
@@ -1386,6 +1387,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
|------|------|------|--------|
| `initial_partitions` | int | account 首次初始化的 metadata partition 数;修改不影响已有 account | `16` |
| `checkpoint_interval_secs` | int | Checkpoint 扫描间隔,最小值为 `60` | `86400` |
+| `reconcile_existing_data` | bool | 首次启用多写时,扫描 primary 已存在文件并写入 V2 metadata。仅在 backup 已对齐或明确跳过历史回填时保持 `false`。 | `false` |
| `provider` | str | Metadata provider:`filesystem` 或复用顶层 CacheRuntime 的 `cache` | `"filesystem"` |
| `items` | array | backup 列表,包含 `name`、`backend`、`params` 和可选 `encryption` | `[]` |
diff --git a/docs/zh/guides/13-multi-write-storage.md b/docs/zh/guides/13-multi-write-storage.md
index 3211082db..82d1abc4d 100644
--- a/docs/zh/guides/13-multi-write-storage.md
+++ b/docs/zh/guides/13-multi-write-storage.md
@@ -14,6 +14,7 @@ V2 多写在 primary 成功后返回,metadata 和 backup 在后台异步更新
"backups": {
"initial_partitions": 16,
"checkpoint_interval_secs": 86400,
+ "reconcile_existing_data": false,
"provider": "filesystem",
"items": [
{
@@ -30,12 +31,13 @@ V2 多写在 primary 成功后返回,metadata 和 backup 在后台异步更新
}
```
-V2 的三个顶层配置是:
+V2 的顶层配置是:
| 字段 | 默认值 | 含义 |
| --- | --- | --- |
| `initial_partitions` | `16` | 为 account 创建的 metadata partition 数量 |
| `checkpoint_interval_secs` | `86400` | Checkpoint 扫描间隔,最小值为 `60` |
+| `reconcile_existing_data` | `false` | 扫描 primary 已存在文件并写入 V2 metadata |
| `provider` | `"filesystem"` | Metadata segment provider:`filesystem` 或 `cache` |
每个 `items[]` 需要 `name`、`backend` 和 backend 专用 `params`。
@@ -66,8 +68,13 @@ redirects 字段,输出 warning 后会在调用 Rust 前删除,字段不生
## 首次启用与回滚
-`.multiwrite.json` 不存在时,mount 先启用前台 V2 事件提交,再在后台导入
-primary 当前文件。导入不会删除 backup 中多余的旧文件。
+`.multiwrite.json` 不存在时,mount 先启用前台 V2 事件提交,并初始化 V2
+metadata。若 `reconcile_existing_data=true`,启动流程还会扫描 primary 当前文件
+并写入 V2 metadata。该流程不会删除 backup 中多余的旧文件。
+
+默认 `reconcile_existing_data=false` 时,primary 上启用前已存在的文件不会作为
+V2 write event 记录。只有在 backup 已与 primary 对齐,或明确跳过历史回填时
+才保持默认值。
旧多写协议和旧迁移字段直接拒绝。启用多写前必须创建完整备份;回滚二进制
前必须先恢复该备份。
diff --git a/examples/ov.conf.example b/examples/ov.conf.example
index ff4766f6d..9ee9dabd6 100644
--- a/examples/ov.conf.example
+++ b/examples/ov.conf.example
@@ -82,6 +82,7 @@
"_comment": "Rename this key to backups to enable V2 multi-write.",
"initial_partitions": 16,
"checkpoint_interval_secs": 86400,
+ "reconcile_existing_data": false,
"provider": "cache",
"items": [{
"name": "local-backup",