Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Feat/rfv reindex (#5416)

Jiahui Zhou0f31ece6b821
diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rs
index c49055cea..23f942a2b 100644
--- a/crates/ov_cli/src/client.rs
+++ b/crates/ov_cli/src/client.rs
@@ -474,7 +474,7 @@ impl HttpClient {
uri: &str,
mode: &str,
wait: bool,
- dry_run: bool,
+ force: bool,
tags: Vec<String>,
tag_mode: &str,
recursive: bool,
@@ -483,8 +483,10 @@ impl HttpClient {
"uri": uri,
"mode": mode,
"wait": wait,
- "dry_run": dry_run,
});
+ if force {
+ body["force"] = serde_json::json!(true);
+ }
if !recursive {
body["recursive"] = serde_json::json!(false);
}
diff --git a/crates/ov_cli/src/commands/content.rs b/crates/ov_cli/src/commands/content.rs
index 8bdf0f231..dfff6781e 100644
--- a/crates/ov_cli/src/commands/content.rs
+++ b/crates/ov_cli/src/commands/content.rs
@@ -87,7 +87,7 @@ pub async fn reindex(
uri: &str,
mode: &str,
wait: bool,
- dry_run: bool,
+ force: bool,
tags: Vec<String>,
tag_mode: &str,
recursive: bool,
@@ -95,7 +95,7 @@ pub async fn reindex(
compact: bool,
) -> Result<()> {
let result = client
- .reindex(uri, mode, wait, dry_run, tags, tag_mode, recursive)
+ .reindex(uri, mode, wait, force, tags, tag_mode, recursive)
.await?;
crate::output::output_success(result, output_format, compact);
Ok(())
diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rs
index 160b51d92..2fb5e905d 100644
--- a/crates/ov_cli/src/handlers.rs
+++ b/crates/ov_cli/src/handlers.rs
@@ -1455,7 +1455,7 @@ pub async fn handle_reindex(
uri: String,
mode: String,
wait: bool,
- dry_run: bool,
+ force: bool,
tags: Vec<String>,
tag_mode: String,
recursive: bool,
@@ -1467,7 +1467,7 @@ pub async fn handle_reindex(
&uri,
&mode,
wait,
- dry_run,
+ force,
tags,
&tag_mode,
recursive,
diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rs
index a81bd6450..c1cc0351c 100644
--- a/crates/ov_cli/src/help_ui.rs
+++ b/crates/ov_cli/src/help_ui.rs
@@ -1388,10 +1388,6 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[
label: "ov reindex viking://projects/acme --mode semantic_and_vectors --recursive=false",
description: "Refresh only the target directory semantics and vectors, without sub-directory.",
},
- HelpItem {
- label: "ov reindex viking://projects/acme --mode prune_orphans --dry-run",
- description: "Preview orphan vector cleanup.",
- },
],
next_steps: &[
HelpItem {
@@ -2956,8 +2952,8 @@ mod tests {
.expect("reindex help should render"),
);
- assert!(rendered.contains("--mode <vectors_only|semantic_and_vectors|prune_orphans>"));
- assert!(rendered.contains("--dry-run"));
+ assert!(rendered.contains("--mode <vectors_only|semantic_and_vectors>"));
+ assert!(rendered.contains("--force"));
assert!(rendered.contains("--recursive <true|false>"));
assert!(rendered.contains("Regenerate semantic artifacts, then vectors."));
}
diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs
index d53c4dcab..d925484fa 100644
--- a/crates/ov_cli/src/main.rs
+++ b/crates/ov_cli/src/main.rs
@@ -1279,11 +1279,11 @@ enum Commands {
/// Viking URI
#[arg(value_name = "uri")]
uri: String,
- /// Reindex mode: vectors_only rebuilds vectors; semantic_and_vectors regenerates semantic artifacts, then vectors; prune_orphans deletes orphan vector records
+ /// Reindex mode: vectors_only rebuilds vectors; semantic_and_vectors regenerates semantic artifacts, then vectors
#[arg(
long,
default_value = "vectors_only",
- value_parser = ["vectors_only", "semantic_and_vectors", "prune_orphans"],
+ value_parser = ["vectors_only", "semantic_and_vectors"],
value_name = "mode",
help_heading = "Common options"
)]
@@ -1297,9 +1297,9 @@ enum Commands {
help_heading = "Common options"
)]
wait: bool,
- /// Preview prune_orphans deletions without mutating vectors
+ /// Rebuild all selected semantic/vector data without comparing fingerprints
#[arg(long, help_heading = "Common options")]
- dry_run: bool,
+ force: bool,
/// Comma-separated k=v retrieval tags for rebuilt vector records
#[arg(long = "tags", value_delimiter = ',', value_name = "k=v", help_heading = "Common options")]
tags: Vec<String>,
@@ -3789,12 +3789,12 @@ async fn main() {
uri,
mode,
wait,
- dry_run,
+ force,
tags,
tag_mode,
recursive,
} => {
- handlers::handle_reindex(uri, mode, wait, dry_run, tags, tag_mode, recursive, ctx).await
+ handlers::handle_reindex(uri, mode, wait, force, tags, tag_mode, recursive, ctx).await
}
Commands::Get { uri, local_path } => handlers::handle_get(uri, local_path, ctx).await,
Commands::Find {
@@ -5849,9 +5849,9 @@ mod tests {
"reindex",
"viking://resources/demo",
"--mode",
- "prune_orphans",
+ "semantic_and_vectors",
"--wait=false",
- "--dry-run",
+ "--force",
"--tags",
"team=search",
"--tag-mode",
@@ -5862,12 +5862,14 @@ mod tests {
let cli = result.expect("reindex command should parse");
match cli.command {
Commands::Reindex {
+ force,
tags,
tag_mode,
recursive,
..
} => {
assert_eq!(tags, vec!["team=search"]);
+ assert!(force);
assert_eq!(tag_mode, "append");
assert!(!recursive);
}
diff --git a/docs/en/api/02-resources.md b/docs/en/api/02-resources.md
index 33c6ee4d6..838fcd484 100644
--- a/docs/en/api/02-resources.md
+++ b/docs/en/api/02-resources.md
@@ -210,7 +210,7 @@ This endpoint is the core entry point for resource management. It supports vario
- When deleting a resource, OpenViking scans the self or peer memories targeted by the current context before deletion, removes the matching resource URI and content introduced by that `reason`, and refreshes the semantic index for the affected memories.
- Other sources with `wait=false` finish source parsing, target resolution, and AGFS writes before returning. Only semantic and embedding queues continue asynchronously.
- `processing_mode=vectors_only` does not call the VLM semantic-understanding stage and does not generate or refresh `.abstract.md` / `.overview.md`. For existing targets, it preserves existing semantic artifacts and existing semantic vectors. It still updates the resource tree, vectorizes current non-hidden files when `build_index=true`, and removes detail vectors for files deleted during refresh.
-- `processing_mode` belongs to `add_resource`. The admin `reindex` API/CLI continues to use `mode` (`vectors_only`, `semantic_and_vectors`, `prune_orphans`) for maintenance operations on already-ingested data.
+- `processing_mode` belongs to `add_resource`. The admin `reindex` API/CLI continues to use `mode` (`vectors_only`, `semantic_and_vectors`) for maintenance operations on already-ingested data.
- When `watch_interval > 0`, the watch task binds to `to` if provided; otherwise it binds to the `root_uri` returned by this import. If no stable `root_uri` is available, the request fails and asks for an explicit `to`.
- For Connector imports, `is_active=false` creates the paused Watch before submission. Native Feishu and Git imports carry `is_active` through the resource queue and create the Watch after resolving the imported resource URI. In both cases, the initial import still runs once and periodic scheduling remains disabled.
- Feishu/Lark app-token imports do not pass `args.feishu_access_token`. OpenViking keeps the existing app credential flow and the SDK obtains an app/tenant token from `app_id` and `app_secret`. This mode supports both one-time imports and `watch_interval > 0`.
diff --git a/docs/en/api/12-content.md b/docs/en/api/12-content.md
index 6f663bce8..267e46206 100644
--- a/docs/en/api/12-content.md
+++ b/docs/en/api/12-content.md
@@ -488,7 +488,7 @@ Content-Disposition: attachment; filename*=UTF-8''logo.png
### set_tags()
-Set explicit `k=v` tags used by retrieval filters. `replace` replaces existing tags, while `append` adds tags. When the target is a directory, `recursive=true` applies the update to files below it.
+Set explicit `k=v` tags used by retrieval filters. `replace` replaces existing tags, `append` adds tags, and `clear` explicitly removes existing tags. When the target is a directory, `recursive=true` applies the update to files below it. Omitting `tags`, or passing an empty list with `replace`, is a no-op; only `clear` removes tags and it ignores any supplied tag values.
**Python SDK**
@@ -583,7 +583,7 @@ ov set-tags viking://resources/project/ \
### reindex()
-Reindex semantic and/or vector artifacts for existing content already stored in OpenViking. This is an operational maintenance API intended for scenarios such as embedding model changes, VLM changes, vector store rebuild, or post-upgrade repair of existing indexes.
+Validate and repair semantic and/or vector artifacts for existing content already stored in OpenViking. Resource and skill targets use incremental RFV (Request / Formal / Vector) convergence by default: unchanged file fingerprints, directory L0/L1 visible-body fingerprints, and request scalars are skipped. Use `force=true` for an unconditional rebuild.
This API operates on existing `viking://...` content. It does not import new files. For normal ingestion, use [Resources](02-resources.md).
@@ -597,14 +597,14 @@ This API operates on existing `viking://...` content. It does not import new fil
| Parameter | Type | Required | Default | Description |
|-----------|------|----------|---------|-------------|
| uri | str | Yes | - | Viking URI to reindex |
-| mode | str | No | `vectors_only` | Reindex mode: `vectors_only`, `semantic_and_vectors`, or `prune_orphans` |
+| mode | str | No | `vectors_only` | Reindex mode: `vectors_only` or `semantic_and_vectors` |
| wait | bool | No | `true` | Whether to wait for completion |
-| dry_run | bool | No | `false` | Only valid with `mode="prune_orphans"`; report orphan vector records without deleting them |
-| recursive | bool | No | `true` | Whether to process descendants recursively; `false` applies only to `semantic_and_vectors` on a `resource`, `memory`, or `skill` directory |
+| force | bool | No | `false` | Skip fingerprint equality and reprocess every available resource/skill file and directory level in scope |
+| recursive | bool | No | `true` | Whether to process descendants recursively; both resource/skill rebuild modes honor this option |
| tags | list[str] | No | `null` | Write tags to every successfully rebuilt vector record. Omitting tags, or passing an empty list with `replace`, preserves existing tags |
| tag_mode | str | No | `replace` | Tag write mode: `replace`, `append`, or `clear`; `clear` removes existing tags without requiring `tags` |
-The HTTP request body rejects unknown fields. `uri` may use OpenViking path variables accepted by other content APIs; it is resolved before validation.
+The HTTP request body ignores unknown fields to preserve compatibility while clients and servers evolve independently. `uri` may use OpenViking path variables accepted by other content APIs; it is resolved before validation.
**Supported URI scopes**
@@ -623,19 +623,18 @@ when reindexing a broader user namespace, session subtrees are skipped.
**Modes**
-- `vectors_only`: rebuilds vector-store records from currently recoverable source data without rewriting `.abstract.md` or `.overview.md`
-- `semantic_and_vectors`: regenerates semantic artifacts first, then rebuilds vectors from the refreshed semantic outputs
-- `prune_orphans`: deletes vector-store records under the requested URI whose source files no longer exist in the filesystem. With `dry_run=true`, it only reports how many records would be deleted.
+- `vectors_only`: compares current source MD5 values with vector-record MD5 values through RFV and rebuilds only missing or stale L0/L1/L2 records; it does not rewrite `.abstract.md` or `.overview.md`
+- `semantic_and_vectors`: uses the same RFV state and `ContextUpdatePlan` to drive semantic repair and L0/L1/L2 updates, without a second manual vector scan
For `resource` and `skill`, `semantic_and_vectors` refreshes directory/file semantic artifacts, including `.abstract.md` and `.overview.md`. For `memory`, it rebuilds the current persisted memory subtree semantics and vectors, but it does not replay historical extraction order.
-For `semantic_and_vectors`, semantic generation and vector rebuilding are sequenced by the reindex executor. The semantic refresh step does not enqueue its own background vectorization work; vectors are rebuilt by the reindex step so `wait=true` reflects the reindex operation itself.
+Resource/skill `vectors_only` and `semantic_and_vectors` share one F traversal and one narrow-field V inventory. Each source is read once during state resolution and that result is reused while executing the plan. `force=true` bypasses MD5 equality but still performs completeness and scope checks.
-For a `resource` or `memory` directory, `recursive=false` regenerates only the target directory's `.abstract.md` and `.overview.md`, then rebuilds only that directory's L0/L1 vectors. Child directories do not regenerate semantic artifacts, and neither child directories nor files are re-vectorized. The target aggregation still reads existing summaries from deterministically sampled child directories; sampled direct files are summarized as inputs to the target aggregation. For a `skill` target, `recursive=false` regenerates the skill directory's L0/L1 semantic artifacts and vectors from `SKILL.md`, but does not rebuild the `SKILL.md` L2 vector. Namespace targets (such as `viking://user/<user_id>` or its `skills` container) do not support `recursive=false` with `semantic_and_vectors`; these requests are rejected before starting reindex work. Select a concrete resource, memory, or skill directory for a non-recursive refresh. This flag does not change existing behavior for `vectors_only` or `prune_orphans`.
+For a resource/skill directory, `recursive=false` limits both modes to the target directory's own L0/L1 records. Direct children are read only as existing REUSE inputs for the target aggregation; descendants are neither scanned nor rebuilt. Namespace containers have no index records of their own, so namespace targets reject `recursive=false` instead of silently becoming a no-op. Memory remains on the existing reindex path.
-For `prune_orphans`, source existence is checked against the filesystem. If an entire directory is missing, vector records for files and semantic sidecars below that directory, such as `.abstract.md` and `.overview.md`, are pruned together. `dry_run` is rejected for other modes.
+When the F snapshot is complete, the RFV diff identifies records present in V but absent from F as orphans and removes them during the same reindex. Deletion fails closed when the F snapshot is incomplete.
-When non-empty `tags` are provided, tags are included in the same upsert as each vector record produced by reindex; reindex does not call `set_tags` afterwards. Directory and namespace reindex operations apply tags to successfully rebuilt directory L0/L1 and leaf L2 records. `replace` overwrites existing tags, while `append` merges by key. `replace` with an empty tag list is a no-op. `clear` removes existing tags and does not require `tags`; if values are supplied with `clear`, they are ignored. `prune_orphans` produces no vectors and ignores both fields.
+When non-empty `tags` are provided, tags are included in the same upsert as each vector record produced by reindex; reindex does not call `set_tags` afterwards. Directory and namespace reindex operations apply tags to successfully rebuilt directory L0/L1 and leaf L2 records. `replace` overwrites existing tags, while `append` merges by key. `replace` with an empty tag list is a no-op. `clear` removes existing tags and does not require `tags`; if values are supplied with `clear`, they are ignored.
Subtree reindex is not transactional. Records skipped because no semantic source is available, or records whose embedding fails, do not receive the new tags.
@@ -658,24 +657,17 @@ print(result)
result = client.reindex(
uri="viking://user/default/skills",
mode="semantic_and_vectors",
+ force=True,
wait=False,
)
print(result["status"])
```
-```python
-result = client.reindex(
- uri="viking://resources",
- mode="prune_orphans",
- dry_run=True,
-)
-print(result["would_delete_records"])
-```
-
**TypeScript SDK**
```typescript
console.log(await client.reindex("viking://resources/docs/", {
+ force: true,
tags: ["team=search"],
tagMode: "append",
}));
@@ -689,6 +681,7 @@ only `opts=nil` applies the SDK default `wait=true`.
```go
result, err := client.Reindex(ctx, "viking://resources", &openviking.ReindexOptions{
Mode: "vectors_only",
+ Force: true,
Tags: []string{"team=search"},
TagMode: "replace",
})
@@ -698,17 +691,6 @@ if err != nil {
fmt.Println(result["status"])
```
-```go
-result, err := client.Reindex(ctx, "viking://resources", &openviking.ReindexOptions{
- Mode: "prune_orphans",
- DryRun: true,
-})
-if err != nil {
- return err
-}
-fmt.Println(result["task_id"])
-```
-
**HTTP API**
```
@@ -726,6 +708,7 @@ curl -X POST http://localhost:1933/api/v1/content/reindex \
"uri": "viking://resources",
"mode": "vectors_only",
"wait": false,
+ "force": true,
"tags": ["team=search", "env=prod"],
"tag_mode": "replace"
}'
@@ -735,7 +718,7 @@ curl -X POST http://localhost:1933/api/v1/content/reindex \
```bash
openviking reindex viking://resources --mode vectors_only \
- --tags team=search,env=prod --tag-mode replace
+ --force --tags team=search,env=prod --tag-mode replace
```
Use `--tag-mode clear` without `--tags` to clear existing tags:
@@ -748,10 +731,6 @@ openviking reindex viking://resources --mode vectors_only --tag-mode clear
openviking reindex viking://user/default/skills --mode semantic_and_vectors --wait false
```
-```bash
-openviking reindex viking://resources --mode prune_orphans --dry-run
-```
-
**Asynchronous response (`wait=false`)**
```json
@@ -794,8 +773,7 @@ Task records are persisted under `/local/{account_id}/_system/tasks/{user_id}/{t
| mode | Effective reindex mode |
| scanned_records | Number of records or semantic sources considered |
| rebuilt_records | Number of vector records successfully rebuilt |
-| deleted_records | Number of vector records deleted by `prune_orphans`; `0` for `dry_run=true` |
-| would_delete_records | Number of vector records that would be deleted by `prune_orphans` in dry-run mode |
+| deleted_records | Number of orphan vector records confirmed and deleted by the RFV diff |
| unsupported_records | Number of records skipped because no usable vector source was available |
| failed_records | Number of records that failed while rebuilding |
| duration_ms | Synchronous run duration in milliseconds |
@@ -805,12 +783,10 @@ Task records are persisted under `/local/{account_id}/_system/tasks/{user_id}/{t
**Behavior notes**
- `vectors_only` and `semantic_and_vectors` are non-destructive. They use rebuild/upsert behavior and do not require dropping the vector collection first.
-- `prune_orphans` is destructive unless `dry_run=true`: it removes vector records whose source files no longer exist.
- `viking://` reindex fans out to supported top-level namespaces and excludes `session`.
- Namespace reindex operations such as `viking://user` propagate to supported child content types.
- `vectors_only` is the right mode when only the embedding model or vector index needs to be refreshed.
- `semantic_and_vectors` is the right mode when semantic artifacts themselves must be regenerated before re-vectorization.
-- `prune_orphans` is the right mode when the filesystem has been changed outside normal APIs and the vector store may still contain records for deleted paths.
- Only one reindex task can run for the same URI and owner at a time. A concurrent request for the same target returns a conflict.
- For resource files, text files can use file content when no summary is available. Non-text files require a generated summary or existing vector record fallback; otherwise they are counted as unsupported.
diff --git a/docs/en/configuration/01-server.md b/docs/en/configuration/01-server.md
index c632bc078..8aec2919d 100644
--- a/docs/en/configuration/01-server.md
+++ b/docs/en/configuration/01-server.md
@@ -233,6 +233,14 @@ This setting controls queue-job concurrency. It is separate from `vlm.media.max_
`max_concurrent` controls independent AddResource jobs. `file_operation_concurrency` controls file commit and fallback comparison work within one AddResource job, while `file_vectorization_concurrency` controls files within one vectors-only directory job.
+### `queue_workers.reindex`
+
+| Field | Type | Default | Description |
+|---|---|---:|---|
+| `max_concurrent` | integer | `4` | Number of complete Reindex root jobs consumed concurrently; must be greater than `0`; requires a server restart after changes |
+
+This setting limits independent asynchronous reindex requests. URI-overlapping requests remain protected by path locks, while VLM and embedding work continue to use their respective concurrency limits.
+
### `queue_workers.session_commit`
| Field | Type | Default | Description |
@@ -262,7 +270,7 @@ When `base_url` is configured, OV sends the current user's OV API key in `X-API-
| Field | Type | Default | Description |
|---|---|---:|---|
-| `file_vectorization_concurrency` | integer | `8` | Number of files concurrently read, prepared, and enqueued by one `vectors_only` reindex task; must be greater than `0`; values above the internal safety limit of `64` are capped; requires a server restart after changes |
+| `file_vectorization_concurrency` | integer | `8` | Number of files concurrently read, fingerprinted, and prepared by one resource/skill reindex task; must be greater than `0`; values above the internal safety limit of `64` are capped; requires a server restart after changes |
## HTTP Server Settings
diff --git a/docs/en/getting-started/05-cli-setup.md b/docs/en/getting-started/05-cli-setup.md
index 7f200d694..b78502f7d 100644
--- a/docs/en/getting-started/05-cli-setup.md
+++ b/docs/en/getting-started/05-cli-setup.md
@@ -489,12 +489,11 @@ ov add-resource --help
## Rebuilding Indexes
-`ov reindex <uri>` rebuilds the indexes of already-imported content. Three modes are available:
+`ov reindex <uri>` validates and repairs indexes for already-imported content. Three modes are available. Resource and skill records whose MD5 fingerprints already match are skipped by default:
- `--mode vectors_only` — refresh vectors only.
- `--mode semantic_and_vectors` — regenerate semantic artifacts (`.abstract.md`, `.overview.md`) before refreshing vectors.
-- `--mode prune_orphans` — remove vector records whose source files no longer exist. Add `--dry-run` to preview the cleanup without applying it.
-`semantic_and_vectors` processes the full subtree by default. If child summaries already exist and only the target directory's `.abstract.md` / `.overview.md` need regeneration, add `--recursive=false`; this refreshes only the target directory semantics and its L0/L1 vectors.
+Both rebuild modes process the full subtree by default; add `--recursive=false` to process only the target itself. Add `--force` to bypass MD5 comparison and rebuild all resource/skill semantic and vector data in scope.
There is no `semantic` or `full` mode alias.
diff --git a/docs/zh/api/02-resources.md b/docs/zh/api/02-resources.md
index e7652c895..ba62ebdda 100644
--- a/docs/zh/api/02-resources.md
+++ b/docs/zh/api/02-resources.md
@@ -203,7 +203,7 @@ URL/文件 Parser TreeBuilder AGFS Summarizer/Vector
- 删除资源时,系统会在删除前扫描本次上下文对应的 self 或 peer 记忆中的 `resource_refs`,清理对应资源 URI 和由该 `reason` 引入的内容,并重新刷新相关记忆的语义索引。
- 其他来源在 `wait=false` 时会在响应前完成来源解析、目标解析和 AGFS 写入,仅 semantic 与 embedding 队列继续异步处理。
- `processing_mode=vectors_only` 不调用 VLM 语义理解阶段,也不会生成或刷新 `.abstract.md` / `.overview.md`。对已存在目标,它会保留旧的语义产物和旧的语义向量;仍会更新资源树,在 `build_index=true` 时向量化当前非隐藏文件,并清理由本次刷新删除的文件 detail 向量。
-- `processing_mode` 只属于 `add_resource`。管理员维护已有数据时,`reindex` API/CLI 仍使用 `mode`(`vectors_only`、`semantic_and_vectors`、`prune_orphans`)。
+- `processing_mode` 只属于 `add_resource`。管理员维护已有数据时,`reindex` API/CLI 仍使用 `mode`(`vectors_only`、`semantic_and_vectors`)。
- `watch_interval > 0` 时,如果指定了 `to`,监控任务绑定该目标;如果未指定 `to`,监控任务绑定本次导入返回的 `root_uri`。如果无法得到稳定 `root_uri`,请求会报错并要求显式传 `to`。
- Connector 导入设置 `is_active=false` 时会在提交前创建暂停状态的 Watch;原生飞书和 Git 导入会通过资源队列透传 `is_active`,解析出最终资源 URI 后再创建 Watch。两种情况下首次导入均执行一次,周期调度保持关闭。
- 飞书/Lark 应用 token 导入不传 `args.feishu_access_token`。OpenViking 保持原有应用凭证流程,由 SDK 使用 `app_id` 和 `app_secret` 自动获取 app/tenant token。该模式支持一次性导入和 `watch_interval > 0`。
diff --git a/docs/zh/api/12-content.md b/docs/zh/api/12-content.md
index e2ffe90df..c1268f927 100644
--- a/docs/zh/api/12-content.md
+++ b/docs/zh/api/12-content.md
@@ -488,7 +488,7 @@ Content-Disposition: attachment; filename*=UTF-8''logo.png
### set_tags()
-设置用于检索过滤的显式 `k=v` 标签。`replace` 替换已有标签,`append` 追加标签;对目录设置 `recursive=true` 时会更新目录下的文件。
+设置用于检索过滤的显式 `k=v` 标签。`replace` 替换已有标签,`append` 追加标签,`clear` 显式清空已有标签;对目录设置 `recursive=true` 时会更新目录下的文件。省略 `tags` 或传入空数组配合 `replace` 都是 no-op;只有 `clear` 会清空,且会忽略同时传入的标签值。
**Python SDK**
@@ -583,7 +583,7 @@ ov set-tags viking://resources/project/ \
### reindex()
-对已经存储在 OpenViking 中的现有内容,重新构建语义产物和/或向量索引。这是一个运维维护接口,适用于 embedding 模型更换、VLM 更换、向量库重刷、版本升级后修复历史索引等场景。
+对已经存储在 OpenViking 中的现有内容,校验并修复语义产物和/或向量索引。resource 和 skill 默认使用 RFV(Request / Formal / Vector)增量收敛:文件内容指纹、目录 L0/L1 可见正文指纹和请求标量都未变化时直接跳过。需要无条件重建时使用 `force=true`。
这个接口面向已有的 `viking://...` 内容,不负责导入新文件。常规导入请使用 [Resources](02-resources.md)。
@@ -597,14 +597,14 @@ ov set-tags viking://resources/project/ \
| 参数 | 类型 | 必填 | 默认值 | 说明 |
|------|------|------|--------|------|
| uri | str | 是 | - | 要重新索引的 Viking URI |
-| mode | str | 否 | `vectors_only` | 重建模式:`vectors_only`、`semantic_and_vectors` 或 `prune_orphans` |
+| mode | str | 否 | `vectors_only` | 重建模式:`vectors_only` 或 `semantic_and_vectors` |
| wait | bool | 否 | `true` | 是否等待任务完成 |
-| dry_run | bool | 否 | `false` | 仅适用于 `mode="prune_orphans"`;只报告 orphan 向量记录,不实际删除 |
-| recursive | bool | 否 | `true` | 是否递归处理下级内容;`false` 仅对 `resource`、`memory` 或 `skill` 目录的 `semantic_and_vectors` 生效 |
+| force | bool | 否 | `false` | 是否跳过内容指纹比较并强制重新处理范围内全部 resource/skill 文件和目录 level |
+| recursive | bool | 否 | `true` | 是否递归处理下级内容;resource/skill 的两种重建模式都遵守该参数 |
| tags | list[str] | 否 | `null` | 写入本次成功重建的全部向量记录。省略或空数组配合 `replace` 时保留已有 tags |
| tag_mode | str | 否 | `replace` | 标签写入模式:`replace`、`append` 或 `clear`;`clear` 不要求传 `tags` 并清空已有标签 |
-HTTP 请求体不接受未知字段。`uri` 可以使用其他 content API 支持的 OpenViking 路径变量,服务端会先解析再校验。
+HTTP 请求体会忽略未知字段,便于客户端与服务端在字段演进期间兼容。`uri` 可以使用其他 content API 支持的 OpenViking 路径变量,服务端会先解析再校验。
**支持的 URI 范围**
@@ -623,19 +623,18 @@ session 子树会被跳过。
**模式说明**
-- `vectors_only`:基于当前仍可恢复的源数据重建向量库记录,不会重写 `.abstract.md` 和 `.overview.md`
-- `semantic_and_vectors`:先重新生成语义产物,再基于新的语义结果重建向量
-- `prune_orphans`:删除请求 URI 范围内源文件已不存在的向量库记录。设置 `dry_run=true` 时,只报告会删除多少记录,不实际删除。
+- `vectors_only`:通过 RFV 对比当前 source MD5 与向量记录 MD5,只重建缺失或过期的 L0/L1/L2;不会重写 `.abstract.md` 和 `.overview.md`
+- `semantic_and_vectors`:通过同一 RFV 状态和 `ContextUpdatePlan` 驱动语义修复及 L0/L1/L2 向量更新,不再执行第二次手工向量扫描
对于 `resource` 和 `skill`,`semantic_and_vectors` 会刷新目录/文件语义产物,包括 `.abstract.md` 和 `.overview.md`。对于 `memory`,它会重建当前已持久化 memory 子树的语义和向量,但不会回放历史记忆抽取顺序。
-对于 `semantic_and_vectors`,语义刷新和向量重建由 reindex executor 串行编排。语义刷新阶段不会再额外向后台 embedding queue 投递自己的向量化任务;向量由 reindex 阶段统一重建,因此 `wait=true` 表示等待 reindex 操作本身完成。
+resource/skill 的 `vectors_only` 和 `semantic_and_vectors` 共用一次 F 遍历和一次窄字段 V inventory。每个 source 在状态解析时读取一次,实际处理复用本次读取结果,不会为了执行 Plan 再读取一次 F。`force=true` 会跳过 MD5 相等判断并重新处理作用域内全部可用 level;完整性检查仍然执行。
-对 `resource` 或 `memory` 目录设置 `recursive=false` 时,只重新生成目标目录的 `.abstract.md`、`.overview.md`,并重建该目录的 L0/L1 向量;下级目录不会重新生成语义产物,下级目录和文件也不会重新向量化。目标目录仍会读取本轮确定性采样命中的既有下级摘要;若采样命中直接文件,仍会为当前目录聚合准备这些文件的摘要。对 `skill` 目标设置 `recursive=false` 时,会根据 `SKILL.md` 重新生成 skill 目录的 L0/L1 语义产物及向量,但不会重建 `SKILL.md` 的 L2 向量。namespace 目标(例如 `viking://user/<user_id>` 或其 `skills` 容器)不支持在 `semantic_and_vectors` 模式下设置 `recursive=false`;这类请求会在启动重建前被拒绝。非递归刷新请选择具体的 resource、memory 或 skill 目录。该参数不改变 `vectors_only` 或 `prune_orphans` 的既有行为。
+对 resource/skill 目录设置 `recursive=false` 时,两种模式都只处理目标目录自身的 L0/L1。直接 children 只作为目标聚合的既有 REUSE 输入读取,不扫描或重建下级内容。namespace 容器自身没有索引记录,因此 namespace 目标不接受 `recursive=false`,避免请求静默成为 no-op。memory 仍使用原有 reindex 链路。
-对于 `prune_orphans`,源文件是否存在以当前文件系统为准。如果整个目录已经不存在,该目录下的正文文件向量和语义 sidecar 向量(例如 `.abstract.md`、`.overview.md`)会一起清理。`dry_run` 用在其他模式时会被拒绝。
+F snapshot 完整时,RFV 差分会把 V 中存在但 F 中不存在的记录识别为 orphan,并在同一次 reindex 中清理;F snapshot 不完整时 fail-closed,不执行删除。
-传入非空 `tags` 时,标签会随 reindex 生成的向量记录在同一次 upsert 中写入,不会在完成后额外调用 `set_tags`。目录或 namespace reindex 会把标签应用到本次成功重建的目录 L0/L1 和叶子 L2 记录。`replace` 覆盖已有标签,`append` 按 key 合并;`replace` 配合空数组时不修改已有标签。`clear` 不要求传 `tags`,会清空已有标签;即使同时传入标签值也会忽略。`prune_orphans` 不生成向量,因此会忽略 `tags` 和 `tag_mode`。
+传入非空 `tags` 时,标签会随 reindex 生成的向量记录在同一次 upsert 中写入,不会在完成后额外调用 `set_tags`。目录或 namespace reindex 会把标签应用到本次成功重建的目录 L0/L1 和叶子 L2 记录。`replace` 覆盖已有标签,`append` 按 key 合并;`replace` 配合空数组时不修改已有标签。`clear` 不要求传 `tags`,会清空已有标签;即使同时传入标签值也会忽略。
子树 reindex 不是事务性操作。如果部分记录因缺少语义来源或 embedding 失败而未重建,只有成功写入的记录会更新标签。
@@ -658,24 +657,17 @@ print(result)
result = client.reindex(
uri="viking://user/default/skills",
mode="semantic_and_vectors",
+ force=True,
wait=False,
)
print(result["status"])
```
-```python
-result = client.reindex(
- uri="viking://resources",
- mode="prune_orphans",
- dry_run=True,
-)
-print(result["would_delete_records"])
-```
-
**TypeScript SDK**
```typescript
console.log(await client.reindex("viking://resources/docs/", {
+ force: true,
tags: ["team=search"],
tagMode: "append",
}));
@@ -689,6 +681,7 @@ console.log(await client.reindex("viking://resources/docs/", {
```go
result, err := client.Reindex(ctx, "viking://resources", &openviking.ReindexOptions{
Mode: "vectors_only",
+ Force: true,
Tags: []string{"team=search"},
TagMode: "replace",
})
@@ -698,17 +691,6 @@ if err != nil {
fmt.Println(result["status"])
```
-```go
-result, err := client.Reindex(ctx, "viking://resources", &openviking.ReindexOptions{
- Mode: "prune_orphans",
- DryRun: true,
-})
-if err != nil {
- return err
-}
-fmt.Println(result["task_id"])
-```
-
**HTTP API**
```
@@ -726,6 +708,7 @@ curl -X POST http://localhost:1933/api/v1/content/reindex \
"uri": "viking://resources",
"mode": "vectors_only",
"wait": false,
+ "force": true,
"tags": ["team=search", "env=prod"],
"tag_mode": "replace"
}'
@@ -735,7 +718,7 @@ curl -X POST http://localhost:1933/api/v1/content/reindex \
```bash
openviking reindex viking://resources --mode vectors_only \
- --tags team=search,env=prod --tag-mode replace
+ --force --tags team=search,env=prod --tag-mode replace
```
使用 `--tag-mode clear` 且无需传 `--tags` 即可清空已有标签:
@@ -748,10 +731,6 @@ openviking reindex viking://resources --mode vectors_only --tag-mode clear
openviking reindex viking://user/default/skills --mode semantic_and_vectors --wait false
```
-```bash
-openviking reindex viking://resources --mode prune_orphans --dry-run
-```
-
**异步响应(`wait=false`)**
```json
@@ -794,8 +773,7 @@ GET /api/v1/tasks?task_type=admin_reindex&resource_id=viking://resources
| mode | 实际执行的 reindex 模式 |
| scanned_records | 被检查的记录或语义源数量 |
| rebuilt_records | 成功重建的向量记录数量 |
-| deleted_records | `prune_orphans` 实际删除的向量记录数量;`dry_run=true` 时为 `0` |
-| would_delete_records | `prune_orphans` dry-run 模式下将会删除的向量记录数量 |
+| deleted_records | RFV 差分确认并删除的 orphan 向量记录数量 |
| unsupported_records | 因没有可用向量来源而跳过的记录数量 |
| failed_records | 重建失败的记录数量 |
| duration_ms | 同步执行耗时,单位毫秒 |
@@ -805,12 +783,10 @@ GET /api/v1/tasks?task_type=admin_reindex&resource_id=viking://resources
**行为说明**
- `vectors_only` 和 `semantic_and_vectors` 是非破坏式的,采用重建/覆盖写入,不需要先 drop 向量集合。
-- `prune_orphans` 除非设置 `dry_run=true`,否则会删除源文件已经不存在的向量记录。
- 对 `viking://` 发起 reindex 时,会向下分发到支持的顶层命名空间,并显式排除 `session`。
- 命名空间级 reindex,例如 `viking://user`,会继续传播到其支持的子内容类型。
- 如果只是 embedding 模型或向量索引需要刷新,应使用 `vectors_only`。
- 如果语义产物本身也需要重建,再做重向量化,应使用 `semantic_and_vectors`。
-- 如果文件系统曾绕过正常 API 发生删除,向量库可能还残留已删除路径的记录,应使用 `prune_orphans`。
- 同一个 URI 和 owner 同时只能运行一个 reindex 任务。对同一目标的并发请求会返回 conflict。
- 对 resource 文件,文本文件在没有 summary 时可以使用文件正文;非文本文件需要已生成的 summary 或已有向量记录 fallback,否则会计为 unsupported。
diff --git a/docs/zh/configuration/01-server.md b/docs/zh/configuration/01-server.md
index 4d835023d..8f43f42a0 100644
--- a/docs/zh/configuration/01-server.md
+++ b/docs/zh/configuration/01-server.md
@@ -232,6 +232,14 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK
`max_concurrent` 控制相互独立的 AddResource 作业并发,`file_operation_concurrency` 控制单个 AddResource 作业内文件提交和 fallback 比较操作的并发,`file_vectorization_concurrency` 控制单个 vectors-only 目录作业内的文件并发。
+### `queue_workers.reindex`
+
+| 字段 | 类型 | 默认值 | 说明 |
+|---|---|---:|---|
+| `max_concurrent` | integer | `4` | 同时消费的完整 Reindex root 作业数,必须大于 `0`;修改后需重启服务 |
+
+该配置限制相互独立的异步 reindex 请求并发。URI 范围重叠的请求仍由 path lock 保护;VLM 和 embedding 仍分别遵守各自的并发限制。
+
### `queue_workers.session_commit`
| 字段 | 类型 | 默认值 | 说明 |
@@ -261,7 +269,7 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK
| 字段 | 类型 | 默认值 | 说明 |
|---|---|---:|---|
-| `file_vectorization_concurrency` | integer | `8` | 单个 `vectors_only` reindex 任务内并发读取、准备并入队的文件数,必须大于 `0`;超过内部安全上限 `64` 的值会被截断;修改后需重启服务 |
+| `file_vectorization_concurrency` | integer | `8` | 单个 resource/skill reindex 任务内并发读取、计算指纹和准备文件的数量,必须大于 `0`;超过内部安全上限 `64` 的值会被截断;修改后需重启服务 |
## HTTP 服务配置
diff --git a/docs/zh/getting-started/05-cli-setup.md b/docs/zh/getting-started/05-cli-setup.md
index 566d0767b..a95684879 100644
--- a/docs/zh/getting-started/05-cli-setup.md
+++ b/docs/zh/getting-started/05-cli-setup.md
@@ -489,12 +489,11 @@ ov add-resource --help
## 重建索引
-`ov reindex <uri>` 用于重建已导入内容的索引,支持三种模式:
+`ov reindex <uri>` 用于校验并修复已导入内容的索引,支持三种模式。resource 和 skill 默认按 MD5 增量跳过已经一致的记录:
- `--mode vectors_only` —— 只刷新向量。
- `--mode semantic_and_vectors` —— 先重新生成语义产物(`.abstract.md`、`.overview.md`),再刷新向量。
-- `--mode prune_orphans` —— 清理源文件已不存在的向量记录,加 `--dry-run` 可预览而不实际执行。
-`semantic_and_vectors` 默认递归处理整个子树。对已经具有下级摘要、只需要重新生成目标目录 `.abstract.md` / `.overview.md` 的场景,可添加 `--recursive=false`;此时仅刷新目标目录语义产物和该目录的 L0/L1 向量。
+两种重建模式默认递归处理整个子树;添加 `--recursive=false` 后只处理目标本身。需要跳过 MD5 比较、强制重建作用域内全部 resource/skill 语义和向量数据时,添加 `--force`。
没有 `semantic` 或 `full` 这样的模式别名。
diff --git a/openviking/server/routers/content.py b/openviking/server/routers/content.py
index d9089a21c..89f1f1423 100644
--- a/openviking/server/routers/content.py
+++ b/openviking/server/routers/content.py
@@ -30,7 +30,7 @@ from openviking.server.telemetry import run_operation
from openviking.storage.acl import AclSpec
from openviking.storage.vector_ids import is_vector_record_id
from openviking.telemetry import TelemetryRequest
-from openviking_cli.exceptions import InvalidArgumentError, NotFoundError, PermissionDeniedError
+from openviking_cli.exceptions import NotFoundError, PermissionDeniedError
from openviking_cli.utils import get_logger
logger = get_logger(__name__)
@@ -84,19 +84,17 @@ class SetTagsRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
uri: str
- tags: list[str]
+ tags: list[str] | None = None
mode: str = "replace"
recursive: bool = False
telemetry: TelemetryRequest = False
class ReindexRequest(BaseModel):
- model_config = ConfigDict(extra="forbid")
-
uri: str
mode: str = "vectors_only"
+ force: bool = False
wait: bool = True
- dry_run: bool = False
recursive: bool = True
tags: list[str] | None = None
tag_mode: str = "replace"
@@ -300,12 +298,13 @@ async def set_tags(
"""Set explicit k=v retrieval tags metadata for a file or directory."""
service = get_service()
uri = validate_request_viking_uri(resolve_path_variables(request.uri), _ctx)
+ tags = [] if request.mode == "clear" else request.tags or []
execution = await run_operation(
operation="content.set_tags",
telemetry=request.telemetry,
fn=lambda: service.fs.set_tags(
uri=uri,
- tags=request.tags,
+ tags=tags,
mode=request.mode,
recursive=request.recursive,
ctx=_ctx,
@@ -324,8 +323,6 @@ async def reindex(
ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN, Role.USER),
):
"""Reindex semantic/vector artifacts for a URI-scoped maintenance target."""
- if body.dry_run and body.mode != "prune_orphans":
- raise InvalidArgumentError("dry_run is only supported for prune_orphans reindex mode.")
uri = validate_request_viking_uri(resolve_path_variables(body.uri), ctx)
uri = _authorize_reindex_uri(uri, ctx)
service = get_service()
@@ -333,9 +330,10 @@ async def reindex(
"uri": uri,
"mode": body.mode,
"wait": body.wait,
- "dry_run": body.dry_run,
"ctx": ctx,
}
+ if body.force:
+ reindex_kwargs["force"] = True
if not body.recursive:
reindex_kwargs["recursive"] = False
if body.tags is not None or body.tag_mode == "clear":
diff --git a/openviking/service/core.py b/openviking/service/core.py
index b94398aed..434ed1b64 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -36,6 +36,7 @@ from openviking.storage.index_consistency import check_index_consistency
from openviking.storage.queuefs.add_resource_processor import AddResourceProcessor
from openviking.storage.queuefs.external_task_processor import ExternalTaskProcessor
from openviking.storage.queuefs.queue_manager import QueueManager, init_queue_manager
+from openviking.storage.queuefs.reindex_processor import ReindexProcessor
from openviking.storage.queuefs.session_commit_processor import SessionCommitProcessor
from openviking.storage.viking_fs import VikingFS, init_viking_fs
from openviking.storage.vikingdb_manager import VikingDBManager
@@ -151,6 +152,7 @@ class OpenVikingService:
max_concurrent_semantic=cluster_vlm.max_concurrent,
max_concurrent_external_parse=config.queue_workers.external_parse.max_concurrent,
max_concurrent_add_resource=config.queue_workers.add_resource.max_concurrent,
+ max_concurrent_reindex=config.queue_workers.reindex.max_concurrent,
max_concurrent_session_commit=config.queue_workers.session_commit.max_concurrent,
max_concurrent_external_task=config.queue_workers.external_task.max_concurrent,
binding_config=binding_config,
@@ -164,6 +166,7 @@ class OpenVikingService:
max_concurrent_semantic: int = 32,
max_concurrent_external_parse: int = 4,
max_concurrent_add_resource: int = 4,
+ max_concurrent_reindex: int = 4,
max_concurrent_session_commit: int = 8,
max_concurrent_external_task: int = 10,
binding_config: Any = None,
@@ -188,6 +191,7 @@ class OpenVikingService:
max_concurrent_semantic=max_concurrent_semantic,
max_concurrent_external_parse=max_concurrent_external_parse,
max_concurrent_add_resource=max_concurrent_add_resource,
+ max_concurrent_reindex=max_concurrent_reindex,
max_concurrent_session_commit=max_concurrent_session_commit,
max_concurrent_external_task=max_concurrent_external_task,
)
@@ -259,11 +263,7 @@ class OpenVikingService:
if manager is None:
return
base_config = self._config.model_copy(
- update={
- "agent_evolution": self._agent_evolution_base_config.model_copy(
- deep=True
- )
- }
+ update={"agent_evolution": self._agent_evolution_base_config.model_copy(deep=True)}
)
await manager.replace_base_config(base_config)
@@ -285,11 +285,7 @@ class OpenVikingService:
if self._agfs_client is None:
raise RuntimeError("AGFS client not initialized")
base_config = self._config.model_copy(
- update={
- "agent_evolution": self._agent_evolution_base_config.model_copy(
- deep=True
- )
- },
+ update={"agent_evolution": self._agent_evolution_base_config.model_copy(deep=True)},
)
manager = build_runtime_config_manager(
AsyncAGFSClient(self._agfs_client),
@@ -451,6 +447,7 @@ class OpenVikingService:
max_concurrent_add_resource=(
self._config.queue_workers.add_resource.max_concurrent
),
+ max_concurrent_reindex=self._config.queue_workers.reindex.max_concurrent,
max_concurrent_session_commit=(
self._config.queue_workers.session_commit.max_concurrent
),
@@ -616,6 +613,11 @@ class OpenVikingService:
),
allow_create=True,
)
+ self._queue_manager.get_queue(
+ self._queue_manager.REINDEX,
+ dequeue_handler=ReindexProcessor(self._viking_fs),
+ allow_create=True,
+ )
self._queue_manager.get_queue(
self._queue_manager.SESSION_COMMIT,
dequeue_handler=SessionCommitProcessor(
@@ -757,8 +759,8 @@ class OpenVikingService:
*,
uri: str,
mode: str = "vectors_only",
+ force: bool = False,
wait: bool = True,
- dry_run: bool = False,
recursive: bool = True,
tags: list[str] | None = None,
tag_mode: str = "replace",
@@ -779,9 +781,10 @@ class OpenVikingService:
"uri": uri,
"mode": mode,
"wait": wait,
- "dry_run": dry_run,
"ctx": effective_ctx,
}
+ if force:
+ execute_kwargs["force"] = True
if not recursive:
execute_kwargs["recursive"] = False
if tags is not None or tag_mode == "clear":
diff --git a/openviking/service/reindex_executor.py b/openviking/service/reindex_executor.py
index 9027517d2..bd8131a67 100644
--- a/openviking/service/reindex_executor.py
+++ b/openviking/service/reindex_executor.py
@@ -5,9 +5,8 @@
from __future__ import annotations
import asyncio
-import re
import time
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
from typing import Any, Iterable, Optional
from openviking.config.vlm import VLMResolver
@@ -23,37 +22,28 @@ from openviking.core.namespace import (
content_owner_context_for_uri,
context_type_for_uri,
is_session_uri,
- owner_fields_for_uri,
owner_space_for_uri,
)
from openviking.server.dependencies import get_service
from openviking.server.identity import RequestContext
from openviking.service.task_tracker import get_task_tracker
-from openviking.service.task_work_index import bind_task_context
from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils
from openviking.storage.abstract_overview import body_for_preview, embedding_text_for_body
-from openviking.storage.errors import ResourceBusyError
-from openviking.storage.expr import And, Eq, Or, PathScope
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
-from openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
from openviking.storage.viking_fs import get_viking_fs
from openviking.telemetry import get_current_telemetry
from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
-from openviking.utils.content_hash import content_md5
from openviking.utils.embedding_input import truncate_embedding_input
from openviking.utils.embedding_utils import (
_apply_ingest_options,
_decode_text_bytes,
_truncate_abstract_bytes,
get_resource_content_type,
- vectorize_directory_meta,
- vectorize_file,
)
from openviking.utils.ingest_options import IngestOptions
from openviking_cli.exceptions import InvalidArgumentError, NotFoundError, OpenVikingError
-from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils import VikingURI, get_logger
from openviking_cli.utils.config import get_openviking_config
from openviking_cli.utils.config.embedding_config import SUMMARY_TEXT_SOURCES
@@ -61,8 +51,6 @@ from openviking_cli.utils.config.embedding_config import SUMMARY_TEXT_SOURCES
logger = get_logger(__name__)
REINDEX_TASK_TYPE = "admin_reindex"
-PRUNE_ORPHAN_CANDIDATE_LIMIT = 100000
-PRUNE_OUTPUT_FIELDS = ["id", "uri", "level", "context_type", "account_id", "owner_user_id"]
_MAX_FILE_VECTORIZATION_CONCURRENCY = 64
@@ -105,7 +93,6 @@ class _ReindexCounters:
scanned_records: int = 0
rebuilt_records: int = 0
deleted_records: int = 0
- would_delete_records: int = 0
unsupported_records: int = 0
failed_records: int = 0
warnings: list[str] = field(default_factory=list)
@@ -114,7 +101,6 @@ class _ReindexCounters:
self.scanned_records += other.scanned_records
self.rebuilt_records += other.rebuilt_records
self.deleted_records += other.deleted_records
- self.would_delete_records += other.would_delete_records
self.unsupported_records += other.unsupported_records
self.failed_records += other.failed_records
self.warnings.extend(other.warnings)
@@ -125,11 +111,16 @@ class _ReindexRunContext:
ctx: RequestContext
counters: _ReindexCounters
lock: dict | None = None
+ lease: dict | None = None
+ lease_handed_off: bool = False
+ lock_root_uri: str | None = None
ingest_options: IngestOptions | None = None
+ force: bool = False
+ root_is_dir: bool | None = None
@dataclass
-class _PruneSourceRead:
+class _SourceRead:
exists: bool
text: str = ""
error: Exception | None = None
@@ -139,12 +130,12 @@ class ReindexExecutor:
"""Non-destructive reindex orchestration for admin maintenance flows."""
SUPPORTED_MODES_BY_TYPE = {
- "global_namespace": {"vectors_only", "semantic_and_vectors", "prune_orphans"},
- "user_namespace": {"vectors_only", "semantic_and_vectors", "prune_orphans"},
- "skill_namespace": {"vectors_only", "semantic_and_vectors", "prune_orphans"},
- "resource": {"vectors_only", "semantic_and_vectors", "prune_orphans"},
- "skill": {"vectors_only", "semantic_and_vectors", "prune_orphans"},
- "memory": {"vectors_only", "semantic_and_vectors", "prune_orphans"},
+ "global_namespace": {"vectors_only", "semantic_and_vectors"},
+ "user_namespace": {"vectors_only", "semantic_and_vectors"},
+ "skill_namespace": {"vectors_only", "semantic_and_vectors"},
+ "resource": {"vectors_only", "semantic_and_vectors"},
+ "skill": {"vectors_only", "semantic_and_vectors"},
+ "memory": {"vectors_only", "semantic_and_vectors"},
}
def __init__(
@@ -157,9 +148,7 @@ class ReindexExecutor:
async def _semantic_processor_for(self, ctx: RequestContext) -> SemanticProcessor:
if self.vlm_resolver is None:
- raise RuntimeError(
- "ReindexExecutor requires a VLM resolver for semantic reindexing"
- )
+ raise RuntimeError("ReindexExecutor requires a VLM resolver for semantic reindexing")
vlm = await self.vlm_resolver.get_vlm(ctx.account_id)
return SemanticProcessor(
max_concurrent_llm=vlm.max_concurrent,
@@ -205,7 +194,7 @@ class ReindexExecutor:
uri: str,
mode: str,
wait: bool,
- dry_run: bool = False,
+ force: bool = False,
recursive: bool = True,
tags: list[str] | None = None,
tag_mode: str = "replace",
@@ -213,17 +202,14 @@ class ReindexExecutor:
) -> dict[str, Any]:
object_type = self._infer_target_type(uri)
self._validate_mode(object_type, mode)
- if (
- mode == "semantic_and_vectors"
- and not recursive
- and object_type in {"global_namespace", "user_namespace", "skill_namespace"}
- ):
+ if not recursive and object_type in {
+ "global_namespace",
+ "user_namespace",
+ "skill_namespace",
+ }:
raise InvalidArgumentError(
- "recursive=false is not supported for namespace reindex targets. "
- "Select a resource, memory, or skill directory instead."
+ f"recursive=false is not supported for namespace reindex target {uri}"
)
- if dry_run and mode != "prune_orphans":
- raise InvalidArgumentError("dry_run is only supported for prune_orphans reindex mode.")
ingest_options = self._resolve_ingest_options(
mode=mode,
tags=tags,
@@ -247,7 +233,7 @@ class ReindexExecutor:
uri=uri,
object_type=object_type,
mode=mode,
- dry_run=dry_run,
+ force=force,
recursive=recursive,
ingest_options=ingest_options,
ctx=ctx,
@@ -266,18 +252,60 @@ class ReindexExecutor:
details={"uri": uri},
)
- asyncio.create_task(
- self._run_tracked(
- task.task_id,
+ from openviking.storage.queuefs import get_queue_manager
+ from openviking.storage.queuefs.reindex_msg import ReindexMsg
+
+ service = None
+ lease = None
+ enqueued = False
+ try:
+ service = get_service()
+ path = service.viking_fs._uri_to_path(uri, ctx=ctx)
+ stat = await service.viking_fs.stat(uri, ctx=ctx, skip_count=True)
+ acquire = (
+ service.viking_fs._async_agfs.pathlock_acquire_tree
+ if stat.get("isDir", stat.get("is_dir"))
+ else service.viking_fs._async_agfs.pathlock_acquire_exact
+ )
+ lease = await acquire(path)
+ handoff = await service.viking_fs._async_agfs.pathlock_to_handoff(lease)
+ msg = ReindexMsg(
+ task_id=task.task_id,
uri=uri,
object_type=object_type,
mode=mode,
- dry_run=dry_run,
+ force=force,
recursive=recursive,
- ingest_options=ingest_options,
- ctx=ctx,
+ tags=tags,
+ tag_mode=tag_mode,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ group_ids=list(ctx.group_ids),
+ role=str(ctx.role),
+ actor_peer_id=ctx.actor_peer_id,
+ bypass_acl=ctx.bypass_acl,
+ telemetry_id=get_current_telemetry().telemetry_id or None,
+ lock_handoff=handoff,
)
- )
+ queue_manager = get_queue_manager()
+ await queue_manager.enqueue(queue_manager.REINDEX, msg.to_dict())
+ enqueued = True
+ await service.viking_fs._async_agfs.pathlock_handoff(lease)
+ lease = None
+ await tracker.update_stage(
+ task.task_id, "queued", account_id=ctx.account_id, user_id=ctx.user.user_id
+ )
+ except BaseException:
+ if lease is not None and service is not None:
+ await service.viking_fs._async_agfs.pathlock_release(lease)
+ if not enqueued:
+ await tracker.fail(
+ task.task_id,
+ "Failed to enqueue reindex processing",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ raise
return {
"task_id": task.task_id,
"status": "accepted",
@@ -293,7 +321,7 @@ class ReindexExecutor:
tags: list[str] | None,
tag_mode: str,
) -> IngestOptions | None:
- if mode == "prune_orphans" or (tags is None and tag_mode != "clear"):
+ if tags is None and tag_mode != "clear":
return None
if tag_mode not in {"replace", "append", "clear"}:
raise InvalidArgumentError(f"unsupported tag mode: {tag_mode}")
@@ -438,6 +466,15 @@ class ReindexExecutor:
if message:
counters.warnings.append(f"Embedding queue failed during reindex: {message}")
+ @staticmethod
+ def _apply_semantic_tree_stats(counters: _ReindexCounters, stats: Any | None) -> None:
+ if stats is None:
+ return
+ counters.rebuilt_records += int(getattr(stats, "indexed_records", 0) or 0)
+ failures = list(getattr(stats, "failures", ()) or ())
+ counters.failed_records += len(failures)
+ counters.warnings.extend(failures)
+
def _is_resource_entry_for_namespace(self, uri: str, target_root: str) -> bool:
if not uri.startswith(self._child_prefix(target_root)):
return False
@@ -455,9 +492,12 @@ class ReindexExecutor:
uri: str,
mode: str,
run: _ReindexRunContext,
+ recursive: bool = True,
) -> None:
counters = run.counters
ctx = run.ctx
+ if not recursive:
+ return
viking_fs = get_viking_fs()
try:
entries = await self._tree_all(viking_fs, uri, show_all_hidden=True, ctx=ctx)
@@ -471,10 +511,19 @@ class ReindexExecutor:
skill_roots.append(entry_uri)
for skill_root in sorted(set(skill_roots)):
- await self._reindex_skill(
+ subtree = [
+ entry
+ for entry in entries
+ if str(entry.get("uri") or "").startswith(skill_root.rstrip("/") + "/")
+ ]
+ await self._reindex_rfv(
uri=skill_root,
mode=mode,
+ context_type="skill",
+ recursive=True,
run=run,
+ root_is_dir=True,
+ formal_entries=subtree,
)
if not skill_roots:
@@ -500,15 +549,16 @@ class ReindexExecutor:
uri: str,
object_type: str,
mode: str,
- dry_run: bool = False,
+ force: bool = False,
recursive: bool = True,
ingest_options: IngestOptions | None = None,
ctx: RequestContext,
+ existing_lease: dict | None = None,
) -> dict[str, Any]:
service = get_service()
if service.viking_fs is None or service.vikingdb_manager is None:
raise RuntimeError("OpenVikingService not initialized")
- if mode != "prune_orphans" and not service.vikingdb_manager.has_queue_manager:
+ if not service.vikingdb_manager.has_queue_manager:
raise OpenVikingError(
"Reindex requires embedding queue",
code="FAILED_PRECONDITION",
@@ -523,19 +573,13 @@ class ReindexExecutor:
if telemetry_id:
wait_tracker.register_request(telemetry_id)
- # prune_orphans only touches the vector store. A missing target has
- # nothing on disk to protect, and acquiring a lock there would create
- # the directory just to hold lock metadata. Refuse only when another
- # owner is mid-write at that name (e.g. add_resource reserving it).
- lease = None
- if mode != "prune_orphans" or await service.viking_fs.exists(uri, ctx=ctx):
- acquire_lock = service.viking_fs._async_agfs.pathlock_acquire_tree
- stat = await service.viking_fs.stat(uri, ctx=ctx, skip_count=True)
- if not stat.get("isDir", stat.get("is_dir")):
- acquire_lock = service.viking_fs._async_agfs.pathlock_acquire_exact
- lease = await acquire_lock(path)
- elif await service.viking_fs._async_agfs.pathlock_is_locked(path):
- raise ResourceBusyError(f"Resource is being processed: {uri}", uri=uri)
+ acquire_lock = service.viking_fs._async_agfs.pathlock_acquire_tree
+ stat = await service.viking_fs.stat(uri, ctx=ctx, skip_count=True)
+ root_is_dir = bool(stat.get("isDir", stat.get("is_dir")))
+ if not root_is_dir:
+ acquire_lock = service.viking_fs._async_agfs.pathlock_acquire_exact
+ lease = existing_lease or await acquire_lock(path)
+ run: _ReindexRunContext | None = None
try:
borrowed = (
await service.viking_fs._async_agfs.pathlock_as_borrowed(lease)
@@ -546,34 +590,27 @@ class ReindexExecutor:
ctx=ctx,
counters=counters,
lock=borrowed,
+ lease=lease,
ingest_options=ingest_options,
+ force=force,
+ root_is_dir=root_is_dir,
+ lock_root_uri=uri.rstrip("/"),
)
- if mode == "prune_orphans":
- await self._prune_orphan_vectors(
- uri=uri,
- object_type=object_type,
- dry_run=dry_run,
- counters=counters,
- ctx=ctx,
- )
- elif object_type == "global_namespace":
- await self._reindex_global_namespace(
- uri=uri,
- mode=mode,
- run=run,
- )
+ if object_type == "global_namespace":
+ namespace_kwargs = {"uri": uri, "mode": mode, "run": run}
+ if not recursive:
+ namespace_kwargs["recursive"] = False
+ await self._reindex_global_namespace(**namespace_kwargs)
elif object_type == "user_namespace":
- await self._reindex_user_namespace(
- uri=uri,
- mode=mode,
- run=run,
- )
+ namespace_kwargs = {"uri": uri, "mode": mode, "run": run}
+ if not recursive:
+ namespace_kwargs["recursive"] = False
+ await self._reindex_user_namespace(**namespace_kwargs)
elif object_type == "skill_namespace":
- await self._reindex_skill_namespace(
- uri=uri,
- mode=mode,
- run=run,
- )
+ namespace_kwargs = {"uri": uri, "mode": mode, "run": run}
+ if not recursive:
+ namespace_kwargs["recursive"] = False
+ await self._reindex_skill_namespace(**namespace_kwargs)
elif object_type == "resource":
resource_kwargs = {"uri": uri, "mode": mode, "run": run}
if not recursive:
@@ -596,15 +633,20 @@ class ReindexExecutor:
details={"uri": uri},
)
- if telemetry_id and mode != "prune_orphans":
+ if telemetry_id:
await wait_tracker.wait_for_request(telemetry_id)
+ self._apply_semantic_tree_stats(
+ counters,
+ SemanticProcessor.consume_tree_stats(telemetry_id=telemetry_id),
+ )
self._apply_embedding_wait_status(
counters,
wait_tracker.build_queue_status(telemetry_id),
)
finally:
- if lease is not None:
- await service.viking_fs._async_agfs.pathlock_release(lease)
+ active_lease = run.lease if run is not None else lease
+ if active_lease is not None and (run is None or not run.lease_handed_off):
+ await service.viking_fs._async_agfs.pathlock_release(active_lease)
if telemetry_id:
wait_tracker.cleanup(telemetry_id)
@@ -616,395 +658,263 @@ class ReindexExecutor:
"scanned_records": counters.scanned_records,
"rebuilt_records": counters.rebuilt_records,
"deleted_records": counters.deleted_records,
- "would_delete_records": counters.would_delete_records,
"unsupported_records": counters.unsupported_records,
"failed_records": counters.failed_records,
"duration_ms": int((time.perf_counter() - started_at) * 1000),
"warnings": counters.warnings,
}
- async def _run_tracked(
+ async def _reindex_resource(
self,
- task_id: str,
*,
uri: str,
- object_type: str,
mode: str,
- dry_run: bool = False,
+ run: _ReindexRunContext,
recursive: bool = True,
- ingest_options: IngestOptions | None = None,
- ctx: RequestContext,
) -> None:
- tracker = get_task_tracker()
- tracker.register_running_task(task_id)
- try:
- await tracker.start(task_id, account_id=ctx.account_id, user_id=ctx.user.user_id)
- with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):
- result = await self._run(
- uri=uri,
- object_type=object_type,
- mode=mode,
- dry_run=dry_run,
- recursive=recursive,
- ingest_options=ingest_options,
- ctx=ctx,
- )
- await tracker.complete(
- task_id,
- result,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- except asyncio.CancelledError:
- # TaskWorkIndex finalizes after this active task and its queue work settle.
- return
- except Exception as exc:
- await tracker.fail(
- task_id,
- str(exc),
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- finally:
- await tracker.unregister_running_task(task_id)
+ await self._reindex_rfv(
+ uri=uri,
+ mode=mode,
+ context_type="resource",
+ recursive=recursive,
+ run=run,
+ )
- async def _prune_orphan_vectors(
+ async def _reindex_rfv(
self,
*,
uri: str,
- object_type: str,
- dry_run: bool,
- counters: _ReindexCounters,
- ctx: RequestContext,
+ mode: str,
+ context_type: str,
+ recursive: bool,
+ run: _ReindexRunContext,
+ root_is_dir: bool | None = None,
+ formal_entries: list[dict[str, Any]] | None = None,
) -> None:
- service = get_service()
- vikingdb = service.vikingdb_manager
- delete_groups: dict[tuple[str, str], tuple[RequestContext, list[str]]] = {}
-
- for context_type in self._prune_context_types(uri=uri, object_type=object_type):
- offset = 0
- while True:
- filter_kwargs: dict[str, Any] = {}
- if offset:
- filter_kwargs["offset"] = offset
- records = await vikingdb.filter(
- filter=And(
- [
- Eq("account_id", ctx.account_id),
- Eq("context_type", context_type),
- Or(
- [
- PathScope("uri", uri, depth=0),
- PathScope("uri", uri, depth=-1),
- ]
- ),
- ]
- ),
- limit=PRUNE_ORPHAN_CANDIDATE_LIMIT,
- output_fields=PRUNE_OUTPUT_FIELDS,
- ctx=ctx,
- **filter_kwargs,
- )
- if not records:
- break
-
- for record in records:
- counters.scanned_records += 1
- if not self._is_supported_prune_record(record, counters):
- continue
- if not await self._is_orphan_vector_record(record, counters=counters, ctx=ctx):
- continue
-
- if dry_run:
- counters.would_delete_records += 1
- continue
-
- delete_ctx = self._delete_ctx_for_prune_record(record, ctx)
- key = (delete_ctx.account_id, delete_ctx.user.user_id)
- if key not in delete_groups:
- delete_groups[key] = (delete_ctx, [])
- delete_groups[key][1].append(str(record["id"]))
-
- if len(records) < PRUNE_ORPHAN_CANDIDATE_LIMIT:
- break
- offset += len(records)
-
- if dry_run:
- return
+ from openviking.storage.context_update_plan import build_rfv_context_update_plan
+ from openviking.storage.resource_rfv import build_rfv_snapshot
+ from openviking.storage.resource_rnfv import RequestIntent
+ from openviking.utils.resource_processor import ResourceProcessor
- for delete_ctx, ids in delete_groups.values():
- try:
- deleted = await vikingdb.delete(ids, ctx=delete_ctx)
- deleted_count = int(deleted if deleted is not None else len(ids))
- counters.deleted_records += deleted_count
- if deleted_count < len(ids):
- failed_count = len(ids) - max(deleted_count, 0)
- counters.failed_records += failed_count
- counters.warnings.append(
- f"Only deleted {deleted_count} of {len(ids)} orphan vectors for owner "
- f"{delete_ctx.user.user_id}"
- )
- except Exception as exc:
- counters.failed_records += len(ids)
- counters.warnings.append(
- f"Failed to delete {len(ids)} orphan vectors for owner "
- f"{delete_ctx.user.user_id}: {exc}"
- )
-
- def _prune_context_types(self, *, uri: str, object_type: str) -> list[str]:
- if object_type == "resource":
- return [ContextType.RESOURCE.value]
- if object_type == "memory":
- return [ContextType.MEMORY.value]
- if object_type in {"skill", "skill_namespace"}:
- return [ContextType.SKILL.value]
- if object_type in {"global_namespace", "user_namespace"}:
- return [
- ContextType.RESOURCE.value,
- ContextType.MEMORY.value,
- ContextType.SKILL.value,
- ]
- return [str(context_type_for_uri(uri))]
-
- def _is_supported_prune_record(
- self,
- record: dict[str, Any],
- counters: _ReindexCounters,
- ) -> bool:
- record_id = record.get("id")
- uri = record.get("uri")
- context_type = str(record.get("context_type") or "")
- level = record.get("level")
- if not record_id or not uri:
- counters.unsupported_records += 1
- counters.warnings.append(f"Skipping unknown prune record without id/uri: {record!r}")
- return False
- if context_type not in {
- ContextType.RESOURCE.value,
- ContextType.MEMORY.value,
- ContextType.SKILL.value,
- }:
- counters.unsupported_records += 1
- counters.warnings.append(f"Skipping unknown context_type for prune record {uri}")
- return False
- try:
- normalized_level = int(level)
- except (TypeError, ValueError):
- counters.unsupported_records += 1
- counters.warnings.append(f"Skipping unknown level for prune record {uri}: {level}")
- return False
- if normalized_level not in {
- int(ContextLevel.ABSTRACT),
- int(ContextLevel.OVERVIEW),
- int(ContextLevel.DETAIL),
- }:
- counters.unsupported_records += 1
- counters.warnings.append(f"Skipping unknown level for prune record {uri}: {level}")
- return False
- record["_prune_level"] = normalized_level
- return True
-
- async def _is_orphan_vector_record(
- self,
- record: dict[str, Any],
- *,
- counters: _ReindexCounters,
- ctx: RequestContext,
- ) -> bool:
- uri = str(record["uri"])
- level = int(record.get("_prune_level", record["level"]))
- context_type = str(record["context_type"])
- owner_ctx = self._delete_ctx_for_prune_record(record, ctx)
-
- if level == int(ContextLevel.ABSTRACT):
- abstract = await self._read_prune_source(
- f"{uri}/.abstract.md",
- ctx=owner_ctx,
+ service = get_service()
+ request = RequestIntent.from_ingest_options(
+ target_uri=uri,
+ processing_mode=mode,
+ ingest_options=run.ingest_options,
+ force=run.force,
+ scalar_target_levels=frozenset({0, 1, 2}),
+ )
+ snapshot = await build_rfv_snapshot(
+ viking_fs=service.viking_fs,
+ vikingdb=service.vikingdb_manager,
+ target_uri=uri,
+ ctx=run.ctx,
+ request_intent=request,
+ recursive=recursive,
+ source_read_concurrency=self._effective_file_vectorization_concurrency(),
+ root_is_dir=run.root_is_dir if root_is_dir is None else root_is_dir,
+ formal_inventory=(
+ (True if root_is_dir is None else root_is_dir, formal_entries, True)
+ if formal_entries is not None
+ else None
+ ),
+ )
+ diff, plan = build_rfv_context_update_plan(
+ snapshot=snapshot,
+ context_type=context_type,
+ account_id=run.ctx.account_id,
+ source_metadata=snapshot.source_metadata,
+ )
+ if (
+ mode == "vectors_only"
+ and not snapshot.formal.entries[""].is_dir
+ and any(
+ action.level == int(ContextLevel.DETAIL)
+ and action.action.value in {"merge", "upsert"}
+ for action in plan.direct_index_actions
+ )
+ ):
+ enriched_snapshot = await self._with_single_file_parent_overview(
+ snapshot,
+ ctx=run.ctx,
+ viking_fs=service.viking_fs,
)
- if abstract.error:
- self._record_prune_source_error(
- counters=counters,
- uri=uri,
- source_uri=f"{uri}/.abstract.md",
- error=abstract.error,
+ if enriched_snapshot is not snapshot:
+ snapshot = enriched_snapshot
+ diff, plan = build_rfv_context_update_plan(
+ snapshot=snapshot,
+ context_type=context_type,
+ account_id=run.ctx.account_id,
+ source_metadata=snapshot.source_metadata,
)
- return False
- return (
- not abstract.exists
- or not abstract.text
- or _is_not_ready_sentinel(abstract.text, _ABSTRACT_NOT_READY_SUFFIX)
+ run.counters.scanned_records += len(snapshot.formal.entries)
+ if plan.direct_index_actions:
+ owner_ctx = self._content_owner_ctx(uri, run.ctx)
+ delete_actions = tuple(
+ action for action in plan.direct_index_actions if action.action.value == "delete"
)
-
- if level == int(ContextLevel.OVERVIEW):
- overview = await self._read_prune_source(
- f"{uri}/.overview.md",
- ctx=owner_ctx,
+ enqueue_actions = tuple(
+ action for action in plan.direct_index_actions if action.action.value != "delete"
)
- if overview.error:
- self._record_prune_source_error(
- counters=counters,
- uri=uri,
- source_uri=f"{uri}/.overview.md",
- error=overview.error,
- )
- return False
- if (
- overview.exists
- and overview.text
- and not _is_not_ready_sentinel(overview.text, _OVERVIEW_NOT_READY_SUFFIX)
- ):
- return False
- if context_type in {ContextType.RESOURCE.value, ContextType.SKILL.value}:
- abstract = await self._read_prune_source(
- f"{uri}/.abstract.md",
+ if delete_actions:
+ run.counters.deleted_records += await service.vikingdb_manager.strict_delete(
+ [action.record_id for action in delete_actions],
ctx=owner_ctx,
)
- if abstract.error:
- self._record_prune_source_error(
- counters=counters,
- uri=uri,
- source_uri=f"{uri}/.abstract.md",
- error=abstract.error,
- )
- return False
- return (
- not abstract.exists
- or not abstract.text
- or _is_not_ready_sentinel(abstract.text, _ABSTRACT_NOT_READY_SUFFIX)
+ if enqueue_actions:
+ processor = getattr(service, "_resource_processor", None)
+ if processor is None:
+ processor = ResourceProcessor(service.vikingdb_manager)
+ run.counters.rebuilt_records += await processor._enqueue_index_actions(
+ enqueue_actions,
+ ctx=owner_ctx,
+ source_contents=dict(snapshot.source_contents),
+ source_metadata=snapshot.source_metadata,
)
- return True
-
- if self._is_hidden_meta_file(uri):
- return False
- exists = await self._prune_source_exists(uri, ctx=owner_ctx)
- if exists.error:
- self._record_prune_source_error(
- counters=counters,
- uri=uri,
- source_uri=uri,
- error=exists.error,
+ if plan.semantic_plan is not None:
+ await self._enqueue_reindex_semantic_plan(
+ semantic_plan=plan.semantic_plan,
+ context_type=context_type,
+ recursive=recursive,
+ run=run,
+ )
+ elif plan.file_refresh is not None:
+ service_processor = getattr(service, "_resource_processor", None)
+ if service_processor is None:
+ service_processor = ResourceProcessor(service.vikingdb_manager)
+ await service_processor._get_summarizer().refresh_file_parent(
+ file_uri=plan.file_refresh.file_uri,
+ ctx=run.ctx,
+ ingest_options=run.ingest_options,
+ file_md5=plan.file_refresh.md5,
)
- return False
- # A real filename can have the same spelling as a virtual chunk URI.
- if exists.exists:
- return False
-
- if "#" in uri:
- # Generated chunks append a zero-padded index to the full base URI.
- chunk_match = re.fullmatch(r"(.+)#chunk_[0-9]{4,}", uri)
- if context_type == ContextType.MEMORY.value and chunk_match:
- base_uri = chunk_match.group(1)
- base = await self._read_prune_source(base_uri, ctx=owner_ctx)
- if base.error:
- self._record_prune_source_error(
- counters=counters,
- uri=uri,
- source_uri=base_uri,
- error=base.error,
- )
- return False
- if not base.exists:
- return True
- expected = {
- chunk_uri for chunk_uri, _chunk in self._chunk_memory_body(base_uri, base.text)
- }
- return uri not in expected
- return False
-
- return True
-
- async def _read_prune_source(self, uri: str, *, ctx: RequestContext) -> _PruneSourceRead:
- viking_fs = get_viking_fs()
- try:
- exists = await viking_fs.exists(uri, ctx=ctx)
- except Exception as exc:
- return _PruneSourceRead(exists=False, error=exc)
- if not exists:
- return _PruneSourceRead(exists=False)
- try:
- content = await viking_fs.read_file(uri, ctx=ctx)
- except Exception as exc:
- return _PruneSourceRead(exists=True, error=exc)
- if isinstance(content, bytes):
- text = content.decode("utf-8", errors="replace")
- else:
- text = str(content or "")
- return _PruneSourceRead(exists=True, text=text)
-
- async def _prune_source_exists(self, uri: str, *, ctx: RequestContext) -> _PruneSourceRead:
- try:
- exists = await get_viking_fs().exists(uri, ctx=ctx)
- except Exception as exc:
- return _PruneSourceRead(exists=False, error=exc)
- return _PruneSourceRead(exists=exists)
- def _record_prune_source_error(
- self,
+ @staticmethod
+ async def _with_single_file_parent_overview(
+ snapshot: Any,
*,
- counters: _ReindexCounters | None,
- uri: str,
- source_uri: str,
- error: Exception,
- ) -> None:
- if counters is None:
- return
- counters.failed_records += 1
- counters.warnings.append(f"Skipped prune for {uri}: failed to read {source_uri}: {error}")
-
- def _delete_ctx_for_prune_record(
- self,
- record: dict[str, Any],
ctx: RequestContext,
- ) -> RequestContext:
- uri = str(record.get("uri") or "")
- owner = record.get("owner_user_id")
- if not owner:
- owner = owner_fields_for_uri(uri).get("owner_user_id")
- if not owner or owner == ctx.user.user_id:
- return ctx
- return RequestContext(
- user=UserIdentifier(ctx.account_id, str(owner)),
- role=ctx.role,
- actor_peer_id=ctx.actor_peer_id,
- from_oauth=ctx.from_oauth,
+ viking_fs: Any,
+ ) -> Any:
+ """Best-effort hydrate a file root's parent overview for L2 summary reuse."""
+ parent = VikingURI(snapshot.request.target_uri).parent
+ if parent is None:
+ return snapshot
+ try:
+ raw = await viking_fs.read_file_bytes(f"{parent.uri}/.overview.md", ctx=ctx)
+ overview = body_for_preview(raw)
+ except Exception:
+ return snapshot
+ if not overview.strip():
+ return snapshot
+ return replace(
+ snapshot,
+ source_contents={
+ **snapshot.source_contents,
+ (parent.uri, int(ContextLevel.OVERVIEW)): overview,
+ },
)
- async def _reindex_resource(
+ async def _enqueue_reindex_semantic_plan(
self,
*,
- uri: str,
- mode: str,
+ semantic_plan: Any,
+ context_type: str,
+ recursive: bool,
run: _ReindexRunContext,
- recursive: bool = True,
) -> None:
- counters = run.counters
- ctx = run.ctx
- if mode == "semantic_and_vectors":
- semantic_kwargs = {
- "uri": uri,
- "context_type": "resource",
- "ctx": ctx,
- "lock": run.lock,
- }
- if not recursive:
- semantic_kwargs["recursive"] = False
- await self._run_semantic_processor(**semantic_kwargs)
- vector_kwargs = {"uri": uri, "counters": counters, "ctx": ctx}
- if not recursive:
- vector_kwargs["recursive"] = False
- await self._reindex_resource_vectors(
- **self._with_ingest_options(
- vector_kwargs,
- run.ingest_options,
- )
- )
+ """Persist one fully planned reindex DAG without persisting its source bytes."""
+ from openviking.storage.queuefs import get_queue_manager
+
+ owner_ctx = self._content_owner_ctx(semantic_plan.root_uri, run.ctx)
+ telemetry_id = get_current_telemetry().telemetry_id
+ msg = SemanticMsg(
+ uri=semantic_plan.root_uri,
+ context_type=context_type,
+ recursive=recursive,
+ account_id=owner_ctx.account_id,
+ user_id=owner_ctx.user.user_id,
+ group_ids=list(owner_ctx.group_ids),
+ peer_id=owner_ctx.user.user_id,
+ role=str(owner_ctx.role),
+ telemetry_id=telemetry_id,
+ generation_trigger="reindex",
+ propagate_to_parent=recursive,
+ plan=semantic_plan,
+ )
+ tracker = get_request_wait_tracker()
+ if telemetry_id:
+ tracker.register_semantic_root(telemetry_id, msg.id)
+
+ queue_manager = get_queue_manager()
+ semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)
+ if run.lease is None:
+ try:
+ enqueue_id = await semantic_queue.enqueue(msg)
+ except Exception:
+ if telemetry_id:
+ tracker.mark_semantic_failed(
+ telemetry_id, msg.id, "semantic plan enqueue failed"
+ )
+ raise
+ if enqueue_id == "deduplicated" and telemetry_id:
+ tracker.mark_semantic_done(telemetry_id, msg.id, processed_delta=0)
return
- await self._reindex_resource_vectors(
- **self._with_ingest_options(
- {"uri": uri, "counters": counters, "ctx": ctx},
- run.ingest_options,
+
+ viking_fs = get_service().viking_fs
+ agfs = viking_fs._async_agfs
+ use_outer_lease = run.lock_root_uri in {None, semantic_plan.root_uri.rstrip("/")}
+ plan_lease = run.lease
+ acquired_plan_lease = False
+ if not use_outer_lease:
+ root_entry = next(
+ entry for entry in semantic_plan.tree.entries if not entry.relative_path
)
- )
+ acquire = (
+ agfs.pathlock_acquire_tree
+ if root_entry.kind == "directory"
+ else agfs.pathlock_acquire_exact
+ )
+ kwargs = {"owner_lease_ref": run.lease} if run.lease is not None else {}
+ plan_lease = await acquire(
+ viking_fs._uri_to_path(semantic_plan.root_uri, ctx=owner_ctx), **kwargs
+ )
+ acquired_plan_lease = True
+ assert plan_lease is not None
+ handoff = await agfs.pathlock_to_handoff(plan_lease)
+ handed_off = False
+ try:
+ await agfs.pathlock_handoff(plan_lease)
+ handed_off = True
+ msg.lock_handoff = handoff
+ enqueue_id = await semantic_queue.enqueue(msg)
+ if enqueue_id == "deduplicated":
+ lock = await agfs.pathlock_adopt(handoff)
+ await agfs.pathlock_release(lock)
+ if not acquired_plan_lease:
+ run.lease_handed_off = True
+ run.lease = None
+ run.lock = None
+ if telemetry_id:
+ tracker.mark_semantic_done(telemetry_id, msg.id, processed_delta=0)
+ return
+ if not acquired_plan_lease:
+ run.lease_handed_off = True
+ run.lease = None
+ run.lock = None
+ except BaseException:
+ if handed_off:
+ recovered_lease = await agfs.pathlock_adopt(handoff)
+ if acquired_plan_lease:
+ await agfs.pathlock_release(recovered_lease)
+ else:
+ run.lease = recovered_lease
+ run.lock = await agfs.pathlock_as_borrowed(run.lease)
+ elif acquired_plan_lease:
+ await agfs.pathlock_release(plan_lease)
+ if telemetry_id:
+ tracker.mark_semantic_failed(telemetry_id, msg.id, "semantic plan enqueue failed")
+ raise
async def _reindex_skill(
self,
@@ -1013,56 +923,19 @@ class ReindexExecutor:
mode: str,
run: _ReindexRunContext,
recursive: bool = True,
+ formal_entries: list[dict[str, Any]] | None = None,
) -> None:
- counters = run.counters
- ctx = run.ctx
- if mode == "semantic_and_vectors":
- if not recursive:
- await self._regenerate_skill_semantics(uri=uri, ctx=ctx, lock=run.lock)
- await self._reindex_skill_vectors(
- uri=uri,
- counters=counters,
- ctx=ctx,
- recursive=False,
- ingest_options=run.ingest_options,
- )
- return
- owner_ctx = self._content_owner_ctx(uri, ctx)
- processor = await self._semantic_processor_for(owner_ctx)
- executor = SemanticTreeExecutor(
- processor=processor,
- context_type="skill",
- max_concurrent_llm=processor.max_concurrent_llm,
- ctx=owner_ctx,
- lock=run.lock,
- generation_trigger="reindex",
- ingest_options=run.ingest_options,
- source={
- "path": (await self._skill_meta(uri=uri, abstract="", ctx=ctx)).get(
- "source_path", ""
- )
- },
- )
- try:
- await executor.run(uri)
- finally:
- # Even a semantic error must settle already submitted Skill
- # vectors before the caller releases its package lease.
- await get_request_wait_tracker().wait_for_embeddings(
- get_current_telemetry().telemetry_id
- )
- stats = executor.get_stats()
- counters.scanned_records += stats.total_nodes
- counters.rebuilt_records += stats.indexed_records
- counters.failed_records += len(stats.failures)
- counters.warnings.extend(stats.failures)
- return
- await self._reindex_skill_vectors(
- **self._with_ingest_options(
- {"uri": uri, "counters": counters, "ctx": ctx},
- run.ingest_options,
- )
- )
+ kwargs: dict[str, Any] = {
+ "uri": uri,
+ "mode": mode,
+ "context_type": "skill",
+ "recursive": recursive,
+ "run": run,
+ }
+ if formal_entries is not None:
+ kwargs["root_is_dir"] = True
+ kwargs["formal_entries"] = formal_entries
+ await self._reindex_rfv(**kwargs)
async def _reindex_memory(
self,
@@ -1075,7 +948,7 @@ class ReindexExecutor:
counters = run.counters
ctx = run.ctx
if mode == "semantic_and_vectors":
- stat = await get_viking_fs().stat(uri, ctx=ctx, skip_count=True)
+ stat = await get_viking_fs().stat(uri, ctx=ctx)
if stat.get("isDir", stat.get("is_dir")):
semantic_kwargs = {
"uri": uri,
@@ -1098,7 +971,12 @@ class ReindexExecutor:
return
await self._reindex_memory_vectors(
**self._with_ingest_options(
- {"uri": uri, "counters": counters, "ctx": ctx},
+ {
+ "uri": uri,
+ "counters": counters,
+ "ctx": ctx,
+ "recursive": recursive,
+ },
run.ingest_options,
)
)
@@ -1129,184 +1007,6 @@ class ReindexExecutor:
)
await processor.on_dequeue({"data": msg.to_json()}, lock=lock)
- async def _reindex_resource_vectors(
- self,
- *,
- uri: str,
- counters: _ReindexCounters,
- ctx: RequestContext,
- recursive: bool = True,
- ingest_options: IngestOptions | None = None,
- ) -> None:
- viking_fs = get_viking_fs()
- try:
- if not await viking_fs.exists(uri, ctx=ctx):
- raise NotFoundError(uri, "resource")
- stat = await viking_fs.stat(uri, ctx=ctx, skip_count=True)
- is_dir = stat.get("isDir", stat.get("is_dir")) if isinstance(stat, dict) else False
- if is_dir and recursive:
- entries = await self._tree_all(viking_fs, uri, show_all_hidden=True, ctx=ctx)
- else:
- entries = []
- except Exception as exc:
- raise NotFoundError(uri, "resource") from exc
-
- if is_dir:
- directories = [uri]
- files: list[str] = []
- for entry in entries:
- entry_uri = entry.get("uri")
- if not entry_uri:
- continue
- if entry.get("isDir"):
- directories.append(entry_uri)
- elif not self._is_hidden_meta_file(entry_uri):
- files.append(entry_uri)
- else:
- directories = []
- files = [uri]
-
- await self._reindex_resource_vectors_from_entries(
- **self._with_ingest_options(
- {
- "root_uri": uri,
- "directories": directories,
- "files": files,
- "counters": counters,
- "ctx": ctx,
- },
- ingest_options,
- )
- )
-
- async def _reindex_resource_vectors_from_entries(
- self,
- *,
- root_uri: str,
- directories: Iterable[str],
- files: Iterable[str],
- counters: _ReindexCounters,
- ctx: RequestContext,
- ingest_options: IngestOptions | None = None,
- ) -> None:
- deduped_directories = []
- seen_directories = set()
- for directory_uri in directories:
- if directory_uri and directory_uri not in seen_directories:
- deduped_directories.append(directory_uri)
- seen_directories.add(directory_uri)
-
- deduped_files = []
- seen_files = set()
- for file_uri in files:
- if file_uri and file_uri not in seen_files:
- deduped_files.append(file_uri)
- seen_files.add(file_uri)
-
- for directory_uri in deduped_directories:
- if directory_uri == "viking://":
- continue
- counters.scanned_records += 1
- abstract = await self._read_directory_abstract(directory_uri, ctx=ctx)
- overview = await self._read_directory_overview(directory_uri, ctx=ctx)
- if not overview:
- overview = abstract
- if not abstract and not overview:
- counters.unsupported_records += 1
- counters.warnings.append(f"No semantic source found for {directory_uri}")
- continue
- if abstract:
- try:
- await self._upsert_context(
- uri=directory_uri,
- parent_uri=VikingURI(directory_uri).parent.uri,
- abstract=abstract,
- vector_text=embedding_text_for_body(
- ContextLevel.ABSTRACT, directory_uri, abstract
- ),
- is_leaf=False,
- context_type=context_type_for_uri(directory_uri),
- level=ContextLevel.ABSTRACT,
- ctx=ctx,
- ingest_options=ingest_options,
- )
- counters.rebuilt_records += 1
- except Exception as exc:
- counters.failed_records += 1
- counters.warnings.append(f"Failed to reindex {directory_uri} L0 vector: {exc}")
- if overview:
- try:
- await self._upsert_context(
- uri=directory_uri,
- parent_uri=VikingURI(directory_uri).parent.uri,
- # L1 abstract scalar carries the overview for Rerank.
- abstract=_truncate_abstract_bytes(overview),
- vector_text=embedding_text_for_body(
- ContextLevel.OVERVIEW, directory_uri, overview
- ),
- is_leaf=False,
- context_type=context_type_for_uri(directory_uri),
- level=ContextLevel.OVERVIEW,
- ctx=ctx,
- ingest_options=ingest_options,
- )
- counters.rebuilt_records += 1
- except Exception as exc:
- counters.failed_records += 1
- counters.warnings.append(f"Failed to reindex {directory_uri} L1 vector: {exc}")
-
- async def process_file(file_uri: str) -> _ReindexCounters:
- file_counters = _ReindexCounters(scanned_records=1)
- parent_uri = VikingURI(file_uri).parent.uri
- try:
- file_bytes = await get_viking_fs().read_file_bytes(file_uri, ctx=ctx)
- except Exception as exc:
- file_counters.failed_records += 1
- file_counters.warnings.append(f"Failed to read {file_uri} for reindex: {exc}")
- return file_counters
- summary = await self._best_file_summary(file_uri, ctx=ctx)
- vector_text = await self._best_resource_file_vector_text(
- file_uri, summary, ctx=ctx, file_content=file_bytes
- )
- if not vector_text:
- file_counters.unsupported_records += 1
- file_counters.warnings.append(f"No vector source found for {file_uri}")
- return file_counters
- abstract = self._prefer_non_empty(summary, vector_text)
- try:
- await self._upsert_context(
- uri=file_uri,
- parent_uri=parent_uri,
- abstract=abstract,
- vector_text=vector_text,
- is_leaf=True,
- context_type=context_type_for_uri(file_uri),
- level=ContextLevel.DETAIL,
- ctx=ctx,
- ingest_options=ingest_options,
- md5=content_md5(file_bytes),
- )
- file_counters.rebuilt_records += 1
- except Exception as exc:
- file_counters.failed_records += 1
- file_counters.warnings.append(f"Failed to reindex {file_uri} vector: {exc}")
- return file_counters
-
- concurrency = self._effective_file_vectorization_concurrency()
- if deduped_files:
- logger.info(
- "Reindex resource file vectorization: root=%s files=%d concurrency=%d",
- root_uri,
- len(deduped_files),
- concurrency,
- )
- await self._run_ordered_counter_batches(
- deduped_files,
- concurrency=concurrency,
- processor=process_file,
- counters=counters,
- )
-
async def reindex_directory_marker(
self, *, dir_uri: str, level: ContextLevel, ctx: RequestContext
) -> None:
@@ -1380,11 +1080,13 @@ class ReindexExecutor:
uri: str,
mode: str,
run: _ReindexRunContext,
+ recursive: bool = True,
) -> None:
- counters = run.counters
ctx = run.ctx
normalized_uri = uri.rstrip("/")
target_root = normalized_uri if normalized_uri else uri
+ if not recursive:
+ return
viking_fs = get_viking_fs()
try:
entries = await self._tree_all(viking_fs, target_root, show_all_hidden=True, ctx=ctx)
@@ -1403,6 +1105,7 @@ class ReindexExecutor:
uri=user_root,
mode=mode,
run=run,
+ recursive=True,
)
return
@@ -1451,31 +1154,53 @@ class ReindexExecutor:
uri=skill_root,
mode=skill_mode,
run=run,
+ recursive=True,
+ formal_entries=[
+ entry
+ for entry in entries
+ if str(entry.get("uri") or "").startswith(skill_root.rstrip("/") + "/")
+ ],
)
- if mode == "semantic_and_vectors":
- (
- resource_directories,
- resource_files,
- ) = await self._refresh_namespace_resource_semantics(
- target_root=target_root,
- directories=resource_directories,
- files=resource_files,
+ resource_roots = [
+ directory_uri
+ for directory_uri in resource_directories
+ if directory_uri.rstrip("/").endswith("/resources")
+ ]
+ covered_prefixes = tuple(root.rstrip("/") + "/" for root in resource_roots)
+ for resource_root in resource_roots:
+ await self._reindex_rfv(
+ uri=resource_root,
+ mode=mode,
+ context_type="resource",
+ recursive=True,
run=run,
+ root_is_dir=True,
+ formal_entries=[
+ entry
+ for entry in entries
+ if str(entry.get("uri") or "").startswith(resource_root.rstrip("/") + "/")
+ ],
)
-
- await self._reindex_resource_vectors_from_entries(
- **self._with_ingest_options(
- {
- "root_uri": target_root,
- "directories": resource_directories,
- "files": resource_files,
- "counters": counters,
- "ctx": ctx,
- },
- run.ingest_options,
+ for file_uri in resource_files:
+ if any(file_uri.startswith(prefix) for prefix in covered_prefixes):
+ continue
+ if mode == "semantic_and_vectors":
+ run.counters.unsupported_records += 1
+ run.counters.warnings.append(
+ f"Skipped {file_uri}: namespace semantic_and_vectors only refreshes "
+ "resource directories"
+ )
+ continue
+ await self._reindex_rfv(
+ uri=file_uri,
+ mode=mode,
+ context_type="resource",
+ recursive=False,
+ run=run,
+ root_is_dir=False,
+ formal_entries=[],
)
- )
async def _reindex_global_namespace(
self,
@@ -1483,10 +1208,12 @@ class ReindexExecutor:
uri: str,
mode: str,
run: _ReindexRunContext,
+ recursive: bool = True,
) -> None:
- counters = run.counters
ctx = run.ctx
target_root = "viking://"
+ if not recursive:
+ return
viking_fs = get_viking_fs()
try:
entries = await self._tree_all(viking_fs, target_root, show_all_hidden=True, ctx=ctx)
@@ -1524,126 +1251,39 @@ class ReindexExecutor:
run=run,
)
- if mode == "semantic_and_vectors":
- (
- resource_directories,
- resource_files,
- ) = await self._refresh_namespace_resource_semantics(
- target_root=target_root,
- directories=resource_directories,
- files=resource_files,
+ if "viking://resources" in resource_directories:
+ await self._reindex_rfv(
+ uri="viking://resources",
+ mode=mode,
+ context_type="resource",
+ recursive=True,
run=run,
+ root_is_dir=True,
+ formal_entries=[
+ entry
+ for entry in entries
+ if str(entry.get("uri") or "").startswith("viking://resources/")
+ ],
)
-
- await self._reindex_resource_vectors_from_entries(
- **self._with_ingest_options(
- {
- "root_uri": target_root,
- "directories": resource_directories,
- "files": resource_files,
- "counters": counters,
- "ctx": ctx,
- },
- run.ingest_options,
- )
- )
-
- async def _reindex_skill_vectors(
- self,
- *,
- uri: str,
- counters: _ReindexCounters,
- ctx: RequestContext,
- recursive: bool = True,
- ingest_options: IngestOptions | None = None,
- ) -> None:
- viking_fs = get_viking_fs()
- owner_ctx = self._content_owner_ctx(uri, ctx)
- pending = [uri]
- while pending:
- directory_uri = pending.pop()
- counters.scanned_records += 1
- abstract = await self._read_directory_abstract(directory_uri, ctx=ctx)
- overview = await self._read_directory_overview(directory_uri, ctx=ctx)
- for level in (0, 1):
- if abstract if level == 0 else overview:
- continue
- record = await self._fetch_existing_record(
- uri=directory_uri, level=level, ctx=owner_ctx
+ for file_uri in resource_files:
+ if file_uri.startswith("viking://resources/"):
+ continue
+ if mode == "semantic_and_vectors":
+ run.counters.unsupported_records += 1
+ run.counters.warnings.append(
+ f"Skipped {file_uri}: namespace semantic_and_vectors only refreshes "
+ "resource directories"
)
- if level == 0:
- abstract = self._record_abstract(record)
- else:
- overview = self._record_abstract(record)
- if abstract or overview:
- try:
- await vectorize_directory_meta(
- directory_uri,
- abstract,
- overview,
- context_type="skill",
- ctx=owner_ctx,
- include_abstract=bool(abstract),
- include_overview=bool(overview),
- ingest_options=ingest_options,
- content_is_body=True,
- **(
- {
- "meta": await self._skill_meta(
- uri=directory_uri, abstract=abstract, ctx=ctx
- )
- }
- if classify_uri(directory_uri).is_skill_root
- else {}
- ),
- )
- counters.rebuilt_records += int(bool(abstract)) + int(bool(overview))
- except Exception as exc:
- counters.failed_records += 1
- counters.warnings.append(f"Failed to reindex {directory_uri}: {exc}")
- else:
- counters.unsupported_records += 1
- counters.warnings.append(f"No semantic source found for {directory_uri}")
-
- if not recursive:
continue
- from openviking.storage.queuefs.semantic_executor import _SKIP_FILENAMES
- from openviking.storage.viking_fs import LS_ALL_NODES
-
- entries = await viking_fs.ls(directory_uri, node_limit=LS_ALL_NODES, ctx=ctx)
- for entry in entries:
- name = entry.get("name", "")
- if not name or name.startswith(".") or name in _SKIP_FILENAMES:
- continue
- child_uri = VikingURI(directory_uri).join(name).uri
- if entry.get("isDir", False):
- pending.append(child_uri)
- continue
- counters.scanned_records += 1
- try:
- record = await self._fetch_existing_record(
- uri=child_uri, level=2, ctx=owner_ctx
- )
- summary = self._record_abstract(record)
- if not summary:
- summary = await self._best_file_summary(child_uri, ctx=ctx)
- enqueued = await vectorize_file(
- file_path=child_uri,
- summary_dict={"name": name, "summary": summary},
- parent_uri=directory_uri,
- context_type="skill",
- ctx=owner_ctx,
- preserve_existing_created_at=True,
- ingest_options=ingest_options,
- )
- if enqueued:
- counters.rebuilt_records += 1
- else:
- counters.unsupported_records += 1
- counters.warnings.append(f"No vector source found for {child_uri}")
- except Exception as exc:
- counters.failed_records += 1
- counters.warnings.append(f"Failed to reindex {child_uri}: {exc}")
+ await self._reindex_rfv(
+ uri=file_uri,
+ mode=mode,
+ context_type="resource",
+ recursive=False,
+ run=run,
+ root_is_dir=False,
+ formal_entries=[],
+ )
async def _reindex_memory_vectors(
self,
@@ -1880,9 +1520,7 @@ class ReindexExecutor:
if content_type == ResourceContentType.TEXT:
if self.vector_config_resolver is None:
raise RuntimeError("ReindexExecutor requires a vector config resolver")
- embedding_config = (
- await self.vector_config_resolver.resolve(ctx.account_id)
- ).embedding
+ embedding_config = (await self.vector_config_resolver.resolve(ctx.account_id)).embedding
text_source = embedding_config.text_source
if text_source in SUMMARY_TEXT_SOURCES and summary:
return summary
@@ -2033,8 +1671,23 @@ class ReindexExecutor:
except Exception:
return ""
- async def _read_memory_body(self, uri: str, *, ctx: RequestContext) -> _PruneSourceRead:
- return await self._read_prune_source(uri, ctx=ctx)
+ async def _read_memory_body(self, uri: str, *, ctx: RequestContext) -> _SourceRead:
+ viking_fs = get_viking_fs()
+ try:
+ exists = await viking_fs.exists(uri, ctx=ctx)
+ except Exception as exc:
+ return _SourceRead(exists=False, error=exc)
+ if not exists:
+ return _SourceRead(exists=False)
+ try:
+ content = await viking_fs.read_file(uri, ctx=ctx)
+ except Exception as exc:
+ return _SourceRead(exists=True, error=exc)
+ if isinstance(content, bytes):
+ text = content.decode("utf-8", errors="replace")
+ else:
+ text = str(content or "")
+ return _SourceRead(exists=True, text=text)
def _chunk_memory_body(self, uri: str, body: str) -> Iterable[tuple[str, str]]:
semantic = get_openviking_config().semantic
diff --git a/openviking/storage/abstract_overview.py b/openviking/storage/abstract_overview.py
index 07a166491..7fa62c6e8 100644
--- a/openviking/storage/abstract_overview.py
+++ b/openviking/storage/abstract_overview.py
@@ -12,7 +12,7 @@ import hashlib
import re
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional, Sequence, TypeVar
-from urllib.parse import quote, unquote
+from urllib.parse import quote, unquote, urlsplit
import yaml
@@ -397,6 +397,62 @@ def body_for_preview(raw: str | bytes) -> str:
return document.body.rstrip("\r\n")
+def parse_overview_file_summaries(overview_content: str | bytes) -> Dict[str, str]:
+ """Extract direct-file summaries from an overview body."""
+ overview = body_for_preview(overview_content)
+ if not overview.strip():
+ return {}
+
+ summaries: Dict[str, str] = {}
+ current_file = ""
+ current_summary_lines: list[str] = []
+
+ def save_current() -> None:
+ if current_file and current_summary_lines:
+ summaries[current_file] = " ".join(current_summary_lines).strip()
+
+ for line in overview.split("\n"):
+ header_match = re.match(r"^###\s+(.+?)\s*$", line)
+ if header_match:
+ save_current()
+ heading = header_match.group(1).strip()
+ file_name = _overview_heading_file_name(heading)
+ parts = file_name.split()
+ current_file = parts[0] if len(parts) >= 2 and parts[0] == parts[1] else file_name
+ current_summary_lines = []
+ continue
+
+ numbered_match = re.match(r"^\[(\d+)\]\s+(.+?):\s*(.+)$", line)
+ if numbered_match:
+ save_current()
+ current_file = numbered_match.group(2).strip()
+ current_summary_lines = [numbered_match.group(3).strip()]
+ continue
+
+ if current_file:
+ stripped = line.strip()
+ if stripped and not stripped.startswith("#"):
+ current_summary_lines.append(stripped)
+
+ save_current()
+ return summaries
+
+
+def _overview_heading_file_name(heading: str) -> str:
+ """Return the direct entry name represented by an overview H3 heading."""
+ if heading.startswith("[") and heading.endswith(")"):
+ destination_start = heading.rfind("](")
+ if destination_start > 0:
+ target = heading[destination_start + 2 : -1].strip()
+ if target.startswith("<") and target.endswith(">"):
+ target = target[1:-1].strip()
+ if target.startswith("viking://"):
+ path = unquote(urlsplit(target).path).rstrip("/")
+ if path:
+ return path.rsplit("/", 1)[-1]
+ return heading
+
+
def body_for_embedding(
raw: str | bytes,
whitelist: Sequence[str] = EMBEDDING_METADATA_FIELDS,
@@ -486,6 +542,7 @@ async def write_abstract_overview(
consume_pending: Optional[int] = None,
lock: Optional[Dict[str, Any]] = None,
log_prefix: str = "[Semantic]",
+ existing_raw: Optional[Mapping[int, str | bytes]] = None,
) -> AbstractOverviewWriteResult:
"""Render and atomically write sidecars while preserving source metadata.
@@ -512,8 +569,20 @@ async def write_abstract_overview(
logger.info("%s Skipping stale semantic write for %s", log_prefix, dir_uri)
return AbstractOverviewWriteResult(wrote=False)
- existing_overview = await _read_existing_document(viking_fs, overview_uri, ctx)
- existing_abstract = await _read_existing_document(viking_fs, abstract_uri, ctx)
+ existing_overview = (
+ parse_abstract_overview(existing_raw[1])
+ if existing_raw is not None and 1 in existing_raw
+ else await _read_existing_document(viking_fs, overview_uri, ctx)
+ if existing_raw is None
+ else None
+ )
+ existing_abstract = (
+ parse_abstract_overview(existing_raw[0])
+ if existing_raw is not None and 0 in existing_raw
+ else await _read_existing_document(viking_fs, abstract_uri, ctx)
+ if existing_raw is None
+ else None
+ )
merged_metadata = dict(metadata or {})
for existing in (existing_overview, existing_abstract):
if existing and "source" in existing.metadata and "source" not in merged_metadata:
@@ -549,8 +618,20 @@ async def write_abstract_overview(
rendered_abstract = render_abstract_overview(
ContextLevel.ABSTRACT, dir_uri, abstract, merged_metadata
)
- current_overview = await _raw_if_exists(viking_fs, overview_uri, ctx)
- current_abstract = await _raw_if_exists(viking_fs, abstract_uri, ctx)
+ current_overview = (
+ existing_raw.get(1)
+ if existing_raw is not None
+ else await _raw_if_exists(viking_fs, overview_uri, ctx)
+ )
+ current_abstract = (
+ existing_raw.get(0)
+ if existing_raw is not None
+ else await _raw_if_exists(viking_fs, abstract_uri, ctx)
+ )
+ if isinstance(current_overview, bytes):
+ current_overview = current_overview.decode("utf-8")
+ if isinstance(current_abstract, bytes):
+ current_abstract = current_abstract.decode("utf-8")
if current_overview != rendered_overview:
await viking_fs.write_file(
@@ -704,6 +785,7 @@ async def read_abstract_overview_pending_snapshot(
dir_uri: str,
ctx: Optional[RequestContext],
lock: Optional[Dict[str, Any]] = None,
+ existing_raw: Optional[Mapping[int, str | bytes]] = None,
) -> int:
"""Read the pending counter captured at the start of an aggregation."""
@@ -720,6 +802,14 @@ async def read_abstract_overview_pending_snapshot(
pending_values.append(int(freshness["pending_child_changes"]))
return max(pending_values, default=0)
+ if existing_raw is not None:
+ return _pending_of(
+ [
+ parse_abstract_overview(existing_raw[level]) if level in existing_raw else None
+ for level in (1, 0)
+ ]
+ )
+
owns_lease = lock is None
snapshot_lease = lock
if owns_lease:
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index b440fe64a..647d8375a 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -588,6 +588,9 @@ class TextEmbeddingHandler(DequeueHandlerBase):
ctx: RequestContext,
) -> str:
inserted_data = embedding_msg.context_data
+ materialized = inserted_data.pop("_materialized_content", None)
+ if isinstance(materialized, str):
+ return materialized[:VIKINGDB_CONTENT_MAX_SIZE]
if inserted_data.get("is_leaf") and inserted_data.get("context_type") in (
ContextType.RESOURCE.value,
ContextType.SKILL.value,
@@ -894,6 +897,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
embedding_msg,
ctx,
)
+ inserted_data.pop("_materialized_content", None)
if embedding_msg.action is IndexAction.MERGE:
field_patch = embedding_msg.field_patch
merge_fields = dict(field_patch.values) if field_patch is not None else {}
diff --git a/openviking/storage/content_write.py b/openviking/storage/content_write.py
index 7572af63d..f06ddbbf1 100644
--- a/openviking/storage/content_write.py
+++ b/openviking/storage/content_write.py
@@ -720,24 +720,45 @@ class ContentWriteCoordinator:
) -> Dict[str, Any]:
self._validate_tag_mode(mode)
normalized_uri = self._validate_uri_path(uri, field_name="uri")
- normalized_tags = normalize_search_tags(tags, discard_invalid=True)
+ requested_mode = mode
+ normalized_tags = (
+ [] if requested_mode == "clear" else normalize_search_tags(tags, discard_invalid=True)
+ )
await self._viking_fs._ensure_access(normalized_uri, ctx, action=AclAction.WRITE)
stat = await self._safe_stat(normalized_uri, ctx=ctx)
+ context_type = context_type_for_uri(normalized_uri)
+ root_uri = await self._resolve_root_uri(normalized_uri, ctx=ctx)
+ if requested_mode == "replace" and not normalized_tags:
+ return self._build_tags_result(
+ uri=normalized_uri,
+ updated_uris=[],
+ skipped_count=1,
+ failed_count=0,
+ root_uri=root_uri,
+ context_type=context_type,
+ tags=normalized_tags,
+ mode=requested_mode,
+ )
+ storage_mode = "replace" if requested_mode == "clear" else requested_mode
if stat.get("isDir"):
- return await self._set_directory_tags(
+ result = await self._set_directory_tags(
uri=normalized_uri,
tags=normalized_tags,
- mode=mode,
+ mode=storage_mode,
recursive=recursive,
ctx=ctx,
)
- return await self._set_single_uri_tags(
- uri=normalized_uri,
- tags=normalized_tags,
- mode=mode,
- recursive=recursive,
- ctx=ctx,
- )
+ else:
+ result = await self._set_single_uri_tags(
+ uri=normalized_uri,
+ tags=normalized_tags,
+ mode=storage_mode,
+ recursive=recursive,
+ ctx=ctx,
+ )
+ result["mode"] = requested_mode
+ result["tags"] = normalized_tags
+ return result
def _build_write_result(
self,
@@ -1083,7 +1104,7 @@ class ContentWriteCoordinator:
raise InvalidArgumentError(f"unsupported batch-write mode: {mode}")
def _validate_tag_mode(self, mode: str) -> None:
- if mode not in {"replace", "append"}:
+ if mode not in {"replace", "append", "clear"}:
raise InvalidArgumentError(f"unsupported tag mode: {mode}")
def _ensure_content_write_policy(self, uri: str) -> None:
diff --git a/openviking/storage/context_update_plan.py b/openviking/storage/context_update_plan.py
index 755959ec4..80a7a9126 100644
--- a/openviking/storage/context_update_plan.py
+++ b/openviking/storage/context_update_plan.py
@@ -11,6 +11,7 @@ from pathlib import PurePosixPath
from typing import Any, Mapping
from openviking.concurrency import bounded_map
+from openviking.storage.abstract_overview import parse_overview_file_summaries
from openviking.storage.index_action import FieldPatch, IndexAction
from openviking.storage.resource_diff import ContentState, IndexState
from openviking.storage.resource_rnfv import (
@@ -52,11 +53,6 @@ class FileVectorSource(str, Enum):
SUMMARY_WHEN_AVAILABLE = "summary_when_available"
-@dataclass(frozen=True)
-class ParentPropagation:
- enabled: bool = True
-
-
@dataclass(frozen=True)
class FileRefreshIntent:
"""Refresh a flat file and its parent after synchronous content commit."""
@@ -247,7 +243,6 @@ class SemanticPlan:
context_type: str
tree: SemanticTreeSnapshot
vectorize: bool = True
- propagation: ParentPropagation = field(default_factory=ParentPropagation)
file_vector_source: FileVectorSource = FileVectorSource.CONTENT
ingest_options: IngestOptions = field(default_factory=IngestOptions)
source_metadata: Mapping[str, str] | None = None
@@ -301,7 +296,6 @@ class SemanticPlan:
context_type=str(data["context_type"]),
tree=SemanticTreeSnapshot.from_dict(data.get("tree", {})),
vectorize=bool(data.get("vectorize", True)),
- propagation=ParentPropagation(**dict(data.get("propagation", {}))),
file_vector_source=FileVectorSource(
data.get("file_vector_source", FileVectorSource.CONTENT.value)
),
@@ -388,6 +382,7 @@ class DirectIndexAction:
upsert_fields: Mapping[str, Any] = field(default_factory=dict)
field_patch: FieldPatch | None = None
md5: str | None = None
+ summary: str = ""
def __post_init__(self) -> None:
object.__setattr__(self, "action", IndexAction(self.action))
@@ -588,11 +583,37 @@ def _field_patch(request: RequestIntent, record: VectorRecordSnapshot | None) ->
return FieldPatch(values, modes) if values else None
+def _rfv_file_summary(
+ snapshot: Any,
+ relative_path: str,
+ record: VectorRecordSnapshot | None,
+ overview_summaries: dict[str, Mapping[str, str]],
+) -> str:
+ file_name = relative_path.rsplit("/", 1)[-1]
+ if not relative_path and not snapshot.formal.entries[""].is_dir:
+ file_uri = snapshot.request.target_uri.rstrip("/")
+ parent_uri = file_uri.rsplit("/", 1)[0]
+ file_name = file_uri.rsplit("/", 1)[-1]
+ else:
+ parent_uri = _uri(snapshot.request.target_uri, _parent(relative_path))
+ overview = snapshot.source_contents.get((parent_uri, 1))
+ if isinstance(overview, (str, bytes)):
+ summaries = overview_summaries.get(parent_uri)
+ if summaries is None:
+ summaries = parse_overview_file_summaries(overview)
+ overview_summaries[parent_uri] = summaries
+ summary = summaries.get(file_name)
+ if summary:
+ return summary
+ return str(record.fields.get("abstract") or "") if record else ""
+
+
def _semantic_closure(
diff: Any,
new_kinds: Mapping[str, str],
*,
repair_indexes: bool = True,
+ input_only_paths: frozenset[str] = frozenset(),
) -> tuple[set[str], set[str], set[str]]:
"""Return active nodes, membership-changing parents, and retained closure.
@@ -612,6 +633,8 @@ def _semantic_closure(
parent = _parent(parent)
for path, entry in diff.entries.items():
+ if path in input_only_paths:
+ continue
content_state = ContentState(entry.content_state)
index_state = IndexState(entry.index_state)
content_requires_semantics = content_state in {
@@ -624,7 +647,12 @@ def _semantic_closure(
or (
repair_indexes
and index_state
- in {IndexState.MISSING, IndexState.PARTIAL, IndexState.LEVEL_CONFLICT}
+ in {
+ IndexState.MISSING,
+ IndexState.PARTIAL,
+ IndexState.LEVEL_CONFLICT,
+ }
+ or (repair_indexes and index_state is IndexState.STALE and bool(entry.level_states))
)
) and entry.new_kind in {"file", "directory"}:
active.add(path)
@@ -645,6 +673,51 @@ def _semantic_closure(
return active, membership_changed, retained
+def _promote_missing_semantic_inputs(
+ *,
+ active: set[str],
+ retained: set[str],
+ new_kinds: Mapping[str, str],
+ records_by_path: Mapping[str, Mapping[int, VectorRecordSnapshot]],
+ input_only_paths: frozenset[str] = frozenset(),
+) -> None:
+ """Promote retained inputs that cannot provide an abstract to a parent.
+
+ An L2 record can be vector-complete after a vectors-only ingest while still
+ lacking the file abstract required by its parent's first aggregation. The
+ same applies to a retained directory without an L0 abstract. Semantic
+ closure therefore tracks reusable abstracts independently from index MD5.
+ """
+ while True:
+ missing_inputs = {
+ path
+ for path in retained - active
+ if new_kinds.get(path) in {"file", "directory"}
+ and (
+ (
+ record := records_by_path.get(path, {}).get(
+ 2 if new_kinds.get(path) == "file" else 0
+ )
+ )
+ is None
+ or not str(record.fields.get("abstract") or "").strip()
+ )
+ }
+ blocked_inputs = missing_inputs & input_only_paths
+ if blocked_inputs:
+ raise ValueError(
+ "non-recursive semantic plan lacks reusable abstract for "
+ + ", ".join(sorted(blocked_inputs))
+ )
+ promoted = missing_inputs - input_only_paths
+ if not promoted:
+ return
+ active.update(promoted)
+ for path in new_kinds:
+ if path and _parent(path) in promoted:
+ retained.add(path)
+
+
async def hydrate_context_plan_records(
*,
diff: Any,
@@ -761,6 +834,7 @@ def build_context_update_plan(
ingest_options: IngestOptions | Mapping[str, Any] | None = None,
source_metadata: Mapping[str, str] | None = None,
closure: tuple[set[str], set[str], set[str]] | None = None,
+ input_only_paths: frozenset[str] = frozenset(),
) -> ContextUpdatePlan:
"""Compile resolved RNFV facts into synchronous and asynchronous actions.
@@ -779,6 +853,7 @@ def build_context_update_plan(
diff,
new_kinds,
repair_indexes=request.vectorize and request.processing_mode != "vectors_only",
+ input_only_paths=input_only_paths,
)
semantic_enabled = request.processing_mode != "vectors_only"
if not semantic_enabled:
@@ -788,8 +863,11 @@ def build_context_update_plan(
direct_actions: list[DirectIndexAction] = [
DirectIndexAction(IndexAction.DELETE, record.uri, record.level, record.record_id)
for record in duplicate_records
+ if record.relative_path not in input_only_paths
]
for path, entry in sorted(diff.entries.items()):
+ if path in input_only_paths:
+ continue
state = ContentState(entry.content_state)
kind = entry.new_kind
if state in {ContentState.ADDED, ContentState.RESTORE, ContentState.MODIFIED}:
@@ -874,11 +952,14 @@ def build_context_update_plan(
),
field_patch=field_patch if index_action is IndexAction.MERGE else None,
md5=entry.md5,
+ summary=str(record.fields.get("abstract") or "") if record else "",
)
)
scheduled_direct_ids = {action.record_id for action in direct_actions}
for path, levels in records_by_path.items():
+ if path in input_only_paths:
+ continue
for record in levels.values():
field_patch = _field_patch(request, record) if request.vectorize else None
patch_values = dict(field_patch.values) if field_patch is not None else {}
@@ -918,6 +999,15 @@ def build_context_update_plan(
)
scheduled_direct_ids.add(record.record_id)
+ if semantic_enabled:
+ _promote_missing_semantic_inputs(
+ active=active,
+ retained=retained,
+ new_kinds=new_kinds,
+ records_by_path=records_by_path,
+ input_only_paths=input_only_paths,
+ )
+
semantic_entries: list[SemanticTreeEntry] = []
for path in sorted(retained, key=lambda value: (value.count("/"), value)):
kind = new_kinds.get(path)
@@ -926,7 +1016,9 @@ def build_context_update_plan(
diff_entry = diff.entries.get(path)
state = ContentState(diff_entry.content_state) if diff_entry else ContentState.UNCHANGED
action = (
- SemanticAction.GENERATE
+ SemanticAction.REUSE
+ if path in input_only_paths
+ else SemanticAction.GENERATE
if kind == "file" and path in active
else SemanticAction.AGGREGATE
if kind == "directory" and path in active
@@ -959,6 +1051,8 @@ def build_context_update_plan(
)
else IndexAction.UPSERT
)
+ if diff_entry and diff_entry.level_states:
+ index_action = IndexAction.MERGE
slots.append(
IndexSlot(
level,
@@ -992,7 +1086,17 @@ def build_context_update_plan(
request.vectorize
and diff_entry
and IndexState(diff_entry.index_state)
- in {IndexState.MISSING, IndexState.PARTIAL, IndexState.LEVEL_CONFLICT}
+ in {
+ IndexState.MISSING,
+ IndexState.PARTIAL,
+ IndexState.LEVEL_CONFLICT,
+ }
+ or (
+ request.vectorize
+ and diff_entry
+ and IndexState(diff_entry.index_state) is IndexState.STALE
+ and bool(diff_entry.level_states)
+ )
),
)
)
@@ -1025,6 +1129,116 @@ def build_context_update_plan(
)
+def build_rfv_context_update_plan(
+ *,
+ snapshot: Any,
+ context_type: str,
+ account_id: str,
+ source_metadata: Mapping[str, str] | None = None,
+) -> tuple[Any, ContextUpdatePlan]:
+ """Compile an R/F/V maintenance snapshot without inventing content writes."""
+ from openviking.storage.resource_rfv import resolve_rfv_state
+
+ diff = resolve_rfv_state(snapshot)
+ if snapshot.request.processing_mode != "vectors_only":
+ plan = build_context_update_plan(
+ root_uri=snapshot.request.target_uri,
+ context_type=context_type,
+ request=snapshot.request,
+ diff=diff,
+ new_kinds={
+ path: "directory" if entry.is_dir else "file"
+ for path, entry in snapshot.formal.entries.items()
+ },
+ artifact_paths={},
+ records=snapshot.vectors.records_by_id,
+ is_code_repo=False,
+ account_id=account_id,
+ source_metadata=source_metadata or snapshot.source_metadata,
+ input_only_paths=snapshot.input_only_paths,
+ )
+ if plan.content_tree_actions:
+ raise ValueError("RFV maintenance plan must not mutate formal content")
+ return diff, plan
+ records_by_path, duplicate_records = _records_by_path(snapshot.vectors.records_by_id)
+ overview_summaries: dict[str, Mapping[str, str]] = {}
+ direct_actions: list[DirectIndexAction] = [
+ DirectIndexAction(IndexAction.DELETE, record.uri, record.level, record.record_id)
+ for record in duplicate_records
+ if record.relative_path not in snapshot.input_only_paths
+ ]
+
+ for path, entry in sorted(diff.entries.items()):
+ if path in snapshot.input_only_paths:
+ continue
+ existing = records_by_path.get(path, {})
+ if entry.new_kind is None and IndexState(entry.index_state) is IndexState.ORPHAN:
+ direct_actions.extend(
+ DirectIndexAction(IndexAction.DELETE, record.uri, level, record.record_id)
+ for level, record in sorted(existing.items())
+ )
+ continue
+
+ expected_levels = {2} if entry.new_kind == "file" else {0, 1}
+ for level, record in sorted(existing.items()):
+ if level not in expected_levels or (
+ snapshot.request.processing_mode == "vectors_only"
+ and IndexState(entry.level_states.get(level, IndexState.ABSENT))
+ is IndexState.ORPHAN
+ ):
+ direct_actions.append(
+ DirectIndexAction(IndexAction.DELETE, record.uri, level, record.record_id)
+ )
+
+ for level in sorted(expected_levels):
+ record = existing.get(level)
+ level_state = IndexState(entry.level_states.get(level, IndexState.MISSING))
+ field_patch = _field_patch(snapshot.request, record)
+ if level_state in {IndexState.MISSING, IndexState.STALE}:
+ if level not in entry.level_md5s:
+ continue
+ uri = _uri(snapshot.request.target_uri, path)
+ direct_actions.append(
+ DirectIndexAction(
+ IndexAction.MERGE,
+ uri,
+ level,
+ (
+ record.record_id
+ if record is not None
+ else vector_record_id(account_id, uri, level)
+ ),
+ field_patch=field_patch,
+ md5=entry.level_md5s.get(level),
+ summary=_rfv_file_summary(snapshot, path, record, overview_summaries),
+ )
+ )
+ elif field_patch is not None and record is not None:
+ direct_actions.append(
+ DirectIndexAction(
+ IndexAction.UPDATE_FIELDS,
+ record.uri,
+ level,
+ record.record_id,
+ field_patch=field_patch.with_seed(
+ {
+ "uri": record.uri,
+ "account_id": account_id,
+ "level": level,
+ **(_portable_existing_fields(record) or {}),
+ **field_patch.resolve(record.fields),
+ }
+ ),
+ )
+ )
+
+ return diff, ContextUpdatePlan(
+ snapshot.request.target_uri,
+ context_type,
+ direct_index_actions=tuple(direct_actions),
+ )
+
+
def _with_directory_root(snapshot: RNFVSnapshot, *, root_preexisting: bool) -> RNFVSnapshot:
"""Add the logical directory root omitted by artifact/tree walks."""
new_entries = {"": NewEntry(is_dir=True), **snapshot.new.entries}
@@ -1200,13 +1414,13 @@ __all__ = [
"IndexAction",
"IndexSlot",
"IndexState",
- "ParentPropagation",
"SemanticAction",
"SemanticPlan",
"SemanticTreeEntry",
"SemanticTreeSnapshot",
"build_context_update_plan",
"build_context_update_plan_from_snapshot",
+ "build_rfv_context_update_plan",
"execute_content_tree_actions",
"hydrate_context_plan_records",
]
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index 43a78b537..e045527b1 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -39,6 +39,7 @@ def init_queue_manager(
max_concurrent_semantic: int = 32,
max_concurrent_external_parse: int = 4,
max_concurrent_add_resource: int = 4,
+ max_concurrent_reindex: int = 4,
max_concurrent_session_commit: int = DEFAULT_MAX_CONCURRENT_SESSION_COMMIT,
max_concurrent_external_task: int = 10,
*,
@@ -54,6 +55,7 @@ def init_queue_manager(
max_concurrent_semantic: Max concurrent semantic node work.
max_concurrent_external_parse: Max concurrent ExternalParse tasks.
max_concurrent_add_resource: Max concurrent AddResource tasks.
+ max_concurrent_reindex: Max concurrent Reindex tasks.
max_concurrent_session_commit: Max concurrent SessionCommit tasks.
middlewares: Additional middleware, fixed at construction for all queues.
"""
@@ -66,6 +68,7 @@ def init_queue_manager(
max_concurrent_semantic=max_concurrent_semantic,
max_concurrent_external_parse=max_concurrent_external_parse,
max_concurrent_add_resource=max_concurrent_add_resource,
+ max_concurrent_reindex=max_concurrent_reindex,
max_concurrent_session_commit=max_concurrent_session_commit,
max_concurrent_external_task=max_concurrent_external_task,
middlewares=middlewares,
@@ -92,6 +95,7 @@ class QueueManager:
# Keep the on-disk name stable so pre-upgrade jobs remain recoverable.
EXTERNAL_PARSE = "ExternalParse"
ADD_RESOURCE = "AddResource"
+ REINDEX = "Reindex"
SESSION_COMMIT = "SessionCommit"
EXTERNAL_TASK = "ExternalTask"
# Account and user cleanup share one consumer. Retain the persisted name
@@ -109,6 +113,7 @@ class QueueManager:
max_concurrent_semantic: int = 32,
max_concurrent_external_parse: int = 4,
max_concurrent_add_resource: int = 4,
+ max_concurrent_reindex: int = 4,
max_concurrent_session_commit: int = DEFAULT_MAX_CONCURRENT_SESSION_COMMIT,
max_concurrent_external_task: int = 10,
*,
@@ -122,6 +127,7 @@ class QueueManager:
self._max_concurrent_semantic = max_concurrent_semantic
self._max_concurrent_external_parse = max_concurrent_external_parse
self._max_concurrent_add_resource = max_concurrent_add_resource
+ self._max_concurrent_reindex = max_concurrent_reindex
self._max_concurrent_session_commit = max_concurrent_session_commit
self._max_concurrent_external_task = max_concurrent_external_task
self._queues: Dict[str, NamedQueue] = {}
@@ -150,9 +156,7 @@ class QueueManager:
if self._started:
return
if self.SEMANTIC in self._queues and self._vlm_resolver is None:
- raise RuntimeError(
- "QueueManager requires a VLM resolver before semantic workers start"
- )
+ raise RuntimeError("QueueManager requires a VLM resolver before semantic workers start")
self._started = True
@@ -262,6 +266,8 @@ class QueueManager:
return self._max_concurrent_external_parse
if queue_name == self.ADD_RESOURCE:
return self._max_concurrent_add_resource
+ if queue_name == self.REINDEX:
+ return self._max_concurrent_reindex
if queue_name == self.SESSION_COMMIT:
return self._max_concurrent_session_commit
if queue_name == self.EXTERNAL_TASK:
diff --git a/openviking/storage/queuefs/reindex_msg.py b/openviking/storage/queuefs/reindex_msg.py
new file mode 100644
index 000000000..0db0397db
--- /dev/null
+++ b/openviking/storage/queuefs/reindex_msg.py
@@ -0,0 +1,66 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Durable root request for asynchronous reindex work."""
+
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, Optional
+
+
+@dataclass(kw_only=True)
+class ReindexMsg:
+ task_id: str
+ uri: str
+ object_type: str
+ mode: str
+ account_id: str
+ user_id: str
+ role: str
+ force: bool = False
+ recursive: bool = True
+ tags: Optional[list[str]] = None
+ tag_mode: str = "replace"
+ group_ids: list[str] = field(default_factory=list)
+ actor_peer_id: Optional[str] = None
+ bypass_acl: bool = False
+ telemetry_id: Optional[str] = None
+ lock_handoff: Optional[Dict[str, Any]] = None
+ lock_handoff_retry: int = 0
+
+ def to_dict(self) -> Dict[str, Any]:
+ return asdict(self)
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> "ReindexMsg":
+ required = ("task_id", "uri", "object_type", "mode", "account_id", "user_id")
+ missing = [name for name in required if not data.get(name)]
+ if missing:
+ raise ValueError(f"Missing required reindex fields: {missing}")
+ try:
+ lock_handoff_retry = max(0, int(data.get("lock_handoff_retry", 0) or 0))
+ except (TypeError, ValueError):
+ lock_handoff_retry = 0
+ return cls(
+ task_id=str(data["task_id"]),
+ uri=str(data["uri"]),
+ object_type=str(data["object_type"]),
+ mode=str(data["mode"]),
+ account_id=str(data["account_id"]),
+ user_id=str(data["user_id"]),
+ role=str(data.get("role") or "root"),
+ force=bool(data.get("force", False)),
+ recursive=bool(data.get("recursive", True)),
+ tags=list(data["tags"]) if isinstance(data.get("tags"), list) else None,
+ tag_mode=str(data.get("tag_mode") or "replace"),
+ group_ids=[str(value) for value in data.get("group_ids", [])],
+ actor_peer_id=data.get("actor_peer_id")
+ if isinstance(data.get("actor_peer_id"), str)
+ else None,
+ bypass_acl=bool(data.get("bypass_acl", False)),
+ telemetry_id=data.get("telemetry_id")
+ if isinstance(data.get("telemetry_id"), str)
+ else None,
+ lock_handoff=data.get("lock_handoff")
+ if isinstance(data.get("lock_handoff"), dict)
+ else None,
+ lock_handoff_retry=lock_handoff_retry,
+ )
diff --git a/openviking/storage/queuefs/reindex_processor.py b/openviking/storage/queuefs/reindex_processor.py
new file mode 100644
index 000000000..a9c15ea03
--- /dev/null
+++ b/openviking/storage/queuefs/reindex_processor.py
@@ -0,0 +1,186 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Queue consumer that owns one durable asynchronous reindex request."""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, Optional
+
+from openviking.server.identity import RequestContext, Role
+from openviking.service.task_tracker import TaskStatus, get_task_tracker
+from openviking.service.task_work_index import bind_task_context, extract_task_metadata
+from openviking.storage.queuefs.named_queue import DequeueHandlerBase
+from openviking.storage.queuefs.process_result import ProcessResult
+from openviking.storage.queuefs.reindex_msg import ReindexMsg
+from openviking.telemetry import (
+ OperationTelemetry,
+ bind_telemetry,
+ register_telemetry,
+ resolve_telemetry,
+ unregister_telemetry,
+)
+from openviking_cli.session.user_id import UserIdentifier
+from openviking_cli.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+class ReindexProcessor(DequeueHandlerBase):
+ def __init__(self, viking_fs: Any) -> None:
+ self._viking_fs = viking_fs
+
+ @staticmethod
+ def _ctx(msg: ReindexMsg) -> RequestContext:
+ return RequestContext(
+ user=UserIdentifier(account_id=msg.account_id, user_id=msg.user_id),
+ role=Role(msg.role),
+ group_ids=tuple(msg.group_ids),
+ actor_peer_id=msg.actor_peer_id,
+ bypass_acl=msg.bypass_acl,
+ )
+
+ async def _adopt_or_reacquire(self, msg: ReindexMsg, ctx: RequestContext) -> Dict[str, Any]:
+ agfs = self._viking_fs._async_agfs
+ if msg.lock_handoff is not None:
+ try:
+ return await agfs.pathlock_adopt(msg.lock_handoff)
+ except Exception:
+ pass
+ stat = await self._viking_fs.stat(msg.uri, ctx=ctx, skip_count=True)
+ acquire = (
+ agfs.pathlock_acquire_tree
+ if stat.get("isDir", stat.get("is_dir"))
+ else agfs.pathlock_acquire_exact
+ )
+ return await acquire(self._viking_fs._uri_to_path(msg.uri, ctx=ctx))
+
+ async def _requeue_lock_handoff(self, msg: ReindexMsg, exc: Exception) -> bool:
+ if msg.lock_handoff_retry >= 2:
+ return False
+
+ from openviking.storage.queuefs import get_queue_manager
+
+ payload = msg.to_dict()
+ payload["lock_handoff_retry"] = msg.lock_handoff_retry + 1
+ queue_manager = get_queue_manager()
+ await queue_manager.enqueue(queue_manager.REINDEX, payload)
+ logger.warning(
+ "[Reindex] Requeued task %s after lock handoff failure: %s",
+ msg.task_id,
+ exc,
+ )
+ return True
+
+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
+ if not data:
+ return ProcessResult.success()
+ payload = data.get("data", data)
+ if isinstance(payload, str):
+ payload = json.loads(payload)
+ msg = ReindexMsg.from_dict(payload)
+ ctx = self._ctx(msg)
+ tracker = get_task_tracker()
+ metadata = extract_task_metadata(data)
+ task = await tracker.get(
+ msg.task_id,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ if task is not None and task.status in {
+ TaskStatus.COMPLETED,
+ TaskStatus.FAILED,
+ TaskStatus.CANCELLED,
+ }:
+ unregister_telemetry(msg.telemetry_id or "")
+ return ProcessResult.success()
+ lease = None
+ try:
+ lease = await self._adopt_or_reacquire(msg, ctx)
+ except Exception as exc:
+ if await self._requeue_lock_handoff(msg, exc):
+ return ProcessResult.requeued()
+ await tracker.fail(
+ msg.task_id,
+ f"Invalid lock_handoff: {exc}",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ unregister_telemetry(msg.telemetry_id or "")
+ return ProcessResult.failed(f"Invalid lock_handoff: {exc}")
+
+ telemetry_id = msg.telemetry_id or ""
+ telemetry = resolve_telemetry(telemetry_id) if telemetry_id else None
+ if telemetry is None:
+ telemetry = OperationTelemetry(operation="reindex_job", enabled=True)
+ if telemetry_id:
+ telemetry.telemetry_id = telemetry_id
+ else:
+ telemetry_id = telemetry.telemetry_id
+ register_telemetry(telemetry)
+ try:
+ await tracker.start(
+ msg.task_id, account_id=ctx.account_id, user_id=ctx.user.user_id, stage="queued"
+ )
+ from openviking.server.dependencies import get_service
+ from openviking.service.reindex_executor import ReindexExecutor
+
+ service = get_service()
+ executor = ReindexExecutor(
+ vlm_resolver=service._vlm_resolver,
+ vector_config_resolver=service._vector_config_resolver,
+ )
+ with (
+ bind_telemetry(telemetry),
+ bind_task_context(msg.task_id, ctx.account_id, ctx.user.user_id),
+ ):
+ # _run owns an existing lease and releases or hands it off on
+ # every exit path. Clear this consumer's reference before the
+ # call so an exception cannot release the same lease twice.
+ executor_lease = lease
+ lease = None
+ result = await executor._run(
+ uri=msg.uri,
+ object_type=msg.object_type,
+ mode=msg.mode,
+ force=msg.force,
+ recursive=msg.recursive,
+ ingest_options=executor._resolve_ingest_options(
+ mode=msg.mode, tags=msg.tags, tag_mode=msg.tag_mode
+ ),
+ ctx=ctx,
+ existing_lease=executor_lease,
+ )
+ if metadata is not None:
+ await tracker.wait_for_descendants(msg.task_id, metadata.work_id)
+ await tracker.complete(
+ msg.task_id, result, account_id=ctx.account_id, user_id=ctx.user.user_id
+ )
+ return ProcessResult.success()
+ except Exception as exc:
+ await tracker.fail(
+ msg.task_id, str(exc), account_id=ctx.account_id, user_id=ctx.user.user_id
+ )
+ return ProcessResult.failed(str(exc))
+ finally:
+ if lease is not None:
+ await self._viking_fs._async_agfs.pathlock_release(lease)
+ unregister_telemetry(telemetry_id)
+
+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
+ """Release the root lock handoff before QueueFS ACKs cancelled work."""
+ try:
+ payload = data.get("data", data) if isinstance(data, dict) else data
+ if isinstance(payload, str):
+ payload = json.loads(payload)
+ msg = ReindexMsg.from_dict(payload)
+ except (json.JSONDecodeError, TypeError, ValueError) as exc:
+ return ProcessResult.failed(str(exc))
+
+ if msg.lock_handoff is not None:
+ try:
+ lock = await self._viking_fs._async_agfs.pathlock_adopt(msg.lock_handoff)
+ await self._viking_fs._async_agfs.pathlock_release(lock)
+ except Exception as exc:
+ logger.warning("[Reindex] Failed to release cancelled lock handoff: %s", exc)
+ return ProcessResult.cancelled()
diff --git a/openviking/storage/queuefs/semantic_executor.py b/openviking/storage/queuefs/semantic_executor.py
index 77ac836cb..f41029520 100644
--- a/openviking/storage/queuefs/semantic_executor.py
+++ b/openviking/storage/queuefs/semantic_executor.py
@@ -7,7 +7,7 @@ import re
import threading
from contextlib import nullcontext
from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Set
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Mapping, Optional, Set
from weakref import WeakKeyDictionary
from openviking.core.namespace import classify_uri
@@ -198,6 +198,9 @@ class SemanticTreeExecutor:
file_abstracts: Optional[Dict[str, str]] = None,
semantic_plan: Optional["SemanticPlan"] = None,
telemetry_id: str | None = None,
+ source_contents: Optional[Mapping[tuple[str, int], str | bytes]] = None,
+ source_raw_contents: Optional[Mapping[tuple[str, int], str | bytes]] = None,
+ materialize_content: bool = False,
):
self._processor = processor
self._context_type = context_type
@@ -216,6 +219,11 @@ class SemanticTreeExecutor:
self._changes_provided = changes is not None
self._changes = changes or {}
self._semantic_plan = semantic_plan
+ self._source_contents = dict(source_contents or {})
+ self._source_raw_contents = (
+ dict(source_raw_contents) if source_raw_contents is not None else None
+ )
+ self._materialize_content = materialize_content
self._semantic_resource_root = (
semantic_plan.root_uri.rstrip("/") if semantic_plan is not None else None
)
@@ -361,6 +369,22 @@ class SemanticTreeExecutor:
self._stats.total_nodes = 1
self._stats.done_nodes = 1
return
+ if isinstance(entry, SemanticTreeEntry) and entry.kind == "file":
+ self._stats.total_nodes = 1
+ self._stats.in_progress_nodes = 1
+ # A file-root maintenance plan has no directory node or
+ # sidecars of its own. Execute its planned L2 work directly;
+ # the caller's parent freshness policy handles its parent.
+ previous_incremental = self._incremental_update
+ self._incremental_update = False
+ try:
+ await self._file_summary_task(root_uri, root_uri)
+ finally:
+ self._incremental_update = previous_incremental
+ self._root_write_result = AbstractOverviewWriteResult(
+ wrote=True, abstract_body_changed=True
+ )
+ return
owner = processing_owner.get()
if owner is not None and self._task_context is not None:
@@ -580,6 +604,16 @@ class SemanticTreeExecutor:
dir_uri=dir_uri,
ctx=self._ctx,
lock=self._lock,
+ existing_raw=(
+ {
+ level: raw
+ for level in (0, 1)
+ if (raw := self._source_raw_contents.get((dir_uri, level)))
+ is not None
+ }
+ if self._source_raw_contents is not None
+ else None
+ ),
)
except LockAcquisitionError:
if self._generation_trigger != "content_write":
@@ -1047,6 +1081,13 @@ class SemanticTreeExecutor:
self, dir_uri: str
) -> tuple[Optional[str], Optional[str]]:
if self._semantic_plan is not None:
+ snapshot_overview = self._source_contents.get((dir_uri.rstrip("/"), 1))
+ snapshot_abstract = self._source_contents.get((dir_uri.rstrip("/"), 0))
+ if snapshot_overview is not None or snapshot_abstract is not None:
+ return (
+ str(snapshot_overview) if snapshot_overview is not None else None,
+ str(snapshot_abstract) if snapshot_abstract is not None else None,
+ )
entry = self._plan_entries_by_uri.get(dir_uri.rstrip("/"))
if entry is None:
return None, None
@@ -1100,8 +1141,10 @@ class SemanticTreeExecutor:
else:
self._file_change_status[file_path] = True
if summary_dict is None:
- file_content = None
- if hasattr(self._viking_fs, "read_file_bytes"):
+ file_content = self._source_contents.get((file_path.rstrip("/"), 2))
+ if file_content is not None and not isinstance(file_content, bytes):
+ raise ValueError(f"file vector source must be bytes: {file_path}")
+ if file_content is None and hasattr(self._viking_fs, "read_file_bytes"):
file_content = await self._viking_fs.read_file_bytes(file_path, ctx=self._ctx)
summary_kwargs: Dict[str, Any] = {
"llm_sem": self._llm_sem,
@@ -1149,6 +1192,8 @@ class SemanticTreeExecutor:
vectorize_kwargs["file_content"] = file_content
if self._telemetry_id is not None:
vectorize_kwargs["telemetry_id"] = self._telemetry_id
+ if self._materialize_content:
+ vectorize_kwargs["materialize_content"] = True
manifest_md5 = self._file_md5s.get(file_path.rstrip("/")) or None
if file_content is not None and not manifest_md5:
file_md5 = content_md5(file_content)
@@ -1175,6 +1220,7 @@ class SemanticTreeExecutor:
)
if enqueued and slot is not None:
self._scheduled_vector_record_ids.add(slot.record_id)
+ self._stats.indexed_records += 1
except Exception as e:
logger.error(
"Failed to schedule vectorization for %s: %s",
@@ -1344,6 +1390,15 @@ class SemanticTreeExecutor:
consume_pending=consume_pending,
lock=self._lock,
log_prefix="[SemanticTree]",
+ existing_raw=(
+ {
+ level: raw
+ for level in (0, 1)
+ if (raw := self._source_raw_contents.get((dir_uri, level))) is not None
+ }
+ if self._source_raw_contents is not None
+ else None
+ ),
)
)
if not wrote.wrote:
@@ -1386,15 +1441,20 @@ class SemanticTreeExecutor:
# The package root describes the skill definition, independently
# of the summaries produced for its attachments.
definition_changed = f"{dir_uri}/SKILL.md" in self._changed_paths
+ plan_entry = self._plan_entries_by_uri.get(dir_uri.rstrip("/"))
+ repair = bool(plan_entry is not None and plan_entry.repair)
overview, abstract = await self._processor._skill_root_semantics(
dir_uri,
ctx=self._ctx,
- regenerate=self._generation_trigger == "reindex" or definition_changed,
+ regenerate=self._generation_trigger == "reindex"
+ or definition_changed
+ or repair,
lock=self._lock,
+ skill_content=self._source_contents.get((f"{dir_uri}/SKILL.md", 2)),
)
should_write = False
- need_vectorize = not self._incremental_update or definition_changed
- children_changed = definition_changed
+ need_vectorize = not self._incremental_update or definition_changed or repair
+ children_changed = definition_changed or repair
elif self._generation_trigger == "content_copy" and not node.transfer_inputs_ready:
need_vectorize = False
should_write = False
@@ -1406,7 +1466,18 @@ class SemanticTreeExecutor:
if not children_changed:
overview, abstract = await self._read_existing_overview_abstract(dir_uri)
should_write = overview is None or abstract is None
- # Rebuilt sidecars must also replace their stale vectors.
+ plan_entry = self._plan_entries_by_uri.get(dir_uri.rstrip("/"))
+ if (
+ self._semantic_plan is not None
+ and plan_entry is not None
+ and plan_entry.repair
+ ):
+ # RFV maintenance uses ``repair`` for both stale semantic
+ # input and force rebuilds. Regenerate through the normal
+ # semantic action before emitting its planned index slots.
+ overview = None
+ abstract = None
+ should_write = True
need_vectorize = should_write
children_changed = should_write
if should_write and (overview is None or abstract is None):
@@ -1479,6 +1550,7 @@ class SemanticTreeExecutor:
entry = self._plan_entries_by_uri.get(dir_uri.rstrip("/"))
slots = {slot.level: slot for slot in entry.index_slots} if entry else {}
+ enqueued_levels: set[int] = set()
if need_vectorize and not self._skip_vectorization:
assert overview is not None and abstract is not None
try:
@@ -1499,7 +1571,7 @@ class SemanticTreeExecutor:
return False
old_body = slot.abstract
new_body = abstract if level == 0 else overview
- return not old_body or old_body != new_body
+ return entry.repair or not old_body or old_body != new_body
include_abstract = should_emit(0)
include_overview = should_emit(1)
@@ -1521,6 +1593,10 @@ class SemanticTreeExecutor:
},
"include_abstract": include_abstract,
"include_overview": include_overview,
+ "md5s": {
+ 0: content_md5(abstract.encode("utf-8")),
+ 1: content_md5(overview.encode("utf-8")),
+ },
}
if self._telemetry_id is not None:
directory_vector_kwargs["telemetry_id"] = self._telemetry_id
@@ -1556,8 +1632,6 @@ class SemanticTreeExecutor:
if self._context_type != "skill":
raise
self._record_skill_failure(dir_uri, e)
- else:
- enqueued_levels = set()
for level, slot in sorted(slots.items()):
if (
@@ -1592,6 +1666,7 @@ class SemanticTreeExecutor:
self._scheduled_vector_record_ids.update(
slot.record_id for slot in slots.values() if slot.level in enqueued_levels
)
+ self._stats.indexed_records += len(enqueued_levels)
finally:
self._stats.done_nodes += 1
self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 60e358227..e28b42d9b 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -8,7 +8,6 @@ import threading
import time
from contextlib import nullcontext
from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple
-from urllib.parse import unquote, urlsplit
from openviking.core.namespace import classify_uri
from openviking.observability.context import (
@@ -45,6 +44,7 @@ from openviking.storage.abstract_overview import (
deterministic_sample,
freshness_metadata,
markdown_safe_viking_uri,
+ parse_overview_file_summaries,
plan_abstract_overview_refresh,
write_abstract_overview,
)
@@ -82,6 +82,7 @@ if TYPE_CHECKING:
logger = get_logger(__name__)
+
class RequestQueueStats:
processed: int = 0
requeue_count: int = 0
@@ -152,15 +153,23 @@ class SemanticProcessor(DequeueHandlerBase):
async def _get_vlm_config(self, ctx: RequestContext) -> "VLMHandle":
if self._vlm_resolver is None:
- raise RuntimeError(
- "SemanticProcessor requires a VLM resolver for account-owned work"
- )
+ raise RuntimeError("SemanticProcessor requires a VLM resolver for account-owned work")
return await self._vlm_resolver.get_vlm(ctx.account_id)
@classmethod
def _cache_tree_stats(cls, telemetry_id: str, uri: str, stats: SemanticTreeStats) -> None:
with cls._stats_lock:
if telemetry_id:
+ previous = cls._tree_stats_by_telemetry_id.get(telemetry_id)
+ if previous is not None:
+ stats = SemanticTreeStats(
+ total_nodes=previous.total_nodes + stats.total_nodes,
+ pending_nodes=previous.pending_nodes + stats.pending_nodes,
+ in_progress_nodes=previous.in_progress_nodes + stats.in_progress_nodes,
+ done_nodes=previous.done_nodes + stats.done_nodes,
+ failures=[*previous.failures, *stats.failures],
+ indexed_records=previous.indexed_records + stats.indexed_records,
+ )
cls._tree_stats_by_telemetry_id[telemetry_id] = stats
cls._tree_stats_by_uri[uri] = stats
cls._tree_stats_order.append((telemetry_id, uri))
@@ -286,8 +295,10 @@ class SemanticProcessor(DequeueHandlerBase):
else:
logger.warning(f"No queue manager available, cannot re-enqueue: {msg.uri}")
- async def _enqueue_skill_retry(self, queue, msg: SemanticMsg, scope: SemanticLockScope) -> None:
- """Transfer the live package lease to a retry before releasing this worker.
+ async def _enqueue_semantic_retry(
+ self, queue, msg: SemanticMsg, scope: SemanticLockScope
+ ) -> None:
+ """Transfer the live semantic lease to a retry before releasing this worker.
Reusing the consumed handoff would require acquiring an unrelated lock,
which conflicts with an update request still waiting under its outer lease.
@@ -307,6 +318,10 @@ class SemanticProcessor(DequeueHandlerBase):
scope._owned = True
raise
+ async def _enqueue_skill_retry(self, queue, msg: SemanticMsg, scope: SemanticLockScope) -> None:
+ """Compatibility wrapper for existing skill retry callers."""
+ await self._enqueue_semantic_retry(queue, msg, scope)
+
async def _requeue_semantic_msg_after_error(
self,
msg: SemanticMsg,
@@ -529,8 +544,7 @@ class SemanticProcessor(DequeueHandlerBase):
if self._vlm_resolver is None:
raise RuntimeError(
- "SemanticProcessor requires a VLM resolver "
- "for account-owned work"
+ "SemanticProcessor requires a VLM resolver for account-owned work"
)
if not await work.acquire_lock(current_ctx):
get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
@@ -557,12 +571,13 @@ class SemanticProcessor(DequeueHandlerBase):
source=msg.plan.source_metadata,
semantic_plan=msg.plan,
telemetry_id=msg.telemetry_id,
+ generation_trigger=msg.generation_trigger,
)
await executor.run(run_uri)
self._cache_tree_stats(
msg.telemetry_id, run_uri, executor.get_stats()
)
- if not executor.stale and msg.plan.propagation.enabled:
+ if not executor.stale:
write_result = getattr(
executor,
"root_write_result",
@@ -1232,6 +1247,7 @@ class SemanticProcessor(DequeueHandlerBase):
llm_sem: asyncio.Semaphore,
ctx: Optional[RequestContext] = None,
file_content: Optional[bytes] = None,
+ materialize_content: bool = False,
) -> Dict[str, Any]:
"""Generate summary for a single text file (code, documentation, or other text)."""
viking_fs = get_viking_fs()
@@ -1463,73 +1479,8 @@ class SemanticProcessor(DequeueHandlerBase):
@classmethod
def _parse_overview_md(cls, overview_content: str) -> Dict[str, str]:
- """Parse overview.md and extract file summaries.
-
- Args:
- overview_content: Content of the overview.md file
-
- Returns:
- Dictionary mapping file names to their summaries
- """
- import re
-
- summaries: Dict[str, str] = {}
-
- overview_content = body_for_preview(overview_content)
- if not overview_content or not overview_content.strip():
- return summaries
-
- lines = overview_content.split("\n")
- current_file = None
- current_summary_lines: List[str] = []
-
- for line in lines:
- header_match = re.match(r"^###\s+(.+?)\s*$", line)
- if header_match:
- if current_file and current_summary_lines:
- summaries[current_file] = " ".join(current_summary_lines).strip()
-
- file_name = cls._overview_heading_cache_key(header_match.group(1).strip())
- parts = file_name.split()
- if len(parts) >= 2 and parts[0] == parts[1]:
- file_name = parts[0]
-
- current_file = file_name
- current_summary_lines = []
- continue
-
- numbered_match = re.match(r"^\[(\d+)\]\s+(.+?):\s*(.+)$", line)
- if numbered_match:
- if current_file and current_summary_lines:
- summaries[current_file] = " ".join(current_summary_lines).strip()
- current_file = numbered_match.group(2).strip()
- current_summary_lines = [numbered_match.group(3).strip()]
- continue
-
- if current_file:
- stripped = line.strip()
- if stripped and not stripped.startswith("#"):
- current_summary_lines.append(stripped)
-
- if current_file and current_summary_lines:
- summaries[current_file] = " ".join(current_summary_lines).strip()
-
- return summaries
-
- @staticmethod
- def _overview_heading_cache_key(heading: str) -> str:
- """Return the entry name represented by a plain or linked H3 heading."""
- if heading.startswith("[") and heading.endswith(")"):
- destination_start = heading.rfind("](")
- if destination_start > 0:
- target = heading[destination_start + 2 : -1].strip()
- if target.startswith("<") and target.endswith(">"):
- target = target[1:-1].strip()
- if target.startswith("viking://"):
- path = unquote(urlsplit(target).path).rstrip("/")
- if path:
- return path.rsplit("/", 1)[-1]
- return heading
+ """Parse overview.md and extract direct-file summaries."""
+ return parse_overview_file_summaries(overview_content)
async def _generate_overview(
self,
@@ -1850,6 +1801,7 @@ class SemanticProcessor(DequeueHandlerBase):
ctx: RequestContext,
regenerate: bool = False,
lock: Optional[Dict[str, Any]] = None,
+ skill_content: str | bytes | None = None,
) -> Tuple[str, str]:
"""Keep the package root tied only to its SKILL.md definition."""
viking_fs = get_viking_fs()
@@ -1866,7 +1818,11 @@ class SemanticProcessor(DequeueHandlerBase):
from openviking.core.skill_loader import SkillLoader
from openviking.utils.skill_processor import SkillProcessor
- content = await viking_fs.read_file(f"{uri}/SKILL.md", ctx=ctx)
+ content = (
+ skill_content
+ if skill_content is not None
+ else await viking_fs.read_file(f"{uri}/SKILL.md", ctx=ctx)
+ )
if isinstance(content, bytes):
content = content.decode("utf-8")
definition = SkillLoader.parse(content)
@@ -1907,6 +1863,7 @@ class SemanticProcessor(DequeueHandlerBase):
include_abstract: bool = True,
include_overview: bool = True,
telemetry_id: str | None = None,
+ md5s: Optional[Dict[int, str]] = None,
) -> set[int]:
"""Create directory Context and enqueue to EmbeddingQueue."""
@@ -1939,6 +1896,7 @@ class SemanticProcessor(DequeueHandlerBase):
include_abstract=include_abstract,
include_overview=include_overview,
telemetry_id=telemetry_id,
+ md5s=md5s,
)
async def _load_transfer_file_summaries(
@@ -2004,6 +1962,7 @@ class SemanticProcessor(DequeueHandlerBase):
ingest_options: IngestOptions | None = None,
file_md5: Optional[str] = None,
file_content: Optional[bytes] = None,
+ materialize_content: bool = False,
scalar_override: Optional[Dict[str, Any]] = None,
field_patch: FieldPatch | None = None,
action: str = "merge",
@@ -2024,6 +1983,7 @@ class SemanticProcessor(DequeueHandlerBase):
ingest_options=ingest_options,
file_md5=file_md5,
file_content=file_content,
+ materialize_content=materialize_content,
scalar_override=scalar_override,
field_patch=field_patch,
action=action,
diff --git a/openviking/storage/queuefs/semantic_work.py b/openviking/storage/queuefs/semantic_work.py
index 553221af5..65daf3124 100644
--- a/openviking/storage/queuefs/semantic_work.py
+++ b/openviking/storage/queuefs/semantic_work.py
@@ -41,6 +41,7 @@ class SemanticMessageWork:
self.msg = msg
self.caller_lock = caller_lock
self.scope: SemanticLockScope | None = None
+ self._lock_closed = False
def start(self) -> None:
pass
@@ -61,22 +62,38 @@ class SemanticMessageWork:
async def finish_processing(self, succeeded: bool) -> None:
assert self.scope is not None
- await self.scope.close()
+ if self.msg.plan is not None and self.msg.lock_handoff is not None:
+ await get_request_wait_tracker().wait_for_embeddings(
+ self.msg.telemetry_id,
+ stop_waiting=getattr(self.processor, "_embedding_worker_stopped", None),
+ )
+ if succeeded:
+ await self.scope.close()
+ self._lock_closed = True
def failure_result(self, stats: SemanticTreeStats | None) -> ProcessResult | None:
return None
async def reenqueue(self) -> None:
- await self.processor._reenqueue_semantic_msg(self.msg)
+ async def enqueue(queue, msg):
+ if self.scope is not None and self.scope.lock is not None and self.scope._owned:
+ await self.processor._enqueue_semantic_retry(queue, msg, self.scope)
+ else:
+ await queue.enqueue(msg)
+
+ await self.processor._reenqueue_semantic_msg(self.msg, enqueue=enqueue)
async def reenqueue_after_error(self) -> None:
await self.reenqueue()
async def skip(self) -> None:
- pass
+ if self.msg.lock_handoff is not None:
+ await self.processor._release_cancelled_semantic_lock(self.msg)
async def cancel(self) -> None:
- pass
+ if self.scope is None:
+ await self.skip()
+ get_request_wait_tracker().mark_semantic_done(self.msg.telemetry_id, self.msg.id)
async def cancel_queued(self) -> None:
if self.msg.telemetry_id and self.msg.id:
@@ -84,7 +101,9 @@ class SemanticMessageWork:
await self.processor._release_cancelled_semantic_lock(self.msg)
async def close(self) -> None:
- pass
+ if self.scope is not None and not self._lock_closed:
+ await self.scope.close()
+ self._lock_closed = True
class SkillSemanticMessageWork(SemanticMessageWork):
diff --git a/openviking/storage/resource_diff.py b/openviking/storage/resource_diff.py
index a60b5a23f..c6a1c9149 100644
--- a/openviking/storage/resource_diff.py
+++ b/openviking/storage/resource_diff.py
@@ -27,7 +27,7 @@ import asyncio
import json
import logging
from collections import Counter
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, Mapping, Tuple
@@ -115,6 +115,8 @@ class ResourceDiffEntry:
old_kind: str | None = None
new_kind: str | None = None
md5: str | None = None
+ level_states: Mapping[int, IndexState] = field(default_factory=dict)
+ level_md5s: Mapping[int, str] = field(default_factory=dict)
@dataclass(frozen=True)
diff --git a/openviking/storage/resource_rfv.py b/openviking/storage/resource_rfv.py
new file mode 100644
index 000000000..8fcd60b08
--- /dev/null
+++ b/openviking/storage/resource_rfv.py
@@ -0,0 +1,419 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Typed R/F/V inputs for planning maintenance reindex operations."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from dataclasses import dataclass, field
+from typing import Any, Mapping
+
+from openviking.concurrency import bounded_map
+from openviking.core.namespace import context_type_for_uri
+from openviking.server.error_mapping import is_not_found_error
+from openviking.storage.abstract_overview import body_for_preview
+from openviking.storage.internal_names import is_storage_internal_name
+from openviking.storage.resource_diff import (
+ ContentState,
+ IndexState,
+ ResourceDiffEntry,
+ ResourceDiffResult,
+)
+from openviking.storage.resource_rnfv import (
+ RequestIntent,
+ VectorIndexSnapshot,
+ VectorRecordSnapshot,
+ canonical_vector_records_by_level,
+)
+from openviking.utils.content_hash import content_md5
+from openviking_cli.exceptions import NotFoundError
+
+
+@dataclass(frozen=True)
+class RFVEntry:
+ """One current formal-tree node and its per-level source fingerprints."""
+
+ uri: str
+ relative_path: str
+ is_dir: bool
+ level_md5s: Mapping[int, str] = field(default_factory=dict)
+
+ def __post_init__(self) -> None:
+ expected = {0, 1} if self.is_dir else {2}
+ invalid = set(self.level_md5s) - expected
+ if invalid:
+ raise ValueError(f"invalid RFV levels for {self.relative_path}: {sorted(invalid)}")
+
+
+@dataclass(frozen=True)
+class RFVFormalSnapshot:
+ entries: Mapping[str, RFVEntry]
+ complete: bool = True
+
+
+@dataclass(frozen=True)
+class RFVSnapshot:
+ request: RequestIntent
+ formal: RFVFormalSnapshot
+ vectors: VectorIndexSnapshot
+ source_contents: Mapping[tuple[str, int], str | bytes] = field(default_factory=dict)
+ source_raw_contents: Mapping[tuple[str, int], str | bytes] = field(default_factory=dict)
+ source_metadata: Mapping[str, Any] | None = None
+ input_only_paths: frozenset[str] = frozenset()
+
+
+def _relative_uri(root: str, uri: str) -> str | None:
+ root = root.rstrip("/")
+ if uri == root:
+ return ""
+ prefix = root + "/"
+ return uri[len(prefix) :] if uri.startswith(prefix) else None
+
+
+def _is_sidecar(relative_path: str) -> bool:
+ return relative_path.rsplit("/", 1)[-1] in {".abstract.md", ".overview.md"}
+
+
+def _is_control_file(relative_path: str) -> bool:
+ name = relative_path.rsplit("/", 1)[-1]
+ return name in {".source.json", ".image_mappings.json", ".artifact_manifest.json"}
+
+
+async def build_rfv_snapshot(
+ *,
+ viking_fs: Any,
+ vikingdb: Any,
+ target_uri: str,
+ ctx: Any,
+ request_intent: RequestIntent,
+ recursive: bool = True,
+ source_read_concurrency: int = 8,
+ root_is_dir: bool | None = None,
+ formal_inventory: tuple[bool, list[Mapping[str, Any]], bool] | None = None,
+) -> RFVSnapshot:
+ """Read one complete F/V inventory and each selected embedding source once."""
+ root = target_uri.rstrip("/")
+ projection = request_intent.required_vector_fields() | {"abstract"}
+ if target_uri.rstrip("/").startswith(("viking://user/", "viking://agent/skills/")):
+ projection = projection | {"name", "description", "tags"}
+
+ resolved_root_is_dir = root_is_dir
+ if formal_inventory is not None:
+ resolved_root_is_dir = bool(formal_inventory[0])
+ if resolved_root_is_dir is None and not recursive:
+ stat = await viking_fs.stat(root, ctx=ctx, skip_count=True)
+ resolved_root_is_dir = bool(stat.get("isDir", stat.get("is_dir")))
+ include_direct_children = bool(resolved_root_is_dir) and not recursive
+
+ async def read_formal_inventory() -> tuple[bool, list[Mapping[str, Any]], bool]:
+ if formal_inventory is not None:
+ return formal_inventory
+ is_dir = resolved_root_is_dir
+ if is_dir is None:
+ stat = await viking_fs.stat(root, ctx=ctx, skip_count=True)
+ is_dir = bool(stat.get("isDir", stat.get("is_dir")))
+ if not is_dir:
+ return is_dir, [], True
+ if not recursive:
+ entries = await viking_fs.ls(
+ root,
+ node_limit=None,
+ show_all_hidden=True,
+ ctx=ctx,
+ )
+ return is_dir, list(entries), True
+ raw_entries = await viking_fs.tree(
+ root,
+ output="original",
+ show_all_hidden=True,
+ node_limit=None,
+ level_limit=None,
+ ctx=ctx,
+ )
+ complete = not any(entry.get("access") == "denied" for entry in raw_entries)
+ return is_dir, list(raw_entries), complete
+
+ formal_task = asyncio.create_task(read_formal_inventory())
+ vector_task = asyncio.create_task(
+ vikingdb.get_incremental_inventory_under_uri(
+ root,
+ ctx=ctx,
+ output_fields=sorted(projection),
+ recursive=recursive,
+ include_direct_children=include_direct_children,
+ )
+ )
+ try:
+ (root_is_dir, raw_entries, complete), inventory = await asyncio.gather(
+ formal_task, vector_task
+ )
+ except BaseException:
+ for task in (formal_task, vector_task):
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(formal_task, vector_task, return_exceptions=True)
+ raise
+ complete = complete and not any(entry.get("access") == "denied" for entry in raw_entries)
+ if not complete:
+ raise ValueError("cannot build RFV snapshot from an incomplete formal tree")
+
+ nodes: dict[str, tuple[str, bool]] = {"": (root, root_is_dir)}
+ sidecars: dict[tuple[str, int], str] = {}
+ source_metadata_uri: str | None = None
+ for raw in raw_entries:
+ if raw.get("access") == "denied":
+ continue
+ uri = str(raw.get("uri") or "")
+ relative = _relative_uri(root, uri)
+ if relative is None:
+ complete = False
+ continue
+ relative_path = relative.strip("/")
+ if not uri or not relative_path:
+ continue
+ name = relative_path.rsplit("/", 1)[-1]
+ if _is_sidecar(relative_path):
+ parent = relative_path.rsplit("/", 1)[0] if "/" in relative_path else ""
+ level = 0 if name == ".abstract.md" else 1
+ sidecars[(parent, level)] = uri
+ continue
+ if relative_path == ".source.json":
+ source_metadata_uri = uri
+ continue
+ if is_storage_internal_name(name) or _is_control_file(relative_path):
+ continue
+ nodes[relative_path] = (uri, bool(raw.get("isDir")))
+
+ if root_is_dir and not recursive:
+ for level, name in ((0, ".abstract.md"), (1, ".overview.md")):
+ sidecars[("", level)] = f"{root}/{name}"
+
+ records_by_uri_level: dict[tuple[str, int], Mapping[str, Any]] = {}
+ for record in inventory.values():
+ uri = str(record.get("uri") or "")
+ try:
+ level = int(record.get("level", -1))
+ except (TypeError, ValueError):
+ continue
+ records_by_uri_level.setdefault((uri, level), record)
+
+ source_contents: dict[tuple[str, int], str | bytes] = {}
+ source_raw_contents: dict[tuple[str, int], str | bytes] = {}
+ level_md5s: dict[str, dict[int, str]] = {path: {} for path in nodes}
+ source_jobs: list[tuple[str, str, int, bool]] = []
+ for path, (uri, is_dir) in nodes.items():
+ if include_direct_children and path:
+ continue
+ if is_dir:
+ for level in (0, 1):
+ source_uri = sidecars.get((path, level))
+ if source_uri:
+ source_jobs.append((path, source_uri, level, True))
+ else:
+ source_jobs.append((path, uri, 2, False))
+
+ async def read_source(job: tuple[str, str, int, bool]) -> None:
+ path, source_uri, level, is_sidecar = job
+ try:
+ raw = await viking_fs.read_file_bytes(source_uri, ctx=ctx)
+ except Exception as exc:
+ if is_sidecar and (
+ isinstance(exc, (FileNotFoundError, KeyError, NotFoundError))
+ or is_not_found_error(exc)
+ ):
+ return
+ raise
+ node_uri = nodes[path][0]
+ if is_sidecar:
+ source_raw_contents[(node_uri, level)] = raw
+ body = body_for_preview(raw)
+ if not body.strip():
+ return
+ source_contents[(node_uri, level)] = body
+ source_md5 = content_md5(body.encode("utf-8"))
+ level_md5s[path][level] = source_md5
+ source = body
+ else:
+ source_md5 = content_md5(raw)
+ level_md5s[path][level] = source_md5
+ source = raw
+ record = records_by_uri_level.get((node_uri, level))
+ keep_file_source = (
+ not is_sidecar
+ and path.rsplit("/", 1)[-1] == "SKILL.md"
+ and context_type_for_uri(root) == "skill"
+ )
+ if keep_file_source or (
+ not is_sidecar
+ and (
+ request_intent.processing_mode != "vectors_only"
+ or request_intent.force
+ or record is None
+ or str(record.get("md5") or "") != source_md5
+ )
+ ):
+ source_contents[(node_uri, level)] = source
+
+ await bounded_map(
+ source_jobs,
+ read_source,
+ concurrency=max(1, source_read_concurrency),
+ )
+ source_metadata = None
+ if source_metadata_uri is not None:
+ try:
+ raw_metadata = await viking_fs.read_file_bytes(source_metadata_uri, ctx=ctx)
+ parsed_metadata = json.loads(raw_metadata)
+ except (UnicodeDecodeError, json.JSONDecodeError):
+ parsed_metadata = None
+ if isinstance(parsed_metadata, dict):
+ kind = parsed_metadata.get("kind") or parsed_metadata.get("type")
+ source_uri = parsed_metadata.get("uri") or parsed_metadata.get("source")
+ if kind and source_uri:
+ source_metadata = {
+ "kind": str(kind),
+ "uri": str(source_uri),
+ "path": str(source_uri),
+ }
+ vector_records: dict[str, VectorRecordSnapshot] = {}
+ for record_id, record in inventory.items():
+ uri = str(record.get("uri") or "")
+ relative_path = _relative_uri(root, uri)
+ if relative_path is None:
+ raise ValueError(f"RFV vector record is outside target: {uri}")
+ level = int(record.get("level", -1))
+ fields = {
+ name: record[name] for name in projection - {"id", "uri", "level"} if name in record
+ }
+ vector_records[str(record_id)] = VectorRecordSnapshot(
+ str(record_id), uri, relative_path, level, fields
+ )
+
+ formal_entries = {
+ path: RFVEntry(uri, path, is_dir, level_md5s[path]) for path, (uri, is_dir) in nodes.items()
+ }
+ return RFVSnapshot(
+ request=request_intent,
+ formal=RFVFormalSnapshot(formal_entries, complete=complete),
+ vectors=VectorIndexSnapshot(vector_records, projection, complete=True),
+ source_contents=source_contents,
+ source_raw_contents=source_raw_contents,
+ source_metadata=source_metadata,
+ input_only_paths=frozenset(
+ path for path in nodes if include_direct_children and path and "/" not in path
+ ),
+ )
+
+
+def _summarize_index_state(
+ *,
+ expected_levels: set[int],
+ level_states: Mapping[int, IndexState],
+ invalid_levels: set[int],
+) -> IndexState:
+ if invalid_levels:
+ return IndexState.LEVEL_CONFLICT
+ values = set(level_states.values())
+ if not expected_levels:
+ return IndexState.ABSENT
+ if not level_states:
+ return IndexState.MISSING
+ if values == {IndexState.ABSENT}:
+ return IndexState.ABSENT
+ if IndexState.ORPHAN in values:
+ return IndexState.ORPHAN
+ if IndexState.STALE in values:
+ return IndexState.STALE
+ if IndexState.MISSING in values:
+ return IndexState.PARTIAL if len(level_states) > 1 else IndexState.MISSING
+ return IndexState.COMPLETE
+
+
+def resolve_rfv_state(snapshot: RFVSnapshot) -> ResourceDiffResult:
+ """Resolve current formal nodes and vectors without inventing an N snapshot."""
+
+ if not snapshot.formal.complete or not snapshot.vectors.complete:
+ raise ValueError("cannot resolve an incomplete RFV snapshot")
+ required = snapshot.request.required_vector_fields()
+ if not snapshot.vectors.projected_fields.issuperset(required):
+ raise ValueError("RFV vector projection misses required fields")
+ root = snapshot.request.target_uri.rstrip("/")
+ for path, formal in snapshot.formal.entries.items():
+ expected_uri = root if not path else f"{root}/{path}"
+ if formal.relative_path != path or formal.uri.rstrip("/") != expected_uri:
+ raise ValueError(f"RFV formal entry is outside target: {formal.uri}")
+ for record in snapshot.vectors.records_by_id.values():
+ expected_uri = root if not record.relative_path else f"{root}/{record.relative_path}"
+ if record.uri.rstrip("/") != expected_uri:
+ raise ValueError(f"RFV vector record is outside target: {record.uri}")
+
+ records_by_path: dict[str, list[VectorRecordSnapshot]] = {}
+ for record in snapshot.vectors.records_by_id.values():
+ records_by_path.setdefault(record.relative_path, []).append(record)
+
+ entries: dict[str, ResourceDiffEntry] = {}
+ all_paths = set(snapshot.formal.entries) | set(records_by_path)
+ for path in sorted(all_paths):
+ formal = snapshot.formal.entries.get(path)
+ records = records_by_path.get(path, [])
+ canonical, _duplicates = canonical_vector_records_by_level(records)
+ if formal is None:
+ entries[path] = ResourceDiffEntry(
+ relative_path=path,
+ content_state=ContentState.ABSENT,
+ index_state=IndexState.ORPHAN,
+ old_kind=None,
+ new_kind=None,
+ )
+ continue
+
+ kind = "directory" if formal.is_dir else "file"
+ expected_levels = {0, 1} if formal.is_dir else {2}
+ invalid_levels = set(canonical) - expected_levels
+ level_states: dict[int, IndexState] = {}
+ for level in sorted(expected_levels):
+ source_md5 = str(formal.level_md5s.get(level) or "")
+ record = canonical.get(level)
+ if formal.is_dir and not source_md5:
+ level_states[level] = (
+ IndexState.MISSING
+ if snapshot.request.processing_mode != "vectors_only"
+ else IndexState.ABSENT
+ )
+ continue
+ if record is None:
+ level_states[level] = IndexState.MISSING
+ continue
+ stored_md5 = str(record.fields.get("md5") or "")
+ level_states[level] = (
+ IndexState.STALE
+ if snapshot.request.force or not source_md5 or stored_md5 != source_md5
+ else IndexState.COMPLETE
+ )
+
+ index_state = _summarize_index_state(
+ expected_levels=expected_levels,
+ level_states=level_states,
+ invalid_levels=invalid_levels,
+ )
+ entries[path] = ResourceDiffEntry(
+ relative_path=path,
+ content_state=ContentState.UNCHANGED,
+ index_state=index_state,
+ old_kind=kind,
+ new_kind=kind,
+ md5=(formal.level_md5s.get(2) if not formal.is_dir else None),
+ level_states=level_states,
+ level_md5s=dict(formal.level_md5s),
+ )
+ return ResourceDiffResult(entries=entries)
+
+
+__all__ = [
+ "RFVEntry",
+ "RFVFormalSnapshot",
+ "RFVSnapshot",
+ "build_rfv_snapshot",
+ "resolve_rfv_state",
+]
diff --git a/openviking/storage/resource_rnfv.py b/openviking/storage/resource_rnfv.py
index f71e95c33..4b6d60a24 100644
--- a/openviking/storage/resource_rnfv.py
+++ b/openviking/storage/resource_rnfv.py
@@ -96,6 +96,7 @@ class RequestIntent:
processing_mode: str
vectorize: bool = True
scalar_intents: tuple[ScalarIntent, ...] = ()
+ force: bool = False
@classmethod
def from_ingest_options(
@@ -105,6 +106,8 @@ class RequestIntent:
processing_mode: Any,
ingest_options: IngestOptions | Mapping[str, Any] | None,
vectorize: bool = True,
+ force: bool = False,
+ scalar_target_levels: frozenset[int] | None = None,
) -> "RequestIntent":
options = IngestOptions.from_value(ingest_options)
scalar_intents: tuple[ScalarIntent, ...] = ()
@@ -115,9 +118,8 @@ class RequestIntent:
field="search_tags",
mode=IngestOptions.vector_search_tag_mode(options.search_tag_mode),
value=tuple(options.search_tags),
- target_levels=(
- frozenset({2}) if str(mode) == "vectors_only" else frozenset({0, 1, 2})
- ),
+ target_levels=scalar_target_levels
+ or (frozenset({2}) if str(mode) == "vectors_only" else frozenset({0, 1, 2})),
),
)
else:
@@ -127,6 +129,7 @@ class RequestIntent:
processing_mode=str(mode),
vectorize=bool(vectorize),
scalar_intents=scalar_intents,
+ force=force,
)
def required_vector_fields(self) -> frozenset[str]:
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index 0dc884add..166cca13b 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -2132,8 +2132,10 @@ class VikingVectorIndexBackend:
ctx: RequestContext,
batch_size: int = 100,
output_fields: Optional[List[str]] = None,
+ recursive: bool = True,
+ include_direct_children: bool = False,
) -> Dict[str, Dict[str, Any]]:
- """Strictly load lightweight L0/L1/L2 metadata below a resource root."""
+ """Strictly load lightweight L0/L1/L2 metadata in a URI scope."""
projection = list(
dict.fromkeys(
[
@@ -2146,7 +2148,11 @@ class VikingVectorIndexBackend:
scope = And(
[
Eq("account_id", ctx.account_id),
- PathScope("uri", canonical_uri, depth=-1),
+ PathScope(
+ "uri",
+ canonical_uri,
+ depth=-1 if recursive else 1 if include_direct_children else 0,
+ ),
In("level", [0, 1, 2]),
]
)
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index 6c58194ff..cb89b045b 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -128,9 +128,7 @@ def _apply_ingest_options(
embedding_msg.context_data.get("search_tags"), incoming_tags
)
embedding_msg.context_data["search_tags"] = incoming_tags
- embedding_msg.context_data.setdefault("_upsert_options", {})["search_tag_mode"] = (
- tag_mode
- )
+ embedding_msg.context_data.setdefault("_upsert_options", {})["search_tag_mode"] = tag_mode
async def _enqueue_embedding_message(
@@ -308,6 +306,7 @@ async def _build_image_data_uri(
file_name: str,
viking_fs,
ctx: Optional[RequestContext],
+ content: Optional[bytes] = None,
) -> Optional[str]:
"""Read an image file and encode it as a base64 ``data:`` URI.
@@ -316,7 +315,8 @@ async def _build_image_data_uri(
Returns None if the image cannot be read.
"""
try:
- content = await viking_fs.read_file_bytes(file_path, ctx=ctx)
+ if content is None:
+ content = await viking_fs.read_file_bytes(file_path, ctx=ctx)
image_config = getattr(get_openviking_config(), "image", None)
return image_bytes_to_model_data_uri(content, file_name, config=image_config)
except Exception as e:
@@ -412,6 +412,7 @@ async def vectorize_directory_meta(
actions: Optional[Dict[int, IndexAction | str]] = None,
field_patches: Optional[Dict[int, FieldPatch]] = None,
telemetry_id: str | None = None,
+ md5s: Optional[Dict[int, str]] = None,
) -> set[int]:
"""
Vectorize directory metadata (.abstract.md and .overview.md).
@@ -461,6 +462,7 @@ async def vectorize_directory_meta(
account_id=ctx.account_id,
owner_space=owner_space,
meta=meta,
+ md5=(md5s or {}).get(int(ContextLevel.ABSTRACT.value)),
)
context_abstract.set_vectorize(
Vectorize(text=embedding_text_for_body(ContextLevel.ABSTRACT, uri, abstract))
@@ -522,6 +524,7 @@ async def vectorize_directory_meta(
account_id=ctx.account_id,
owner_space=owner_space,
meta=meta,
+ md5=(md5s or {}).get(int(ContextLevel.OVERVIEW.value)),
)
context_overview.set_vectorize(
Vectorize(text=embedding_text_for_body(ContextLevel.OVERVIEW, uri, overview))
@@ -590,6 +593,7 @@ async def vectorize_file(
ingest_options: IngestOptions | None = None,
file_md5: Optional[str] = None,
file_content: Optional[bytes] = None,
+ materialize_content: bool = False,
action: str = "merge",
telemetry_id: str | None = None,
) -> bool:
@@ -701,7 +705,9 @@ async def vectorize_file(
context.set_vectorize(Vectorize(text=embedding_text))
elif content_type == ResourceContentType.IMAGE:
# Multimodal embedders consume both parts; text-only embedders fall back to summary.
- image_uri = await _build_image_data_uri(file_path, file_name, viking_fs, ctx)
+ image_uri = await _build_image_data_uri(
+ file_path, file_name, viking_fs, ctx, content=file_content
+ )
if image_uri:
context.set_vectorize(Vectorize(text=summary, images=[image_uri]))
elif summary:
@@ -739,6 +745,21 @@ async def vectorize_file(
_apply_ingest_options(embedding_msg, ingest_options)
_apply_scalar_overrides(embedding_msg, scalar_override)
+ if (
+ materialize_content
+ and file_content is not None
+ and (
+ content_type is ResourceContentType.TEXT
+ or (content_type is None and is_text_file(file_name))
+ )
+ ):
+ from openviking.storage.viking_vector_index_backend import (
+ VIKINGDB_CONTENT_MAX_SIZE,
+ )
+
+ embedding_msg.context_data["_materialized_content"] = _coerce_text_file_content(
+ file_content
+ )[:VIKINGDB_CONTENT_MAX_SIZE]
_apply_planned_field_patch(embedding_msg, field_patch)
enqueued = await _enqueue_embedding_message(
embedding_queue,
diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py
index 5ce437ca4..2b3512d4b 100644
--- a/openviking/utils/resource_processor.py
+++ b/openviking/utils/resource_processor.py
@@ -38,7 +38,11 @@ from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.telemetry import get_current_telemetry
from openviking.utils import is_github_url
-from openviking.utils.embedding_utils import index_resource, vectorize_file
+from openviking.utils.embedding_utils import (
+ index_resource,
+ vectorize_directory_meta,
+ vectorize_file,
+)
from openviking.utils.git_auth import is_git_https_url
from openviking.utils.ingest_options import IngestOptions
from openviking.utils.log_correlation import log_correlation
@@ -182,26 +186,20 @@ class ResourceProcessor:
if self._summarizer is not None:
return self._summarizer
if self.vlm_resolver is None:
- raise RuntimeError(
- "ResourceProcessor requires a VLM resolver for account-owned work"
- )
+ raise RuntimeError("ResourceProcessor requires a VLM resolver for account-owned work")
return Summarizer(await self._vlm_processor_for(ctx))
def _get_vlm_processor(self) -> "VLMProcessor":
"""Return an explicitly configured standalone VLM processor."""
if self._vlm_processor is None:
- raise RuntimeError(
- "ResourceProcessor requires an explicitly configured VLMProcessor"
- )
+ raise RuntimeError("ResourceProcessor requires an explicitly configured VLMProcessor")
return self._vlm_processor
async def _vlm_processor_for(self, ctx: RequestContext) -> "VLMProcessor":
from openviking.parse.vlm import VLMProcessor
if self.vlm_resolver is None:
- raise RuntimeError(
- "ResourceProcessor requires a VLM resolver for account-owned work"
- )
+ raise RuntimeError("ResourceProcessor requires a VLM resolver for account-owned work")
return VLMProcessor(vlm=await self.vlm_resolver.get_vlm(ctx.account_id))
def _get_media_processor(self):
@@ -1257,9 +1255,7 @@ class ResourceProcessor:
if vectors_only and target_preexisting and not root_is_file:
diff = await SemanticProcessor(
vlm_resolver=self.vlm_resolver
- )._sync_topdown_recursive(
- temp_uri, root_uri, ctx=ctx, lock=resource_lock
- )
+ )._sync_topdown_recursive(temp_uri, root_uri, ctx=ctx, lock=resource_lock)
sync_deleted_files = list(getattr(diff, "deleted_files", []))
sync_deleted_dirs = list(getattr(diff, "deleted_dirs", []))
else:
@@ -1381,20 +1377,44 @@ class ResourceProcessor:
return {"kind": kind, "uri": str(path)}
async def _enqueue_index_actions(
- self, actions: Any, *, ctx: RequestContext, ingest_options: IngestOptions | None = None
- ) -> None:
+ self,
+ actions: Any,
+ *,
+ ctx: RequestContext,
+ ingest_options: IngestOptions | None = None,
+ source_contents: Optional[Dict[tuple[str, int], str | bytes]] = None,
+ source_metadata: Optional[Dict[str, Any]] = None,
+ ) -> int:
from collections import Counter
+ from openviking.core.skill_loader import SkillLoader
from openviking.storage.index_action import IndexAction
from openviking.storage.queuefs import get_queue_manager
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
from openviking.telemetry import get_current_telemetry
from openviking.utils.embedding_utils import _enqueue_embedding_message
+ if source_contents is not None:
+ missing_sources = [
+ (action.uri, action.level)
+ for action in actions
+ if action.action in {IndexAction.UPSERT, IndexAction.MERGE}
+ and (action.uri, action.level) not in source_contents
+ ]
+ if missing_sources:
+ uri, level = missing_sources[0]
+ source_name = (
+ f"directory L{level}"
+ if level in {int(ContextLevel.ABSTRACT), int(ContextLevel.OVERVIEW)}
+ else "file"
+ )
+ raise ValueError(f"{source_name} vector source is unavailable: {uri}")
+
queue_manager = get_queue_manager()
embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING, allow_create=True)
telemetry_id = get_current_telemetry().telemetry_id
action_counts = Counter(action.action.value for action in actions)
+ completed_actions = 0
delete_ids = [action.record_id for action in actions if action.action == IndexAction.DELETE]
if delete_ids:
message = EmbeddingMsg.for_delete(
@@ -1411,22 +1431,37 @@ class ResourceProcessor:
message,
failure_message="Failed to enqueue planned vector deletes",
)
+ completed_actions += len(delete_ids)
+ directory_actions: dict[str, dict[int, Any]] = {}
for action in actions:
if action.action in {IndexAction.UPSERT, IndexAction.MERGE}:
- if action.level != int(ContextLevel.DETAIL):
- raise ValueError("Direct index upsert only supports file detail records")
- await self._vectorize_resource_file(
- action.uri,
- ctx=ctx,
- file_md5=action.md5,
- ingest_options=ingest_options,
- scalar_override={
+ if action.level in {
+ int(ContextLevel.ABSTRACT),
+ int(ContextLevel.OVERVIEW),
+ }:
+ directory_actions.setdefault(action.uri, {})[action.level] = action
+ continue
+ file_kwargs = {
+ "ctx": ctx,
+ "file_md5": action.md5,
+ "ingest_options": ingest_options,
+ "scalar_override": {
**dict(action.upsert_fields),
"_record_id": action.record_id,
},
- action=action.action.value,
- field_patch=action.field_patch,
- )
+ "action": action.action.value,
+ "field_patch": action.field_patch,
+ "summary": action.summary,
+ }
+ source = (source_contents or {}).get((action.uri, action.level))
+ if source is not None:
+ if not isinstance(source, bytes):
+ raise ValueError(f"file vector source must be bytes: {action.uri}")
+ file_kwargs["file_content"] = source
+ if getattr(self.vikingdb, "uses_content_field", False):
+ file_kwargs["materialize_content"] = True
+ if await self._vectorize_resource_file(action.uri, **file_kwargs):
+ completed_actions += 1
continue
if action.action != IndexAction.UPDATE_FIELDS:
continue
@@ -1442,11 +1477,78 @@ class ResourceProcessor:
},
telemetry_id=telemetry_id,
)
- await _enqueue_embedding_message(
+ if await _enqueue_embedding_message(
embedding_queue,
message,
failure_message=f"Failed to enqueue scalar update for {action.uri}",
+ ):
+ completed_actions += 1
+ for uri, levels in directory_actions.items():
+ abstract = (source_contents or {}).get((uri, int(ContextLevel.ABSTRACT)), "")
+ overview = (source_contents or {}).get((uri, int(ContextLevel.OVERVIEW)), "")
+ if isinstance(abstract, bytes):
+ abstract = abstract.decode("utf-8")
+ if isinstance(overview, bytes):
+ overview = overview.decode("utf-8")
+ if int(ContextLevel.ABSTRACT) in levels and not isinstance(abstract, str):
+ raise ValueError(f"directory L0 vector source is unavailable: {uri}")
+ if int(ContextLevel.OVERVIEW) in levels and not isinstance(overview, str):
+ raise ValueError(f"directory L1 vector source is unavailable: {uri}")
+ meta = None
+ if context_type_for_uri(uri) == "skill" and abstract:
+ try:
+ skill = SkillLoader.parse(str(abstract))
+ except ValueError:
+ skill = None
+ if skill is not None:
+ meta = {
+ "name": skill["name"],
+ "description": skill.get("description", ""),
+ "tags": skill.get("tags", []),
+ "allowed_tools": skill.get("allowed_tools", []),
+ }
+ else:
+ try:
+ import yaml
+
+ parsed = yaml.safe_load(str(abstract))
+ except Exception:
+ parsed = None
+ if isinstance(parsed, dict):
+ meta = {
+ key: parsed.get(key, [] if key in {"tags", "allowed_tools"} else "")
+ for key in ("name", "description", "tags", "allowed_tools")
+ }
+ if meta is not None and source_metadata is not None:
+ meta["source_path"] = str(source_metadata.get("uri") or "")
+ enqueued_levels = await vectorize_directory_meta(
+ uri,
+ abstract,
+ overview,
+ context_type=context_type_for_uri(uri),
+ ctx=ctx,
+ include_abstract=int(ContextLevel.ABSTRACT) in levels,
+ include_overview=int(ContextLevel.OVERVIEW) in levels,
+ content_is_body=True,
+ actions={level: action.action.value for level, action in levels.items()},
+ scalar_overrides={
+ level: {
+ **dict(action.upsert_fields),
+ "_record_id": action.record_id,
+ }
+ for level, action in levels.items()
+ },
+ field_patches={
+ level: action.field_patch
+ for level, action in levels.items()
+ if action.field_patch is not None
+ },
+ md5s={
+ level: action.md5 for level, action in levels.items() if action.md5 is not None
+ },
+ **({"meta": meta} if meta is not None else {}),
)
+ completed_actions += len(enqueued_levels)
logger.debug(
"[DirectIndexActions] %s root=%s action_counts=%s action_count=%d",
log_correlation(),
@@ -1454,6 +1556,7 @@ class ResourceProcessor:
dict(action_counts),
len(actions),
)
+ return completed_actions
async def _delete_removed_resource_vectors(
self,
@@ -1581,14 +1684,17 @@ class ResourceProcessor:
scalar_override: Optional[Dict[str, Any]] = None,
field_patch: FieldPatch | None = None,
action: str = "merge",
- ) -> None:
+ file_content: bytes | None = None,
+ materialize_content: bool = False,
+ summary: str = "",
+ ) -> bool:
parent = VikingURI(file_uri).parent
if parent is None:
- return
+ return False
name = file_uri.rsplit("/", 1)[-1]
- await vectorize_file(
+ return await vectorize_file(
file_path=file_uri,
- summary_dict={"name": name, "summary": ""},
+ summary_dict={"name": name, "summary": summary},
parent_uri=parent.uri,
context_type=context_type_for_uri(file_uri),
ctx=ctx,
@@ -1597,6 +1703,8 @@ class ResourceProcessor:
scalar_override=scalar_override,
field_patch=field_patch,
action=action,
+ file_content=file_content,
+ materialize_content=materialize_content,
)
async def reserve_unique_candidate(
diff --git a/openviking_cli/utils/config/queue_worker_config.py b/openviking_cli/utils/config/queue_worker_config.py
index 3ebb7f01a..d7f7453e0 100644
--- a/openviking_cli/utils/config/queue_worker_config.py
+++ b/openviking_cli/utils/config/queue_worker_config.py
@@ -36,6 +36,7 @@ class QueueWorkersConfig(BaseModel):
external_parse: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)
add_resource: AddResourceQueueWorkerConfig = Field(default_factory=AddResourceQueueWorkerConfig)
+ reindex: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)
session_commit: QueueWorkerConfig = Field(
default_factory=lambda: QueueWorkerConfig(max_concurrent=8)
)
diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go
index 2e027a197..88ab56b92 100644
--- a/sdk/go/client_test.go
+++ b/sdk/go/client_test.go
@@ -366,7 +366,7 @@ func TestFindSendsImageQuery(t *testing.T) {
}
}
-func TestReindexSendsDryRun(t *testing.T) {
+func TestReindexSendsForceAndRecursive(t *testing.T) {
client, closeServer := testClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/content/reindex" {
t.Fatalf("path = %s", r.URL.Path)
@@ -378,14 +378,14 @@ func TestReindexSendsDryRun(t *testing.T) {
if got := body["uri"]; got != "viking://resources/demo" {
t.Fatalf("uri = %#v", got)
}
- if got := body["mode"]; got != "prune_orphans" {
+ if got := body["mode"]; got != "vectors_only" {
t.Fatalf("mode = %#v", got)
}
if got := body["wait"]; got != false {
t.Fatalf("wait = %#v", got)
}
- if got := body["dry_run"]; got != true {
- t.Fatalf("dry_run = %#v", got)
+ if got := body["force"]; got != true {
+ t.Fatalf("force = %#v", got)
}
if got := body["recursive"]; got != false {
t.Fatalf("recursive = %#v", got)
@@ -395,9 +395,9 @@ func TestReindexSendsDryRun(t *testing.T) {
defer closeServer()
if _, err := client.Reindex(context.Background(), "resources/demo", &ReindexOptions{
- Mode: "prune_orphans",
+ Mode: "vectors_only",
Wait: false,
- DryRun: true,
+ Force: true,
Recursive: Bool(false),
}); err != nil {
t.Fatal(err)
diff --git a/sdk/go/filesystem.go b/sdk/go/filesystem.go
index ed5ca9014..fe256b5fc 100644
--- a/sdk/go/filesystem.go
+++ b/sdk/go/filesystem.go
@@ -295,8 +295,9 @@ func (c *Client) BatchWrite(
}
// SetTags sets explicit k=v retrieval tags metadata for a file or directory.
-// Valid modes are "replace" (default) and "append"; Recursive applies the tags
-// to every file under a directory URI.
+// Valid modes are "replace" (default), "append", and "clear". An empty
+// replace request is a no-op; clear removes existing tags. Recursive applies
+// the update to every file under a directory URI.
func (c *Client) SetTags(ctx context.Context, uri string, tags []string, opts *SetTagsOptions) (map[string]any, error) {
if opts == nil {
opts = &SetTagsOptions{Mode: "replace"}
@@ -305,9 +306,8 @@ func (c *Client) SetTags(ctx context.Context, uri string, tags []string, opts *S
if mode == "" {
mode = "replace"
}
- // The server contract is tags:list[str]; a nil slice would marshal to JSON
- // null and fail validation, so normalize to an empty list. With mode
- // "replace" an empty list clears all tags.
+ // Normalize nil to an empty list. The server treats replace + [] as a no-op
+ // and clear as the explicit request to remove existing tags.
if tags == nil {
tags = []string{}
}
@@ -339,9 +339,11 @@ func (c *Client) Reindex(ctx context.Context, uri string, opts *ReindexOptions)
"uri": NormalizeURI(uri),
"mode": mode,
"wait": opts.Wait,
- "dry_run": opts.DryRun,
"recursive": boolValue(opts.Recursive, true),
}
+ if opts.Force {
+ payload["force"] = true
+ }
if opts.Tags != nil || opts.TagMode == "clear" {
if opts.Tags != nil {
payload["tags"] = opts.Tags
@@ -352,7 +354,7 @@ func (c *Client) Reindex(ctx context.Context, uri string, opts *ReindexOptions)
}
payload["tag_mode"] = tagMode
}
- if err := mergeExtraProtected(payload, opts.Extra, "tags", "tag_mode"); err != nil {
+ if err := mergeExtraProtected(payload, opts.Extra, "force", "tags", "tag_mode"); err != nil {
return nil, err
}
var result map[string]any
diff --git a/sdk/go/types.go b/sdk/go/types.go
index de26f5d93..f0847deed 100644
--- a/sdk/go/types.go
+++ b/sdk/go/types.go
@@ -268,7 +268,7 @@ type SetTagsOptions struct {
type ReindexOptions struct {
Mode string
Wait bool
- DryRun bool
+ Force bool
Recursive *bool
Tags []string
TagMode string
diff --git a/sdk/python/openviking_sdk/client.py b/sdk/python/openviking_sdk/client.py
index 2e29ea2ef..82d1a09e6 100644
--- a/sdk/python/openviking_sdk/client.py
+++ b/sdk/python/openviking_sdk/client.py
@@ -1303,20 +1303,22 @@ class AsyncHTTPClient:
async def set_tags(
self,
uri: str,
- tags: List[str],
+ tags: Optional[List[str]] = None,
mode: str = "replace",
recursive: bool = False,
options: Optional[SetTagsOptions] = None,
) -> Dict[str, Any]:
+ fixed: Dict[str, Any] = {
+ "uri": VikingURI.normalize(uri),
+ "mode": mode,
+ "recursive": recursive,
+ }
+ if tags is not None:
+ fixed["tags"] = tags
payload = self._build_options_payload(
options,
SetTagsOptions,
- fixed={
- "uri": VikingURI.normalize(uri),
- "tags": tags,
- "mode": mode,
- "recursive": recursive,
- },
+ fixed=fixed,
)
response = await self._request(
"POST",
@@ -1778,8 +1780,8 @@ class AsyncHTTPClient:
uri: str,
mode: str = "vectors_only",
wait: bool = True,
- dry_run: bool = False,
recursive: bool = True,
+ force: bool = False,
options: Optional[ReindexOptions] = None,
) -> Dict[str, Any]:
payload = self._build_options_payload(
@@ -1789,10 +1791,12 @@ class AsyncHTTPClient:
"uri": VikingURI.normalize(uri),
"mode": mode,
"wait": wait,
- "dry_run": dry_run,
+ "force": force,
"recursive": recursive,
},
)
+ if not payload.get("force"):
+ payload.pop("force", None)
response = await self._request(
"POST",
"/api/v1/content/reindex",
@@ -2077,24 +2081,16 @@ class AsyncHTTPClient:
)
return self._handle_response(response)
- def queue_status(
- self, format: Optional[Literal["table", "json"]] = None
- ) -> Dict[str, Any]:
+ def queue_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:
return run_async(self._get_queue_status(format=format))
- def vikingdb_status(
- self, format: Optional[Literal["table", "json"]] = None
- ) -> Dict[str, Any]:
+ def vikingdb_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:
return run_async(self._get_vikingdb_status(format=format))
- def models_status(
- self, format: Optional[Literal["table", "json"]] = None
- ) -> Dict[str, Any]:
+ def models_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:
return run_async(self._get_models_status(format=format))
- def get_status(
- self, format: Optional[Literal["table", "json"]] = None
- ) -> Dict[str, Any]:
+ def get_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:
return run_async(self._get_system_status(format=format))
def is_healthy(self) -> bool:
@@ -2589,7 +2585,7 @@ class SyncHTTPClient:
def set_tags(
self,
uri: str,
- tags: List[str],
+ tags: Optional[List[str]] = None,
mode: str = "replace",
recursive: bool = False,
options: Optional[SetTagsOptions] = None,
@@ -2862,20 +2858,19 @@ class SyncHTTPClient:
uri: str,
mode: str = "vectors_only",
wait: bool = True,
- dry_run: bool = False,
recursive: bool = True,
+ force: bool = False,
options: Optional[ReindexOptions] = None,
) -> Dict[str, Any]:
- return run_async(
- self._async_client.reindex(
- uri,
- mode=mode,
- wait=wait,
- dry_run=dry_run,
- recursive=recursive,
- options=options,
- )
- )
+ kwargs: Dict[str, Any] = {
+ "mode": mode,
+ "wait": wait,
+ "recursive": recursive,
+ "options": options,
+ }
+ if force:
+ kwargs["force"] = True
+ return run_async(self._async_client.reindex(uri, **kwargs))
def admin_create_account(
self,
@@ -3019,24 +3014,16 @@ class SyncHTTPClient:
) -> Dict[str, Any]:
return run_async(self._async_client.preflight_openviking_asset(name, repo_url, options))
- def queue_status(
- self, format: Optional[Literal["table", "json"]] = None
- ) -> Dict[str, Any]:
+ def queue_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:
return self._async_client.queue_status(format=format)
- def vikingdb_status(
- self, format: Optional[Literal["table", "json"]] = None
- ) -> Dict[str, Any]:
+ def vikingdb_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:
return self._async_client.vikingdb_status(format=format)
- def models_status(
- self, format: Optional[Literal["table", "json"]] = None
- ) -> Dict[str, Any]:
+ def models_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:
return self._async_client.models_status(format=format)
- def get_status(
- self, format: Optional[Literal["table", "json"]] = None
- ) -> Dict[str, Any]:
+ def get_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:
return self._async_client.get_status(format=format)
def is_healthy(self) -> bool:
diff --git a/sdk/python/openviking_sdk/options.py b/sdk/python/openviking_sdk/options.py
index 63841b525..0358bfd55 100644
--- a/sdk/python/openviking_sdk/options.py
+++ b/sdk/python/openviking_sdk/options.py
@@ -112,6 +112,7 @@ class SetTagsOptions(_ExtraOptions, total=False):
class ReindexOptions(_ExtraOptions, total=False):
+ force: bool
tags: List[str]
tag_mode: Literal["replace", "append", "clear"]
diff --git a/sdk/python/tests/test_async_client_behaviors.py b/sdk/python/tests/test_async_client_behaviors.py
index 2d11ef909..5c12856fa 100644
--- a/sdk/python/tests/test_async_client_behaviors.py
+++ b/sdk/python/tests/test_async_client_behaviors.py
@@ -310,9 +310,9 @@ async def test_async_http_client_reindex_posts_content_reindex():
result = await client.reindex(
"viking://resources/demo",
- mode="prune_orphans",
+ mode="vectors_only",
wait=False,
- dry_run=True,
+ force=True,
options=None,
)
@@ -321,9 +321,9 @@ async def test_async_http_client_reindex_posts_content_reindex():
"/api/v1/content/reindex",
json={
"uri": "viking://resources/demo",
- "mode": "prune_orphans",
+ "mode": "vectors_only",
"wait": False,
- "dry_run": True,
+ "force": True,
"recursive": True,
},
)
@@ -474,18 +474,16 @@ def test_sync_http_client_reindex_forwards_to_async_client():
) as mock_run:
result = client.reindex(
"viking://resources/demo",
- mode="prune_orphans",
+ mode="vectors_only",
wait=False,
- dry_run=True,
)
assert result == {"status": "accepted"}
assert mock_run.called
mock_reindex.assert_called_once_with(
"viking://resources/demo",
- mode="prune_orphans",
+ mode="vectors_only",
wait=False,
- dry_run=True,
recursive=True,
options=None,
)
@@ -1339,6 +1337,25 @@ async def test_grep_omits_unset_tags_and_forwards_explicit_tags():
assert fake_http.post.await_args_list[1].kwargs["json"]["tags"] == ["env=prod"]
+async def test_set_tags_clear_omits_tags_from_sdk_request():
+ client = AsyncHTTPClient(url="http://localhost:1933")
+ fake_http = SimpleNamespace(post=AsyncMock(return_value=object()))
+ client._http = fake_http
+ client._handle_response_data = lambda _response: {"result": {"tags": []}}
+
+ await client.set_tags("viking://resources/demo.md", mode="clear")
+
+ fake_http.post.assert_awaited_once_with(
+ "/api/v1/fs/attrs/set_tags",
+ json={
+ "uri": "viking://resources/demo.md",
+ "mode": "clear",
+ "recursive": False,
+ },
+ )
+
+
@pytest.mark.asyncio
async def test_glob_normalizes_scope_uri():
client = AsyncHTTPClient(url="http://localhost:1933")
diff --git a/sdk/typescript/src/client.ts b/sdk/typescript/src/client.ts
index 181fa1e0b..ea7fb1666 100644
--- a/sdk/typescript/src/client.ts
+++ b/sdk/typescript/src/client.ts
@@ -682,7 +682,7 @@ export class OpenVikingClient {
/** Set retrieval tags. */
setTags(
uri: string,
- tags: string[],
+ tags?: string[],
options: SetTagsOptions = {},
): Promise<JsonObject> {
const body = compact({
@@ -708,7 +708,7 @@ export class OpenVikingClient {
uri: normalizeURI(uri),
mode: options.mode ?? "vectors_only",
wait: options.wait ?? true,
- dry_run: options.dryRun ?? false,
+ force: options.force || undefined,
recursive: options.recursive ?? true,
tags: options.tags,
tag_mode:
@@ -717,7 +717,7 @@ export class OpenVikingClient {
: undefined,
});
return this.request("POST", "/api/v1/content/reindex", {
- body: mergeExtra(body, options.extra, ["tags", "tag_mode"]),
+ body: mergeExtra(body, options.extra, ["force", "tags", "tag_mode"]),
});
}
diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.ts
index 197d51bbe..77c5dbac2 100644
--- a/sdk/typescript/src/types.ts
+++ b/sdk/typescript/src/types.ts
@@ -131,7 +131,7 @@ export interface CompileOptions {
}
/** Retrieval tag update options. */
export interface SetTagsOptions {
- mode?: "replace" | "append";
+ mode?: "replace" | "append" | "clear";
recursive?: boolean;
telemetry?: unknown;
extra?: JsonObject;
@@ -140,7 +140,7 @@ export interface SetTagsOptions {
export interface ReindexOptions {
mode?: string;
wait?: boolean;
- dryRun?: boolean;
+ force?: boolean;
recursive?: boolean;
tags?: string[];
tagMode?: "replace" | "append" | "clear";
diff --git a/sdk/typescript/tests/client.test.ts b/sdk/typescript/tests/client.test.ts
index 93814c1ea..e11dc8a25 100644
--- a/sdk/typescript/tests/client.test.ts
+++ b/sdk/typescript/tests/client.test.ts
@@ -176,12 +176,7 @@ describe("OpenVikingClient", () => {
[{ role: "user", content: "hello" }],
telemetry,
);
- await client.commitSession(
- "session-1",
- 2,
- telemetry,
- ["team=platform"],
- );
+ await client.commitSession("session-1", 2, telemetry, ["team=platform"]);
expect(JSON.parse(String(fetcher.mock.calls[0]![1]?.body))).toEqual({
messages: [{ role: "user", content: "hello" }],
@@ -323,7 +318,7 @@ describe("OpenVikingClient", () => {
});
});
- it("sends dry_run for prune_orphans reindex requests", async () => {
+ it("sends force and recursive for reindex requests", async () => {
const fetcher = vi
.fn<typeof fetch>()
.mockResolvedValue(ok({ status: "completed" }));
@@ -333,9 +328,9 @@ describe("OpenVikingClient", () => {
});
await client.reindex("resources", {
- mode: "prune_orphans",
+ mode: "vectors_only",
wait: true,
- dryRun: true,
+ force: true,
recursive: false,
});
@@ -343,9 +338,9 @@ describe("OpenVikingClient", () => {
expect(String(url)).toBe("https://example.com/api/v1/content/reindex");
expect(JSON.parse(String(init?.body))).toEqual({
uri: "viking://resources",
- mode: "prune_orphans",
+ mode: "vectors_only",
wait: true,
- dry_run: true,
+ force: true,
recursive: false,
});
});
@@ -439,20 +434,49 @@ describe("OpenVikingClient", () => {
});
it("sends explicit tags for write, list, tree, and grep", async () => {
- const fetcher = vi.fn<typeof fetch>().mockImplementation(async () => ok({}));
- const client = new OpenVikingClient({ baseUrl: "https://example.com", fetch: fetcher });
+ const fetcher = vi
+ .fn<typeof fetch>()
+ .mockImplementation(async () => ok({}));
+ const client = new OpenVikingClient({
+ baseUrl: "https://example.com",
+ fetch: fetcher,
+ });
- await client.write("resources/demo.md", "updated", { tags: [], tagMode: "replace" });
+ await client.write("resources/demo.md", "updated", {
+ tags: [],
+ tagMode: "replace",
+ });
await client.list("resources", { tags: ["env=prod"], includeTags: true });
await client.tree("resources", { tags: ["env=prod"], includeTags: true });
- await client.grep("resources", "needle", { tags: ["env=prod"], includeTags: true });
-
- expect(JSON.parse(String(fetcher.mock.calls[0]![1]?.body))).toMatchObject({ tags: [], tag_mode: "replace" });
- expect(new URL(String(fetcher.mock.calls[1]![0])).searchParams.get("tags")).toBe("env=prod");
- expect(new URL(String(fetcher.mock.calls[1]![0])).searchParams.get("include_tags")).toBe("true");
- expect(new URL(String(fetcher.mock.calls[2]![0])).searchParams.get("tags")).toBe("env=prod");
- expect(new URL(String(fetcher.mock.calls[2]![0])).searchParams.get("include_tags")).toBe("true");
- expect(JSON.parse(String(fetcher.mock.calls[3]![1]?.body))).toMatchObject({ tags: ["env=prod"], include_tags: true });
+ await client.grep("resources", "needle", {
+ tags: ["env=prod"],
+ includeTags: true,
+ });
+
+ expect(JSON.parse(String(fetcher.mock.calls[0]![1]?.body))).toMatchObject({
+ tags: [],
+ tag_mode: "replace",
+ });
+ expect(
+ new URL(String(fetcher.mock.calls[1]![0])).searchParams.get("tags"),
+ ).toBe("env=prod");
+ expect(
+ new URL(String(fetcher.mock.calls[1]![0])).searchParams.get(
+ "include_tags",
+ ),
+ ).toBe("true");
+ expect(
+ new URL(String(fetcher.mock.calls[2]![0])).searchParams.get("tags"),
+ ).toBe("env=prod");
+ expect(
+ new URL(String(fetcher.mock.calls[2]![0])).searchParams.get(
+ "include_tags",
+ ),
+ ).toBe("true");
+ expect(JSON.parse(String(fetcher.mock.calls[3]![1]?.body))).toMatchObject({
+ tags: ["env=prod"],
+ include_tags: true,
+ });
});
it("sends clear tag mode without tags", async () => {
@@ -672,6 +696,22 @@ describe("OpenVikingClient", () => {
).toThrow("extra cannot override uri");
});
+ it("omits tags when clear is requested", async () => {
+ const fetcher = vi.fn<typeof fetch>().mockResolvedValue(ok({ updated: 1 }));
+ const client = new OpenVikingClient({
+ baseUrl: "https://example.com",
+ fetch: fetcher,
+ });
+
+ await client.setTags("resources/demo.md", undefined, { mode: "clear" });
+
+ expect(JSON.parse(String(fetcher.mock.calls[0]![1]?.body))).toEqual({
+ uri: "viking://resources/demo.md",
+ mode: "clear",
+ recursive: false,
+ });
+ });
+
it("converts an existing Node.js image path to a data URI", async () => {
const directory = await mkdtemp(join(tmpdir(), "openviking-sdk-image-"));
const path = join(directory, "photo.png");
@@ -1264,7 +1304,9 @@ describe("OpenVikingClient", () => {
});
await expect(client.getStatus()).resolves.toEqual({ is_healthy: true });
- await expect(client.queueStatus("json")).resolves.toEqual({ name: "queue" });
+ await expect(client.queueStatus("json")).resolves.toEqual({
+ name: "queue",
+ });
await expect(client.modelsStatus("table")).resolves.toEqual({
name: "models",
});
diff --git a/tests/client/test_rebuild_clients.py b/tests/client/test_rebuild_clients.py
index b2aa201a8..f0bba6969 100644
--- a/tests/client/test_rebuild_clients.py
+++ b/tests/client/test_rebuild_clients.py
@@ -130,9 +130,9 @@ async def test_async_http_client_reindex_posts_content_reindex():
) as mock_handle:
result = await client.reindex(
"viking://resources/demo",
- mode="prune_orphans",
+ mode="vectors_only",
wait=False,
- dry_run=True,
+ force=True,
)
assert result == {"status": "completed"}
@@ -140,9 +140,10 @@ async def test_async_http_client_reindex_posts_content_reindex():
"/api/v1/content/reindex",
json={
"uri": "viking://resources/demo",
- "mode": "prune_orphans",
+ "mode": "vectors_only",
"wait": False,
- "dry_run": True,
+ "force": True,
+ "recursive": True,
},
)
assert mock_handle.called
@@ -161,18 +162,20 @@ def test_sync_http_client_reindex_forwards_to_async_client():
) as mock_run:
result = client.reindex(
"viking://resources/demo",
- mode="prune_orphans",
+ mode="vectors_only",
wait=False,
- dry_run=True,
+ force=True,
)
assert result == {"status": "accepted"}
assert mock_run.called
mock_reindex.assert_called_once_with(
- uri="viking://resources/demo",
- mode="prune_orphans",
+ "viking://resources/demo",
+ mode="vectors_only",
wait=False,
- dry_run=True,
+ recursive=True,
+ force=True,
+ options=None,
)
@@ -193,7 +196,6 @@ async def test_async_http_client_reindex_sends_non_recursive_flag():
"uri": "viking://resources/demo",
"mode": "semantic_and_vectors",
"wait": True,
- "dry_run": False,
"recursive": False,
},
)
@@ -207,9 +209,7 @@ def test_sync_http_client_reindex_forwards_non_recursive_flag():
new_callable=Mock,
return_value={"status": "completed"},
) as mock_reindex:
- with patch(
- "openviking_sdk.client.run_async", return_value={"status": "completed"}
- ):
+ with patch("openviking_sdk.client.run_async", return_value={"status": "completed"}):
client.reindex(
"viking://resources/demo",
mode="semantic_and_vectors",
@@ -217,11 +217,11 @@ def test_sync_http_client_reindex_forwards_non_recursive_flag():
)
mock_reindex.assert_called_once_with(
- uri="viking://resources/demo",
+ "viking://resources/demo",
mode="semantic_and_vectors",
wait=True,
- dry_run=False,
recursive=False,
+ options=None,
)
diff --git a/tests/server/test_admin_rebuild_api.py b/tests/server/test_admin_rebuild_api.py
index ef0cc0dc7..57bece0d1 100644
--- a/tests/server/test_admin_rebuild_api.py
+++ b/tests/server/test_admin_rebuild_api.py
@@ -1,6 +1,5 @@
"""Tests for reindex admin endpoint and executor behavior."""
-import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
@@ -65,8 +64,8 @@ async def test_reindex_user_can_only_target_own_user_scope(monkeypatch):
seen = {}
class FakeService:
- async def reindex(self, *, uri, mode, wait, dry_run=False, ctx):
- seen.update(uri=uri, mode=mode, wait=wait, dry_run=dry_run, ctx=ctx)
+ async def reindex(self, *, uri, mode, wait, ctx):
+ seen.update(uri=uri, mode=mode, wait=wait, ctx=ctx)
return {"status": "completed", "uri": uri, "mode": mode}
monkeypatch.setattr("openviking.server.routers.content.get_service", lambda: FakeService())
@@ -148,17 +147,20 @@ async def test_reindex_rejects_session_uri(admin_client: httpx.AsyncClient, uri:
assert body["status"] == "error"
-async def test_reindex_rejects_reason_field(admin_client: httpx.AsyncClient):
- resp = await admin_client.post(
- "/api/v1/content/reindex",
- json={
+def test_reindex_ignores_unknown_request_fields():
+ from openviking.server.routers.content import ReindexRequest
+
+ request = ReindexRequest.model_validate(
+ {
"uri": "viking://resources/demo",
"mode": "vectors_only",
"reason": "unused",
- },
- headers=ROOT_ACCOUNT_HEADERS,
+ }
)
- assert resp.status_code == 403
+
+ assert request.uri == "viking://resources/demo"
+ assert request.mode == "vectors_only"
+ assert "reason" not in request.model_dump()
async def test_reindex_root_requires_explicit_account(admin_client: httpx.AsyncClient):
@@ -186,7 +188,6 @@ async def test_reindex_resource_vectors_only_wait_true(monkeypatch):
mode,
wait,
ctx,
- dry_run=False,
tags=None,
tag_mode="replace",
):
@@ -282,7 +283,7 @@ async def test_reindex_resource_vectors_only_wait_false(monkeypatch):
from openviking.server.routers.content import ReindexRequest, reindex
class FakeService:
- async def reindex(self, *, uri, mode, wait, ctx, dry_run=False):
+ async def reindex(self, *, uri, mode, wait, ctx):
return {
"task_id": "rbld_123",
"status": "accepted",
@@ -337,81 +338,36 @@ async def test_reindex_passes_non_recursive_to_service(monkeypatch):
@pytest.mark.asyncio
-async def test_reindex_prune_orphans_passes_dry_run_to_service(monkeypatch):
+async def test_reindex_passes_force_to_service(monkeypatch):
from openviking.server.routers.content import ReindexRequest, reindex
seen = {}
class FakeService:
- async def reindex(self, *, uri, mode, wait, ctx, dry_run=False):
- seen["uri"] = uri
- seen["mode"] = mode
- seen["wait"] = wait
- seen["dry_run"] = dry_run
- return {
- "status": "completed",
- "uri": uri,
- "object_type": "resource",
- "mode": mode,
- "scanned_records": 1,
- "deleted_records": 0,
- "would_delete_records": 1,
- "unsupported_records": 0,
- "failed_records": 0,
- "duration_ms": 1,
- "warnings": [],
- }
+ async def reindex(self, **kwargs):
+ seen.update(kwargs)
+ return {"status": "completed"}
ctx = RequestContext(
user=UserIdentifier(account_id="test", user_id="alice"),
role=Role.ROOT,
)
- request = ReindexRequest(
- uri="viking://resources/demo",
- mode="prune_orphans",
- wait=True,
- dry_run=True,
- )
-
monkeypatch.setattr("openviking.server.routers.content.get_service", lambda: FakeService())
- response = await reindex(body=request, ctx=ctx)
-
- assert response.status == "ok"
- assert response.result["would_delete_records"] == 1
- assert seen == {
- "uri": "viking://resources/demo",
- "mode": "prune_orphans",
- "wait": True,
- "dry_run": True,
- }
-
-async def test_reindex_rejects_dry_run_for_non_prune_mode(monkeypatch):
- from openviking.server.routers.content import ReindexRequest, reindex
-
- class FakeService:
- async def reindex(self, **kwargs):
- raise AssertionError("service should not be called")
-
- ctx = RequestContext(
- user=UserIdentifier(account_id="test", user_id="alice"),
- role=Role.ROOT,
- )
- request = ReindexRequest(
- uri="viking://resources/demo",
- mode="vectors_only",
- wait=True,
- dry_run=True,
+ await reindex(
+ body=ReindexRequest(
+ uri="viking://resources/demo",
+ mode="vectors_only",
+ force=True,
+ ),
+ ctx=ctx,
)
- monkeypatch.setattr("openviking.server.routers.content.get_service", lambda: FakeService())
- with pytest.raises(OpenVikingError, match="dry_run"):
- await reindex(body=request, ctx=ctx)
+ assert seen["force"] is True
@pytest.mark.asyncio
-async def test_reindex_executor_rejects_dry_run_for_non_prune_mode_direct():
+async def test_reindex_rejects_non_recursive_namespace_before_work_starts():
from openviking.service.reindex_executor import ReindexExecutor
ctx = RequestContext(
@@ -419,12 +375,12 @@ async def test_reindex_executor_rejects_dry_run_for_non_prune_mode_direct():
role=Role.ROOT,
)
- with pytest.raises(OpenVikingError, match="dry_run"):
+ with pytest.raises(OpenVikingError, match="recursive=false"):
await ReindexExecutor().execute(
- uri="viking://resources/demo",
+ uri="viking://user/alice",
mode="vectors_only",
wait=True,
- dry_run=True,
+ recursive=False,
ctx=ctx,
)
@@ -443,44 +399,6 @@ async def test_reindex_executor_discards_invalid_tags_before_creating_background
assert ingest_options.search_tags == ["team=search"]
-async def test_reindex_executor_ignores_tags_for_prune_orphans(monkeypatch):
- from openviking.service.reindex_executor import ReindexExecutor
-
- executor = ReindexExecutor()
- seen = {}
-
- async def fake_run(**kwargs):
- seen.update(kwargs)
- return {"status": "completed"}
-
- class FakeTracker:
- async def has_running(self, *args, **kwargs):
- return False
-
- monkeypatch.setattr(executor, "_run", fake_run)
- monkeypatch.setattr(
- "openviking.service.reindex_executor.get_task_tracker",
- lambda: FakeTracker(),
- )
- ctx = RequestContext(
- user=UserIdentifier(account_id="test", user_id="alice"),
- role=Role.ROOT,
- )
-
- result = await executor.execute(
- uri="viking://resources/demo",
- mode="prune_orphans",
- wait=True,
- tags=["invalid"],
- tag_mode="invalid",
- ctx=ctx,
- )
-
- assert result == {"status": "completed"}
- assert seen["ingest_options"] is None
-
-
@pytest.mark.asyncio
async def test_reindex_file_target_uses_exact_lock(monkeypatch):
from types import SimpleNamespace
@@ -510,7 +428,7 @@ async def test_reindex_file_target_uses_exact_lock(monkeypatch):
def _uri_to_path(self, uri, ctx):
return "/local/default/resources/demo.md"
- async def stat(self, uri, ctx, skip_count=None):
+ async def stat(self, uri, ctx, skip_count=True):
return {"isDir": False}
executor = ReindexExecutor()
@@ -546,188 +464,152 @@ async def test_reindex_file_target_uses_exact_lock(monkeypatch):
@pytest.mark.asyncio
-async def test_reindex_prune_existing_file_uses_exact_lock(monkeypatch):
- from types import SimpleNamespace
-
+async def test_reindex_executor_passes_tags_to_background_run(monkeypatch):
from openviking.service.reindex_executor import ReindexExecutor
+ from openviking.storage.queuefs.reindex_msg import ReindexMsg
- calls = []
-
- class FakeAGFS:
- async def pathlock_acquire_exact(self, path):
- calls.append(("exact", path))
- return {"id": "lease-1"}
-
- async def pathlock_acquire_tree(self, path):
- calls.append(("tree", path))
- raise AssertionError("existing file target must not acquire a tree lock")
-
- async def pathlock_as_borrowed(self, lease):
- return lease
-
- async def pathlock_release(self, lease):
- calls.append(("release", lease["id"]))
-
- class FakeFS:
- _async_agfs = FakeAGFS()
-
- def _uri_to_path(self, uri, ctx):
- return "/local/default/resources/demo.md"
-
- async def exists(self, uri, ctx):
- return True
-
- async def stat(self, uri, ctx, skip_count=None):
- return {"isDir": False}
-
- executor = ReindexExecutor()
-
- async def fake_prune_orphan_vectors(**kwargs):
- return None
-
- monkeypatch.setattr(executor, "_prune_orphan_vectors", fake_prune_orphan_vectors)
- monkeypatch.setattr(
- "openviking.service.reindex_executor.get_service",
- lambda: SimpleNamespace(
- viking_fs=FakeFS(),
- vikingdb_manager=SimpleNamespace(has_queue_manager=True),
- ),
- )
- ctx = RequestContext(
- user=UserIdentifier(account_id="test", user_id="alice"),
- role=Role.ROOT,
- )
+ seen = {"enqueued": []}
- result = await executor._run(
- uri="viking://resources/demo.md",
- object_type="resource",
- mode="prune_orphans",
- dry_run=True,
- ctx=ctx,
- )
-
- assert result["status"] == "completed"
- assert calls == [
- ("exact", "/local/default/resources/demo.md"),
- ("release", "lease-1"),
- ]
-
-
[email protected]("locked", [False, True])
-async def test_reindex_prune_missing_target_does_not_lock(monkeypatch, locked):
- """A missing prune target must not acquire a lock: doing so creates the directory."""
- from types import SimpleNamespace
+ class FakeTask:
+ task_id = "task-1"
- from openviking.service.reindex_executor import ReindexExecutor
- from openviking.storage.errors import ResourceBusyError
+ class FakeTracker:
+ async def create_if_no_running(self, *args, **kwargs):
+ return FakeTask()
- calls = []
+ update_stage = AsyncMock()
+ fail = AsyncMock()
class FakeAGFS:
- async def pathlock_acquire_exact(self, path):
- raise AssertionError("missing target must not acquire a lock")
-
async def pathlock_acquire_tree(self, path):
- raise AssertionError("missing target must not acquire a lock")
+ seen["path"] = path
+ return {"lease": "root"}
- async def pathlock_is_locked(self, path):
- calls.append(("is_locked", path))
- return locked
+ async def pathlock_to_handoff(self, lease):
+ assert lease == {"lease": "root"}
+ return {"handoff": "root"}
- async def pathlock_as_borrowed(self, lease):
- raise AssertionError("no lease to borrow")
+ async def pathlock_handoff(self, lease):
+ assert lease == {"lease": "root"}
async def pathlock_release(self, lease):
- raise AssertionError("no lease to release")
+ raise AssertionError(f"unexpected release: {lease}")
- class FakeFS:
+ class FakeVikingFS:
_async_agfs = FakeAGFS()
- def _uri_to_path(self, uri, ctx):
- return "/local/default/resources/missing"
+ async def stat(self, uri, *, ctx, skip_count):
+ assert uri == "viking://resources/demo"
+ assert skip_count is True
+ return {"isDir": True}
- async def exists(self, uri, ctx):
- return False
+ def _uri_to_path(self, uri, *, ctx):
+ return "/resources/demo"
- executor = ReindexExecutor()
+ class FakeQueueManager:
+ REINDEX = "Reindex"
- async def fake_prune_orphan_vectors(**kwargs):
- return None
+ async def enqueue(self, queue_name, data):
+ seen["enqueued"].append((queue_name, data))
+ return "message-1"
- monkeypatch.setattr(executor, "_prune_orphan_vectors", fake_prune_orphan_vectors)
+ executor = ReindexExecutor()
+ monkeypatch.setattr(
+ "openviking.service.reindex_executor.get_task_tracker",
+ lambda: FakeTracker(),
+ )
monkeypatch.setattr(
"openviking.service.reindex_executor.get_service",
- lambda: SimpleNamespace(
- viking_fs=FakeFS(),
- vikingdb_manager=SimpleNamespace(has_queue_manager=True),
- ),
+ lambda: SimpleNamespace(viking_fs=FakeVikingFS()),
)
+ monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: FakeQueueManager())
ctx = RequestContext(
user=UserIdentifier(account_id="test", user_id="alice"),
role=Role.ROOT,
)
- run = executor._run(
- uri="viking://resources/missing",
- object_type="resource",
- mode="prune_orphans",
- dry_run=True,
+ result = await executor.execute(
+ uri="viking://resources/demo",
+ mode="vectors_only",
+ wait=False,
+ tags=["Team=Search"],
+ tag_mode="append",
ctx=ctx,
)
- if locked:
- with pytest.raises(ResourceBusyError):
- await run
- else:
- assert (await run)["status"] == "completed"
- assert calls == [("is_locked", "/local/default/resources/missing")]
-
-
-async def test_reindex_executor_passes_tags_to_background_run(monkeypatch):
- import asyncio
+ assert result["status"] == "accepted"
+ assert seen["path"] == "/resources/demo"
+ assert len(seen["enqueued"]) == 1
+ queue_name, payload = seen["enqueued"][0]
+ assert queue_name == "Reindex"
+ message = ReindexMsg.from_dict(payload)
+ assert message.task_id == "task-1"
+ assert message.tags == ["Team=Search"]
+ assert message.tag_mode == "append"
+ assert message.lock_handoff == {"handoff": "root"}
+ assert message.actor_peer_id == ctx.actor_peer_id
+ assert message.bypass_acl is ctx.bypass_acl
+
+
[email protected]("failure", ["stat", "lock"])
+async def test_reindex_wait_false_marks_task_failed_when_submit_preparation_fails(
+ monkeypatch, failure
+):
from openviking.service.reindex_executor import ReindexExecutor
- seen = {}
-
class FakeTask:
task_id = "task-1"
- class FakeTracker:
- async def create_if_no_running(self, *args, **kwargs):
- return FakeTask()
+ tracker = SimpleNamespace(
+ create_if_no_running=AsyncMock(return_value=FakeTask()),
+ fail=AsyncMock(),
+ )
- executor = ReindexExecutor()
+ class FakeAGFS:
+ async def pathlock_acquire_tree(self, _path):
+ if failure == "lock":
+ raise RuntimeError("root lock is busy")
+ raise AssertionError("lock must not be acquired after stat failure")
- async def fake_run_tracked(task_id, **kwargs):
- seen["task_id"] = task_id
- seen.update(kwargs)
+ class FakeVikingFS:
+ _async_agfs = FakeAGFS()
+
+ async def stat(self, _uri, *, ctx, skip_count):
+ del ctx, skip_count
+ if failure == "stat":
+ raise RuntimeError("root stat is unavailable")
+ return {"isDir": True}
+
+ def _uri_to_path(self, _uri, *, ctx):
+ del ctx
+ return "/resources/demo"
- monkeypatch.setattr(executor, "_run_tracked", fake_run_tracked)
+ monkeypatch.setattr("openviking.service.reindex_executor.get_task_tracker", lambda: tracker)
monkeypatch.setattr(
- "openviking.service.reindex_executor.get_task_tracker",
- lambda: FakeTracker(),
+ "openviking.service.reindex_executor.get_service",
+ lambda: SimpleNamespace(viking_fs=FakeVikingFS()),
)
ctx = RequestContext(
user=UserIdentifier(account_id="test", user_id="alice"),
role=Role.ROOT,
)
- result = await executor.execute(
- uri="viking://resources/demo",
- mode="vectors_only",
- wait=False,
- tags=["Team=Search"],
- tag_mode="append",
- ctx=ctx,
- )
- await asyncio.sleep(0)
+ with pytest.raises(RuntimeError, match=f"root {failure}"):
+ await ReindexExecutor().execute(
+ uri="viking://resources/demo",
+ mode="vectors_only",
+ wait=False,
+ ctx=ctx,
+ )
- assert result["status"] == "accepted"
- assert seen["task_id"] == "task-1"
- assert seen["ingest_options"].search_tags == ["team=search"]
- assert seen["ingest_options"].search_tag_mode == "append"
+ tracker.fail.assert_awaited_once_with(
+ "task-1",
+ "Failed to enqueue reindex processing",
+ account_id="test",
+ user_id="alice",
+ )
@pytest.mark.asyncio
@@ -741,13 +623,13 @@ async def test_reindex_resource_passes_run_tags_to_vector_rebuild(monkeypatch):
seen = {}
- async def fake_reindex_resource_vectors(self, **kwargs):
+ async def fake_reindex_rfv(self, **kwargs):
seen.update(kwargs)
monkeypatch.setattr(
ReindexExecutor,
- "_reindex_resource_vectors",
- fake_reindex_resource_vectors,
+ "_reindex_rfv",
+ fake_reindex_rfv,
)
ctx = RequestContext(
user=UserIdentifier(account_id="test", user_id="alice"),
@@ -765,641 +647,466 @@ async def test_reindex_resource_passes_run_tags_to_vector_rebuild(monkeypatch):
),
)
- assert seen["ingest_options"] is ingest_options
-
-
-async def test_reindex_upsert_uses_uri_owner_for_user_scoped_records(monkeypatch):
- from openviking.service.reindex_executor import ReindexExecutor
-
- captured = {}
-
- class FakeVikingDB:
- async def enqueue_embedding_msg(self, msg):
- captured["msg"] = msg
- return True
-
- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()
- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)
-
- service = ReindexExecutor()
- ctx = RequestContext(
- user=UserIdentifier(account_id="acct", user_id="admin"),
- role=Role.ROOT,
- )
-
- await service._upsert_context(
- uri="viking://user/bob/memories/preferences/theme.md",
- parent_uri="viking://user/bob/memories/preferences",
- abstract="theme",
- vector_text="theme",
- is_leaf=True,
- context_type="memory",
- level=ContextLevel.DETAIL,
- ctx=ctx,
- )
-
- data = captured["msg"].context_data
- assert data["user"]["user_id"] == "bob"
- assert data["owner_user_id"] == "bob"
- assert data["owner_space"] == "bob"
-
-
-async def test_reindex_semantic_processor_uses_uri_owner_for_user_scoped_records(
- monkeypatch, semantic_config
-):
- from openviking.service.reindex_executor import ReindexExecutor
-
- captured = {}
-
- class FakeSemanticProcessor:
- def __init__(self, **_kwargs):
- pass
-
- async def on_dequeue(self, payload, lock=None):
- captured["payload"] = payload
- captured["lock"] = lock
- return ProcessResult.success()
-
- monkeypatch.setattr(
- "openviking.service.reindex_executor.SemanticProcessor",
- FakeSemanticProcessor,
- )
-
- service = ReindexExecutor(vlm_resolver=semantic_config)
- ctx = RequestContext(
- user=UserIdentifier(account_id="acct", user_id="admin"),
- role=Role.ROOT,
- )
-
- await service._run_semantic_processor(
- uri="viking://user/bob/memories/preferences",
- context_type="memory",
- ctx=ctx,
- )
-
- data = captured["payload"]["data"]
- assert '"user_id": "bob"' in data
- assert '"peer_id": "bob"' in data
- assert '"account_id": "acct"' in data
-
-
-async def test_prune_orphans_candidate_filter_includes_target_uri(monkeypatch):
- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
- from openviking.storage.expr import And, Eq, Or, PathScope
-
- target_uri = "viking://resources/demo/missing.txt"
-
- def matches_target_self(filter_expr):
- if isinstance(filter_expr, And):
- return all(matches_target_self(cond) for cond in filter_expr.conds)
- if isinstance(filter_expr, Or):
- return any(matches_target_self(cond) for cond in filter_expr.conds)
- if isinstance(filter_expr, Eq):
- return True
- if isinstance(filter_expr, PathScope):
- return filter_expr.path == target_uri and filter_expr.depth == 0
- return False
-
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return False
-
- class FakeVikingDB:
- async def filter(self, *, filter, limit, output_fields, ctx):
- if not matches_target_self(filter):
- return []
- return [
- {
- "id": "target",
- "uri": target_uri,
- "level": 2,
- "context_type": "resource",
- "account_id": "test",
- }
- ]
-
- async def delete(self, ids, *, ctx):
- raise AssertionError("no records should be deleted")
-
- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()
- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
-
- counters = _ReindexCounters()
- ctx = RequestContext(
- user=UserIdentifier(account_id="test", user_id="admin"),
- role=Role.ROOT,
- )
-
- await ReindexExecutor()._prune_orphan_vectors(
- uri=target_uri,
- object_type="resource",
- dry_run=True,
- counters=counters,
- ctx=ctx,
- )
-
- assert counters.scanned_records == 1
- assert counters.would_delete_records == 1
+ assert seen["run"].ingest_options is ingest_options
@pytest.mark.asyncio
-async def test_prune_orphans_dry_run_preserves_resource_l1_fallback_and_skips_unknown(
- monkeypatch,
-):
[email protected]("mode", ["vectors_only", "semantic_and_vectors"])
+async def test_reindex_resource_routes_both_modes_through_rfv_plan(monkeypatch, mode):
from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- if uri == "viking://resources/demo/.abstract.md":
- return True
- if uri == "viking://resources/demo/.overview.md":
- return False
- return False
-
- async def read_file(self, uri, ctx=None):
- if uri == "viking://resources/demo/.abstract.md":
- return "resource abstract"
- raise FileNotFoundError(uri)
-
- class FakeVikingDB:
- async def filter(self, *, filter, limit, output_fields, ctx):
- assert limit == 100000
- assert ctx.account_id == "test"
- return [
- {
- "id": "keep_l1",
- "uri": "viking://resources/demo",
- "level": 1,
- "context_type": "resource",
- "account_id": "test",
- },
- {
- "id": "delete_l2",
- "uri": "viking://resources/demo/missing.txt",
- "level": 2,
- "context_type": "resource",
- "account_id": "test",
- },
- {
- "id": "skip_unknown",
- "uri": "viking://resources/demo/unknown",
- "level": 99,
- "context_type": "resource",
- "account_id": "test",
- },
- ]
-
- async def delete(self, ids, *, ctx):
- raise AssertionError("dry_run should not delete")
+ seen = {}
- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()
- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
+ async def fake_reindex_rfv(self, **kwargs):
+ seen.update(kwargs)
- service = ReindexExecutor()
- counters = _ReindexCounters()
+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)
ctx = RequestContext(
- user=UserIdentifier(account_id="test", user_id="admin"),
+ user=UserIdentifier(account_id="test", user_id="alice"),
role=Role.ROOT,
)
+ run = _make_reindex_run(ctx, _ReindexCounters())
+ run.force = True
- await service._prune_orphan_vectors(
+ await ReindexExecutor()._reindex_resource(
uri="viking://resources/demo",
- object_type="resource",
- dry_run=True,
- counters=counters,
- ctx=ctx,
+ mode=mode,
+ recursive=False,
+ run=run,
)
- assert counters.scanned_records == 3
- assert counters.deleted_records == 0
- assert counters.would_delete_records == 1
- assert counters.unsupported_records == 1
- assert any("unknown" in warning for warning in counters.warnings)
+ assert seen == {
+ "uri": "viking://resources/demo",
+ "mode": mode,
+ "context_type": "resource",
+ "recursive": False,
+ "run": run,
+ }
@pytest.mark.asyncio
[email protected]("filename", ["theme.md", "report#chunk_notes.md", "report#chunk_0000"])
-async def test_prune_orphans_deletes_stale_memory_chunk_with_uri_owner_ctx(monkeypatch, filename):
[email protected]("mode", ["vectors_only", "semantic_and_vectors"])
+async def test_reindex_skill_routes_both_modes_through_rfv_plan(monkeypatch, mode):
from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
- base_uri = f"viking://user/bob/memories/preferences/{filename}"
- deleted = {}
-
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return uri == base_uri
-
- async def read_file(self, uri, ctx=None):
- assert uri == base_uri
- return "memory body"
-
- class FakeVikingDB:
- async def filter(self, *, filter, limit, output_fields, ctx):
- assert limit == 100000
- assert ctx.account_id == "acct"
- return [
- {
- "id": "keep_base",
- "uri": base_uri,
- "level": 2,
- "context_type": "memory",
- "account_id": "acct",
- "owner_user_id": "bob",
- },
- {
- "id": "keep_chunk",
- "uri": f"{base_uri}#chunk_0000",
- "level": 2,
- "context_type": "memory",
- "account_id": "acct",
- "owner_user_id": "bob",
- },
- {
- "id": "delete_chunk",
- "uri": f"{base_uri}#chunk_0001",
- "level": 2,
- "context_type": "memory",
- "account_id": "acct",
- "owner_user_id": "bob",
- },
- ]
-
- async def delete(self, ids, *, ctx):
- deleted["ids"] = ids
- deleted["ctx"] = ctx
- return len(ids)
+ seen = {}
- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()
- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
- monkeypatch.setattr(
- ReindexExecutor,
- "_chunk_memory_body",
- lambda self, uri, body: [(f"{uri}#chunk_0000", body)],
- )
+ async def fake_reindex_rfv(self, **kwargs):
+ seen.update(kwargs)
- service = ReindexExecutor()
- counters = _ReindexCounters()
+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)
ctx = RequestContext(
- user=UserIdentifier(account_id="acct", user_id="admin"),
+ user=UserIdentifier(account_id="test", user_id="alice"),
role=Role.ROOT,
)
+ run = _make_reindex_run(ctx, _ReindexCounters())
- await service._prune_orphan_vectors(
- uri="viking://user/bob/memories",
- object_type="memory",
- dry_run=False,
- counters=counters,
- ctx=ctx,
+ await ReindexExecutor()._reindex_skill(
+ uri="viking://user/alice/skills/demo",
+ mode=mode,
+ recursive=False,
+ run=run,
)
- assert deleted["ids"] == ["delete_chunk"]
- assert deleted["ctx"].user.user_id == "bob"
- assert counters.deleted_records == 1
- assert counters.would_delete_records == 0
+ assert seen == {
+ "uri": "viking://user/alice/skills/demo",
+ "mode": mode,
+ "context_type": "skill",
+ "recursive": False,
+ "run": run,
+ }
@pytest.mark.asyncio
-async def test_prune_orphans_skips_resource_l1_when_sidecar_read_fails(monkeypatch):
- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
-
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return uri in {
- "viking://resources/demo/.overview.md",
- "viking://resources/demo/.abstract.md",
- }
-
- async def read_file(self, uri, ctx=None):
- raise OSError("temporary storage failure")
-
- class FakeVikingDB:
- async def filter(self, *, filter, limit, output_fields, ctx):
- return [
- {
- "id": "keep_l1",
- "uri": "viking://resources/demo",
- "level": 1,
- "context_type": "resource",
- "account_id": "test",
- }
- ]
-
- async def delete(self, ids, *, ctx):
- raise AssertionError("read failures must not delete vectors")
-
- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()
- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
-
- counters = _ReindexCounters()
- ctx = RequestContext(
- user=UserIdentifier(account_id="test", user_id="admin"),
- role=Role.ROOT,
- )
-
- await ReindexExecutor()._prune_orphan_vectors(
- uri="viking://resources/demo",
- object_type="resource",
- dry_run=False,
- counters=counters,
- ctx=ctx,
- )
-
- assert counters.deleted_records == 0
- assert counters.failed_records == 1
- assert any("failed to read" in warning for warning in counters.warnings)
+async def test_reindex_rfv_skips_complete_file_without_enqueue_or_second_read(monkeypatch):
+ from unittest.mock import AsyncMock
-
-async def test_prune_orphans_skips_memory_chunks_when_base_read_fails(monkeypatch):
from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
+ from openviking.utils.content_hash import content_md5
- base_uri = "viking://user/bob/memories/preferences/theme.md"
-
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return uri == base_uri
+ uri = "viking://resources/demo.md"
+ body = b"same body"
- async def read_file(self, uri, ctx=None):
- raise OSError("temporary storage failure")
+ class FakeFS:
+ stat = AsyncMock(return_value={"isDir": False})
+ read_file_bytes = AsyncMock(return_value=body)
- class FakeVikingDB:
- async def filter(self, *, filter, limit, output_fields, ctx):
- return [
- {
- "id": "keep_chunk",
- "uri": f"{base_uri}#chunk_0001",
+ class FakeDB:
+ async def get_incremental_inventory_under_uri(
+ self, target, *, ctx, output_fields, recursive=True, include_direct_children=False
+ ):
+ assert target == uri
+ assert recursive is False
+ assert include_direct_children is False
+ return {
+ "file-l2": {
+ "id": "file-l2",
+ "uri": uri,
"level": 2,
- "context_type": "memory",
- "account_id": "acct",
- "owner_user_id": "bob",
+ "md5": content_md5(body),
}
- ]
-
- async def delete(self, ids, *, ctx):
- raise AssertionError("base read failures must not delete chunks")
-
- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()
- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
+ }
- counters = _ReindexCounters()
+ resource_processor = SimpleNamespace(_enqueue_index_actions=AsyncMock())
+ service = SimpleNamespace(
+ viking_fs=FakeFS(),
+ vikingdb_manager=FakeDB(),
+ _resource_processor=resource_processor,
+ )
+ monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: service)
ctx = RequestContext(
- user=UserIdentifier(account_id="acct", user_id="admin"),
+ user=UserIdentifier(account_id="test", user_id="alice"),
role=Role.ROOT,
)
+ counters = _ReindexCounters()
- await ReindexExecutor()._prune_orphan_vectors(
- uri="viking://user/bob/memories",
- object_type="memory",
- dry_run=False,
- counters=counters,
- ctx=ctx,
+ await ReindexExecutor()._reindex_rfv(
+ uri=uri,
+ mode="vectors_only",
+ context_type="resource",
+ recursive=False,
+ run=_make_reindex_run(ctx, counters),
)
- assert counters.deleted_records == 0
- assert counters.failed_records == 1
- assert any("failed to read" in warning for warning in counters.warnings)
+ resource_processor._enqueue_index_actions.assert_not_awaited()
+ service.viking_fs.read_file_bytes.assert_awaited_once_with(uri, ctx=ctx)
+ assert counters.scanned_records == 1
+ assert counters.rebuilt_records == 0
@pytest.mark.asyncio
-async def test_prune_orphans_deletes_missing_file_with_string_level(monkeypatch):
- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
+async def test_reindex_rfv_deletes_orphans_synchronously_with_owner_context(monkeypatch):
+ from unittest.mock import AsyncMock
- deleted = {}
+ from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return False
+ root = "viking://user/bob/resources/demo"
- class FakeVikingDB:
- async def filter(self, *, filter, limit, output_fields, ctx):
- return [
- {
- "id": "delete_l2",
- "uri": "viking://resources/demo/missing.txt",
- "level": "2",
- "context_type": "resource",
- "account_id": "test",
- }
- ]
+ class FakeFS:
+ stat = AsyncMock(return_value={"isDir": True})
+ tree = AsyncMock(return_value=[])
- async def delete(self, ids, *, ctx):
- deleted["ids"] = ids
- return len(ids)
+ class FakeDB:
+ strict_delete = AsyncMock(return_value=1)
- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()
- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
+ async def get_incremental_inventory_under_uri(
+ self, target, *, ctx, output_fields, recursive=True, include_direct_children=False
+ ):
+ assert include_direct_children is False
+ return {
+ "orphan-l2": {
+ "id": "orphan-l2",
+ "uri": f"{root}/old.md",
+ "level": 2,
+ "md5": "old",
+ }
+ }
- counters = _ReindexCounters()
- ctx = RequestContext(
+ db = FakeDB()
+ service = SimpleNamespace(viking_fs=FakeFS(), vikingdb_manager=db)
+ monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: service)
+ admin_ctx = RequestContext(
user=UserIdentifier(account_id="test", user_id="admin"),
role=Role.ROOT,
)
+ counters = _ReindexCounters()
- await ReindexExecutor()._prune_orphan_vectors(
- uri="viking://resources/demo",
- object_type="resource",
- dry_run=False,
- counters=counters,
- ctx=ctx,
+ await ReindexExecutor()._reindex_rfv(
+ uri=root,
+ mode="vectors_only",
+ context_type="resource",
+ recursive=True,
+ run=_make_reindex_run(admin_ctx, counters),
)
- assert deleted["ids"] == ["delete_l2"]
+ db.strict_delete.assert_awaited_once()
+ assert db.strict_delete.await_args.kwargs["ctx"].user.user_id == "bob"
assert counters.deleted_records == 1
- assert counters.unsupported_records == 0
@pytest.mark.asyncio
-async def test_prune_orphans_continues_after_delete_group_failure(monkeypatch):
+async def test_reindex_rfv_enqueues_semantic_plan_without_snapshot_source_bytes(monkeypatch):
+ import json
+
+ import openviking.service.reindex_executor as reindex_mod
from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
+ from openviking.storage.context_update_plan import (
+ ContextUpdatePlan,
+ SemanticPlan,
+ SemanticTreeEntry,
+ SemanticTreeSnapshot,
+ )
+
+ root = "viking://resources/demo"
+ lease = {"id": "reindex-lease"}
+ plan = SemanticPlan(
+ root,
+ "resource",
+ SemanticTreeSnapshot((SemanticTreeEntry("", "directory", "unchanged", "aggregate"),)),
+ )
+ snapshot = SimpleNamespace(
+ formal=SimpleNamespace(entries={"": object()}),
+ source_contents={(root, 2): b"reindex source must stay in memory"},
+ source_raw_contents={(root, 2): b"raw source must stay in memory"},
+ source_metadata={"uri": "s3://source"},
+ )
+ handoff = {"owner_id": "semantic-worker", "covered_paths": []}
+ calls = []
- deleted = {}
- filter_calls = 0
+ class PathLock:
+ async def pathlock_to_handoff(self, current):
+ assert current == lease
+ calls.append("to_handoff")
+ return handoff
+
+ async def pathlock_handoff(self, current):
+ assert current == lease
+ calls.append("handoff")
+
+ class Queue:
+ async def enqueue(self, msg):
+ calls.append("enqueue")
+ self.message = msg
+ return "queued"
+
+ queue = Queue()
+ queue_manager = SimpleNamespace(SEMANTIC="semantic", get_queue=lambda *args, **kwargs: queue)
+ service = SimpleNamespace(
+ viking_fs=SimpleNamespace(_async_agfs=PathLock()),
+ vikingdb_manager=SimpleNamespace(uses_content_field=False),
+ )
+ monkeypatch.setattr(reindex_mod, "get_service", lambda: service)
+ monkeypatch.setattr(
+ "openviking.storage.resource_rfv.build_rfv_snapshot",
+ AsyncMock(return_value=snapshot),
+ )
+ monkeypatch.setattr(
+ "openviking.storage.context_update_plan.build_rfv_context_update_plan",
+ lambda **kwargs: (None, ContextUpdatePlan(root, "resource", semantic_plan=plan)),
+ )
+ monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: queue_manager)
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return False
+ ctx = RequestContext(
+ user=UserIdentifier(account_id="test", user_id="alice"),
+ role=Role.ROOT,
+ )
+ run = _make_reindex_run(ctx, _ReindexCounters())
+ run.lease = lease
- class FakeVikingDB:
- async def filter(self, *, filter, limit, output_fields, ctx):
- nonlocal filter_calls
- filter_calls += 1
- if filter_calls > 1:
- return []
- return [
- {
- "id": "fail_delete",
- "uri": "viking://user/alice/docs/missing.txt",
- "level": 2,
- "context_type": "resource",
- "account_id": "acct",
- "owner_user_id": "alice",
- },
- {
- "id": "ok_delete",
- "uri": "viking://user/bob/docs/missing.txt",
- "level": 2,
- "context_type": "resource",
- "account_id": "acct",
- "owner_user_id": "bob",
- },
- ]
+ await ReindexExecutor()._reindex_rfv(
+ uri=root,
+ mode="semantic_and_vectors",
+ context_type="resource",
+ recursive=False,
+ run=run,
+ )
- async def delete(self, ids, *, ctx):
- if ctx.user.user_id == "alice":
- raise RuntimeError("delete backend unavailable")
- deleted["ids"] = ids
- deleted["ctx"] = ctx
- return len(ids)
+ assert calls == ["to_handoff", "handoff", "enqueue"]
+ assert run.lease_handed_off is True
+ assert queue.message.plan == plan
+ assert queue.message.propagate_to_parent is False
+ assert queue.message.generation_trigger == "reindex"
+ assert queue.message.lock_handoff == handoff
+ assert b"reindex source must stay in memory" not in json.dumps(queue.message.to_dict()).encode()
- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()
- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
- counters = _ReindexCounters()
+async def test_single_file_vectors_only_reindex_reads_parent_overview_only_for_rebuild(monkeypatch):
+ import openviking.service.reindex_executor as reindex_mod
+ from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot
+ from openviking.storage.resource_rnfv import RequestIntent, VectorIndexSnapshot
+
+ file_uri = "viking://resources/demo/a.md"
+ parent_uri = "viking://resources/demo"
+ snapshot = RFVSnapshot(
+ request=RequestIntent(file_uri, "vectors_only", force=True),
+ formal=RFVFormalSnapshot({"": RFVEntry(file_uri, "", False, {2: "a-md5"})}),
+ vectors=VectorIndexSnapshot({}, frozenset({"id", "uri", "level", "md5", "abstract"})),
+ source_contents={(file_uri, 2): b"current body"},
+ )
+ fs = SimpleNamespace(
+ read_file_bytes=AsyncMock(return_value=b"### a.md\nOverview summary."),
+ )
+ service = SimpleNamespace(
+ viking_fs=fs,
+ vikingdb_manager=SimpleNamespace(uses_content_field=False),
+ _resource_processor=None,
+ )
+ enqueued = AsyncMock(return_value=1)
+ monkeypatch.setattr(reindex_mod, "get_service", lambda: service)
+ monkeypatch.setattr(
+ "openviking.storage.resource_rfv.build_rfv_snapshot",
+ AsyncMock(return_value=snapshot),
+ )
+ monkeypatch.setattr(
+ "openviking.utils.resource_processor.ResourceProcessor._enqueue_index_actions",
+ enqueued,
+ )
ctx = RequestContext(
- user=UserIdentifier(account_id="acct", user_id="admin"),
+ user=UserIdentifier(account_id="test", user_id="alice"),
role=Role.ROOT,
)
+ run = _make_reindex_run(ctx, _ReindexCounters())
+ run.root_is_dir = False
- await ReindexExecutor()._prune_orphan_vectors(
- uri="viking://user",
- object_type="user_namespace",
- dry_run=False,
- counters=counters,
- ctx=ctx,
+ await ReindexExecutor()._reindex_rfv(
+ uri=file_uri,
+ mode="vectors_only",
+ context_type="resource",
+ recursive=False,
+ run=run,
)
- assert deleted["ids"] == ["ok_delete"]
- assert deleted["ctx"].user.user_id == "bob"
- assert counters.deleted_records == 1
- assert counters.failed_records == 1
- assert any("Failed to delete" in warning for warning in counters.warnings)
+ fs.read_file_bytes.assert_awaited_once_with(f"{parent_uri}/.overview.md", ctx=ctx)
+ actions = enqueued.await_args.args[0]
+ assert len(actions) == 1
+ assert actions[0].summary == "Overview summary."
@pytest.mark.asyncio
-async def test_prune_orphans_pages_past_candidate_limit(monkeypatch):
- from openviking.service import reindex_executor
+async def test_single_file_vectors_only_noop_does_not_read_parent_overview(monkeypatch):
+ import openviking.service.reindex_executor as reindex_mod
from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot
+ from openviking.storage.resource_rnfv import (
+ RequestIntent,
+ VectorIndexSnapshot,
+ VectorRecordSnapshot,
+ )
+
+ file_uri = "viking://resources/demo/a.md"
+ snapshot = RFVSnapshot(
+ request=RequestIntent(file_uri, "vectors_only"),
+ formal=RFVFormalSnapshot({"": RFVEntry(file_uri, "", False, {2: "a-md5"})}),
+ vectors=VectorIndexSnapshot(
+ {
+ "a-l2": VectorRecordSnapshot(
+ "a-l2", file_uri, "", 2, {"md5": "a-md5", "abstract": "summary"}
+ )
+ },
+ frozenset({"id", "uri", "level", "md5", "abstract"}),
+ ),
+ )
+ fs = SimpleNamespace(read_file_bytes=AsyncMock())
+ service = SimpleNamespace(
+ viking_fs=fs,
+ vikingdb_manager=SimpleNamespace(uses_content_field=False),
+ _resource_processor=None,
+ )
+ monkeypatch.setattr(reindex_mod, "get_service", lambda: service)
+ monkeypatch.setattr(
+ "openviking.storage.resource_rfv.build_rfv_snapshot",
+ AsyncMock(return_value=snapshot),
+ )
+ ctx = RequestContext(
+ user=UserIdentifier(account_id="test", user_id="alice"),
+ role=Role.ROOT,
+ )
+ run = _make_reindex_run(ctx, _ReindexCounters())
+ run.root_is_dir = False
- filter_offsets = []
- deleted = {}
+ await ReindexExecutor()._reindex_rfv(
+ uri=file_uri,
+ mode="vectors_only",
+ context_type="resource",
+ recursive=False,
+ run=run,
+ )
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return not uri.endswith("missing.txt")
+ fs.read_file_bytes.assert_not_awaited()
- class FakeVikingDB:
- async def filter(self, *, filter, limit, output_fields, ctx, offset=0):
- filter_offsets.append(offset)
- records = [
- {
- "id": "keep_1",
- "uri": "viking://resources/demo/keep-1.txt",
- "level": 2,
- "context_type": "resource",
- "account_id": "acct",
- },
- {
- "id": "keep_2",
- "uri": "viking://resources/demo/keep-2.txt",
- "level": 2,
- "context_type": "resource",
- "account_id": "acct",
- },
- {
- "id": "delete_3",
- "uri": "viking://resources/demo/missing.txt",
- "level": 2,
- "context_type": "resource",
- "account_id": "acct",
- },
- ]
- return records[offset : offset + limit]
- async def delete(self, ids, *, ctx):
- deleted["ids"] = ids
- return len(ids)
+def test_reindex_merges_persisted_semantic_plan_stats_into_response_counters():
+ from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
+ from openviking.storage.queuefs.semantic_executor import SemanticTreeStats
+
+ counters = _ReindexCounters(rebuilt_records=2)
+ ReindexExecutor._apply_semantic_tree_stats(
+ counters,
+ SemanticTreeStats(indexed_records=3, failures=["summary failed"]),
+ )
+
+ assert counters.rebuilt_records == 5
+ assert counters.failed_records == 1
+ assert counters.warnings == ["summary failed"]
+
+
+async def test_reindex_upsert_uses_uri_owner_for_user_scoped_records(monkeypatch):
+ from openviking.service.reindex_executor import ReindexExecutor
+
+ captured = {}
+
+ class FakeVikingDB:
+ async def enqueue_embedding_msg(self, msg):
+ captured["msg"] = msg
+ return True
fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()
monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
- monkeypatch.setattr(reindex_executor, "PRUNE_ORPHAN_CANDIDATE_LIMIT", 2)
- counters = _ReindexCounters()
+ service = ReindexExecutor()
ctx = RequestContext(
user=UserIdentifier(account_id="acct", user_id="admin"),
role=Role.ROOT,
)
- await ReindexExecutor()._prune_orphan_vectors(
- uri="viking://resources/demo",
- object_type="resource",
- dry_run=False,
- counters=counters,
+ await service._upsert_context(
+ uri="viking://user/bob/memories/preferences/theme.md",
+ parent_uri="viking://user/bob/memories/preferences",
+ abstract="theme",
+ vector_text="theme",
+ is_leaf=True,
+ context_type="memory",
+ level=ContextLevel.DETAIL,
ctx=ctx,
)
- assert filter_offsets == [0, 2]
- assert counters.scanned_records == 3
- assert deleted["ids"] == ["delete_3"]
- assert counters.deleted_records == 1
+ data = captured["msg"].context_data
+ assert data["user"]["user_id"] == "bob"
+ assert data["owner_user_id"] == "bob"
+ assert data["owner_space"] == "bob"
@pytest.mark.asyncio
-async def test_prune_orphans_reports_zero_delete_count_as_failure(monkeypatch):
- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
+async def test_reindex_semantic_processor_uses_uri_owner_for_user_scoped_records(
+ monkeypatch, semantic_config
+):
+ from openviking.service.reindex_executor import ReindexExecutor
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return False
+ captured = {}
- class FakeVikingDB:
- async def filter(self, *, filter, limit, output_fields, ctx, offset=0):
- if offset:
- return []
- return [
- {
- "id": "stale",
- "uri": "viking://resources/demo/missing.txt",
- "level": 2,
- "context_type": "resource",
- "account_id": "acct",
- }
- ]
+ class FakeSemanticProcessor:
+ def __init__(self, **_kwargs):
+ pass
- async def delete(self, ids, *, ctx):
- return 0
+ async def on_dequeue(self, payload, lock=None):
+ captured["payload"] = payload
+ captured["lock"] = lock
+ return ProcessResult.success()
- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()
- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
+ monkeypatch.setattr(
+ "openviking.service.reindex_executor.SemanticProcessor",
+ FakeSemanticProcessor,
+ )
- counters = _ReindexCounters()
+ service = ReindexExecutor(vlm_resolver=semantic_config)
ctx = RequestContext(
user=UserIdentifier(account_id="acct", user_id="admin"),
role=Role.ROOT,
)
- await ReindexExecutor()._prune_orphan_vectors(
- uri="viking://resources/demo",
- object_type="resource",
- dry_run=False,
- counters=counters,
+ await service._run_semantic_processor(
+ uri="viking://user/bob/memories/preferences",
+ context_type="memory",
ctx=ctx,
)
- assert counters.deleted_records == 0
- assert counters.failed_records == 1
- assert any("Only deleted 0 of 1" in warning for warning in counters.warnings)
+ data = captured["payload"]["data"]
+ assert '"user_id": "bob"' in data
+ assert '"peer_id": "bob"' in data
+ assert '"account_id": "acct"' in data
@pytest.mark.asyncio
@@ -1433,9 +1140,7 @@ async def test_reindex_semantic_processor_uses_configured_vlm_concurrency(
user=UserIdentifier(account_id="test", user_id="alice"),
role=Role.ROOT,
)
- await reindex_mod.ReindexExecutor(
- vlm_resolver=semantic_config
- )._run_semantic_processor(
+ await reindex_mod.ReindexExecutor(vlm_resolver=semantic_config)._run_semantic_processor(
uri="viking://resources/demo",
context_type="resource",
ctx=ctx,
@@ -1457,7 +1162,7 @@ async def test_reindex_memory_semantic_and_vectors_rebuilds_full_subtree(monkeyp
seen["vectors"].append(uri)
class FakeVikingFS:
- async def stat(self, uri, ctx=None, skip_count=None):
+ async def stat(self, uri, ctx=None, skip_count=True):
return {"isDir": True}
monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)
@@ -1521,14 +1226,10 @@ async def test_reindex_resource_non_recursive_limits_semantics_and_vectors(monke
seen = {}
- async def fake_run_semantic_processor(self, **kwargs):
- seen["semantic"] = kwargs
-
- async def fake_reindex_resource_vectors(self, **kwargs):
- seen["vectors"] = kwargs
+ async def fake_reindex_rfv(self, **kwargs):
+ seen.update(kwargs)
- monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)
- monkeypatch.setattr(ReindexExecutor, "_reindex_resource_vectors", fake_reindex_resource_vectors)
+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)
ctx = RequestContext(
user=UserIdentifier(account_id="test", user_id="alice"),
@@ -1541,8 +1242,8 @@ async def test_reindex_resource_non_recursive_limits_semantics_and_vectors(monke
run=_make_reindex_run(ctx, _ReindexCounters()),
)
- assert seen["semantic"]["recursive"] is False
- assert seen["vectors"]["recursive"] is False
+ assert seen["recursive"] is False
+ assert seen["mode"] == "semantic_and_vectors"
@pytest.mark.asyncio
@@ -1558,7 +1259,7 @@ async def test_reindex_memory_non_recursive_limits_semantics_and_vectors(monkeyp
seen["vectors"] = kwargs
class FakeVikingFS:
- async def stat(self, uri, ctx=None, skip_count=None):
+ async def stat(self, uri, ctx=None, skip_count=True):
return {"isDir": True}
monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)
@@ -1586,16 +1287,10 @@ async def test_reindex_skill_non_recursive_limits_vectors_to_directory(monkeypat
seen = {}
- async def fake_regenerate_skill_semantics(self, **kwargs):
- seen["semantic"] = kwargs
-
- async def fake_reindex_skill_vectors(self, **kwargs):
- seen["vectors"] = kwargs
+ async def fake_reindex_rfv(self, **kwargs):
+ seen.update(kwargs)
- monkeypatch.setattr(
- ReindexExecutor, "_regenerate_skill_semantics", fake_regenerate_skill_semantics
- )
- monkeypatch.setattr(ReindexExecutor, "_reindex_skill_vectors", fake_reindex_skill_vectors)
+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)
ctx = RequestContext(
user=UserIdentifier(account_id="test", user_id="alice"),
@@ -1608,20 +1303,20 @@ async def test_reindex_skill_non_recursive_limits_vectors_to_directory(monkeypat
run=_make_reindex_run(ctx, _ReindexCounters()),
)
- assert seen["semantic"]["uri"] == "viking://user/alice/skills/demo"
- assert seen["vectors"]["recursive"] is False
+ assert seen["uri"] == "viking://user/alice/skills/demo"
+ assert seen["recursive"] is False
@pytest.mark.asyncio
-async def test_reindex_skill_vectors_only_ignores_non_recursive_flag(monkeypatch):
+async def test_reindex_skill_vectors_only_honors_non_recursive_flag(monkeypatch):
from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
seen = {}
- async def fake_reindex_skill_vectors(self, **kwargs):
+ async def fake_reindex_rfv(self, **kwargs):
seen.update(kwargs)
- monkeypatch.setattr(ReindexExecutor, "_reindex_skill_vectors", fake_reindex_skill_vectors)
+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)
ctx = RequestContext(
user=UserIdentifier(account_id="test", user_id="alice"),
@@ -1634,49 +1329,37 @@ async def test_reindex_skill_vectors_only_ignores_non_recursive_flag(monkeypatch
run=_make_reindex_run(ctx, _ReindexCounters()),
)
- assert "recursive" not in seen
+ assert seen["recursive"] is False
@pytest.mark.asyncio
-async def test_reindex_resource_vectors_non_recursive_skips_tree(monkeypatch):
+async def test_reindex_resource_vectors_only_honors_non_recursive(monkeypatch):
from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
seen = {}
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return True
-
- async def stat(self, uri, ctx=None, skip_count=True):
- return {"isDir": True}
-
- async def fail_tree_all(*args, **kwargs):
- raise AssertionError("non-recursive vector rebuild must not walk descendants")
-
- async def fake_reindex_from_entries(self, **kwargs):
+ async def fake_reindex_rfv(self, **kwargs):
seen.update(kwargs)
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
- monkeypatch.setattr(ReindexExecutor, "_tree_all", fail_tree_all)
monkeypatch.setattr(
ReindexExecutor,
- "_reindex_resource_vectors_from_entries",
- fake_reindex_from_entries,
+ "_reindex_rfv",
+ fake_reindex_rfv,
)
-
ctx = RequestContext(
user=UserIdentifier(account_id="test", user_id="alice"),
role=Role.ROOT,
)
- await ReindexExecutor()._reindex_resource_vectors(
+
+ await ReindexExecutor()._reindex_resource(
uri="viking://resources/demo",
- counters=_ReindexCounters(),
- ctx=ctx,
+ mode="vectors_only",
recursive=False,
+ run=_make_reindex_run(ctx, _ReindexCounters()),
)
- assert seen["directories"] == ["viking://resources/demo"]
- assert seen["files"] == []
+ assert seen["recursive"] is False
@pytest.mark.asyncio
@@ -1777,27 +1460,17 @@ async def test_reindex_user_namespace_semantic_and_vectors_promotes_memory_mode(
{"uri": "viking://user/default/resources", "isDir": True},
]
- seen = {"memory_modes": [], "semantic_calls": [], "resource_calls": []}
+ seen = {"memory_modes": [], "rfv_calls": []}
async def fake_reindex_memory(self, *, uri, mode, run):
seen["memory_modes"].append((uri, mode))
- async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):
- seen["semantic_calls"].append((uri, context_type))
-
- async def fake_reindex_resource_vectors_from_entries(
- self, *, root_uri, directories, files, counters, ctx
- ):
- seen["resource_calls"].append((root_uri, list(directories), list(files)))
+ async def fake_reindex_rfv(self, **kwargs):
+ seen["rfv_calls"].append(kwargs)
monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
monkeypatch.setattr(ReindexExecutor, "_reindex_memory", fake_reindex_memory)
- monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)
- monkeypatch.setattr(
- ReindexExecutor,
- "_reindex_resource_vectors_from_entries",
- fake_reindex_resource_vectors_from_entries,
- )
+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)
service = ReindexExecutor()
counters = _ReindexCounters()
@@ -1813,8 +1486,7 @@ async def test_reindex_user_namespace_semantic_and_vectors_promotes_memory_mode(
)
assert seen["memory_modes"] == [("viking://user/default/memories", "semantic_and_vectors")]
- assert seen["semantic_calls"] == [("viking://user/default/resources", "resource")]
- assert seen["resource_calls"]
+ assert [call["uri"] for call in seen["rfv_calls"]] == ["viking://user/default/resources"]
@pytest.mark.asyncio
@@ -1839,27 +1511,17 @@ async def test_reindex_user_namespace_semantic_and_vectors_does_not_reprocess_me
{"uri": "viking://user/default/resources", "isDir": True},
]
- seen = {"memory_modes": [], "semantic_calls": []}
+ seen = {"memory_modes": [], "rfv_calls": []}
async def fake_reindex_memory(self, *, uri, mode, run):
seen["memory_modes"].append((uri, mode))
- async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):
- seen["semantic_calls"].append((uri, context_type))
-
- async def fake_reindex_resource_vectors_from_entries(
- self, *, root_uri, directories, files, counters, ctx
- ):
- return None
+ async def fake_reindex_rfv(self, **kwargs):
+ seen["rfv_calls"].append(kwargs)
monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
monkeypatch.setattr(ReindexExecutor, "_reindex_memory", fake_reindex_memory)
- monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)
- monkeypatch.setattr(
- ReindexExecutor,
- "_reindex_resource_vectors_from_entries",
- fake_reindex_resource_vectors_from_entries,
- )
+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)
service = ReindexExecutor()
counters = _ReindexCounters()
@@ -1875,7 +1537,7 @@ async def test_reindex_user_namespace_semantic_and_vectors_does_not_reprocess_me
)
assert seen["memory_modes"] == [("viking://user/default/memories", "semantic_and_vectors")]
- assert seen["semantic_calls"] == [("viking://user/default/resources", "resource")]
+ assert [call["uri"] for call in seen["rfv_calls"]] == ["viking://user/default/resources"]
@pytest.mark.asyncio
@@ -1900,23 +1562,13 @@ async def test_reindex_user_namespace_semantic_and_vectors_skips_uncovered_root_
{"uri": "viking://user/default/profile.md", "isDir": False},
]
- seen = {"semantic_calls": [], "resource_files": []}
-
- async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):
- seen["semantic_calls"].append((uri, context_type))
+ seen = {"rfv_calls": []}
- async def fake_reindex_resource_vectors_from_entries(
- self, *, root_uri, directories, files, counters, ctx
- ):
- seen["resource_files"] = list(files)
+ async def fake_reindex_rfv(self, **kwargs):
+ seen["rfv_calls"].append(kwargs)
monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
- monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)
- monkeypatch.setattr(
- ReindexExecutor,
- "_reindex_resource_vectors_from_entries",
- fake_reindex_resource_vectors_from_entries,
- )
+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)
service = ReindexExecutor()
counters = _ReindexCounters()
@@ -1931,8 +1583,7 @@ async def test_reindex_user_namespace_semantic_and_vectors_skips_uncovered_root_
run=_make_reindex_run(ctx, counters),
)
- assert seen["semantic_calls"] == [("viking://user/default/resources", "resource")]
- assert seen["resource_files"] == ["viking://user/default/resources/doc.md"]
+ assert [call["uri"] for call in seen["rfv_calls"]] == ["viking://user/default/resources"]
assert counters.unsupported_records == 1
assert "viking://user/default/profile.md" in counters.warnings[0]
@@ -1961,11 +1612,11 @@ async def test_reindex_skill_namespace_reindexes_only_skill_roots(monkeypatch):
seen = []
- async def fake_reindex_skill(self, *, uri, mode, run):
- seen.append((uri, mode))
+ async def fake_reindex_rfv(self, **kwargs):
+ seen.append(kwargs)
monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
- monkeypatch.setattr(ReindexExecutor, "_reindex_skill", fake_reindex_skill)
+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)
service = ReindexExecutor()
counters = _ReindexCounters()
@@ -1980,7 +1631,13 @@ async def test_reindex_skill_namespace_reindexes_only_skill_roots(monkeypatch):
run=_make_reindex_run(ctx, counters),
)
- assert seen == [("viking://user/default/skills/my_skill", "semantic_and_vectors")]
+ assert [(call["uri"], call["context_type"]) for call in seen] == [
+ ("viking://user/default/skills/my_skill", "skill")
+ ]
+ assert {entry["uri"] for entry in seen[0]["formal_entries"]} == {
+ "viking://user/default/skills/my_skill/assets",
+ "viking://user/default/skills/my_skill/SKILL.md",
+ }
@pytest.mark.asyncio
@@ -2005,27 +1662,17 @@ async def test_reindex_global_namespace_semantic_and_vectors_propagates_to_child
{"uri": "viking://resources", "isDir": True},
]
- seen = {"user_modes": [], "semantic_calls": [], "resource_calls": []}
+ seen = {"user_modes": [], "rfv_calls": []}
async def fake_reindex_user_namespace(self, *, uri, mode, run):
seen["user_modes"].append((uri, mode))
- async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):
- seen["semantic_calls"].append((uri, context_type))
-
- async def fake_reindex_resource_vectors_from_entries(
- self, *, root_uri, directories, files, counters, ctx
- ):
- seen["resource_calls"].append((root_uri, list(directories), list(files)))
+ async def fake_reindex_rfv(self, **kwargs):
+ seen["rfv_calls"].append(kwargs)
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
- monkeypatch.setattr(ReindexExecutor, "_reindex_user_namespace", fake_reindex_user_namespace)
- monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)
- monkeypatch.setattr(
- ReindexExecutor,
- "_reindex_resource_vectors_from_entries",
- fake_reindex_resource_vectors_from_entries,
- )
+ monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
+ monkeypatch.setattr(ReindexExecutor, "_reindex_user_namespace", fake_reindex_user_namespace)
+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)
service = ReindexExecutor()
counters = _ReindexCounters()
@@ -2041,8 +1688,7 @@ async def test_reindex_global_namespace_semantic_and_vectors_propagates_to_child
)
assert seen["user_modes"] == [("viking://user/default", "semantic_and_vectors")]
- assert seen["semantic_calls"] == [("viking://resources", "resource")]
- assert seen["resource_calls"]
+ assert [call["uri"] for call in seen["rfv_calls"]] == ["viking://resources"]
@pytest.mark.asyncio
@@ -2137,103 +1783,6 @@ async def test_reindex_upsert_context_omits_search_tags_without_ingest_options(m
@pytest.mark.asyncio
-async def test_reindex_resource_vectors_parallelize_files_and_isolate_failures(monkeypatch):
- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
- from openviking_cli.exceptions import OpenVikingError
-
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return True
-
- async def stat(self, uri, ctx=None, skip_count=True):
- return {"isDir": True}
-
- async def read_file_bytes(self, uri, ctx=None):
- return uri.encode()
-
- async def tree(
- self,
- uri,
- output="original",
- show_all_hidden=True,
- node_limit=1000,
- level_limit=3,
- ctx=None,
- ):
- assert node_limit is None
- assert level_limit is None
- return [
- {"uri": "viking://resources/demo/bad.txt", "isDir": False},
- {"uri": "viking://resources/demo/good.txt", "isDir": False},
- ]
-
- async def fake_read_directory_abstract(self, uri, *, ctx):
- return ""
-
- async def fake_read_directory_overview(self, uri, *, ctx):
- return ""
-
- async def fake_best_file_summary(self, uri, *, ctx):
- return f"summary:{uri.rsplit('/', 1)[-1]}"
-
- async def fake_best_resource_file_vector_text(self, uri, summary, ctx, file_content=None):
- return summary
-
- seen = []
- both_entered = asyncio.Event()
- release = asyncio.Event()
-
- async def fake_upsert_context(self, **kwargs):
- seen.append(kwargs["uri"])
- if len(seen) == 2:
- both_entered.set()
- await release.wait()
- if kwargs["uri"].endswith("bad.txt"):
- raise OpenVikingError("boom", code="PROCESSING_ERROR")
-
- monkeypatch.setattr(
- "openviking.service.reindex_executor.get_openviking_config",
- lambda: SimpleNamespace(reindex=SimpleNamespace(file_vectorization_concurrency=8)),
- )
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
- monkeypatch.setattr(ReindexExecutor, "_read_directory_abstract", fake_read_directory_abstract)
- monkeypatch.setattr(ReindexExecutor, "_read_directory_overview", fake_read_directory_overview)
- monkeypatch.setattr(ReindexExecutor, "_best_file_summary", fake_best_file_summary)
- monkeypatch.setattr(
- ReindexExecutor,
- "_best_resource_file_vector_text",
- fake_best_resource_file_vector_text,
- )
- monkeypatch.setattr(ReindexExecutor, "_upsert_context", fake_upsert_context)
-
- service = ReindexExecutor()
- counters = _ReindexCounters()
- ctx = RequestContext(
- user=UserIdentifier(account_id="test", user_id="alice"),
- role=Role.ROOT,
- )
-
- task = asyncio.create_task(
- service._reindex_resource_vectors(
- uri="viking://resources/demo",
- counters=counters,
- ctx=ctx,
- )
- )
- try:
- await asyncio.wait_for(both_entered.wait(), timeout=1)
- finally:
- release.set()
- await task
-
- assert set(seen) == {
- "viking://resources/demo/bad.txt",
- "viking://resources/demo/good.txt",
- }
- assert counters.failed_records == 1
- assert counters.rebuilt_records == 1
-
-
@pytest.mark.asyncio
async def test_reindex_semantic_processor_runs_with_skip_vectorization(
monkeypatch, semantic_config
@@ -2356,122 +1905,7 @@ async def test_reindex_semantic_processor_sets_memory_aggregation_policy(
@pytest.mark.asyncio
-async def test_reindex_resource_l2_falls_back_to_vector_text_when_summary_missing(monkeypatch):
- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
-
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return True
-
- async def stat(self, uri, ctx=None, skip_count=True):
- return {"isDir": True}
-
- async def read_file_bytes(self, uri, ctx=None):
- return b"raw file body"
-
- async def tree(
- self,
- uri,
- output="original",
- show_all_hidden=True,
- node_limit=1000,
- level_limit=3,
- ctx=None,
- ):
- assert node_limit is None
- assert level_limit is None
- return [{"uri": "viking://resources/demo/file.txt", "isDir": False}]
-
- seen = {}
-
- async def fake_read_directory_abstract(self, uri, *, ctx):
- return "skill abstract"
-
- async def fake_read_directory_overview(self, uri, *, ctx):
- return ""
-
- async def fake_best_file_summary(self, uri, *, ctx):
- return ""
-
- async def fake_best_resource_file_vector_text(self, uri, summary, ctx, file_content=None):
- return "raw file body"
-
- async def fake_upsert_context(self, **kwargs):
- seen[kwargs["uri"]] = kwargs
-
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
- monkeypatch.setattr(ReindexExecutor, "_read_directory_abstract", fake_read_directory_abstract)
- monkeypatch.setattr(ReindexExecutor, "_read_directory_overview", fake_read_directory_overview)
- monkeypatch.setattr(ReindexExecutor, "_best_file_summary", fake_best_file_summary)
- monkeypatch.setattr(
- ReindexExecutor,
- "_best_resource_file_vector_text",
- fake_best_resource_file_vector_text,
- )
- monkeypatch.setattr(ReindexExecutor, "_upsert_context", fake_upsert_context)
-
- service = ReindexExecutor()
- counters = _ReindexCounters()
- ctx = RequestContext(
- user=UserIdentifier(account_id="test", user_id="alice"),
- role=Role.ROOT,
- )
-
- await service._reindex_resource_vectors(
- uri="viking://resources/demo",
- counters=counters,
- ctx=ctx,
- )
-
- assert seen["viking://resources/demo/file.txt"]["abstract"] == "raw file body"
-
-
@pytest.mark.asyncio
-async def test_reindex_resource_file_writes_md5_from_same_bytes(monkeypatch):
- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
- from openviking.utils.content_hash import content_md5
-
- file_uri = "viking://resources/demo/file.txt"
- raw = b"current body"
-
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return True
-
- async def stat(self, uri, ctx=None, skip_count=True):
- return {"isDir": False}
-
- async def read_file_bytes(self, uri, ctx=None):
- assert uri == file_uri
- return raw
-
- seen = {}
-
- async def fake_best_file_summary(self, uri, *, ctx):
- return "summary"
-
- async def fake_best_resource_file_vector_text(self, uri, summary, ctx, file_content=None):
- return "current body"
-
- async def fake_upsert_context(self, **kwargs):
- seen.update(kwargs)
-
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
- monkeypatch.setattr(ReindexExecutor, "_best_file_summary", fake_best_file_summary)
- monkeypatch.setattr(
- ReindexExecutor,
- "_best_resource_file_vector_text",
- fake_best_resource_file_vector_text,
- )
- monkeypatch.setattr(ReindexExecutor, "_upsert_context", fake_upsert_context)
-
- counters = _ReindexCounters()
- ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="alice"), role=Role.ROOT)
- await ReindexExecutor()._reindex_resource_vectors(uri=file_uri, counters=counters, ctx=ctx)
-
- assert seen["md5"] == content_md5(raw)
-
-
@pytest.mark.asyncio
async def test_reindex_resource_vector_text_uses_existing_record_for_non_text(monkeypatch):
from openviking.service.reindex_executor import ReindexExecutor
@@ -2512,9 +1946,13 @@ async def test_reindex_resource_vector_text_summary_first_skips_content_read(mon
monkeypatch.setattr(ReindexExecutor, "_safe_read_text", fail_if_content_read)
monkeypatch.setattr(ReindexExecutor, "_fetch_existing_record", fake_fetch_existing_record)
- resolver = SimpleNamespace(resolve=AsyncMock(return_value=SimpleNamespace(
- embedding=SimpleNamespace(text_source="summary_first"),
- )))
+ resolver = SimpleNamespace(
+ resolve=AsyncMock(
+ return_value=SimpleNamespace(
+ embedding=SimpleNamespace(text_source="summary_first"),
+ )
+ )
+ )
service = ReindexExecutor(vector_config_resolver=resolver)
ctx = RequestContext(
user=UserIdentifier(account_id="test", user_id="alice"),
@@ -2602,8 +2040,8 @@ async def test_reindex_file_summary_reads_existing_record_as_uri_owner(monkeypat
async def test_reindex_memory_fallback_reads_existing_record_as_uri_owner(monkeypatch):
from openviking.service.reindex_executor import (
ReindexExecutor,
- _PruneSourceRead,
_ReindexCounters,
+ _SourceRead,
)
captured = {}
@@ -2613,11 +2051,11 @@ async def test_reindex_memory_fallback_reads_existing_record_as_uri_owner(monkey
async def exists(self, uri, ctx=None):
return True
- async def stat(self, uri, ctx=None, skip_count=None):
+ async def stat(self, uri, ctx=None, skip_count=True):
return {"isDir": False}
async def fake_read_memory_body(self, uri, *, ctx):
- return _PruneSourceRead(exists=True, text="")
+ return _SourceRead(exists=True, text="")
async def fake_fetch_existing_record(self, *, uri, level, ctx):
captured["ctx"] = ctx
@@ -2661,7 +2099,7 @@ async def test_reindex_memory_skips_fallback_when_body_read_fails(monkeypatch):
async def exists(self, uri, ctx=None):
return True
- async def stat(self, uri, ctx=None, skip_count=None):
+ async def stat(self, uri, ctx=None, skip_count=True):
return {"isDir": False}
async def read_file(self, uri, ctx=None):
@@ -2728,79 +2166,25 @@ async def test_reindex_resource_vector_text_skips_non_text_body_without_summary(
@pytest.mark.asyncio
-async def test_reindex_resource_vectors_accepts_single_file_uri(monkeypatch):
- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
-
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- return True
-
- async def stat(self, uri, ctx=None, skip_count=True):
- return {"isDir": False}
-
- async def read_file_bytes(self, uri, ctx=None):
- return b"file summary"
-
- async def tree(self, *args, **kwargs):
- raise AssertionError("single-file reindex should not call tree")
-
- seen = {}
-
- async def fake_best_file_summary(self, uri, *, ctx):
- return "file summary"
-
- async def fake_best_resource_file_vector_text(self, uri, summary, ctx, file_content=None):
- return summary
-
- async def fake_upsert_context(self, **kwargs):
- seen[kwargs["uri"]] = kwargs
-
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
- monkeypatch.setattr(ReindexExecutor, "_best_file_summary", fake_best_file_summary)
- monkeypatch.setattr(
- ReindexExecutor,
- "_best_resource_file_vector_text",
- fake_best_resource_file_vector_text,
- )
- monkeypatch.setattr(ReindexExecutor, "_upsert_context", fake_upsert_context)
-
- service = ReindexExecutor()
- counters = _ReindexCounters()
- ctx = RequestContext(
- user=UserIdentifier(account_id="test", user_id="alice"),
- role=Role.ROOT,
- )
-
- await service._reindex_resource_vectors(
- uri="viking://resources/demo/file.txt",
- counters=counters,
- ctx=ctx,
- )
-
- assert list(seen) == ["viking://resources/demo/file.txt"]
- assert counters.scanned_records == 1
- assert counters.rebuilt_records == 1
-
-
@pytest.mark.asyncio
async def test_reindex_memory_l2_falls_back_to_body_when_abstract_missing(monkeypatch):
from openviking.service.reindex_executor import (
ReindexExecutor,
- _PruneSourceRead,
_ReindexCounters,
+ _SourceRead,
)
class FakeVikingFS:
async def exists(self, uri, ctx=None):
return True
- async def stat(self, uri, ctx=None, skip_count=None):
+ async def stat(self, uri, ctx=None, skip_count=True):
return {"isDir": False}
seen = {}
async def fake_read_memory_body(self, uri, *, ctx):
- return _PruneSourceRead(exists=True, text="memory body text")
+ return _SourceRead(exists=True, text="memory body text")
async def fake_fetch_existing_record(self, *, uri, level, ctx):
return None
@@ -2837,15 +2221,15 @@ async def test_reindex_memory_l2_falls_back_to_body_when_abstract_missing(monkey
async def test_reindex_memory_l2_strips_memory_fields_from_abstract(monkeypatch):
from openviking.service.reindex_executor import (
ReindexExecutor,
- _PruneSourceRead,
_ReindexCounters,
+ _SourceRead,
)
class FakeVikingFS:
async def exists(self, uri, ctx=None):
return True
- async def stat(self, uri, ctx=None, skip_count=None):
+ async def stat(self, uri, ctx=None, skip_count=True):
return {"isDir": False}
seen = {}
@@ -2855,7 +2239,7 @@ async def test_reindex_memory_l2_strips_memory_fields_from_abstract(monkeypatch)
)
async def fake_read_memory_body(self, uri, *, ctx):
- return _PruneSourceRead(exists=True, text=raw_body)
+ return _SourceRead(exists=True, text=raw_body)
async def fake_fetch_existing_record(self, *, uri, level, ctx):
return None
@@ -2899,15 +2283,15 @@ async def test_reindex_memory_l2_strips_memory_fields_from_abstract(monkeypatch)
async def test_reindex_memory_vectors_walks_deep_subtree(monkeypatch):
from openviking.service.reindex_executor import (
ReindexExecutor,
- _PruneSourceRead,
_ReindexCounters,
+ _SourceRead,
)
class FakeVikingFS:
async def exists(self, uri, ctx=None):
return True
- async def stat(self, uri, ctx=None, skip_count=None):
+ async def stat(self, uri, ctx=None, skip_count=True):
return {"isDir": True}
async def tree(
@@ -2934,7 +2318,7 @@ async def test_reindex_memory_vectors_walks_deep_subtree(monkeypatch):
seen = {}
async def fake_read_memory_body(self, uri, *, ctx):
- return _PruneSourceRead(exists=True, text="likes spicy food")
+ return _SourceRead(exists=True, text="likes spicy food")
async def fake_fetch_existing_record(self, *, uri, level, ctx):
return None
@@ -2974,15 +2358,15 @@ async def test_reindex_memory_vectors_rebuilds_directory_levels_without_regenera
from openviking.core.context import ContextLevel
from openviking.service.reindex_executor import (
ReindexExecutor,
- _PruneSourceRead,
_ReindexCounters,
+ _SourceRead,
)
class FakeVikingFS:
async def exists(self, uri, ctx=None):
return True
- async def stat(self, uri, ctx=None, skip_count=None):
+ async def stat(self, uri, ctx=None, skip_count=True):
return {"isDir": True}
async def tree(
@@ -3017,7 +2401,7 @@ async def test_reindex_memory_vectors_rebuilds_directory_levels_without_regenera
return ""
async def fake_read_memory_body(self, uri, *, ctx):
- return _PruneSourceRead(exists=True, text="likes spicy food")
+ return _SourceRead(exists=True, text="likes spicy food")
async def fake_fetch_existing_record(self, *, uri, level, ctx):
return None
@@ -3095,34 +2479,17 @@ async def test_reindex_user_namespace_partitions_memory_skill_and_resource(monke
{"uri": "viking://user/default/skills/my_skill/SKILL.md", "isDir": False},
]
- seen = {"memory": [], "skill": [], "resource_dirs": [], "resource_files": []}
+ seen = {"memory": [], "rfv": []}
async def fake_reindex_memory(self, *, uri, mode, run):
seen["memory"].append((uri, mode))
- async def fake_reindex_skill(self, *, uri, mode, run):
- seen["skill"].append((uri, mode))
-
- async def fake_reindex_resource_vectors_from_entries(
- self,
- *,
- root_uri,
- directories,
- files,
- counters,
- ctx,
- ):
- seen["resource_dirs"] = list(directories)
- seen["resource_files"] = list(files)
+ async def fake_reindex_rfv(self, **kwargs):
+ seen["rfv"].append(kwargs)
monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
monkeypatch.setattr(ReindexExecutor, "_reindex_memory", fake_reindex_memory)
- monkeypatch.setattr(ReindexExecutor, "_reindex_skill", fake_reindex_skill)
- monkeypatch.setattr(
- ReindexExecutor,
- "_reindex_resource_vectors_from_entries",
- fake_reindex_resource_vectors_from_entries,
- )
+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)
service = ReindexExecutor()
counters = _ReindexCounters()
@@ -3138,14 +2505,10 @@ async def test_reindex_user_namespace_partitions_memory_skill_and_resource(monke
)
assert seen["memory"] == [("viking://user/default/memories", "vectors_only")]
- assert seen["skill"] == [("viking://user/default/skills/my_skill", "vectors_only")]
- assert "viking://user/default/resources" in seen["resource_dirs"]
- assert "viking://user/default/memories" not in seen["resource_dirs"]
- assert "viking://user/default/skills" not in seen["resource_dirs"]
- assert "viking://user/default/sessions" not in seen["resource_dirs"]
- assert seen["resource_files"] == [
- "viking://user/default/resources/doc.md",
- "viking://user/default/profile.md",
+ assert [(call["uri"], call["context_type"]) for call in seen["rfv"]] == [
+ ("viking://user/default/skills/my_skill", "skill"),
+ ("viking://user/default/resources", "resource"),
+ ("viking://user/default/profile.md", "resource"),
]
@@ -3177,30 +2540,17 @@ async def test_reindex_global_namespace_partitions_user_and_resources(monkeypatc
{"uri": "viking://README.md", "isDir": False},
]
- seen = {"user": [], "resource_dirs": [], "resource_files": []}
+ seen = {"user": [], "rfv": []}
async def fake_reindex_user_namespace(self, *, uri, mode, run):
seen["user"].append((uri, mode))
- async def fake_reindex_resource_vectors_from_entries(
- self,
- *,
- root_uri,
- directories,
- files,
- counters,
- ctx,
- ):
- seen["resource_dirs"] = list(directories)
- seen["resource_files"] = list(files)
+ async def fake_reindex_rfv(self, **kwargs):
+ seen["rfv"].append(kwargs)
monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
monkeypatch.setattr(ReindexExecutor, "_reindex_user_namespace", fake_reindex_user_namespace)
- monkeypatch.setattr(
- ReindexExecutor,
- "_reindex_resource_vectors_from_entries",
- fake_reindex_resource_vectors_from_entries,
- )
+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)
service = ReindexExecutor()
counters = _ReindexCounters()
@@ -3216,70 +2566,12 @@ async def test_reindex_global_namespace_partitions_user_and_resources(monkeypatc
)
assert seen["user"] == [("viking://user/default", "vectors_only")]
- assert "viking://resources" in seen["resource_dirs"]
- assert "viking://" not in seen["resource_dirs"]
- assert "viking://user" not in seen["resource_dirs"]
- assert "viking://session" not in seen["resource_dirs"]
- assert "viking://session/default" not in seen["resource_dirs"]
- assert seen["resource_files"] == ["viking://resources/demo.txt"]
-
-
-async def test_reindex_skill_vectors_non_recursive_skips_skill_detail(monkeypatch):
- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
-
- class FakeVikingFS:
- async def exists(self, uri, ctx=None):
- raise AssertionError(f"must not inspect skill detail: {uri}")
-
- async def fake_read_directory_abstract(self, uri, *, ctx):
- return "skill abstract"
-
- async def fake_read_directory_overview(self, uri, *, ctx):
- return "skill overview"
-
- async def fake_safe_read_text(self, uri, *, ctx):
- raise AssertionError(f"must not read skill detail: {uri}")
-
- async def fake_skill_meta(self, *, uri, abstract, ctx):
- return {"name": "my_skill", "description": abstract}
-
- upserts = []
-
- async def fake_vectorize_directory(uri, abstract, overview, **kwargs):
- upserts.extend([(uri, ContextLevel.ABSTRACT), (uri, ContextLevel.OVERVIEW)])
-
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
- monkeypatch.setattr(ReindexExecutor, "_read_directory_abstract", fake_read_directory_abstract)
- monkeypatch.setattr(ReindexExecutor, "_read_directory_overview", fake_read_directory_overview)
- monkeypatch.setattr(ReindexExecutor, "_safe_read_text", fake_safe_read_text)
- monkeypatch.setattr(ReindexExecutor, "_skill_meta", fake_skill_meta)
- monkeypatch.setattr(
- "openviking.service.reindex_executor.vectorize_directory_meta", fake_vectorize_directory
- )
-
- counters = _ReindexCounters()
- ctx = RequestContext(
- user=UserIdentifier(account_id="test", user_id="alice"),
- role=Role.ROOT,
- )
- uri = "viking://user/skills/my_skill"
-
- await ReindexExecutor()._reindex_skill_vectors(
- uri=uri,
- counters=counters,
- ctx=ctx,
- recursive=False,
- )
-
- assert upserts == [
- (uri, ContextLevel.ABSTRACT),
- (uri, ContextLevel.OVERVIEW),
+ assert [(call["uri"], call["context_type"]) for call in seen["rfv"]] == [
+ ("viking://resources", "resource")
]
- assert counters.scanned_records == 1
- assert counters.rebuilt_records == 2
@pytest.mark.asyncio
async def test_reindex_upsert_ignores_empty_replace_tags(monkeypatch):
from types import SimpleNamespace
@@ -3377,7 +2669,7 @@ async def test_openviking_service_reindex_uses_default_root_context(monkeypatch)
seen["vlm_resolver"] = vlm_resolver
seen["vector_config_resolver"] = vector_config_resolver
- async def execute(self, *, uri, mode, wait, ctx, dry_run=False):
+ async def execute(self, *, uri, mode, wait, ctx):
seen["uri"] = uri
seen["mode"] = mode
seen["wait"] = wait
@@ -3424,7 +2716,7 @@ async def test_openviking_service_reindex_keeps_canonical_user_id(monkeypatch):
seen["vlm_resolver"] = vlm_resolver
seen["vector_config_resolver"] = vector_config_resolver
- async def execute(self, *, uri, mode, wait, ctx, dry_run=False):
+ async def execute(self, *, uri, mode, wait, ctx):
seen["uri"] = uri
seen["ctx"] = ctx
return {"status": "completed", "uri": uri}
@@ -3450,9 +2742,8 @@ async def test_openviking_service_reindex_keeps_canonical_user_id(monkeypatch):
result = await OpenVikingService.reindex(
service,
uri="viking://user/resources/memories",
- mode="prune_orphans",
+ mode="vectors_only",
wait=True,
- dry_run=True,
ctx=ctx,
)
diff --git a/tests/server/test_api_content_write.py b/tests/server/test_api_content_write.py
index d1834806a..9fba532cc 100644
--- a/tests/server/test_api_content_write.py
+++ b/tests/server/test_api_content_write.py
@@ -302,11 +302,72 @@ async def test_api_create_mode_regression_replace_unchanged(client_with_resource
assert read_resp.json()["result"] == "# Updated\n\nFresh content."
-async def test_set_tags_requires_tags_field(client_with_resource):
- client, uri = client_with_resource
- file_uri = await _first_file_uri(client, uri)
- resp = await client.post("/api/v1/fs/attrs/set_tags", json={"uri": file_uri})
- assert resp.status_code == 400
+async def test_set_tags_omitted_tags_is_noop(client, service, monkeypatch):
+ captured = {}
+
+ async def fake_set_tags(
+ *,
+ uri,
+ tags,
+ ctx,
+ mode="replace",
+ recursive=False,
+ ):
+ del ctx
+ captured["uri"] = uri
+ captured["tags"] = tags
+ captured["mode"] = mode
+ captured["recursive"] = recursive
+ return {"uri": uri, "tags": tags, "mode": mode}
+
+ monkeypatch.setattr(service.fs, "set_tags", fake_set_tags)
+
+ resp = await client.post(
+ "/api/v1/fs/attrs/set_tags",
+ json={"uri": "viking://resources/demo/file.md"},
+ )
+
+ assert resp.status_code == 200
+ assert captured == {
+ "uri": "viking://resources/demo/file.md",
+ "tags": [],
+ "mode": "replace",
+ "recursive": False,
+ }
+
+
+async def test_set_tags_clear_allows_omitted_tags(client, service, monkeypatch):
+ captured = {}
+
+ async def fake_set_tags(
+ *,
+ uri,
+ tags,
+ ctx,
+ mode="replace",
+ recursive=False,
+ ):
+ del ctx
+ captured["uri"] = uri
+ captured["tags"] = tags
+ captured["mode"] = mode
+ captured["recursive"] = recursive
+ return {"uri": uri, "tags": tags, "mode": mode}
+
+ monkeypatch.setattr(service.fs, "set_tags", fake_set_tags)
+
+ resp = await client.post(
+ "/api/v1/fs/attrs/set_tags",
+ json={"uri": "viking://resources/demo/file.md", "mode": "clear"},
+ )
+
+ assert resp.status_code == 200
+ assert captured == {
+ "uri": "viking://resources/demo/file.md",
+ "tags": [],
+ "mode": "clear",
+ "recursive": False,
+ }
async def test_set_tags_passes_tags_to_service(client, service, monkeypatch):
diff --git a/tests/server/test_content_write_service.py b/tests/server/test_content_write_service.py
index da44b0ddc..32fc479ff 100644
--- a/tests/server/test_content_write_service.py
+++ b/tests/server/test_content_write_service.py
@@ -1551,6 +1551,68 @@ async def test_set_tags_append_merges_existing_tags(monkeypatch):
assert fake_store.update_calls == [(file_uri, ["env=prod", "team=search"], "append")]
+async def test_set_tags_clear_ignores_values_and_clears_existing_tags(monkeypatch):
+ file_uri = "viking://resources/demo/doc.md"
+ root_uri = "viking://resources/demo"
+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
+ fake_vfs = _FakeVikingFS(file_uri=file_uri, root_uri=root_uri)
+ coordinator = ContentWriteCoordinator(viking_fs=fake_vfs)
+
+ class _FakeVectorStore:
+ def __init__(self):
+ self.update_calls = []
+
+ async def update_search_tags(self, uri: str, tags, *, mode: str, levels=None, ctx=None):
+ del ctx
+ assert levels is None
+ self.update_calls.append((uri, list(tags), mode))
+ return [{"uri": uri}]
+
+ fake_store = _FakeVectorStore()
+ fake_vfs.vector_store = fake_store
+
+ result = await coordinator.set_tags(
+ uri=file_uri,
+ tags=["team=ignored"],
+ mode="clear",
+ ctx=ctx,
+ )
+
+ assert result["mode"] == "clear"
+ assert result["tags"] == []
+ assert result["tags_updated"] is True
+ assert fake_store.update_calls == [(file_uri, [], "replace")]
+
+
+async def test_set_tags_empty_replace_is_noop(monkeypatch):
+ file_uri = "viking://resources/demo/doc.md"
+ root_uri = "viking://resources/demo"
+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
+ fake_vfs = _FakeVikingFS(file_uri=file_uri, root_uri=root_uri)
+ coordinator = ContentWriteCoordinator(viking_fs=fake_vfs)
+
+ class _FakeVectorStore:
+ async def update_search_tags(self, *args, **kwargs):
+ raise AssertionError("replace with [] must not update stored tags")
+
+ fake_vfs.vector_store = _FakeVectorStore()
+
+ result = await coordinator.set_tags(
+ uri=file_uri,
+ tags=[],
+ mode="replace",
+ ctx=ctx,
+ )
+
+ assert result["mode"] == "replace"
+ assert result["tags"] == []
+ assert result["tags_updated"] is False
+ assert result["success_count"] == 0
+ assert result["skipped_count"] == 1
+
+
@pytest.mark.asyncio
async def test_set_tags_discards_non_kv_tags(monkeypatch):
file_uri = "viking://resources/demo/doc.md"
diff --git a/tests/service/test_skill_package_reindex.py b/tests/service/test_skill_package_reindex.py
deleted file mode 100644
index 5d983dc68..000000000
--- a/tests/service/test_skill_package_reindex.py
+++ /dev/null
@@ -1,97 +0,0 @@
-from types import SimpleNamespace
-from unittest.mock import AsyncMock
-
-import pytest
-
-from openviking.server.identity import RequestContext, Role
-from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
-from openviking_cli.session.user_id import UserIdentifier
-
-
-async def test_vectors_only_reindexes_entire_skill_with_common_media_inputs(monkeypatch):
- root = "viking://agent/skills/demo"
- tree = {
- root: [
- {"name": "SKILL.md"},
- {"name": "reference", "isDir": True},
- {"name": ".source.json"},
- ],
- root + "/reference": [
- {"name": "image.png"},
- {"name": "video.mp4"},
- {"name": "nested", "isDir": True},
- ],
- root + "/reference/nested": [
- {"name": "api.md"},
- {"name": ".abstract.md"},
- {"name": "messages.jsonl"},
- ],
- }
- fs = SimpleNamespace(ls=AsyncMock(side_effect=lambda uri, **kw: tree[uri]))
- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: fs)
- executor = ReindexExecutor()
- executor._read_directory_abstract = AsyncMock(return_value="name: demo\ndescription: Demo")
- executor._read_directory_overview = AsyncMock(return_value="overview")
- executor._fetch_existing_record = AsyncMock(
- side_effect=lambda **kw: {"abstract": "media summary", "meta": {"source_path": "/tmp/demo"}}
- )
- directories = AsyncMock()
- files = AsyncMock(return_value=True)
- monkeypatch.setattr("openviking.service.reindex_executor.vectorize_directory_meta", directories)
- monkeypatch.setattr("openviking.service.reindex_executor.vectorize_file", files)
- monkeypatch.setattr(
- "openviking.service.reindex_executor.SemanticProcessor",
- lambda **kw: pytest.fail("vectors_only must not generate summaries"),
- )
- counters = _ReindexCounters()
- ctx = RequestContext(user=UserIdentifier("acc", "alice"), role=Role.USER)
- await executor._reindex_skill_vectors(uri=root, ctx=ctx, counters=counters)
- assert {call.args[0] for call in directories.await_args_list} == set(tree)
- expected = {
- root + "/SKILL.md",
- root + "/reference/image.png",
- root + "/reference/video.mp4",
- root + "/reference/nested/api.md",
- }
- assert {call.kwargs["file_path"] for call in files.await_args_list} == expected
- assert all(call.kwargs["context_type"] == "skill" for call in files.await_args_list)
- assert all(
- call.kwargs["summary_dict"]["summary"] == "media summary" for call in files.await_args_list
- )
- assert directories.await_args_list[0].kwargs["meta"]["source_path"] == "/tmp/demo"
- assert counters.scanned_records == 7 and counters.rebuilt_records == 10
-
-
- "abstract",
- [
- "Manage backups.\nUse when: a backup needs recovery.",
- "Use when: recovery is needed",
- "A plain description",
- ],
-)
-async def test_vectors_only_preserves_legacy_skill_descriptions(monkeypatch, abstract):
- executor = ReindexExecutor()
- executor._read_directory_abstract = AsyncMock(return_value=abstract)
- executor._read_directory_overview = AsyncMock(return_value="Overview")
- previous_meta = {
- "name": "demo",
- "tags": ["backup"],
- "allowed_tools": ["Read"],
- "source_path": "/demo",
- }
- executor._fetch_existing_record = AsyncMock(return_value={"meta": previous_meta})
- monkeypatch.setattr(
- "openviking.service.reindex_executor.get_viking_fs", lambda: SimpleNamespace()
- )
- vectorize = AsyncMock()
- monkeypatch.setattr("openviking.service.reindex_executor.vectorize_directory_meta", vectorize)
- counters = _ReindexCounters()
- ctx = RequestContext(user=UserIdentifier("acc", "alice"), role=Role.USER)
- await executor._reindex_skill_vectors(
- uri="viking://agent/skills/demo", counters=counters, ctx=ctx, recursive=False
- )
- assert counters.failed_records == 0 and counters.rebuilt_records == 2
- assert vectorize.await_args.kwargs["meta"] == {**previous_meta, "description": abstract}
diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.py
index 805262634..19c8f8f35 100644
--- a/tests/storage/test_collection_schemas.py
+++ b/tests/storage/test_collection_schemas.py
@@ -86,7 +86,8 @@ class TextEmbeddingHandler(ProductionTextEmbeddingHandler):
provider = AccountEmbeddingProvider(AccountVectorConfigResolver(manager), manager)
vikingdb.account_uses_content_field = AsyncMock(
return_value=getattr(
- vikingdb, "uses_content_field",
+ vikingdb,
+ "uses_content_field",
config.storage.vectordb.backend in {"volcengine", "vikingdb"},
)
)
@@ -836,9 +837,10 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_w
collection_schemas.logger.removeHandler(caplog.handler)
warnings = [record.message for record in caplog.records if record.levelno == logging.WARNING]
- assert warnings.count(
- "Embedding circuit breaker is open; re-enqueueing messages account=default"
- ) == 1
+ assert (
+ warnings.count("Embedding circuit breaker is open; re-enqueueing messages account=default")
+ == 1
+ )
for result in (first_result, second_result):
assert result.outcome is ProcessOutcome.REQUEUED
assert result.value is None
@@ -999,6 +1001,33 @@ async def test_embedding_handler_materialize_content_keeps_inline(monkeypatch):
assert content == "already inline"
+async def test_embedding_handler_materialize_content_uses_preloaded_source(monkeypatch):
+ class _DummyVikingDB:
+ is_closing = False
+
+ class _BrokenFS:
+ async def read_file(self, uri, *, ctx):
+ raise AssertionError(f"must not reread {uri}")
+
+ monkeypatch.setattr("openviking.storage.viking_fs.get_viking_fs", lambda: _BrokenFS())
+ handler = TextEmbeddingHandler(_DummyVikingDB())
+ msg = EmbeddingMsg(
+ "embedding text",
+ {
+ "uri": "viking://resources/current.txt",
+ "abstract": "abstract",
+ "is_leaf": True,
+ "context_type": "resource",
+ "account_id": "default",
+ "_materialized_content": "current body",
+ },
+ )
+ ctx = RequestContext(user=UserIdentifier("default", "default"), role=Role.ROOT)
+
+ assert await handler._materialize_content(msg, ctx) == "current body"
+
+
@pytest.mark.asyncio
async def test_embedding_handler_propagates_account_id_on_error(monkeypatch):
class _DummyVikingDB:
diff --git a/tests/storage/test_context_update_plan.py b/tests/storage/test_context_update_plan.py
index 2d2869550..3e4cdcacd 100644
--- a/tests/storage/test_context_update_plan.py
+++ b/tests/storage/test_context_update_plan.py
@@ -306,6 +306,25 @@ def test_semantic_plan_validates_ancestors_without_rescanning_entries():
assert entries.iterations <= 4
+def test_semantic_plan_serialization_ignores_legacy_parent_propagation():
+ from openviking.storage.context_update_plan import (
+ SemanticPlan,
+ SemanticTreeEntry,
+ SemanticTreeSnapshot,
+ )
+
+ plan = SemanticPlan(
+ "viking://resources/repo",
+ "resource",
+ SemanticTreeSnapshot((SemanticTreeEntry("", "directory", "unchanged", "aggregate"),)),
+ )
+
+ serialized = plan.to_dict()
+
+ assert "propagation" not in serialized
+ assert SemanticPlan.from_dict({**serialized, "propagation": {"enabled": False}}) == plan
+
+
def test_semantic_plan_rejects_missing_higher_ancestor_independent_of_entry_order():
from openviking.storage.context_update_plan import (
IndexSlot,
@@ -971,6 +990,53 @@ def test_healthy_noop_produces_no_actions_or_semantic_plan():
assert plan.is_noop()
assert plan.content_tree_actions == ()
assert plan.semantic_plan is None
+
+
+def test_shared_semantic_closure_generates_complete_file_without_abstract():
+ """RNFV add_resources and RFV reindex share this closure rule."""
+ from openviking.storage.context_update_plan import (
+ ContentState,
+ IndexState,
+ SemanticAction,
+ build_context_update_plan,
+ )
+ from openviking.storage.resource_diff import ResourceDiffEntry, ResourceDiffResult
+ from openviking.storage.resource_rnfv import RequestIntent, VectorRecordSnapshot
+
+ root = "viking://resources/repo"
+ plan = build_context_update_plan(
+ root_uri=root,
+ context_type="resource",
+ request=RequestIntent(root, "semantic_and_vectors"),
+ diff=ResourceDiffResult(
+ {
+ "": ResourceDiffEntry(
+ "",
+ ContentState.UNCHANGED,
+ IndexState.MISSING,
+ old_kind="directory",
+ new_kind="directory",
+ ),
+ "a.md": ResourceDiffEntry(
+ "a.md",
+ ContentState.UNCHANGED,
+ IndexState.COMPLETE,
+ old_kind="file",
+ new_kind="file",
+ md5="same",
+ ),
+ }
+ ),
+ new_kinds={"": "directory", "a.md": "file"},
+ artifact_paths={"a.md": "repository/a.md"},
+ records={"a-l2": VectorRecordSnapshot("a-l2", f"{root}/a.md", "a.md", 2, {"md5": "same"})},
+ is_code_repo=False,
+ account_id="acc",
+ )
+
+ entries = {entry.relative_path: entry for entry in plan.semantic_plan.tree.entries}
+ assert entries[""].semantic_action is SemanticAction.AGGREGATE
+ assert entries["a.md"].semantic_action is SemanticAction.GENERATE
assert plan.direct_index_actions == ()
@@ -1137,12 +1203,8 @@ def test_healthy_directory_rnfv_clear_updates_all_vector_levels():
ingest_options=IngestOptions.from_search_tags(None, mode="clear"),
)
records = {
- "root-l0": VectorRecordSnapshot(
- "root-l0", root, "", 0, {"search_tags": ["scope=old"]}
- ),
- "root-l1": VectorRecordSnapshot(
- "root-l1", root, "", 1, {"search_tags": ["scope=old"]}
- ),
+ "root-l0": VectorRecordSnapshot("root-l0", root, "", 0, {"search_tags": ["scope=old"]}),
+ "root-l1": VectorRecordSnapshot("root-l1", root, "", 1, {"search_tags": ["scope=old"]}),
"a-l2": VectorRecordSnapshot(
"a-l2",
root + "/a.py",
@@ -1237,11 +1299,7 @@ def test_healthy_rnfv_clear_is_noop_when_vector_has_no_tags(existing_tags):
),
new_kinds={"a.py": "file"},
artifact_paths={"a.py": "repository/a.py"},
- records={
- "a-l2": VectorRecordSnapshot(
- "a-l2", root + "/a.py", "a.py", 2, fields
- )
- },
+ records={"a-l2": VectorRecordSnapshot("a-l2", root + "/a.py", "a.py", 2, fields)},
is_code_repo=False,
account_id="acc",
)
@@ -1297,11 +1355,7 @@ def test_healthy_rnfv_non_empty_replace_compares_against_vector_tags(
),
new_kinds={"a.py": "file"},
artifact_paths={"a.py": "repository/a.py"},
- records={
- "a-l2": VectorRecordSnapshot(
- "a-l2", root + "/a.py", "a.py", 2, fields
- )
- },
+ records={"a-l2": VectorRecordSnapshot("a-l2", root + "/a.py", "a.py", 2, fields)},
is_code_repo=False,
account_id="acc",
)
@@ -2583,6 +2637,11 @@ async def test_resource_processor_dispatches_direct_index_actions_without_semant
)
monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: queue_manager)
monkeypatch.setattr("openviking.utils.embedding_utils._enqueue_embedding_message", enqueue)
+ vectorize_directory = AsyncMock(return_value={0, 1})
+ monkeypatch.setattr(
+ "openviking.utils.resource_processor.vectorize_directory_meta",
+ vectorize_directory,
+ )
processor = ResourceProcessor(SimpleNamespace(get_embedder=lambda: None))
processor._vectorize_resource_file = AsyncMock()
ctx = RequestContext(UserIdentifier("acc", "user"), Role.USER)
@@ -2619,9 +2678,33 @@ async def test_resource_processor_dispatches_direct_index_actions_without_semant
{"search_tags": "append"},
),
md5="new-md5",
+ summary="existing summary",
+ ),
+ DirectIndexAction(
+ "merge",
+ "viking://resources/repo",
+ 0,
+ "repo-l0",
+ field_patch=FieldPatch(
+ {"search_tags": ["scope=new"]},
+ {"search_tags": "replace"},
+ ),
+ md5="abstract-md5",
+ ),
+ DirectIndexAction(
+ "upsert",
+ "viking://resources/repo",
+ 1,
+ "repo-l1",
+ md5="overview-md5",
),
),
ctx=ctx,
+ source_contents={
+ ("viking://resources/repo/c.py", 2): b"print('current')",
+ ("viking://resources/repo", 0): "current abstract",
+ ("viking://resources/repo", 1): "current overview",
+ },
)
assert [msg.action.value for msg in enqueued] == [
"delete",
@@ -2644,7 +2727,93 @@ async def test_resource_processor_dispatches_direct_index_actions_without_semant
{"search_tags": ["scope=new"]},
{"search_tags": "append"},
),
+ summary="existing summary",
+ file_content=b"print('current')",
)
+ vectorize_directory.assert_awaited_once_with(
+ "viking://resources/repo",
+ "current abstract",
+ "current overview",
+ context_type="resource",
+ ctx=ctx,
+ include_abstract=True,
+ include_overview=True,
+ content_is_body=True,
+ actions={0: "merge", 1: "upsert"},
+ scalar_overrides={
+ 0: {"_record_id": "repo-l0"},
+ 1: {"_record_id": "repo-l1"},
+ },
+ field_patches={
+ 0: FieldPatch(
+ {"search_tags": ["scope=new"]},
+ {"search_tags": "replace"},
+ )
+ },
+ md5s={0: "abstract-md5", 1: "overview-md5"},
+ )
+
+
+async def test_resource_processor_rejects_direct_upsert_without_snapshot_source():
+ from openviking.server.identity import RequestContext, Role
+ from openviking.storage.context_update_plan import DirectIndexAction
+ from openviking.utils.resource_processor import ResourceProcessor
+ from openviking_cli.session.user_id import UserIdentifier
+
+ processor = ResourceProcessor(SimpleNamespace(get_embedder=lambda: None))
+ ctx = RequestContext(UserIdentifier("acc", "user"), Role.USER)
+
+ with pytest.raises(ValueError, match="directory L0 vector source is unavailable"):
+ await processor._enqueue_index_actions(
+ (
+ DirectIndexAction(
+ "upsert",
+ "viking://resources/repo",
+ 0,
+ "repo-l0",
+ md5="abstract-md5",
+ ),
+ ),
+ ctx=ctx,
+ source_contents={},
+ )
+
+
+async def test_resource_processor_preserves_skill_metadata_in_direct_directory_merge(monkeypatch):
+ from openviking.server.identity import RequestContext, Role
+ from openviking.storage.context_update_plan import DirectIndexAction
+ from openviking.utils.resource_processor import ResourceProcessor
+ from openviking_cli.session.user_id import UserIdentifier
+
+ vectorize_directory = AsyncMock(return_value={0})
+ monkeypatch.setattr(
+ "openviking.utils.resource_processor.vectorize_directory_meta", vectorize_directory
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.get_queue_manager",
+ lambda: SimpleNamespace(EMBEDDING="embedding", get_queue=lambda *a, **k: object()),
+ )
+ processor = ResourceProcessor(SimpleNamespace(get_embedder=lambda: None))
+ ctx = RequestContext(UserIdentifier("acc", "owner"), Role.USER)
+ uri = "viking://user/owner/skills/demo"
+
+ await processor._enqueue_index_actions(
+ (DirectIndexAction("merge", uri, 0, "skill-l0", md5="new"),),
+ ctx=ctx,
+ source_contents={
+ (uri, 0): "name: demo\ndescription: Demo\ntags: [tool]\nallowed_tools: [Read]",
+ },
+ )
+
+ meta = vectorize_directory.await_args.kwargs["meta"]
+ assert meta == {
+ "name": "demo",
+ "description": "Demo",
+ "tags": ["tool"],
+ "allowed_tools": ["Read"],
+ }
def test_semantic_message_roundtrip_uses_explicit_plan():
@@ -3377,7 +3546,7 @@ async def test_semantic_processor_runs_only_plan_execution_roots(monkeypatch):
root_write_result = None
def __init__(self, **kwargs):
- calls.append(("init", kwargs["semantic_plan"]))
+ calls.append(("init", kwargs))
async def run(self, uri):
calls.append(("run", uri))
@@ -3464,11 +3633,13 @@ async def test_semantic_processor_runs_only_plan_execution_roots(monkeypatch):
user_id="user",
role="user",
plan=plan,
+ generation_trigger="reindex",
)
await processor.on_dequeue(msg.to_dict())
assert [value for kind, value in calls if kind == "run"] == [root]
+ assert calls[0][1]["generation_trigger"] == "reindex"
@pytest.mark.asyncio
diff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.py
index 55d7e79ff..662569a10 100644
--- a/tests/storage/test_queue_manager.py
+++ b/tests/storage/test_queue_manager.py
@@ -40,11 +40,13 @@ def test_queue_concurrency_uses_separate_configured_values() -> None:
agfs=object(),
max_concurrent_external_parse=9,
max_concurrent_add_resource=7,
+ max_concurrent_reindex=6,
max_concurrent_session_commit=5,
)
assert manager._max_concurrent_for_queue(manager.EXTERNAL_PARSE) == 9
assert manager._max_concurrent_for_queue(manager.ADD_RESOURCE) == 7
+ assert manager._max_concurrent_for_queue(manager.REINDEX) == 6
assert manager._max_concurrent_for_queue(manager.SESSION_COMMIT) == 5
@@ -143,9 +145,7 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(
object(), max_concurrent_semantic=concurrency, max_concurrent_embedding=concurrency
)
manager._poll_interval = 0.001
- manager.set_vlm_resolver(
- SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace()))
- )
+ manager.set_vlm_resolver(SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace())))
manager.setup_standard_queues(object(), start=False)
semantic = manager._queues[manager.SEMANTIC]._dequeue_handler
manager._queues = {
diff --git a/tests/storage/test_reindex_msg.py b/tests/storage/test_reindex_msg.py
new file mode 100644
index 000000000..578982af6
--- /dev/null
+++ b/tests/storage/test_reindex_msg.py
@@ -0,0 +1,362 @@
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from openviking.service.task_tracker import TaskStatus
+from openviking.storage.errors import LockAcquisitionError
+from openviking.storage.queuefs.reindex_msg import ReindexMsg
+
+
+def test_reindex_msg_round_trips_only_request_descriptor():
+ message = ReindexMsg(
+ task_id="task-1",
+ uri="viking://resources/demo",
+ object_type="resource",
+ mode="semantic_and_vectors",
+ force=True,
+ recursive=False,
+ tags=["team=search"],
+ tag_mode="append",
+ account_id="account-1",
+ user_id="user-1",
+ group_ids=["group-1"],
+ role="admin",
+ actor_peer_id="peer-1",
+ bypass_acl=True,
+ lock_handoff={"lease_ref": "lease-1"},
+ )
+
+ restored = ReindexMsg.from_dict(message.to_dict())
+
+ assert restored == message
+ assert "source_contents" not in restored.to_dict()
+ assert "snapshot" not in restored.to_dict()
+
+
+def test_reindex_processor_restores_actor_peer_and_acl_bypass_context():
+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor
+
+ message = ReindexMsg(
+ task_id="task-1",
+ uri="viking://resources/demo",
+ object_type="resource",
+ mode="vectors_only",
+ account_id="account-1",
+ user_id="user-1",
+ role="admin",
+ actor_peer_id="peer-1",
+ bypass_acl=True,
+ )
+
+ ctx = ReindexProcessor._ctx(message)
+
+ assert ctx.actor_peer_id == "peer-1"
+ assert ctx.bypass_acl is True
+
+
+def test_reindex_msg_rejects_missing_descriptor_fields():
+ with pytest.raises(ValueError, match="task_id"):
+ ReindexMsg.from_dict({"uri": "viking://resources/demo"})
+
+
+async def test_reindex_processor_terminalizes_root_after_descendants(monkeypatch):
+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor
+
+ tracker = SimpleNamespace(
+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.RUNNING)),
+ start=AsyncMock(),
+ complete=AsyncMock(),
+ fail=AsyncMock(),
+ wait_for_descendants=AsyncMock(),
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)
+ )
+ monkeypatch.setattr(
+ "openviking.server.dependencies.get_service",
+ lambda: SimpleNamespace(_vlm_resolver=None, _vector_config_resolver=None),
+ )
+
+ result = {"status": "completed", "rebuilt_records": 3}
+ monkeypatch.setattr(
+ "openviking.service.reindex_executor.ReindexExecutor._run", AsyncMock(return_value=result)
+ )
+ viking_fs = SimpleNamespace(_async_agfs=SimpleNamespace(pathlock_release=AsyncMock()))
+ processor = ReindexProcessor(viking_fs)
+ processor._adopt_or_reacquire = AsyncMock(return_value={"lease": "root"})
+ message = ReindexMsg(
+ task_id="task-1",
+ uri="viking://resources/demo",
+ object_type="resource",
+ mode="vectors_only",
+ account_id="account-1",
+ user_id="user-1",
+ role="admin",
+ )
+
+ outcome = await processor.on_dequeue(
+ {"id": "queue-1", "data": {**message.to_dict(), "_task_work_id": "root-work"}}
+ )
+
+ assert outcome.outcome.value == "success"
+ tracker.wait_for_descendants.assert_awaited_once_with("task-1", "root-work")
+ tracker.complete.assert_awaited_once_with(
+ "task-1", result, account_id="account-1", user_id="user-1"
+ )
+
+
+async def test_reindex_processor_restores_root_telemetry_for_executor(monkeypatch):
+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor
+ from openviking.telemetry import (
+ OperationTelemetry,
+ get_current_telemetry,
+ register_telemetry,
+ unregister_telemetry,
+ )
+
+ tracker = SimpleNamespace(
+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.RUNNING)),
+ start=AsyncMock(),
+ complete=AsyncMock(),
+ fail=AsyncMock(),
+ wait_for_descendants=AsyncMock(),
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)
+ )
+ monkeypatch.setattr(
+ "openviking.server.dependencies.get_service",
+ lambda: SimpleNamespace(_vlm_resolver=None, _vector_config_resolver=None),
+ )
+ telemetry = OperationTelemetry(operation="content_reindex", enabled=True)
+ telemetry_id = telemetry.telemetry_id
+ register_telemetry(telemetry)
+ seen = {}
+
+ async def capture_telemetry(_self, **_kwargs):
+ seen["telemetry_id"] = get_current_telemetry().telemetry_id
+ return {"status": "completed"}
+
+ monkeypatch.setattr(
+ "openviking.service.reindex_executor.ReindexExecutor._run",
+ capture_telemetry,
+ )
+ processor = ReindexProcessor(
+ SimpleNamespace(_async_agfs=SimpleNamespace(pathlock_release=AsyncMock()))
+ )
+ processor._adopt_or_reacquire = AsyncMock(return_value={"lease": "root"})
+ message = ReindexMsg(
+ task_id="task-1",
+ uri="viking://resources/demo",
+ object_type="resource",
+ mode="semantic_and_vectors",
+ account_id="account-1",
+ user_id="user-1",
+ role="admin",
+ telemetry_id=telemetry_id,
+ )
+
+ try:
+ outcome = await processor.on_dequeue({"id": "queue-1", "data": message.to_dict()})
+ finally:
+ unregister_telemetry(telemetry_id)
+
+ assert outcome.outcome.value == "success"
+ assert seen["telemetry_id"] == telemetry_id
+
+
+async def test_reindex_processor_replay_of_completed_task_skips_force_rebuild(monkeypatch):
+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor
+
+ tracker = SimpleNamespace(
+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.COMPLETED)),
+ start=AsyncMock(),
+ complete=AsyncMock(),
+ fail=AsyncMock(),
+ wait_for_descendants=AsyncMock(),
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)
+ )
+ monkeypatch.setattr(
+ "openviking.server.dependencies.get_service",
+ lambda: SimpleNamespace(_vlm_resolver=None, _vector_config_resolver=None),
+ )
+ monkeypatch.setattr(
+ "openviking.service.reindex_executor.ReindexExecutor._run",
+ AsyncMock(return_value={"status": "completed"}),
+ )
+ viking_fs = SimpleNamespace(_async_agfs=SimpleNamespace(pathlock_release=AsyncMock()))
+ processor = ReindexProcessor(viking_fs)
+ processor._adopt_or_reacquire = AsyncMock(return_value={"lease": "must-not-adopt"})
+ message = ReindexMsg(
+ task_id="task-1",
+ uri="viking://resources/demo",
+ object_type="resource",
+ mode="semantic_and_vectors",
+ force=True,
+ account_id="account-1",
+ user_id="user-1",
+ role="admin",
+ lock_handoff={"lease_ref": "old"},
+ )
+
+ outcome = await processor.on_dequeue(
+ {"id": "queue-1", "data": {**message.to_dict(), "_task_work_id": "root-work"}}
+ )
+
+ assert outcome.outcome.value == "success"
+ processor._adopt_or_reacquire.assert_not_awaited()
+ tracker.start.assert_not_awaited()
+ tracker.complete.assert_not_awaited()
+ tracker.fail.assert_not_awaited()
+
+
+async def test_reindex_processor_requeues_lock_handoff_failure_before_ack(monkeypatch):
+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor
+
+ tracker = SimpleNamespace(
+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.RUNNING)),
+ start=AsyncMock(),
+ complete=AsyncMock(),
+ fail=AsyncMock(),
+ wait_for_descendants=AsyncMock(),
+ )
+ queue_manager = SimpleNamespace(REINDEX="Reindex", enqueue=AsyncMock(return_value="retry-1"))
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)
+ )
+ monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: queue_manager)
+ processor = ReindexProcessor(SimpleNamespace(_async_agfs=SimpleNamespace()))
+ processor._adopt_or_reacquire = AsyncMock(side_effect=LockAcquisitionError("busy"))
+ message = ReindexMsg(
+ task_id="task-1",
+ uri="viking://resources/demo",
+ object_type="resource",
+ mode="vectors_only",
+ account_id="account-1",
+ user_id="user-1",
+ role="admin",
+ )
+
+ outcome = await processor.on_dequeue(
+ {"id": "queue-1", "data": {**message.to_dict(), "_task_work_id": "root-work"}}
+ )
+
+ assert outcome.outcome.value == "requeued"
+ queue_manager.enqueue.assert_awaited_once_with(
+ "Reindex", {**message.to_dict(), "lock_handoff_retry": 1}
+ )
+ tracker.fail.assert_not_awaited()
+
+
+async def test_reindex_processor_fails_after_lock_handoff_retry_limit(monkeypatch):
+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor
+
+ tracker = SimpleNamespace(
+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.RUNNING)),
+ start=AsyncMock(),
+ complete=AsyncMock(),
+ fail=AsyncMock(),
+ wait_for_descendants=AsyncMock(),
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)
+ )
+ processor = ReindexProcessor(SimpleNamespace(_async_agfs=SimpleNamespace()))
+ processor._adopt_or_reacquire = AsyncMock(side_effect=LockAcquisitionError("busy"))
+ message = ReindexMsg(
+ task_id="task-1",
+ uri="viking://resources/demo",
+ object_type="resource",
+ mode="vectors_only",
+ account_id="account-1",
+ user_id="user-1",
+ role="admin",
+ lock_handoff_retry=2,
+ )
+
+ outcome = await processor.on_dequeue({"id": "queue-1", "data": message.to_dict()})
+
+ assert outcome.outcome.value == "failed"
+ tracker.fail.assert_awaited_once()
+
+
+async def test_reindex_processor_cancellation_releases_root_lock_handoff():
+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor
+
+ agfs = SimpleNamespace(
+ pathlock_adopt=AsyncMock(return_value={"lease": "root"}),
+ pathlock_release=AsyncMock(),
+ )
+ processor = ReindexProcessor(SimpleNamespace(_async_agfs=agfs))
+ message = ReindexMsg(
+ task_id="task-1",
+ uri="viking://resources/demo",
+ object_type="resource",
+ mode="vectors_only",
+ account_id="account-1",
+ user_id="user-1",
+ role="admin",
+ lock_handoff={"lease_ref": "root-handoff"},
+ )
+
+ outcome = await processor.on_cancelled({"data": message.to_dict()})
+
+ assert outcome.outcome.value == "cancelled"
+ agfs.pathlock_adopt.assert_awaited_once_with({"lease_ref": "root-handoff"})
+ agfs.pathlock_release.assert_awaited_once_with({"lease": "root"})
+
+
+async def test_reindex_processor_does_not_release_lease_owned_by_failed_executor(monkeypatch):
+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor
+
+ tracker = SimpleNamespace(
+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.RUNNING)),
+ start=AsyncMock(),
+ complete=AsyncMock(),
+ fail=AsyncMock(),
+ wait_for_descendants=AsyncMock(),
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)
+ )
+ monkeypatch.setattr(
+ "openviking.server.dependencies.get_service",
+ lambda: SimpleNamespace(_vlm_resolver=None, _vector_config_resolver=None),
+ )
+ agfs = SimpleNamespace(pathlock_release=AsyncMock())
+
+ async def fail_after_releasing(_self, *, existing_lease, **_kwargs):
+ await agfs.pathlock_release(existing_lease)
+ raise RuntimeError("semantic plan enqueue failed")
+
+ monkeypatch.setattr(
+ "openviking.service.reindex_executor.ReindexExecutor._run",
+ fail_after_releasing,
+ )
+ processor = ReindexProcessor(SimpleNamespace(_async_agfs=agfs))
+ processor._adopt_or_reacquire = AsyncMock(return_value={"lease": "root"})
+ message = ReindexMsg(
+ task_id="task-1",
+ uri="viking://resources/demo",
+ object_type="resource",
+ mode="semantic_and_vectors",
+ account_id="account-1",
+ user_id="user-1",
+ role="admin",
+ )
+
+ outcome = await processor.on_dequeue({"id": "queue-1", "data": message.to_dict()})
+
+ assert outcome.outcome.value == "failed"
+ agfs.pathlock_release.assert_awaited_once_with({"lease": "root"})
+ tracker.fail.assert_awaited_once()
diff --git a/tests/storage/test_resource_rfv.py b/tests/storage/test_resource_rfv.py
new file mode 100644
index 000000000..b27bfd328
--- /dev/null
+++ b/tests/storage/test_resource_rfv.py
@@ -0,0 +1,1065 @@
+from types import SimpleNamespace
+from unittest.mock import ANY, AsyncMock
+
+import pytest
+
+from openviking.storage.resource_diff import IndexState
+from openviking.storage.resource_rnfv import (
+ RequestIntent,
+ VectorIndexSnapshot,
+ VectorRecordSnapshot,
+)
+
+
+def _record(record_id: str, uri: str, relative_path: str, level: int, md5: str):
+ return VectorRecordSnapshot(
+ record_id=record_id,
+ uri=uri,
+ relative_path=relative_path,
+ level=level,
+ fields={"md5": md5},
+ )
+
+
+def test_rfv_resolver_skips_complete_file_and_marks_changed_or_forced_file_stale():
+ from openviking.storage.resource_rfv import (
+ RFVEntry,
+ RFVFormalSnapshot,
+ RFVSnapshot,
+ resolve_rfv_state,
+ )
+
+ root = "viking://resources/demo"
+ record = _record("readme-l2", f"{root}/README.md", "README.md", 2, "same")
+ formal = RFVFormalSnapshot(
+ entries={
+ "": RFVEntry(uri=root, relative_path="", is_dir=True),
+ "README.md": RFVEntry(
+ uri=f"{root}/README.md",
+ relative_path="README.md",
+ is_dir=False,
+ level_md5s={2: "same"},
+ ),
+ }
+ )
+ vectors = VectorIndexSnapshot(
+ records_by_id={record.record_id: record},
+ projected_fields=frozenset({"id", "uri", "level", "md5"}),
+ )
+
+ normal = resolve_rfv_state(RFVSnapshot(RequestIntent(root, "vectors_only"), formal, vectors))
+ assert normal.entries["README.md"].content_state.value == "unchanged"
+ assert normal.entries["README.md"].index_state == IndexState.COMPLETE
+ assert normal.entries["README.md"].level_states == {2: IndexState.COMPLETE}
+
+ changed_formal = RFVFormalSnapshot(
+ entries={
+ **formal.entries,
+ "README.md": RFVEntry(
+ uri=f"{root}/README.md",
+ relative_path="README.md",
+ is_dir=False,
+ level_md5s={2: "changed"},
+ ),
+ }
+ )
+ changed = resolve_rfv_state(
+ RFVSnapshot(RequestIntent(root, "vectors_only"), changed_formal, vectors)
+ )
+ assert changed.entries["README.md"].index_state == IndexState.STALE
+
+ forced = resolve_rfv_state(
+ RFVSnapshot(RequestIntent(root, "vectors_only", force=True), formal, vectors)
+ )
+ assert forced.entries["README.md"].index_state == IndexState.STALE
+
+
+def test_rfv_resolver_tracks_directory_levels_independently():
+ from openviking.storage.resource_rfv import (
+ RFVEntry,
+ RFVFormalSnapshot,
+ RFVSnapshot,
+ resolve_rfv_state,
+ )
+
+ root = "viking://resources/demo"
+ records = {
+ "root-l0": _record("root-l0", root, "", 0, "abstract-same"),
+ "root-l1": _record("root-l1", root, "", 1, "overview-old"),
+ }
+ snapshot = RFVSnapshot(
+ RequestIntent(root, "vectors_only"),
+ RFVFormalSnapshot(
+ entries={
+ "": RFVEntry(
+ uri=root,
+ relative_path="",
+ is_dir=True,
+ level_md5s={0: "abstract-same", 1: "overview-new"},
+ )
+ }
+ ),
+ VectorIndexSnapshot(
+ records_by_id=records,
+ projected_fields=frozenset({"id", "uri", "level", "md5"}),
+ ),
+ )
+
+ resolved = resolve_rfv_state(snapshot)
+ assert resolved.entries[""].index_state == IndexState.STALE
+ assert resolved.entries[""].level_states == {
+ 0: IndexState.COMPLETE,
+ 1: IndexState.STALE,
+ }
+
+
+def test_rfv_resolver_marks_missing_and_orphan_records():
+ from openviking.storage.resource_rfv import (
+ RFVEntry,
+ RFVFormalSnapshot,
+ RFVSnapshot,
+ resolve_rfv_state,
+ )
+
+ root = "viking://resources/demo"
+ orphan = _record("old-l2", f"{root}/old.md", "old.md", 2, "old")
+ snapshot = RFVSnapshot(
+ RequestIntent(root, "vectors_only"),
+ RFVFormalSnapshot(
+ entries={
+ "": RFVEntry(uri=root, relative_path="", is_dir=True),
+ "new.md": RFVEntry(
+ uri=f"{root}/new.md",
+ relative_path="new.md",
+ is_dir=False,
+ level_md5s={2: "new"},
+ ),
+ }
+ ),
+ VectorIndexSnapshot(
+ records_by_id={orphan.record_id: orphan},
+ projected_fields=frozenset({"id", "uri", "level", "md5"}),
+ ),
+ )
+
+ resolved = resolve_rfv_state(snapshot)
+ assert resolved.entries["new.md"].index_state == IndexState.MISSING
+ assert resolved.entries["old.md"].index_state == IndexState.ORPHAN
+ assert resolved.entries["old.md"].content_state.value == "absent"
+
+
+def test_rfv_single_file_semantic_plan_uses_generate_instead_of_file_refresh():
+ from openviking.storage.context_update_plan import SemanticAction, build_rfv_context_update_plan
+ from openviking.storage.resource_rfv import (
+ RFVEntry,
+ RFVFormalSnapshot,
+ RFVSnapshot,
+ )
+
+ uri = "viking://resources/report.md"
+ record = _record("report-l2", uri, "", 2, "old")
+ snapshot = RFVSnapshot(
+ RequestIntent(uri, "semantic_and_vectors"),
+ RFVFormalSnapshot({"": RFVEntry(uri, "", False, {2: "new"})}),
+ VectorIndexSnapshot(
+ {record.record_id: record},
+ frozenset({"id", "uri", "level", "md5", "abstract"}),
+ ),
+ source_contents={(uri, 2): b"new"},
+ )
+
+ _, plan = build_rfv_context_update_plan(
+ snapshot=snapshot,
+ context_type="resource",
+ account_id="acc",
+ )
+
+ assert plan.file_refresh is None
+ assert plan.semantic_plan is not None
+ entry = plan.semantic_plan.tree.entries[0]
+ assert entry.kind == "file"
+ assert entry.semantic_action is SemanticAction.GENERATE
+ assert entry.slot(2).record_id == "report-l2"
+
+
+def test_rfv_missing_directory_sidecar_is_deleted_for_vectors_only_but_repaired_for_semantic():
+ from openviking.storage.resource_rfv import (
+ RFVEntry,
+ RFVFormalSnapshot,
+ RFVSnapshot,
+ resolve_rfv_state,
+ )
+
+ root = "viking://resources/demo"
+ l0 = _record("root-l0", root, "", 0, "old")
+ formal = RFVFormalSnapshot({"": RFVEntry(root, "", True, {})})
+ vectors = VectorIndexSnapshot({l0.record_id: l0}, frozenset({"id", "uri", "level", "md5"}))
+
+ vectors_only = resolve_rfv_state(
+ RFVSnapshot(RequestIntent(root, "vectors_only"), formal, vectors)
+ )
+ semantic = resolve_rfv_state(
+ RFVSnapshot(RequestIntent(root, "semantic_and_vectors"), formal, vectors)
+ )
+
+ assert vectors_only.entries[""].level_states == {
+ 0: IndexState.ABSENT,
+ 1: IndexState.ABSENT,
+ }
+ assert semantic.entries[""].level_states == {
+ 0: IndexState.MISSING,
+ 1: IndexState.MISSING,
+ }
+
+
+def test_rfv_plan_compiles_per_level_repairs_and_orphan_cleanup_without_content_actions():
+ from openviking.storage.context_update_plan import build_rfv_context_update_plan
+ from openviking.storage.index_action import IndexAction
+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot
+
+ root = "viking://resources/demo"
+ records = {
+ "root-l0": _record("root-l0", root, "", 0, "root-old"),
+ "same-l2": _record("same-l2", f"{root}/same.md", "same.md", 2, "same"),
+ "stale-l2": _record("stale-l2", f"{root}/stale.md", "stale.md", 2, "old"),
+ "orphan-l2": _record("orphan-l2", f"{root}/orphan.md", "orphan.md", 2, "orphan"),
+ }
+ snapshot = RFVSnapshot(
+ RequestIntent(root, "vectors_only"),
+ RFVFormalSnapshot(
+ entries={
+ "": RFVEntry(
+ uri=root,
+ relative_path="",
+ is_dir=True,
+ level_md5s={0: "root-new", 1: "overview-new"},
+ ),
+ "same.md": RFVEntry(
+ uri=f"{root}/same.md",
+ relative_path="same.md",
+ is_dir=False,
+ level_md5s={2: "same"},
+ ),
+ "stale.md": RFVEntry(
+ uri=f"{root}/stale.md",
+ relative_path="stale.md",
+ is_dir=False,
+ level_md5s={2: "new"},
+ ),
+ }
+ ),
+ VectorIndexSnapshot(
+ records_by_id=records,
+ projected_fields=frozenset({"id", "uri", "level", "md5"}),
+ ),
+ )
+
+ diff, plan = build_rfv_context_update_plan(
+ snapshot=snapshot,
+ context_type="resource",
+ account_id="acc",
+ )
+
+ assert plan.content_tree_actions == ()
+ assert plan.semantic_plan is None
+ assert diff.entries["same.md"].level_states == {2: IndexState.COMPLETE}
+ assert all(
+ action.action is IndexAction.MERGE and action.upsert_fields == {}
+ for action in plan.direct_index_actions
+ if action.action is not IndexAction.DELETE
+ )
+ assert {
+ (action.action, action.uri, action.level, action.record_id, action.md5)
+ for action in plan.direct_index_actions
+ } == {
+ (IndexAction.MERGE, root, 0, "root-l0", "root-new"),
+ (IndexAction.MERGE, root, 1, action_record_id(root, 1), "overview-new"),
+ (IndexAction.MERGE, f"{root}/stale.md", 2, "stale-l2", "new"),
+ (IndexAction.DELETE, f"{root}/orphan.md", 2, "orphan-l2", None),
+ }
+
+
+def test_rfv_semantic_plan_reuses_existing_semantic_actions_for_stale_nodes():
+ from openviking.storage.context_update_plan import (
+ SemanticAction,
+ build_rfv_context_update_plan,
+ )
+ from openviking.storage.index_action import IndexAction
+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot
+
+ root = "viking://resources/demo"
+ records = {
+ "root-l0": VectorRecordSnapshot(
+ "root-l0", root, "", 0, {"md5": "old-l0", "abstract": "old root"}
+ ),
+ "root-l1": VectorRecordSnapshot(
+ "root-l1", root, "", 1, {"md5": "old-l1", "abstract": "old overview"}
+ ),
+ "a-l2": VectorRecordSnapshot(
+ "a-l2",
+ f"{root}/a.md",
+ "a.md",
+ 2,
+ {"md5": "old-file", "abstract": "old file"},
+ ),
+ }
+ snapshot = RFVSnapshot(
+ RequestIntent(root, "semantic_and_vectors"),
+ RFVFormalSnapshot(
+ {
+ "": RFVEntry(root, "", True, {0: "new-l0", 1: "new-l1"}),
+ "a.md": RFVEntry(f"{root}/a.md", "a.md", False, {2: "new-file"}),
+ }
+ ),
+ VectorIndexSnapshot(
+ records,
+ frozenset({"id", "uri", "level", "md5", "abstract"}),
+ ),
+ )
+
+ _diff, plan = build_rfv_context_update_plan(
+ snapshot=snapshot, context_type="resource", account_id="acc"
+ )
+
+ assert plan.content_tree_actions == ()
+ assert plan.direct_index_actions == ()
+ entries = {entry.relative_path: entry for entry in plan.semantic_plan.tree.entries}
+ assert entries[""].semantic_action is SemanticAction.AGGREGATE
+ assert entries["a.md"].semantic_action is SemanticAction.GENERATE
+ assert entries[""].repair is True
+ assert entries["a.md"].repair is True
+ assert [(slot.level, slot.action) for slot in entries[""].index_slots] == [
+ (0, IndexAction.MERGE),
+ (1, IndexAction.MERGE),
+ ]
+ assert [(slot.level, slot.action) for slot in entries["a.md"].index_slots] == [
+ (2, IndexAction.MERGE)
+ ]
+
+
+def test_rfv_semantic_plan_generates_complete_file_without_reusable_abstract():
+ """A vectors-only L2 cannot satisfy a parent's first semantic aggregation."""
+ from openviking.storage.context_update_plan import (
+ SemanticAction,
+ build_rfv_context_update_plan,
+ )
+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot
+
+ root = "viking://resources/demo"
+ snapshot = RFVSnapshot(
+ RequestIntent(root, "semantic_and_vectors"),
+ RFVFormalSnapshot(
+ {
+ "": RFVEntry(root, "", True, {}),
+ "a.md": RFVEntry(f"{root}/a.md", "a.md", False, {2: "same"}),
+ }
+ ),
+ VectorIndexSnapshot(
+ {"a-l2": _record("a-l2", f"{root}/a.md", "a.md", 2, "same")},
+ frozenset({"id", "uri", "level", "md5", "abstract"}),
+ ),
+ source_contents={(f"{root}/a.md", 2): b"current file body"},
+ )
+
+ _diff, plan = build_rfv_context_update_plan(
+ snapshot=snapshot, context_type="resource", account_id="acc"
+ )
+
+ entries = {entry.relative_path: entry for entry in plan.semantic_plan.tree.entries}
+ assert entries[""].semantic_action is SemanticAction.AGGREGATE
+ assert entries["a.md"].semantic_action is SemanticAction.GENERATE
+
+
+def test_rfv_semantic_file_uses_minimal_plan_without_duplicate_direct_upsert():
+ from openviking.storage.context_update_plan import build_rfv_context_update_plan
+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot
+
+ uri = "viking://resources/demo.md"
+ snapshot = RFVSnapshot(
+ RequestIntent(uri, "semantic_and_vectors", force=True),
+ RFVFormalSnapshot({"": RFVEntry(uri, "", False, {2: "current"})}),
+ VectorIndexSnapshot(
+ {"file-l2": _record("file-l2", uri, "", 2, "current")},
+ frozenset({"id", "uri", "level", "md5", "abstract"}),
+ ),
+ )
+
+ _diff, plan = build_rfv_context_update_plan(
+ snapshot=snapshot, context_type="resource", account_id="acc"
+ )
+
+ assert plan.semantic_plan is not None
+ assert plan.direct_index_actions == ()
+ assert plan.file_refresh is None
+ entry = plan.semantic_plan.tree.entries[0]
+ assert entry.semantic_action.value == "generate"
+ assert entry.slot(2).record_id == "file-l2"
+
+
+def test_rfv_semantic_file_is_noop_when_fingerprint_and_scalars_are_complete():
+ from openviking.storage.context_update_plan import build_rfv_context_update_plan
+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot
+
+ uri = "viking://resources/demo.md"
+ snapshot = RFVSnapshot(
+ RequestIntent(uri, "semantic_and_vectors"),
+ RFVFormalSnapshot({"": RFVEntry(uri, "", False, {2: "same"})}),
+ VectorIndexSnapshot(
+ {"file-l2": _record("file-l2", uri, "", 2, "same")},
+ frozenset({"id", "uri", "level", "md5", "abstract"}),
+ ),
+ )
+
+ _diff, plan = build_rfv_context_update_plan(
+ snapshot=snapshot, context_type="resource", account_id="acc"
+ )
+
+ assert plan.is_noop()
+
+
+def test_rfv_semantic_file_applies_scalar_only_update_without_refresh():
+ from openviking.storage.context_update_plan import build_rfv_context_update_plan
+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot
+ from openviking.storage.resource_rnfv import ScalarIntent
+
+ uri = "viking://resources/demo.md"
+ snapshot = RFVSnapshot(
+ RequestIntent(
+ uri,
+ "semantic_and_vectors",
+ scalar_intents=(ScalarIntent("search_tags", "replace", ("team=search",)),),
+ ),
+ RFVFormalSnapshot({"": RFVEntry(uri, "", False, {2: "same"})}),
+ VectorIndexSnapshot(
+ {"file-l2": _record("file-l2", uri, "", 2, "same")},
+ frozenset({"id", "uri", "level", "md5", "abstract", "search_tags"}),
+ ),
+ )
+
+ _diff, plan = build_rfv_context_update_plan(
+ snapshot=snapshot, context_type="resource", account_id="acc"
+ )
+
+ assert plan.file_refresh is None
+ assert [action.action.value for action in plan.direct_index_actions] == ["update_fields"]
+
+
+def test_rfv_semantic_file_missing_l2_uses_minimal_plan_with_request_scalars():
+ from openviking.storage.context_update_plan import build_rfv_context_update_plan
+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot
+ from openviking.storage.resource_rnfv import ScalarIntent
+
+ uri = "viking://resources/demo.md"
+ snapshot = RFVSnapshot(
+ RequestIntent(
+ uri,
+ "semantic_and_vectors",
+ scalar_intents=(ScalarIntent("search_tags", "replace", ("team=search",)),),
+ ),
+ RFVFormalSnapshot({"": RFVEntry(uri, "", False, {2: "same"})}),
+ VectorIndexSnapshot(
+ {},
+ frozenset({"id", "uri", "level", "md5", "abstract", "search_tags"}),
+ ),
+ )
+
+ _diff, plan = build_rfv_context_update_plan(
+ snapshot=snapshot, context_type="resource", account_id="acc"
+ )
+
+ assert plan.file_refresh is None
+ assert plan.semantic_plan is not None
+ assert plan.direct_index_actions == ()
+ entry = plan.semantic_plan.tree.entries[0]
+ assert entry.semantic_action.value == "generate"
+ assert entry.slot(2).field_patch.values == {"search_tags": ["team=search"]}
+ assert entry.slot(2).field_patch.modes == {"search_tags": "replace"}
+
+
+def action_record_id(uri: str, level: int) -> str:
+ from openviking.storage.vector_ids import vector_record_id
+
+ return vector_record_id("acc", uri, level)
+
+
+class _SnapshotFS:
+ def __init__(self, root: str):
+ self.root = root
+ self.tree = AsyncMock(
+ return_value=[
+ {"uri": f"{root}/docs", "rel_path": "docs", "isDir": True},
+ {
+ "uri": f"{root}/docs/.abstract.md",
+ "rel_path": "docs/.abstract.md",
+ "isDir": False,
+ },
+ {
+ "uri": f"{root}/docs/.overview.md",
+ "rel_path": "docs/.overview.md",
+ "isDir": False,
+ },
+ {"uri": f"{root}/docs/a.md", "rel_path": "docs/a.md", "isDir": False},
+ ]
+ )
+ self.stat = AsyncMock(return_value={"isDir": True})
+ self.read_file_bytes = AsyncMock(
+ side_effect=lambda uri, ctx=None: {
+ f"{root}/docs/.abstract.md": b"docs abstract",
+ f"{root}/docs/.overview.md": b"docs overview",
+ f"{root}/docs/a.md": b"file body",
+ }[uri]
+ )
+
+
+class _SnapshotDB:
+ def __init__(self):
+ self.get_incremental_inventory_under_uri = AsyncMock(return_value={})
+
+
+async def test_build_rfv_snapshot_reads_one_f_inventory_one_v_inventory_and_each_source_once():
+ from openviking.storage.resource_rfv import build_rfv_snapshot
+ from openviking.utils.content_hash import content_md5
+
+ root = "viking://resources/demo"
+ fs = _SnapshotFS(root)
+ db = _SnapshotDB()
+
+ snapshot = await build_rfv_snapshot(
+ viking_fs=fs,
+ vikingdb=db,
+ target_uri=root,
+ ctx=object(),
+ request_intent=RequestIntent(root, "vectors_only"),
+ recursive=True,
+ root_is_dir=True,
+ )
+
+ fs.tree.assert_awaited_once()
+ fs.stat.assert_not_awaited()
+ db.get_incremental_inventory_under_uri.assert_awaited_once()
+ assert fs.read_file_bytes.await_count == 3
+ assert snapshot.formal.entries["docs"].level_md5s == {
+ 0: content_md5(b"docs abstract"),
+ 1: content_md5(b"docs overview"),
+ }
+ assert snapshot.formal.entries["docs/a.md"].level_md5s == {2: content_md5(b"file body")}
+ assert snapshot.source_contents == {
+ (f"{root}/docs", 0): "docs abstract",
+ (f"{root}/docs", 1): "docs overview",
+ (f"{root}/docs/a.md", 2): b"file body",
+ }
+
+
+async def test_build_rfv_snapshot_non_recursive_directory_reads_only_direct_children():
+ from openviking.storage.resource_rfv import build_rfv_snapshot
+
+ root = "viking://resources/demo"
+ fs = _SnapshotFS(root)
+ fs.ls = AsyncMock(
+ return_value=[
+ {"uri": f"{root}/docs", "name": "docs", "isDir": True},
+ {"uri": f"{root}/a.md", "name": "a.md", "isDir": False},
+ {"uri": f"{root}/.abstract.md", "name": ".abstract.md", "isDir": False},
+ {"uri": f"{root}/.overview.md", "name": ".overview.md", "isDir": False},
+ ]
+ )
+ db = _SnapshotDB()
+
+ snapshot = await build_rfv_snapshot(
+ viking_fs=fs,
+ vikingdb=db,
+ target_uri=root,
+ ctx=object(),
+ request_intent=RequestIntent(root, "vectors_only"),
+ recursive=False,
+ )
+
+ fs.tree.assert_not_awaited()
+ fs.ls.assert_awaited_once_with(root, node_limit=None, show_all_hidden=True, ctx=ANY)
+ assert set(snapshot.formal.entries) == {"", "docs", "a.md"}
+ assert snapshot.input_only_paths == frozenset({"docs", "a.md"})
+ assert db.get_incremental_inventory_under_uri.await_args.kwargs["recursive"] is False
+ assert (
+ db.get_incremental_inventory_under_uri.await_args.kwargs["include_direct_children"] is True
+ )
+
+
+async def test_vectors_only_force_plan_preserves_existing_file_summary():
+ from openviking.storage.context_update_plan import (
+ ContextUpdatePlan,
+ IndexAction,
+ build_rfv_context_update_plan,
+ )
+ from openviking.storage.resource_rfv import build_rfv_snapshot
+
+ uri = "viking://resources/demo/a.md"
+ fs = SimpleNamespace(
+ stat=AsyncMock(return_value={"isDir": False}),
+ read_file_bytes=AsyncMock(return_value=b"current body"),
+ )
+ db = SimpleNamespace(
+ get_incremental_inventory_under_uri=AsyncMock(
+ return_value={
+ "a-l2": {
+ "id": "a-l2",
+ "uri": uri,
+ "level": 2,
+ "md5": "old-md5",
+ "abstract": "existing summary",
+ }
+ }
+ )
+ )
+
+ snapshot = await build_rfv_snapshot(
+ viking_fs=fs,
+ vikingdb=db,
+ target_uri=uri,
+ ctx=object(),
+ request_intent=RequestIntent(uri, "vectors_only", force=True),
+ recursive=False,
+ )
+ _, plan = build_rfv_context_update_plan(
+ snapshot=snapshot,
+ context_type="resource",
+ account_id="acc",
+ )
+
+ assert db.get_incremental_inventory_under_uri.await_args.kwargs["output_fields"] == [
+ "abstract",
+ "id",
+ "level",
+ "md5",
+ "uri",
+ ]
+ assert len(plan.direct_index_actions) == 1
+ action = plan.direct_index_actions[0]
+ assert action.action is IndexAction.MERGE
+ assert action.record_id == "a-l2"
+ assert action.summary == "existing summary"
+ assert (
+ ContextUpdatePlan.from_dict(plan.to_dict()).direct_index_actions[0].summary
+ == "existing summary"
+ )
+
+
+def test_vectors_only_missing_file_uses_parent_overview_summary():
+ from openviking.storage.context_update_plan import IndexAction, build_rfv_context_update_plan
+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot
+
+ root = "viking://resources/demo"
+ file_uri = f"{root}/a.md"
+ snapshot = RFVSnapshot(
+ request=RequestIntent(root, "vectors_only"),
+ formal=RFVFormalSnapshot(
+ {
+ "": RFVEntry(root, "", True, {0: "root-l0", 1: "root-l1"}),
+ "a.md": RFVEntry(file_uri, "a.md", False, {2: "a-md5"}),
+ }
+ ),
+ vectors=VectorIndexSnapshot(
+ {
+ "root-l0": VectorRecordSnapshot(
+ "root-l0", root, "", 0, {"md5": "root-l0", "abstract": "root abstract"}
+ ),
+ "root-l1": VectorRecordSnapshot(
+ "root-l1", root, "", 1, {"md5": "root-l1", "abstract": "root overview"}
+ ),
+ },
+ frozenset({"id", "uri", "level", "md5"}),
+ ),
+ source_contents={(root, 1): "### a.md\nCurrent overview summary."},
+ )
+
+ _, plan = build_rfv_context_update_plan(
+ snapshot=snapshot,
+ context_type="resource",
+ account_id="acc",
+ )
+
+ assert len(plan.direct_index_actions) == 1
+ action = plan.direct_index_actions[0]
+ assert action.action is IndexAction.MERGE
+ assert action.uri == file_uri
+ assert action.summary == "Current overview summary."
+
+
+def test_vectors_only_reuses_one_overview_parse_per_parent(monkeypatch):
+ from openviking.storage import context_update_plan
+ from openviking.storage.context_update_plan import build_rfv_context_update_plan
+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot
+
+ root = "viking://resources/demo"
+ snapshot = RFVSnapshot(
+ request=RequestIntent(root, "vectors_only"),
+ formal=RFVFormalSnapshot(
+ {
+ "": RFVEntry(root, "", True, {0: "root-l0", 1: "root-l1"}),
+ "a.md": RFVEntry(f"{root}/a.md", "a.md", False, {2: "a-md5"}),
+ "b.md": RFVEntry(f"{root}/b.md", "b.md", False, {2: "b-md5"}),
+ }
+ ),
+ vectors=VectorIndexSnapshot(
+ {
+ "root-l0": VectorRecordSnapshot("root-l0", root, "", 0, {"md5": "root-l0"}),
+ "root-l1": VectorRecordSnapshot("root-l1", root, "", 1, {"md5": "root-l1"}),
+ },
+ frozenset({"id", "uri", "level", "md5"}),
+ ),
+ source_contents={(root, 1): "already loaded overview"},
+ )
+ calls = []
+
+ def parse(overview):
+ calls.append(overview)
+ return {"a.md": "a summary", "b.md": "b summary"}
+
+ monkeypatch.setattr(context_update_plan, "parse_overview_file_summaries", parse)
+
+ _, plan = build_rfv_context_update_plan(
+ snapshot=snapshot,
+ context_type="resource",
+ account_id="acc",
+ )
+
+ assert [action.summary for action in plan.direct_index_actions] == ["a summary", "b summary"]
+ assert calls == ["already loaded overview"]
+
+
+async def test_non_recursive_semantic_plan_reuses_direct_child_summaries_without_rebuilding_them():
+ from openviking.storage.context_update_plan import (
+ SemanticAction,
+ build_rfv_context_update_plan,
+ )
+ from openviking.storage.resource_rfv import build_rfv_snapshot
+
+ root = "viking://resources/demo"
+
+ class _OneLevelFS:
+ stat = AsyncMock(return_value={"isDir": True})
+ tree = AsyncMock(side_effect=AssertionError("non-recursive plan must not walk descendants"))
+ ls = AsyncMock(
+ return_value=[
+ {"uri": f"{root}/a.md", "name": "a.md", "isDir": False},
+ {"uri": f"{root}/child", "name": "child", "isDir": True},
+ {"uri": f"{root}/.abstract.md", "name": ".abstract.md", "isDir": False},
+ {"uri": f"{root}/.overview.md", "name": ".overview.md", "isDir": False},
+ ]
+ )
+ read_file_bytes = AsyncMock(
+ side_effect=lambda uri, ctx=None: {
+ f"{root}/.abstract.md": b"root abstract",
+ f"{root}/.overview.md": b"root overview",
+ }[uri]
+ )
+
+ records = {
+ "root-l0": {
+ "id": "root-l0",
+ "uri": root,
+ "level": 0,
+ "md5": "root-a",
+ "abstract": "root abstract",
+ },
+ "root-l1": {
+ "id": "root-l1",
+ "uri": root,
+ "level": 1,
+ "md5": "root-o",
+ "abstract": "root overview",
+ },
+ "a-l2": {
+ "id": "a-l2",
+ "uri": f"{root}/a.md",
+ "level": 2,
+ "md5": "a-md5",
+ "abstract": "a summary",
+ },
+ "child-l0": {
+ "id": "child-l0",
+ "uri": f"{root}/child",
+ "level": 0,
+ "md5": "child-a",
+ "abstract": "child abstract",
+ },
+ }
+ db = SimpleNamespace(get_incremental_inventory_under_uri=AsyncMock(return_value=records))
+
+ snapshot = await build_rfv_snapshot(
+ viking_fs=_OneLevelFS(),
+ vikingdb=db,
+ target_uri=root,
+ ctx=object(),
+ request_intent=RequestIntent(root, "semantic_and_vectors", force=True),
+ recursive=False,
+ )
+ _, plan = build_rfv_context_update_plan(
+ snapshot=snapshot, context_type="resource", account_id="acc"
+ )
+
+ assert db.get_incremental_inventory_under_uri.await_args.kwargs == {
+ "ctx": ANY,
+ "output_fields": ["abstract", "id", "level", "md5", "uri"],
+ "recursive": False,
+ "include_direct_children": True,
+ }
+ entries = {entry.relative_path: entry for entry in plan.semantic_plan.tree.entries}
+ assert entries[""].semantic_action is SemanticAction.AGGREGATE
+ assert entries["a.md"].semantic_action is SemanticAction.REUSE
+ assert entries["child"].semantic_action is SemanticAction.REUSE
+ assert entries["a.md"].slot(2).record_id == "a-l2"
+ assert entries["child"].slot(0).record_id == "child-l0"
+
+
+async def test_non_recursive_semantic_plan_rejects_direct_child_without_reusable_abstract():
+ from openviking.storage.context_update_plan import build_rfv_context_update_plan
+ from openviking.storage.resource_rfv import build_rfv_snapshot
+
+ root = "viking://resources/demo"
+
+ class _OneLevelFS:
+ stat = AsyncMock(return_value={"isDir": True})
+ ls = AsyncMock(
+ return_value=[
+ {"uri": f"{root}/child", "name": "child", "isDir": True},
+ {"uri": f"{root}/.abstract.md", "name": ".abstract.md", "isDir": False},
+ {"uri": f"{root}/.overview.md", "name": ".overview.md", "isDir": False},
+ ]
+ )
+ read_file_bytes = AsyncMock(
+ side_effect=lambda uri, ctx=None: {
+ f"{root}/.abstract.md": b"root abstract",
+ f"{root}/.overview.md": b"root overview",
+ }[uri]
+ )
+
+ db = SimpleNamespace(
+ get_incremental_inventory_under_uri=AsyncMock(
+ return_value={
+ "root-l0": {
+ "id": "root-l0",
+ "uri": root,
+ "level": 0,
+ "md5": "root-a",
+ "abstract": "root abstract",
+ },
+ "root-l1": {
+ "id": "root-l1",
+ "uri": root,
+ "level": 1,
+ "md5": "root-o",
+ "abstract": "root overview",
+ },
+ "child-l0": {
+ "id": "child-l0",
+ "uri": f"{root}/child",
+ "level": 0,
+ "md5": "child-a",
+ "abstract": "",
+ },
+ }
+ )
+ )
+ snapshot = await build_rfv_snapshot(
+ viking_fs=_OneLevelFS(),
+ vikingdb=db,
+ target_uri=root,
+ ctx=object(),
+ request_intent=RequestIntent(root, "semantic_and_vectors", force=True),
+ recursive=False,
+ )
+
+ with pytest.raises(
+ ValueError, match="non-recursive semantic plan lacks reusable abstract for child"
+ ):
+ build_rfv_context_update_plan(snapshot=snapshot, context_type="resource", account_id="acc")
+
+
+async def test_non_recursive_plan_keeps_hidden_children_and_never_mutates_input_only_nodes():
+ from openviking.storage.context_update_plan import build_rfv_context_update_plan
+ from openviking.storage.resource_rfv import build_rfv_snapshot
+ from openviking.utils.ingest_options import IngestOptions
+
+ root = "viking://resources/demo"
+
+ class _OneLevelFS:
+ stat = AsyncMock(return_value={"isDir": True})
+ ls = AsyncMock(
+ return_value=[
+ {"uri": f"{root}/.env", "name": ".env", "isDir": False},
+ {"uri": f"{root}/a.md", "name": "a.md", "isDir": False},
+ {"uri": f"{root}/.abstract.md", "name": ".abstract.md", "isDir": False},
+ {"uri": f"{root}/.overview.md", "name": ".overview.md", "isDir": False},
+ ]
+ )
+ read_file_bytes = AsyncMock(
+ side_effect=lambda uri, ctx=None: {
+ f"{root}/.abstract.md": b"root abstract",
+ f"{root}/.overview.md": b"root overview",
+ }[uri]
+ )
+
+ db = SimpleNamespace(
+ get_incremental_inventory_under_uri=AsyncMock(
+ return_value={
+ "root-l0": {"id": "root-l0", "uri": root, "level": 0, "abstract": "old"},
+ "root-l1": {"id": "root-l1", "uri": root, "level": 1, "abstract": "old"},
+ "env-l2": {
+ "id": "env-l2",
+ "uri": f"{root}/.env",
+ "level": 2,
+ "md5": "old-env",
+ "abstract": "env summary",
+ "search_tags": ["team=old"],
+ },
+ "a-l2": {
+ "id": "a-l2",
+ "uri": f"{root}/a.md",
+ "level": 2,
+ "md5": "old-a",
+ "abstract": "a summary",
+ "search_tags": ["team=old"],
+ },
+ }
+ )
+ )
+ request = RequestIntent.from_ingest_options(
+ target_uri=root,
+ processing_mode="semantic_and_vectors",
+ force=True,
+ ingest_options=IngestOptions.from_search_tags(["team=new"], mode="replace"),
+ )
+
+ snapshot = await build_rfv_snapshot(
+ viking_fs=_OneLevelFS(),
+ vikingdb=db,
+ target_uri=root,
+ ctx=object(),
+ request_intent=request,
+ recursive=False,
+ )
+ _, plan = build_rfv_context_update_plan(
+ snapshot=snapshot,
+ context_type="resource",
+ account_id="acc",
+ )
+
+ assert {".env", "a.md"}.issubset(snapshot.formal.entries)
+ assert snapshot.input_only_paths == frozenset({".env", "a.md"})
+ assert {action.record_id for action in plan.direct_index_actions}.isdisjoint({"env-l2", "a-l2"})
+
+
+async def test_build_rfv_snapshot_keeps_dotfiles_and_excludes_control_files():
+ from openviking.storage.resource_rfv import build_rfv_snapshot
+
+ root = "viking://resources/demo"
+ fs = _SnapshotFS(root)
+ fs.tree.return_value = [
+ {"uri": f"{root}/.env", "rel_path": ".env", "isDir": False},
+ {"uri": f"{root}/.source.json", "rel_path": ".source.json", "isDir": False},
+ {"uri": f"{root}/.path.ovlock", "rel_path": ".path.ovlock", "isDir": False},
+ ]
+ fs.read_file_bytes.side_effect = lambda uri, ctx=None: b"KEY=value"
+
+ snapshot = await build_rfv_snapshot(
+ viking_fs=fs,
+ vikingdb=_SnapshotDB(),
+ target_uri=root,
+ ctx=object(),
+ request_intent=RequestIntent(root, "vectors_only"),
+ )
+
+ assert set(snapshot.formal.entries) == {"", ".env"}
+
+
+async def test_build_rfv_snapshot_reads_skill_source_metadata_from_same_f_inventory():
+ from openviking.storage.resource_rfv import build_rfv_snapshot
+
+ root = "viking://user/alice/skills/demo"
+ fs = _SnapshotFS(root)
+ fs.tree.return_value = [
+ {"uri": f"{root}/.source.json", "rel_path": ".source.json", "isDir": False}
+ ]
+ fs.read_file_bytes.side_effect = lambda uri, ctx=None: (
+ b'{"type":"git","source":"https://example/repo"}'
+ )
+
+ snapshot = await build_rfv_snapshot(
+ viking_fs=fs,
+ vikingdb=_SnapshotDB(),
+ target_uri=root,
+ ctx=object(),
+ request_intent=RequestIntent(root, "semantic_and_vectors"),
+ )
+
+ assert snapshot.source_metadata == {
+ "kind": "git",
+ "uri": "https://example/repo",
+ "path": "https://example/repo",
+ }
+ assert ".source.json" not in snapshot.formal.entries
+
+
+def test_rfv_resolver_rejects_formal_entry_outside_request_scope():
+ from openviking.storage.resource_rfv import (
+ RFVEntry,
+ RFVFormalSnapshot,
+ RFVSnapshot,
+ resolve_rfv_state,
+ )
+
+ root = "viking://resources/demo"
+ snapshot = RFVSnapshot(
+ RequestIntent(root, "vectors_only"),
+ RFVFormalSnapshot(
+ {
+ "outside.md": RFVEntry(
+ "viking://resources/other.md",
+ "outside.md",
+ False,
+ {2: "fingerprint"},
+ )
+ }
+ ),
+ VectorIndexSnapshot({}, frozenset({"id", "uri", "level", "md5"})),
+ )
+
+ with pytest.raises(ValueError, match="outside target"):
+ resolve_rfv_state(snapshot)
+
+
+async def test_build_rfv_snapshot_rejects_denied_reused_formal_inventory():
+ from openviking.storage.resource_rfv import build_rfv_snapshot
+
+ root = "viking://resources/demo"
+ fs = _SnapshotFS(root)
+
+ with pytest.raises(ValueError, match="incomplete formal tree"):
+ await build_rfv_snapshot(
+ viking_fs=fs,
+ vikingdb=_SnapshotDB(),
+ target_uri=root,
+ ctx=object(),
+ request_intent=RequestIntent(root, "vectors_only"),
+ root_is_dir=True,
+ formal_inventory=(
+ True,
+ [
+ {
+ "uri": f"{root}/private.md",
+ "isDir": False,
+ "access": "denied",
+ }
+ ],
+ True,
+ ),
+ )
diff --git a/tests/storage/test_semantic_executor_incremental.py b/tests/storage/test_semantic_executor_incremental.py
index 84ddb057d..74213eb4b 100644
--- a/tests/storage/test_semantic_executor_incremental.py
+++ b/tests/storage/test_semantic_executor_incremental.py
@@ -138,15 +138,19 @@ class _FakeProcessor:
ingest_options=None,
file_md5=None,
file_content=None,
+ materialize_content=False,
scalar_override=None,
action="merge",
+ field_patch=None,
):
self.vectorized_files.append(file_path)
self.file_ingest_options[file_path] = ingest_options
self.file_md5s[file_path] = file_md5
self.file_contents[("vector", file_path)] = file_content
+ self.file_contents[("materialize_content", file_path)] = materialize_content
self.file_contents[("action", file_path)] = action
self.file_contents[("scalar_override", file_path)] = scalar_override
+ self.file_contents[("field_patch", file_path)] = field_patch
return True
async def _vectorize_directory(
@@ -157,15 +161,20 @@ class _FakeProcessor:
overview,
ctx=None,
ingest_options=None,
+ skill_source_path="",
scalar_overrides=None,
actions=None,
+ field_patches=None,
include_abstract=True,
include_overview=True,
+ md5s=None,
):
self.directory_ingest_options[uri] = ingest_options
self.vectorized_dirs.append(uri)
self.file_contents[("actions", uri)] = actions
self.file_contents[("scalar_overrides", uri)] = scalar_overrides
+ self.file_contents[("field_patches", uri)] = field_patches
+ self.file_contents[("md5s", uri)] = md5s
return {
level for level, included in ((0, include_abstract), (1, include_overview)) if included
}
@@ -211,6 +220,172 @@ class _FakeProcessor:
)
+async def test_file_root_semantic_plan_runs_file_node_without_listing_or_sidecars(monkeypatch):
+ from openviking.storage.context_update_plan import (
+ IndexSlot,
+ SemanticPlan,
+ SemanticTreeEntry,
+ SemanticTreeSnapshot,
+ )
+
+ file_uri = "viking://resources/report.md"
+ fake_fs = _FakeVikingFS(tree={}, file_contents={file_uri: b"report body"})
+ fake_fs.ls = AsyncMock(side_effect=AssertionError("semantic plan must not list file root"))
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
+ )
+ processor = _FakeProcessor(fake_fs)
+ plan = SemanticPlan(
+ file_uri,
+ "resource",
+ SemanticTreeSnapshot(
+ (
+ SemanticTreeEntry(
+ "",
+ "file",
+ "modified",
+ "generate",
+ md5="fresh-md5",
+ index_slots=(IndexSlot(2, "report-l2", action="upsert"),),
+ ),
+ )
+ ),
+ )
+ executor = SemanticTreeExecutor(
+ processor=processor,
+ context_type="resource",
+ max_concurrent_llm=1,
+ ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
+ semantic_plan=plan,
+ )
+
+ await executor.run(file_uri)
+
+ assert processor.summarized_files == [file_uri]
+ assert processor.vectorized_files == [file_uri]
+ assert fake_fs.writes == []
+ assert executor.get_stats().indexed_records == 1
+
+
+async def test_semantic_plan_repair_regenerates_and_reembeds_directory(monkeypatch):
+ from openviking.storage.context_update_plan import (
+ IndexSlot,
+ SemanticPlan,
+ SemanticTreeEntry,
+ SemanticTreeSnapshot,
+ )
+ from openviking.utils.content_hash import content_md5
+
+ root_uri = "viking://resources/root"
+ fake_fs = _FakeVikingFS(tree={root_uri: []}, file_contents={})
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_executor.get_openviking_config",
+ lambda: SimpleNamespace(semantic=SimpleNamespace(overview_sample_limit=32)),
+ )
+ processor = _FakeProcessor(fake_fs)
+ plan = SemanticPlan(
+ root_uri,
+ "resource",
+ SemanticTreeSnapshot(
+ (
+ SemanticTreeEntry(
+ "",
+ "directory",
+ "unchanged",
+ "aggregate",
+ index_slots=(
+ IndexSlot(0, "root-l0", {"abstract": "stale abstract"}, action="upsert"),
+ IndexSlot(1, "root-l1", {"abstract": "stale overview"}, action="upsert"),
+ ),
+ repair=True,
+ ),
+ )
+ ),
+ )
+ executor = SemanticTreeExecutor(
+ processor=processor,
+ context_type="resource",
+ max_concurrent_llm=1,
+ ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
+ semantic_plan=plan,
+ generation_trigger="reindex",
+ source_contents={
+ (root_uri, 0): "current abstract",
+ (root_uri, 1): "current overview",
+ },
+ )
+
+ await executor.run(root_uri)
+
+ assert processor.generated_overviews == [root_uri]
+ assert processor.vectorized_dirs == [root_uri]
+ assert processor.file_contents[("md5s", root_uri)] == {
+ 0: content_md5(b"abstract"),
+ 1: content_md5(b"FILES:"),
+ }
+
+
+async def test_skill_plan_repair_regenerates_and_reembeds_skill_root(monkeypatch):
+ from openviking.storage.context_update_plan import (
+ IndexSlot,
+ SemanticPlan,
+ SemanticTreeEntry,
+ SemanticTreeSnapshot,
+ )
+
+ root_uri = "viking://user/alice/skills/demo"
+ fake_fs = _FakeVikingFS(tree={root_uri: []}, file_contents={})
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_executor.get_openviking_config",
+ lambda: SimpleNamespace(semantic=SimpleNamespace(overview_sample_limit=32)),
+ )
+ processor = _FakeProcessor(fake_fs)
+ processor._skill_root_semantics = AsyncMock(return_value=("skill overview", "skill abstract"))
+ plan = SemanticPlan(
+ root_uri,
+ "skill",
+ SemanticTreeSnapshot(
+ (
+ SemanticTreeEntry(
+ "",
+ "directory",
+ "unchanged",
+ "aggregate",
+ index_slots=(
+ IndexSlot(0, "skill-l0", {"abstract": "old"}, action="upsert"),
+ IndexSlot(1, "skill-l1", {"abstract": "old"}, action="upsert"),
+ ),
+ repair=True,
+ ),
+ )
+ ),
+ )
+ executor = SemanticTreeExecutor(
+ processor=processor,
+ context_type="skill",
+ max_concurrent_llm=1,
+ ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
+ semantic_plan=plan,
+ generation_trigger="content_write",
+ source_contents={(f"{root_uri}/SKILL.md", 2): b"name: demo"},
+ )
+
+ await executor.run(root_uri)
+
+ processor._skill_root_semantics.assert_awaited_once()
+ assert processor._skill_root_semantics.await_args.kwargs["regenerate"] is True
+ assert processor.vectorized_dirs == [root_uri]
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("sidecar_state", ["valid", "malformed_overview", "malformed_abstract"])
async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypatch, sidecar_state):
diff --git a/tests/storage/test_semantic_processor_lock_ownership.py b/tests/storage/test_semantic_processor_lock_ownership.py
index ded8fd46f..322144086 100644
--- a/tests/storage/test_semantic_processor_lock_ownership.py
+++ b/tests/storage/test_semantic_processor_lock_ownership.py
@@ -9,8 +9,11 @@ from unittest.mock import AsyncMock
import pytest
from openviking.storage.queuefs.process_result import ProcessOutcome
+from openviking.storage.queuefs.semantic_executor import SemanticTreeStats
+from openviking.storage.queuefs.semantic_lock import SemanticLockScope
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
+from openviking.storage.queuefs.semantic_work import SemanticMessageWork
def _processor():
@@ -68,6 +71,73 @@ async def test_memory_semantic_directory_does_not_release_borrowed_lock(monkeypa
assert pathlock.release_calls == []
+async def test_durable_plan_waits_for_embeddings_before_releasing_handoff_lock(monkeypatch):
+ from openviking.storage.context_update_plan import (
+ SemanticPlan,
+ SemanticTreeEntry,
+ SemanticTreeSnapshot,
+ )
+
+ events = []
+ pathlock = _FakePathLock()
+ lease = {"id": "durable-plan-lock"}
+ plan = SemanticPlan(
+ "viking://resources/demo",
+ "resource",
+ SemanticTreeSnapshot((SemanticTreeEntry("", "directory", "unchanged", "aggregate"),)),
+ )
+ msg = SemanticMsg(
+ uri=plan.root_uri,
+ context_type="resource",
+ telemetry_id="durable-plan",
+ lock_handoff={"owner_id": "producer"},
+ plan=plan,
+ )
+
+ class Tracker:
+ async def wait_for_embeddings(self, telemetry_id, **kwargs):
+ assert telemetry_id == msg.telemetry_id
+ events.append("embeddings-settled")
+
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_work.get_request_wait_tracker", lambda: Tracker()
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_lock.get_viking_fs",
+ lambda: _FakeVikingFS(pathlock),
+ )
+
+ work = SemanticMessageWork(SimpleNamespace(), msg, caller_lock=None)
+ work.scope = SemanticLockScope(lease, _owned=True)
+
+ await work.finish_processing(True)
+
+ assert events == ["embeddings-settled"]
+ assert pathlock.release_calls == ["durable-plan-lock"]
+
+
+def test_semantic_tree_stats_aggregate_multiple_plans_for_one_request():
+ telemetry_id = "reindex-request-stats"
+ SemanticProcessor._cache_tree_stats(
+ telemetry_id,
+ "viking://resources/one",
+ SemanticTreeStats(total_nodes=2, done_nodes=2, indexed_records=2, failures=["one failed"]),
+ )
+ SemanticProcessor._cache_tree_stats(
+ telemetry_id,
+ "viking://resources/two",
+ SemanticTreeStats(total_nodes=3, done_nodes=3, indexed_records=3, failures=["two failed"]),
+ )
+
+ stats = SemanticProcessor.consume_tree_stats(telemetry_id=telemetry_id)
+
+ assert stats.total_nodes == 5
+ assert stats.done_nodes == 5
+ assert stats.indexed_records == 5
+ assert stats.failures == ["one failed", "two failed"]
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("context_type", ["resource", "memory", "skill"])
async def test_missing_root_is_acked_before_lock_scope_is_resolved(monkeypatch, context_type):
diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py
index c6cefcce5..bf5042242 100644
--- a/tests/test_config_loader.py
+++ b/tests/test_config_loader.py
@@ -130,6 +130,7 @@ def test_runtime_concurrency_uses_scope_specific_defaults():
assert config.queue_workers.external_parse.max_concurrent == 4
assert config.queue_workers.add_resource.max_concurrent == 4
+ assert config.queue_workers.reindex.max_concurrent == 4
assert config.queue_workers.add_resource.file_operation_concurrency == 16
assert config.queue_workers.add_resource.file_vectorization_concurrency == 8
assert config.queue_workers.session_commit.max_concurrent == 8
@@ -154,6 +155,7 @@ def test_runtime_concurrency_accepts_separate_values():
"file_operation_concurrency": 20,
"file_vectorization_concurrency": 12,
},
+ "reindex": {"max_concurrent": 6},
"session_commit": {"max_concurrent": 50},
"external_task": {"max_concurrent": 11},
},
@@ -163,6 +165,7 @@ def test_runtime_concurrency_accepts_separate_values():
assert config.queue_workers.external_parse.max_concurrent == 9
assert config.queue_workers.add_resource.max_concurrent == 7
+ assert config.queue_workers.reindex.max_concurrent == 6
assert config.queue_workers.add_resource.file_operation_concurrency == 20
assert config.queue_workers.add_resource.file_vectorization_concurrency == 12
assert config.queue_workers.session_commit.max_concurrent == 50
@@ -173,7 +176,7 @@ def test_runtime_concurrency_accepts_separate_values():
@pytest.mark.parametrize("value", [0, -1])
def test_queue_worker_concurrency_rejects_non_positive_value(value):
with pytest.raises(ValueError) as exc_info:
- QueueWorkersConfig(add_resource={"max_concurrent": value})
+ QueueWorkersConfig(reindex={"max_concurrent": value})
assert exc_info.value.errors()[0]["type"] == "greater_than"
diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.py
index e64b7263c..4af2395f0 100644
--- a/tests/unit/test_vectorize_file_strategy.py
+++ b/tests/unit/test_vectorize_file_strategy.py
@@ -325,6 +325,33 @@ async def test_vectorize_image_downsamples_large_embedding_input(monkeypatch):
assert fs.read_file_bytes_calls == 1
+async def test_vectorize_image_uses_supplied_bytes_without_second_read(monkeypatch):
+ queue = DummyQueue()
+ original = _jpeg_bytes(20, 20)
+ fs = DummyFS(b"must not be read")
+ monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))
+ monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs)
+ monkeypatch.setattr(
+ embedding_utils,
+ "get_openviking_config",
+ lambda: types.SimpleNamespace(
+ embedding=types.SimpleNamespace(text_source="content_only", max_input_tokens=1000),
+ image=ImageConfig(preview_max_dimension=64, max_file_size_mb=100.0),
+ ),
+ )
+
+ await embedding_utils.vectorize_file(
+ file_path="viking://resources/docs/image.jpg",
+ summary_dict={"name": "image.jpg", "summary": "image"},
+ parent_uri="viking://resources/docs",
+ ctx=DummyReq(),
+ file_content=original,
+ )
+
+ assert fs.read_file_bytes_calls == 0
+
+
@pytest.mark.asyncio
async def test_vectorize_file_registers_request_wait_with_embedding_msg_id(monkeypatch):
queue = DummyQueueWithId()
@@ -678,6 +705,26 @@ async def test_vectorize_directory_meta_applies_merge_per_level(monkeypatch):
assert queue.items[1].action.value == "upsert"
+async def test_vectorize_directory_meta_writes_per_level_source_md5(monkeypatch):
+ queue = DummyQueue()
+ monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))
+ monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: DummyFS("ignored"))
+
+ await embedding_utils.vectorize_directory_meta(
+ uri="viking://user/default/resources/demo",
+ abstract="demo abstract",
+ overview="demo overview",
+ ctx=DummyReq(),
+ md5s={0: "abstract-md5", 1: "overview-md5"},
+ )
+
+ assert [item.context_data["md5"] for item in queue.items] == [
+ "abstract-md5",
+ "overview-md5",
+ ]
+
+
@pytest.mark.asyncio
async def test_vectorize_directory_meta_l1_abstract_is_overview(monkeypatch):
"""L1 records must carry the overview in the abstract scalar so Rerank