Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Feat/rfv reindex (#5416)
Jiahui Zhou0f31ece6b821
diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rsindex c49055cea..23f942a2b 100644--- a/crates/ov_cli/src/client.rs+++ b/crates/ov_cli/src/client.rs@@ -474,7 +474,7 @@ impl HttpClient {uri: &str,mode: &str,wait: bool,- dry_run: bool,+ force: bool,tags: Vec<String>,tag_mode: &str,recursive: bool,@@ -483,8 +483,10 @@ impl HttpClient {"uri": uri,"mode": mode,"wait": wait,- "dry_run": dry_run,});+ if force {+ body["force"] = serde_json::json!(true);+ }if !recursive {body["recursive"] = serde_json::json!(false);}diff --git a/crates/ov_cli/src/commands/content.rs b/crates/ov_cli/src/commands/content.rsindex 8bdf0f231..dfff6781e 100644--- a/crates/ov_cli/src/commands/content.rs+++ b/crates/ov_cli/src/commands/content.rs@@ -87,7 +87,7 @@ pub async fn reindex(uri: &str,mode: &str,wait: bool,- dry_run: bool,+ force: bool,tags: Vec<String>,tag_mode: &str,recursive: bool,@@ -95,7 +95,7 @@ pub async fn reindex(compact: bool,) -> Result<()> {let result = client- .reindex(uri, mode, wait, dry_run, tags, tag_mode, recursive)+ .reindex(uri, mode, wait, force, tags, tag_mode, recursive).await?;crate::output::output_success(result, output_format, compact);Ok(())diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rsindex 160b51d92..2fb5e905d 100644--- a/crates/ov_cli/src/handlers.rs+++ b/crates/ov_cli/src/handlers.rs@@ -1455,7 +1455,7 @@ pub async fn handle_reindex(uri: String,mode: String,wait: bool,- dry_run: bool,+ force: bool,tags: Vec<String>,tag_mode: String,recursive: bool,@@ -1467,7 +1467,7 @@ pub async fn handle_reindex(&uri,&mode,wait,- dry_run,+ force,tags,&tag_mode,recursive,diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rsindex a81bd6450..c1cc0351c 100644--- a/crates/ov_cli/src/help_ui.rs+++ b/crates/ov_cli/src/help_ui.rs@@ -1388,10 +1388,6 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[label: "ov reindex viking://projects/acme --mode semantic_and_vectors --recursive=false",description: "Refresh only the target directory semantics and vectors, without sub-directory.",},- HelpItem {- label: "ov reindex viking://projects/acme --mode prune_orphans --dry-run",- description: "Preview orphan vector cleanup.",- },],next_steps: &[HelpItem {@@ -2956,8 +2952,8 @@ mod tests {.expect("reindex help should render"),);- assert!(rendered.contains("--mode <vectors_only|semantic_and_vectors|prune_orphans>"));- assert!(rendered.contains("--dry-run"));+ assert!(rendered.contains("--mode <vectors_only|semantic_and_vectors>"));+ assert!(rendered.contains("--force"));assert!(rendered.contains("--recursive <true|false>"));assert!(rendered.contains("Regenerate semantic artifacts, then vectors."));}diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rsindex d53c4dcab..d925484fa 100644--- a/crates/ov_cli/src/main.rs+++ b/crates/ov_cli/src/main.rs@@ -1279,11 +1279,11 @@ enum Commands {/// Viking URI#[arg(value_name = "uri")]uri: String,- /// Reindex mode: vectors_only rebuilds vectors; semantic_and_vectors regenerates semantic artifacts, then vectors; prune_orphans deletes orphan vector records+ /// Reindex mode: vectors_only rebuilds vectors; semantic_and_vectors regenerates semantic artifacts, then vectors#[arg(long,default_value = "vectors_only",- value_parser = ["vectors_only", "semantic_and_vectors", "prune_orphans"],+ value_parser = ["vectors_only", "semantic_and_vectors"],value_name = "mode",help_heading = "Common options")]@@ -1297,9 +1297,9 @@ enum Commands {help_heading = "Common options")]wait: bool,- /// Preview prune_orphans deletions without mutating vectors+ /// Rebuild all selected semantic/vector data without comparing fingerprints#[arg(long, help_heading = "Common options")]- dry_run: bool,+ force: bool,/// Comma-separated k=v retrieval tags for rebuilt vector records#[arg(long = "tags", value_delimiter = ',', value_name = "k=v", help_heading = "Common options")]tags: Vec<String>,@@ -3789,12 +3789,12 @@ async fn main() {uri,mode,wait,- dry_run,+ force,tags,tag_mode,recursive,} => {- handlers::handle_reindex(uri, mode, wait, dry_run, tags, tag_mode, recursive, ctx).await+ handlers::handle_reindex(uri, mode, wait, force, tags, tag_mode, recursive, ctx).await}Commands::Get { uri, local_path } => handlers::handle_get(uri, local_path, ctx).await,Commands::Find {@@ -5849,9 +5849,9 @@ mod tests {"reindex","viking://resources/demo","--mode",- "prune_orphans",+ "semantic_and_vectors","--wait=false",- "--dry-run",+ "--force","--tags","team=search","--tag-mode",@@ -5862,12 +5862,14 @@ mod tests {let cli = result.expect("reindex command should parse");match cli.command {Commands::Reindex {+ force,tags,tag_mode,recursive,..} => {assert_eq!(tags, vec!["team=search"]);+ assert!(force);assert_eq!(tag_mode, "append");assert!(!recursive);}diff --git a/docs/en/api/02-resources.md b/docs/en/api/02-resources.mdindex 33c6ee4d6..838fcd484 100644--- a/docs/en/api/02-resources.md+++ b/docs/en/api/02-resources.md@@ -210,7 +210,7 @@ This endpoint is the core entry point for resource management. It supports vario- When deleting a resource, OpenViking scans the self or peer memories targeted by the current context before deletion, removes the matching resource URI and content introduced by that `reason`, and refreshes the semantic index for the affected memories.- Other sources with `wait=false` finish source parsing, target resolution, and AGFS writes before returning. Only semantic and embedding queues continue asynchronously.- `processing_mode=vectors_only` does not call the VLM semantic-understanding stage and does not generate or refresh `.abstract.md` / `.overview.md`. For existing targets, it preserves existing semantic artifacts and existing semantic vectors. It still updates the resource tree, vectorizes current non-hidden files when `build_index=true`, and removes detail vectors for files deleted during refresh.-- `processing_mode` belongs to `add_resource`. The admin `reindex` API/CLI continues to use `mode` (`vectors_only`, `semantic_and_vectors`, `prune_orphans`) for maintenance operations on already-ingested data.+- `processing_mode` belongs to `add_resource`. The admin `reindex` API/CLI continues to use `mode` (`vectors_only`, `semantic_and_vectors`) for maintenance operations on already-ingested data.- When `watch_interval > 0`, the watch task binds to `to` if provided; otherwise it binds to the `root_uri` returned by this import. If no stable `root_uri` is available, the request fails and asks for an explicit `to`.- For Connector imports, `is_active=false` creates the paused Watch before submission. Native Feishu and Git imports carry `is_active` through the resource queue and create the Watch after resolving the imported resource URI. In both cases, the initial import still runs once and periodic scheduling remains disabled.- Feishu/Lark app-token imports do not pass `args.feishu_access_token`. OpenViking keeps the existing app credential flow and the SDK obtains an app/tenant token from `app_id` and `app_secret`. This mode supports both one-time imports and `watch_interval > 0`.diff --git a/docs/en/api/12-content.md b/docs/en/api/12-content.mdindex 6f663bce8..267e46206 100644--- a/docs/en/api/12-content.md+++ b/docs/en/api/12-content.md@@ -488,7 +488,7 @@ Content-Disposition: attachment; filename*=UTF-8''logo.png### set_tags()-Set explicit `k=v` tags used by retrieval filters. `replace` replaces existing tags, while `append` adds tags. When the target is a directory, `recursive=true` applies the update to files below it.+Set explicit `k=v` tags used by retrieval filters. `replace` replaces existing tags, `append` adds tags, and `clear` explicitly removes existing tags. When the target is a directory, `recursive=true` applies the update to files below it. Omitting `tags`, or passing an empty list with `replace`, is a no-op; only `clear` removes tags and it ignores any supplied tag values.**Python SDK**@@ -583,7 +583,7 @@ ov set-tags viking://resources/project/ \### reindex()-Reindex semantic and/or vector artifacts for existing content already stored in OpenViking. This is an operational maintenance API intended for scenarios such as embedding model changes, VLM changes, vector store rebuild, or post-upgrade repair of existing indexes.+Validate and repair semantic and/or vector artifacts for existing content already stored in OpenViking. Resource and skill targets use incremental RFV (Request / Formal / Vector) convergence by default: unchanged file fingerprints, directory L0/L1 visible-body fingerprints, and request scalars are skipped. Use `force=true` for an unconditional rebuild.This API operates on existing `viking://...` content. It does not import new files. For normal ingestion, use [Resources](02-resources.md).@@ -597,14 +597,14 @@ This API operates on existing `viking://...` content. It does not import new fil| Parameter | Type | Required | Default | Description ||-----------|------|----------|---------|-------------|| uri | str | Yes | - | Viking URI to reindex |-| mode | str | No | `vectors_only` | Reindex mode: `vectors_only`, `semantic_and_vectors`, or `prune_orphans` |+| mode | str | No | `vectors_only` | Reindex mode: `vectors_only` or `semantic_and_vectors` || wait | bool | No | `true` | Whether to wait for completion |-| dry_run | bool | No | `false` | Only valid with `mode="prune_orphans"`; report orphan vector records without deleting them |-| recursive | bool | No | `true` | Whether to process descendants recursively; `false` applies only to `semantic_and_vectors` on a `resource`, `memory`, or `skill` directory |+| force | bool | No | `false` | Skip fingerprint equality and reprocess every available resource/skill file and directory level in scope |+| recursive | bool | No | `true` | Whether to process descendants recursively; both resource/skill rebuild modes honor this option || tags | list[str] | No | `null` | Write tags to every successfully rebuilt vector record. Omitting tags, or passing an empty list with `replace`, preserves existing tags || tag_mode | str | No | `replace` | Tag write mode: `replace`, `append`, or `clear`; `clear` removes existing tags without requiring `tags` |-The HTTP request body rejects unknown fields. `uri` may use OpenViking path variables accepted by other content APIs; it is resolved before validation.+The HTTP request body ignores unknown fields to preserve compatibility while clients and servers evolve independently. `uri` may use OpenViking path variables accepted by other content APIs; it is resolved before validation.**Supported URI scopes**@@ -623,19 +623,18 @@ when reindexing a broader user namespace, session subtrees are skipped.**Modes**-- `vectors_only`: rebuilds vector-store records from currently recoverable source data without rewriting `.abstract.md` or `.overview.md`-- `semantic_and_vectors`: regenerates semantic artifacts first, then rebuilds vectors from the refreshed semantic outputs-- `prune_orphans`: deletes vector-store records under the requested URI whose source files no longer exist in the filesystem. With `dry_run=true`, it only reports how many records would be deleted.+- `vectors_only`: compares current source MD5 values with vector-record MD5 values through RFV and rebuilds only missing or stale L0/L1/L2 records; it does not rewrite `.abstract.md` or `.overview.md`+- `semantic_and_vectors`: uses the same RFV state and `ContextUpdatePlan` to drive semantic repair and L0/L1/L2 updates, without a second manual vector scanFor `resource` and `skill`, `semantic_and_vectors` refreshes directory/file semantic artifacts, including `.abstract.md` and `.overview.md`. For `memory`, it rebuilds the current persisted memory subtree semantics and vectors, but it does not replay historical extraction order.-For `semantic_and_vectors`, semantic generation and vector rebuilding are sequenced by the reindex executor. The semantic refresh step does not enqueue its own background vectorization work; vectors are rebuilt by the reindex step so `wait=true` reflects the reindex operation itself.+Resource/skill `vectors_only` and `semantic_and_vectors` share one F traversal and one narrow-field V inventory. Each source is read once during state resolution and that result is reused while executing the plan. `force=true` bypasses MD5 equality but still performs completeness and scope checks.-For a `resource` or `memory` directory, `recursive=false` regenerates only the target directory's `.abstract.md` and `.overview.md`, then rebuilds only that directory's L0/L1 vectors. Child directories do not regenerate semantic artifacts, and neither child directories nor files are re-vectorized. The target aggregation still reads existing summaries from deterministically sampled child directories; sampled direct files are summarized as inputs to the target aggregation. For a `skill` target, `recursive=false` regenerates the skill directory's L0/L1 semantic artifacts and vectors from `SKILL.md`, but does not rebuild the `SKILL.md` L2 vector. Namespace targets (such as `viking://user/<user_id>` or its `skills` container) do not support `recursive=false` with `semantic_and_vectors`; these requests are rejected before starting reindex work. Select a concrete resource, memory, or skill directory for a non-recursive refresh. This flag does not change existing behavior for `vectors_only` or `prune_orphans`.+For a resource/skill directory, `recursive=false` limits both modes to the target directory's own L0/L1 records. Direct children are read only as existing REUSE inputs for the target aggregation; descendants are neither scanned nor rebuilt. Namespace containers have no index records of their own, so namespace targets reject `recursive=false` instead of silently becoming a no-op. Memory remains on the existing reindex path.-For `prune_orphans`, source existence is checked against the filesystem. If an entire directory is missing, vector records for files and semantic sidecars below that directory, such as `.abstract.md` and `.overview.md`, are pruned together. `dry_run` is rejected for other modes.+When the F snapshot is complete, the RFV diff identifies records present in V but absent from F as orphans and removes them during the same reindex. Deletion fails closed when the F snapshot is incomplete.-When non-empty `tags` are provided, tags are included in the same upsert as each vector record produced by reindex; reindex does not call `set_tags` afterwards. Directory and namespace reindex operations apply tags to successfully rebuilt directory L0/L1 and leaf L2 records. `replace` overwrites existing tags, while `append` merges by key. `replace` with an empty tag list is a no-op. `clear` removes existing tags and does not require `tags`; if values are supplied with `clear`, they are ignored. `prune_orphans` produces no vectors and ignores both fields.+When non-empty `tags` are provided, tags are included in the same upsert as each vector record produced by reindex; reindex does not call `set_tags` afterwards. Directory and namespace reindex operations apply tags to successfully rebuilt directory L0/L1 and leaf L2 records. `replace` overwrites existing tags, while `append` merges by key. `replace` with an empty tag list is a no-op. `clear` removes existing tags and does not require `tags`; if values are supplied with `clear`, they are ignored.Subtree reindex is not transactional. Records skipped because no semantic source is available, or records whose embedding fails, do not receive the new tags.@@ -658,24 +657,17 @@ print(result)result = client.reindex(uri="viking://user/default/skills",mode="semantic_and_vectors",+ force=True,wait=False,)print(result["status"])```-```python-result = client.reindex(- uri="viking://resources",- mode="prune_orphans",- dry_run=True,-)-print(result["would_delete_records"])-```-**TypeScript SDK**```typescriptconsole.log(await client.reindex("viking://resources/docs/", {+ force: true,tags: ["team=search"],tagMode: "append",}));@@ -689,6 +681,7 @@ only `opts=nil` applies the SDK default `wait=true`.```goresult, err := client.Reindex(ctx, "viking://resources", &openviking.ReindexOptions{Mode: "vectors_only",+ Force: true,Tags: []string{"team=search"},TagMode: "replace",})@@ -698,17 +691,6 @@ if err != nil {fmt.Println(result["status"])```-```go-result, err := client.Reindex(ctx, "viking://resources", &openviking.ReindexOptions{- Mode: "prune_orphans",- DryRun: true,-})-if err != nil {- return err-}-fmt.Println(result["task_id"])-```-**HTTP API**```@@ -726,6 +708,7 @@ curl -X POST http://localhost:1933/api/v1/content/reindex \"uri": "viking://resources","mode": "vectors_only","wait": false,+ "force": true,"tags": ["team=search", "env=prod"],"tag_mode": "replace"}'@@ -735,7 +718,7 @@ curl -X POST http://localhost:1933/api/v1/content/reindex \```bashopenviking reindex viking://resources --mode vectors_only \- --tags team=search,env=prod --tag-mode replace+ --force --tags team=search,env=prod --tag-mode replace```Use `--tag-mode clear` without `--tags` to clear existing tags:@@ -748,10 +731,6 @@ openviking reindex viking://resources --mode vectors_only --tag-mode clearopenviking reindex viking://user/default/skills --mode semantic_and_vectors --wait false```-```bash-openviking reindex viking://resources --mode prune_orphans --dry-run-```-**Asynchronous response (`wait=false`)**```json@@ -794,8 +773,7 @@ Task records are persisted under `/local/{account_id}/_system/tasks/{user_id}/{t| mode | Effective reindex mode || scanned_records | Number of records or semantic sources considered || rebuilt_records | Number of vector records successfully rebuilt |-| deleted_records | Number of vector records deleted by `prune_orphans`; `0` for `dry_run=true` |-| would_delete_records | Number of vector records that would be deleted by `prune_orphans` in dry-run mode |+| deleted_records | Number of orphan vector records confirmed and deleted by the RFV diff || unsupported_records | Number of records skipped because no usable vector source was available || failed_records | Number of records that failed while rebuilding || duration_ms | Synchronous run duration in milliseconds |@@ -805,12 +783,10 @@ Task records are persisted under `/local/{account_id}/_system/tasks/{user_id}/{t**Behavior notes**- `vectors_only` and `semantic_and_vectors` are non-destructive. They use rebuild/upsert behavior and do not require dropping the vector collection first.-- `prune_orphans` is destructive unless `dry_run=true`: it removes vector records whose source files no longer exist.- `viking://` reindex fans out to supported top-level namespaces and excludes `session`.- Namespace reindex operations such as `viking://user` propagate to supported child content types.- `vectors_only` is the right mode when only the embedding model or vector index needs to be refreshed.- `semantic_and_vectors` is the right mode when semantic artifacts themselves must be regenerated before re-vectorization.-- `prune_orphans` is the right mode when the filesystem has been changed outside normal APIs and the vector store may still contain records for deleted paths.- Only one reindex task can run for the same URI and owner at a time. A concurrent request for the same target returns a conflict.- For resource files, text files can use file content when no summary is available. Non-text files require a generated summary or existing vector record fallback; otherwise they are counted as unsupported.diff --git a/docs/en/configuration/01-server.md b/docs/en/configuration/01-server.mdindex c632bc078..8aec2919d 100644--- a/docs/en/configuration/01-server.md+++ b/docs/en/configuration/01-server.md@@ -233,6 +233,14 @@ This setting controls queue-job concurrency. It is separate from `vlm.media.max_`max_concurrent` controls independent AddResource jobs. `file_operation_concurrency` controls file commit and fallback comparison work within one AddResource job, while `file_vectorization_concurrency` controls files within one vectors-only directory job.+### `queue_workers.reindex`++| Field | Type | Default | Description |+|---|---|---:|---|+| `max_concurrent` | integer | `4` | Number of complete Reindex root jobs consumed concurrently; must be greater than `0`; requires a server restart after changes |++This setting limits independent asynchronous reindex requests. URI-overlapping requests remain protected by path locks, while VLM and embedding work continue to use their respective concurrency limits.+### `queue_workers.session_commit`| Field | Type | Default | Description |@@ -262,7 +270,7 @@ When `base_url` is configured, OV sends the current user's OV API key in `X-API-| Field | Type | Default | Description ||---|---|---:|---|-| `file_vectorization_concurrency` | integer | `8` | Number of files concurrently read, prepared, and enqueued by one `vectors_only` reindex task; must be greater than `0`; values above the internal safety limit of `64` are capped; requires a server restart after changes |+| `file_vectorization_concurrency` | integer | `8` | Number of files concurrently read, fingerprinted, and prepared by one resource/skill reindex task; must be greater than `0`; values above the internal safety limit of `64` are capped; requires a server restart after changes |## HTTP Server Settingsdiff --git a/docs/en/getting-started/05-cli-setup.md b/docs/en/getting-started/05-cli-setup.mdindex 7f200d694..b78502f7d 100644--- a/docs/en/getting-started/05-cli-setup.md+++ b/docs/en/getting-started/05-cli-setup.md@@ -489,12 +489,11 @@ ov add-resource --help## Rebuilding Indexes-`ov reindex <uri>` rebuilds the indexes of already-imported content. Three modes are available:+`ov reindex <uri>` validates and repairs indexes for already-imported content. Three modes are available. Resource and skill records whose MD5 fingerprints already match are skipped by default:- `--mode vectors_only` — refresh vectors only.- `--mode semantic_and_vectors` — regenerate semantic artifacts (`.abstract.md`, `.overview.md`) before refreshing vectors.-- `--mode prune_orphans` — remove vector records whose source files no longer exist. Add `--dry-run` to preview the cleanup without applying it.-`semantic_and_vectors` processes the full subtree by default. If child summaries already exist and only the target directory's `.abstract.md` / `.overview.md` need regeneration, add `--recursive=false`; this refreshes only the target directory semantics and its L0/L1 vectors.+Both rebuild modes process the full subtree by default; add `--recursive=false` to process only the target itself. Add `--force` to bypass MD5 comparison and rebuild all resource/skill semantic and vector data in scope.There is no `semantic` or `full` mode alias.diff --git a/docs/zh/api/02-resources.md b/docs/zh/api/02-resources.mdindex e7652c895..ba62ebdda 100644--- a/docs/zh/api/02-resources.md+++ b/docs/zh/api/02-resources.md@@ -203,7 +203,7 @@ URL/文件 Parser TreeBuilder AGFS Summarizer/Vector- 删除资源时,系统会在删除前扫描本次上下文对应的 self 或 peer 记忆中的 `resource_refs`,清理对应资源 URI 和由该 `reason` 引入的内容,并重新刷新相关记忆的语义索引。- 其他来源在 `wait=false` 时会在响应前完成来源解析、目标解析和 AGFS 写入,仅 semantic 与 embedding 队列继续异步处理。- `processing_mode=vectors_only` 不调用 VLM 语义理解阶段,也不会生成或刷新 `.abstract.md` / `.overview.md`。对已存在目标,它会保留旧的语义产物和旧的语义向量;仍会更新资源树,在 `build_index=true` 时向量化当前非隐藏文件,并清理由本次刷新删除的文件 detail 向量。-- `processing_mode` 只属于 `add_resource`。管理员维护已有数据时,`reindex` API/CLI 仍使用 `mode`(`vectors_only`、`semantic_and_vectors`、`prune_orphans`)。+- `processing_mode` 只属于 `add_resource`。管理员维护已有数据时,`reindex` API/CLI 仍使用 `mode`(`vectors_only`、`semantic_and_vectors`)。- `watch_interval > 0` 时,如果指定了 `to`,监控任务绑定该目标;如果未指定 `to`,监控任务绑定本次导入返回的 `root_uri`。如果无法得到稳定 `root_uri`,请求会报错并要求显式传 `to`。- Connector 导入设置 `is_active=false` 时会在提交前创建暂停状态的 Watch;原生飞书和 Git 导入会通过资源队列透传 `is_active`,解析出最终资源 URI 后再创建 Watch。两种情况下首次导入均执行一次,周期调度保持关闭。- 飞书/Lark 应用 token 导入不传 `args.feishu_access_token`。OpenViking 保持原有应用凭证流程,由 SDK 使用 `app_id` 和 `app_secret` 自动获取 app/tenant token。该模式支持一次性导入和 `watch_interval > 0`。diff --git a/docs/zh/api/12-content.md b/docs/zh/api/12-content.mdindex e2ffe90df..c1268f927 100644--- a/docs/zh/api/12-content.md+++ b/docs/zh/api/12-content.md@@ -488,7 +488,7 @@ Content-Disposition: attachment; filename*=UTF-8''logo.png### set_tags()-设置用于检索过滤的显式 `k=v` 标签。`replace` 替换已有标签,`append` 追加标签;对目录设置 `recursive=true` 时会更新目录下的文件。+设置用于检索过滤的显式 `k=v` 标签。`replace` 替换已有标签,`append` 追加标签,`clear` 显式清空已有标签;对目录设置 `recursive=true` 时会更新目录下的文件。省略 `tags` 或传入空数组配合 `replace` 都是 no-op;只有 `clear` 会清空,且会忽略同时传入的标签值。**Python SDK**@@ -583,7 +583,7 @@ ov set-tags viking://resources/project/ \### reindex()-对已经存储在 OpenViking 中的现有内容,重新构建语义产物和/或向量索引。这是一个运维维护接口,适用于 embedding 模型更换、VLM 更换、向量库重刷、版本升级后修复历史索引等场景。+对已经存储在 OpenViking 中的现有内容,校验并修复语义产物和/或向量索引。resource 和 skill 默认使用 RFV(Request / Formal / Vector)增量收敛:文件内容指纹、目录 L0/L1 可见正文指纹和请求标量都未变化时直接跳过。需要无条件重建时使用 `force=true`。这个接口面向已有的 `viking://...` 内容,不负责导入新文件。常规导入请使用 [Resources](02-resources.md)。@@ -597,14 +597,14 @@ ov set-tags viking://resources/project/ \| 参数 | 类型 | 必填 | 默认值 | 说明 ||------|------|------|--------|------|| uri | str | 是 | - | 要重新索引的 Viking URI |-| mode | str | 否 | `vectors_only` | 重建模式:`vectors_only`、`semantic_and_vectors` 或 `prune_orphans` |+| mode | str | 否 | `vectors_only` | 重建模式:`vectors_only` 或 `semantic_and_vectors` || wait | bool | 否 | `true` | 是否等待任务完成 |-| dry_run | bool | 否 | `false` | 仅适用于 `mode="prune_orphans"`;只报告 orphan 向量记录,不实际删除 |-| recursive | bool | 否 | `true` | 是否递归处理下级内容;`false` 仅对 `resource`、`memory` 或 `skill` 目录的 `semantic_and_vectors` 生效 |+| force | bool | 否 | `false` | 是否跳过内容指纹比较并强制重新处理范围内全部 resource/skill 文件和目录 level |+| recursive | bool | 否 | `true` | 是否递归处理下级内容;resource/skill 的两种重建模式都遵守该参数 || tags | list[str] | 否 | `null` | 写入本次成功重建的全部向量记录。省略或空数组配合 `replace` 时保留已有 tags || tag_mode | str | 否 | `replace` | 标签写入模式:`replace`、`append` 或 `clear`;`clear` 不要求传 `tags` 并清空已有标签 |-HTTP 请求体不接受未知字段。`uri` 可以使用其他 content API 支持的 OpenViking 路径变量,服务端会先解析再校验。+HTTP 请求体会忽略未知字段,便于客户端与服务端在字段演进期间兼容。`uri` 可以使用其他 content API 支持的 OpenViking 路径变量,服务端会先解析再校验。**支持的 URI 范围**@@ -623,19 +623,18 @@ session 子树会被跳过。**模式说明**-- `vectors_only`:基于当前仍可恢复的源数据重建向量库记录,不会重写 `.abstract.md` 和 `.overview.md`-- `semantic_and_vectors`:先重新生成语义产物,再基于新的语义结果重建向量-- `prune_orphans`:删除请求 URI 范围内源文件已不存在的向量库记录。设置 `dry_run=true` 时,只报告会删除多少记录,不实际删除。+- `vectors_only`:通过 RFV 对比当前 source MD5 与向量记录 MD5,只重建缺失或过期的 L0/L1/L2;不会重写 `.abstract.md` 和 `.overview.md`+- `semantic_and_vectors`:通过同一 RFV 状态和 `ContextUpdatePlan` 驱动语义修复及 L0/L1/L2 向量更新,不再执行第二次手工向量扫描对于 `resource` 和 `skill`,`semantic_and_vectors` 会刷新目录/文件语义产物,包括 `.abstract.md` 和 `.overview.md`。对于 `memory`,它会重建当前已持久化 memory 子树的语义和向量,但不会回放历史记忆抽取顺序。-对于 `semantic_and_vectors`,语义刷新和向量重建由 reindex executor 串行编排。语义刷新阶段不会再额外向后台 embedding queue 投递自己的向量化任务;向量由 reindex 阶段统一重建,因此 `wait=true` 表示等待 reindex 操作本身完成。+resource/skill 的 `vectors_only` 和 `semantic_and_vectors` 共用一次 F 遍历和一次窄字段 V inventory。每个 source 在状态解析时读取一次,实际处理复用本次读取结果,不会为了执行 Plan 再读取一次 F。`force=true` 会跳过 MD5 相等判断并重新处理作用域内全部可用 level;完整性检查仍然执行。-对 `resource` 或 `memory` 目录设置 `recursive=false` 时,只重新生成目标目录的 `.abstract.md`、`.overview.md`,并重建该目录的 L0/L1 向量;下级目录不会重新生成语义产物,下级目录和文件也不会重新向量化。目标目录仍会读取本轮确定性采样命中的既有下级摘要;若采样命中直接文件,仍会为当前目录聚合准备这些文件的摘要。对 `skill` 目标设置 `recursive=false` 时,会根据 `SKILL.md` 重新生成 skill 目录的 L0/L1 语义产物及向量,但不会重建 `SKILL.md` 的 L2 向量。namespace 目标(例如 `viking://user/<user_id>` 或其 `skills` 容器)不支持在 `semantic_and_vectors` 模式下设置 `recursive=false`;这类请求会在启动重建前被拒绝。非递归刷新请选择具体的 resource、memory 或 skill 目录。该参数不改变 `vectors_only` 或 `prune_orphans` 的既有行为。+对 resource/skill 目录设置 `recursive=false` 时,两种模式都只处理目标目录自身的 L0/L1。直接 children 只作为目标聚合的既有 REUSE 输入读取,不扫描或重建下级内容。namespace 容器自身没有索引记录,因此 namespace 目标不接受 `recursive=false`,避免请求静默成为 no-op。memory 仍使用原有 reindex 链路。-对于 `prune_orphans`,源文件是否存在以当前文件系统为准。如果整个目录已经不存在,该目录下的正文文件向量和语义 sidecar 向量(例如 `.abstract.md`、`.overview.md`)会一起清理。`dry_run` 用在其他模式时会被拒绝。+F snapshot 完整时,RFV 差分会把 V 中存在但 F 中不存在的记录识别为 orphan,并在同一次 reindex 中清理;F snapshot 不完整时 fail-closed,不执行删除。-传入非空 `tags` 时,标签会随 reindex 生成的向量记录在同一次 upsert 中写入,不会在完成后额外调用 `set_tags`。目录或 namespace reindex 会把标签应用到本次成功重建的目录 L0/L1 和叶子 L2 记录。`replace` 覆盖已有标签,`append` 按 key 合并;`replace` 配合空数组时不修改已有标签。`clear` 不要求传 `tags`,会清空已有标签;即使同时传入标签值也会忽略。`prune_orphans` 不生成向量,因此会忽略 `tags` 和 `tag_mode`。+传入非空 `tags` 时,标签会随 reindex 生成的向量记录在同一次 upsert 中写入,不会在完成后额外调用 `set_tags`。目录或 namespace reindex 会把标签应用到本次成功重建的目录 L0/L1 和叶子 L2 记录。`replace` 覆盖已有标签,`append` 按 key 合并;`replace` 配合空数组时不修改已有标签。`clear` 不要求传 `tags`,会清空已有标签;即使同时传入标签值也会忽略。子树 reindex 不是事务性操作。如果部分记录因缺少语义来源或 embedding 失败而未重建,只有成功写入的记录会更新标签。@@ -658,24 +657,17 @@ print(result)result = client.reindex(uri="viking://user/default/skills",mode="semantic_and_vectors",+ force=True,wait=False,)print(result["status"])```-```python-result = client.reindex(- uri="viking://resources",- mode="prune_orphans",- dry_run=True,-)-print(result["would_delete_records"])-```-**TypeScript SDK**```typescriptconsole.log(await client.reindex("viking://resources/docs/", {+ force: true,tags: ["team=search"],tagMode: "append",}));@@ -689,6 +681,7 @@ console.log(await client.reindex("viking://resources/docs/", {```goresult, err := client.Reindex(ctx, "viking://resources", &openviking.ReindexOptions{Mode: "vectors_only",+ Force: true,Tags: []string{"team=search"},TagMode: "replace",})@@ -698,17 +691,6 @@ if err != nil {fmt.Println(result["status"])```-```go-result, err := client.Reindex(ctx, "viking://resources", &openviking.ReindexOptions{- Mode: "prune_orphans",- DryRun: true,-})-if err != nil {- return err-}-fmt.Println(result["task_id"])-```-**HTTP API**```@@ -726,6 +708,7 @@ curl -X POST http://localhost:1933/api/v1/content/reindex \"uri": "viking://resources","mode": "vectors_only","wait": false,+ "force": true,"tags": ["team=search", "env=prod"],"tag_mode": "replace"}'@@ -735,7 +718,7 @@ curl -X POST http://localhost:1933/api/v1/content/reindex \```bashopenviking reindex viking://resources --mode vectors_only \- --tags team=search,env=prod --tag-mode replace+ --force --tags team=search,env=prod --tag-mode replace```使用 `--tag-mode clear` 且无需传 `--tags` 即可清空已有标签:@@ -748,10 +731,6 @@ openviking reindex viking://resources --mode vectors_only --tag-mode clearopenviking reindex viking://user/default/skills --mode semantic_and_vectors --wait false```-```bash-openviking reindex viking://resources --mode prune_orphans --dry-run-```-**异步响应(`wait=false`)**```json@@ -794,8 +773,7 @@ GET /api/v1/tasks?task_type=admin_reindex&resource_id=viking://resources| mode | 实际执行的 reindex 模式 || scanned_records | 被检查的记录或语义源数量 || rebuilt_records | 成功重建的向量记录数量 |-| deleted_records | `prune_orphans` 实际删除的向量记录数量;`dry_run=true` 时为 `0` |-| would_delete_records | `prune_orphans` dry-run 模式下将会删除的向量记录数量 |+| deleted_records | RFV 差分确认并删除的 orphan 向量记录数量 || unsupported_records | 因没有可用向量来源而跳过的记录数量 || failed_records | 重建失败的记录数量 || duration_ms | 同步执行耗时,单位毫秒 |@@ -805,12 +783,10 @@ GET /api/v1/tasks?task_type=admin_reindex&resource_id=viking://resources**行为说明**- `vectors_only` 和 `semantic_and_vectors` 是非破坏式的,采用重建/覆盖写入,不需要先 drop 向量集合。-- `prune_orphans` 除非设置 `dry_run=true`,否则会删除源文件已经不存在的向量记录。- 对 `viking://` 发起 reindex 时,会向下分发到支持的顶层命名空间,并显式排除 `session`。- 命名空间级 reindex,例如 `viking://user`,会继续传播到其支持的子内容类型。- 如果只是 embedding 模型或向量索引需要刷新,应使用 `vectors_only`。- 如果语义产物本身也需要重建,再做重向量化,应使用 `semantic_and_vectors`。-- 如果文件系统曾绕过正常 API 发生删除,向量库可能还残留已删除路径的记录,应使用 `prune_orphans`。- 同一个 URI 和 owner 同时只能运行一个 reindex 任务。对同一目标的并发请求会返回 conflict。- 对 resource 文件,文本文件在没有 summary 时可以使用文件正文;非文本文件需要已生成的 summary 或已有向量记录 fallback,否则会计为 unsupported。diff --git a/docs/zh/configuration/01-server.md b/docs/zh/configuration/01-server.mdindex 4d835023d..8f43f42a0 100644--- a/docs/zh/configuration/01-server.md+++ b/docs/zh/configuration/01-server.md@@ -232,6 +232,14 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK`max_concurrent` 控制相互独立的 AddResource 作业并发,`file_operation_concurrency` 控制单个 AddResource 作业内文件提交和 fallback 比较操作的并发,`file_vectorization_concurrency` 控制单个 vectors-only 目录作业内的文件并发。+### `queue_workers.reindex`++| 字段 | 类型 | 默认值 | 说明 |+|---|---|---:|---|+| `max_concurrent` | integer | `4` | 同时消费的完整 Reindex root 作业数,必须大于 `0`;修改后需重启服务 |++该配置限制相互独立的异步 reindex 请求并发。URI 范围重叠的请求仍由 path lock 保护;VLM 和 embedding 仍分别遵守各自的并发限制。+### `queue_workers.session_commit`| 字段 | 类型 | 默认值 | 说明 |@@ -261,7 +269,7 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK| 字段 | 类型 | 默认值 | 说明 ||---|---|---:|---|-| `file_vectorization_concurrency` | integer | `8` | 单个 `vectors_only` reindex 任务内并发读取、准备并入队的文件数,必须大于 `0`;超过内部安全上限 `64` 的值会被截断;修改后需重启服务 |+| `file_vectorization_concurrency` | integer | `8` | 单个 resource/skill reindex 任务内并发读取、计算指纹和准备文件的数量,必须大于 `0`;超过内部安全上限 `64` 的值会被截断;修改后需重启服务 |## HTTP 服务配置diff --git a/docs/zh/getting-started/05-cli-setup.md b/docs/zh/getting-started/05-cli-setup.mdindex 566d0767b..a95684879 100644--- a/docs/zh/getting-started/05-cli-setup.md+++ b/docs/zh/getting-started/05-cli-setup.md@@ -489,12 +489,11 @@ ov add-resource --help## 重建索引-`ov reindex <uri>` 用于重建已导入内容的索引,支持三种模式:+`ov reindex <uri>` 用于校验并修复已导入内容的索引,支持三种模式。resource 和 skill 默认按 MD5 增量跳过已经一致的记录:- `--mode vectors_only` —— 只刷新向量。- `--mode semantic_and_vectors` —— 先重新生成语义产物(`.abstract.md`、`.overview.md`),再刷新向量。-- `--mode prune_orphans` —— 清理源文件已不存在的向量记录,加 `--dry-run` 可预览而不实际执行。-`semantic_and_vectors` 默认递归处理整个子树。对已经具有下级摘要、只需要重新生成目标目录 `.abstract.md` / `.overview.md` 的场景,可添加 `--recursive=false`;此时仅刷新目标目录语义产物和该目录的 L0/L1 向量。+两种重建模式默认递归处理整个子树;添加 `--recursive=false` 后只处理目标本身。需要跳过 MD5 比较、强制重建作用域内全部 resource/skill 语义和向量数据时,添加 `--force`。没有 `semantic` 或 `full` 这样的模式别名。diff --git a/openviking/server/routers/content.py b/openviking/server/routers/content.pyindex d9089a21c..89f1f1423 100644--- a/openviking/server/routers/content.py+++ b/openviking/server/routers/content.py@@ -30,7 +30,7 @@ from openviking.server.telemetry import run_operationfrom openviking.storage.acl import AclSpecfrom openviking.storage.vector_ids import is_vector_record_idfrom openviking.telemetry import TelemetryRequest-from openviking_cli.exceptions import InvalidArgumentError, NotFoundError, PermissionDeniedError+from openviking_cli.exceptions import NotFoundError, PermissionDeniedErrorfrom openviking_cli.utils import get_loggerlogger = get_logger(__name__)@@ -84,19 +84,17 @@ class SetTagsRequest(BaseModel):model_config = ConfigDict(extra="forbid")uri: str- tags: list[str]+ tags: list[str] | None = Nonemode: str = "replace"recursive: bool = Falsetelemetry: TelemetryRequest = Falseclass ReindexRequest(BaseModel):- model_config = ConfigDict(extra="forbid")-uri: strmode: str = "vectors_only"+ force: bool = Falsewait: bool = True- dry_run: bool = Falserecursive: bool = Truetags: list[str] | None = Nonetag_mode: str = "replace"@@ -300,12 +298,13 @@ async def set_tags("""Set explicit k=v retrieval tags metadata for a file or directory."""service = get_service()uri = validate_request_viking_uri(resolve_path_variables(request.uri), _ctx)+ tags = [] if request.mode == "clear" else request.tags or []execution = await run_operation(operation="content.set_tags",telemetry=request.telemetry,fn=lambda: service.fs.set_tags(uri=uri,- tags=request.tags,+ tags=tags,mode=request.mode,recursive=request.recursive,ctx=_ctx,@@ -324,8 +323,6 @@ async def reindex(ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN, Role.USER),):"""Reindex semantic/vector artifacts for a URI-scoped maintenance target."""- if body.dry_run and body.mode != "prune_orphans":- raise InvalidArgumentError("dry_run is only supported for prune_orphans reindex mode.")uri = validate_request_viking_uri(resolve_path_variables(body.uri), ctx)uri = _authorize_reindex_uri(uri, ctx)service = get_service()@@ -333,9 +330,10 @@ async def reindex("uri": uri,"mode": body.mode,"wait": body.wait,- "dry_run": body.dry_run,"ctx": ctx,}+ if body.force:+ reindex_kwargs["force"] = Trueif not body.recursive:reindex_kwargs["recursive"] = Falseif body.tags is not None or body.tag_mode == "clear":diff --git a/openviking/service/core.py b/openviking/service/core.pyindex b94398aed..434ed1b64 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -36,6 +36,7 @@ from openviking.storage.index_consistency import check_index_consistencyfrom openviking.storage.queuefs.add_resource_processor import AddResourceProcessorfrom openviking.storage.queuefs.external_task_processor import ExternalTaskProcessorfrom openviking.storage.queuefs.queue_manager import QueueManager, init_queue_manager+from openviking.storage.queuefs.reindex_processor import ReindexProcessorfrom openviking.storage.queuefs.session_commit_processor import SessionCommitProcessorfrom openviking.storage.viking_fs import VikingFS, init_viking_fsfrom openviking.storage.vikingdb_manager import VikingDBManager@@ -151,6 +152,7 @@ class OpenVikingService:max_concurrent_semantic=cluster_vlm.max_concurrent,max_concurrent_external_parse=config.queue_workers.external_parse.max_concurrent,max_concurrent_add_resource=config.queue_workers.add_resource.max_concurrent,+ max_concurrent_reindex=config.queue_workers.reindex.max_concurrent,max_concurrent_session_commit=config.queue_workers.session_commit.max_concurrent,max_concurrent_external_task=config.queue_workers.external_task.max_concurrent,binding_config=binding_config,@@ -164,6 +166,7 @@ class OpenVikingService:max_concurrent_semantic: int = 32,max_concurrent_external_parse: int = 4,max_concurrent_add_resource: int = 4,+ max_concurrent_reindex: int = 4,max_concurrent_session_commit: int = 8,max_concurrent_external_task: int = 10,binding_config: Any = None,@@ -188,6 +191,7 @@ class OpenVikingService:max_concurrent_semantic=max_concurrent_semantic,max_concurrent_external_parse=max_concurrent_external_parse,max_concurrent_add_resource=max_concurrent_add_resource,+ max_concurrent_reindex=max_concurrent_reindex,max_concurrent_session_commit=max_concurrent_session_commit,max_concurrent_external_task=max_concurrent_external_task,)@@ -259,11 +263,7 @@ class OpenVikingService:if manager is None:returnbase_config = self._config.model_copy(- update={- "agent_evolution": self._agent_evolution_base_config.model_copy(- deep=True- )- }+ update={"agent_evolution": self._agent_evolution_base_config.model_copy(deep=True)})await manager.replace_base_config(base_config)@@ -285,11 +285,7 @@ class OpenVikingService:if self._agfs_client is None:raise RuntimeError("AGFS client not initialized")base_config = self._config.model_copy(- update={- "agent_evolution": self._agent_evolution_base_config.model_copy(- deep=True- )- },+ update={"agent_evolution": self._agent_evolution_base_config.model_copy(deep=True)},)manager = build_runtime_config_manager(AsyncAGFSClient(self._agfs_client),@@ -451,6 +447,7 @@ class OpenVikingService:max_concurrent_add_resource=(self._config.queue_workers.add_resource.max_concurrent),+ max_concurrent_reindex=self._config.queue_workers.reindex.max_concurrent,max_concurrent_session_commit=(self._config.queue_workers.session_commit.max_concurrent),@@ -616,6 +613,11 @@ class OpenVikingService:),allow_create=True,)+ self._queue_manager.get_queue(+ self._queue_manager.REINDEX,+ dequeue_handler=ReindexProcessor(self._viking_fs),+ allow_create=True,+ )self._queue_manager.get_queue(self._queue_manager.SESSION_COMMIT,dequeue_handler=SessionCommitProcessor(@@ -757,8 +759,8 @@ class OpenVikingService:*,uri: str,mode: str = "vectors_only",+ force: bool = False,wait: bool = True,- dry_run: bool = False,recursive: bool = True,tags: list[str] | None = None,tag_mode: str = "replace",@@ -779,9 +781,10 @@ class OpenVikingService:"uri": uri,"mode": mode,"wait": wait,- "dry_run": dry_run,"ctx": effective_ctx,}+ if force:+ execute_kwargs["force"] = Trueif not recursive:execute_kwargs["recursive"] = Falseif tags is not None or tag_mode == "clear":diff --git a/openviking/service/reindex_executor.py b/openviking/service/reindex_executor.pyindex 9027517d2..bd8131a67 100644--- a/openviking/service/reindex_executor.py+++ b/openviking/service/reindex_executor.py@@ -5,9 +5,8 @@from __future__ import annotationsimport asyncio-import reimport time-from dataclasses import dataclass, field+from dataclasses import dataclass, field, replacefrom typing import Any, Iterable, Optionalfrom openviking.config.vlm import VLMResolver@@ -23,37 +22,28 @@ from openviking.core.namespace import (content_owner_context_for_uri,context_type_for_uri,is_session_uri,- owner_fields_for_uri,owner_space_for_uri,)from openviking.server.dependencies import get_servicefrom openviking.server.identity import RequestContextfrom openviking.service.task_tracker import get_task_tracker-from openviking.service.task_work_index import bind_task_contextfrom openviking.session.memory.utils.memory_file_utils import MemoryFileUtilsfrom openviking.storage.abstract_overview import body_for_preview, embedding_text_for_body-from openviking.storage.errors import ResourceBusyError-from openviking.storage.expr import And, Eq, Or, PathScopefrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter-from openviking.storage.queuefs.semantic_executor import SemanticTreeExecutorfrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessorfrom openviking.storage.viking_fs import get_viking_fsfrom openviking.telemetry import get_current_telemetryfrom openviking.telemetry.request_wait_tracker import get_request_wait_tracker-from openviking.utils.content_hash import content_md5from openviking.utils.embedding_input import truncate_embedding_inputfrom openviking.utils.embedding_utils import (_apply_ingest_options,_decode_text_bytes,_truncate_abstract_bytes,get_resource_content_type,- vectorize_directory_meta,- vectorize_file,)from openviking.utils.ingest_options import IngestOptionsfrom openviking_cli.exceptions import InvalidArgumentError, NotFoundError, OpenVikingError-from openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils import VikingURI, get_loggerfrom openviking_cli.utils.config import get_openviking_configfrom openviking_cli.utils.config.embedding_config import SUMMARY_TEXT_SOURCES@@ -61,8 +51,6 @@ from openviking_cli.utils.config.embedding_config import SUMMARY_TEXT_SOURCESlogger = get_logger(__name__)REINDEX_TASK_TYPE = "admin_reindex"-PRUNE_ORPHAN_CANDIDATE_LIMIT = 100000-PRUNE_OUTPUT_FIELDS = ["id", "uri", "level", "context_type", "account_id", "owner_user_id"]_MAX_FILE_VECTORIZATION_CONCURRENCY = 64@@ -105,7 +93,6 @@ class _ReindexCounters:scanned_records: int = 0rebuilt_records: int = 0deleted_records: int = 0- would_delete_records: int = 0unsupported_records: int = 0failed_records: int = 0warnings: list[str] = field(default_factory=list)@@ -114,7 +101,6 @@ class _ReindexCounters:self.scanned_records += other.scanned_recordsself.rebuilt_records += other.rebuilt_recordsself.deleted_records += other.deleted_records- self.would_delete_records += other.would_delete_recordsself.unsupported_records += other.unsupported_recordsself.failed_records += other.failed_recordsself.warnings.extend(other.warnings)@@ -125,11 +111,16 @@ class _ReindexRunContext:ctx: RequestContextcounters: _ReindexCounterslock: dict | None = None+ lease: dict | None = None+ lease_handed_off: bool = False+ lock_root_uri: str | None = Noneingest_options: IngestOptions | None = None+ force: bool = False+ root_is_dir: bool | None = None@dataclass-class _PruneSourceRead:+class _SourceRead:exists: booltext: str = ""error: Exception | None = None@@ -139,12 +130,12 @@ class ReindexExecutor:"""Non-destructive reindex orchestration for admin maintenance flows."""SUPPORTED_MODES_BY_TYPE = {- "global_namespace": {"vectors_only", "semantic_and_vectors", "prune_orphans"},- "user_namespace": {"vectors_only", "semantic_and_vectors", "prune_orphans"},- "skill_namespace": {"vectors_only", "semantic_and_vectors", "prune_orphans"},- "resource": {"vectors_only", "semantic_and_vectors", "prune_orphans"},- "skill": {"vectors_only", "semantic_and_vectors", "prune_orphans"},- "memory": {"vectors_only", "semantic_and_vectors", "prune_orphans"},+ "global_namespace": {"vectors_only", "semantic_and_vectors"},+ "user_namespace": {"vectors_only", "semantic_and_vectors"},+ "skill_namespace": {"vectors_only", "semantic_and_vectors"},+ "resource": {"vectors_only", "semantic_and_vectors"},+ "skill": {"vectors_only", "semantic_and_vectors"},+ "memory": {"vectors_only", "semantic_and_vectors"},}def __init__(@@ -157,9 +148,7 @@ class ReindexExecutor:async def _semantic_processor_for(self, ctx: RequestContext) -> SemanticProcessor:if self.vlm_resolver is None:- raise RuntimeError(- "ReindexExecutor requires a VLM resolver for semantic reindexing"- )+ raise RuntimeError("ReindexExecutor requires a VLM resolver for semantic reindexing")vlm = await self.vlm_resolver.get_vlm(ctx.account_id)return SemanticProcessor(max_concurrent_llm=vlm.max_concurrent,@@ -205,7 +194,7 @@ class ReindexExecutor:uri: str,mode: str,wait: bool,- dry_run: bool = False,+ force: bool = False,recursive: bool = True,tags: list[str] | None = None,tag_mode: str = "replace",@@ -213,17 +202,14 @@ class ReindexExecutor:) -> dict[str, Any]:object_type = self._infer_target_type(uri)self._validate_mode(object_type, mode)- if (- mode == "semantic_and_vectors"- and not recursive- and object_type in {"global_namespace", "user_namespace", "skill_namespace"}- ):+ if not recursive and object_type in {+ "global_namespace",+ "user_namespace",+ "skill_namespace",+ }:raise InvalidArgumentError(- "recursive=false is not supported for namespace reindex targets. "- "Select a resource, memory, or skill directory instead."+ f"recursive=false is not supported for namespace reindex target {uri}")- if dry_run and mode != "prune_orphans":- raise InvalidArgumentError("dry_run is only supported for prune_orphans reindex mode.")ingest_options = self._resolve_ingest_options(mode=mode,tags=tags,@@ -247,7 +233,7 @@ class ReindexExecutor:uri=uri,object_type=object_type,mode=mode,- dry_run=dry_run,+ force=force,recursive=recursive,ingest_options=ingest_options,ctx=ctx,@@ -266,18 +252,60 @@ class ReindexExecutor:details={"uri": uri},)- asyncio.create_task(- self._run_tracked(- task.task_id,+ from openviking.storage.queuefs import get_queue_manager+ from openviking.storage.queuefs.reindex_msg import ReindexMsg++ service = None+ lease = None+ enqueued = False+ try:+ service = get_service()+ path = service.viking_fs._uri_to_path(uri, ctx=ctx)+ stat = await service.viking_fs.stat(uri, ctx=ctx, skip_count=True)+ acquire = (+ service.viking_fs._async_agfs.pathlock_acquire_tree+ if stat.get("isDir", stat.get("is_dir"))+ else service.viking_fs._async_agfs.pathlock_acquire_exact+ )+ lease = await acquire(path)+ handoff = await service.viking_fs._async_agfs.pathlock_to_handoff(lease)+ msg = ReindexMsg(+ task_id=task.task_id,uri=uri,object_type=object_type,mode=mode,- dry_run=dry_run,+ force=force,recursive=recursive,- ingest_options=ingest_options,- ctx=ctx,+ tags=tags,+ tag_mode=tag_mode,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ group_ids=list(ctx.group_ids),+ role=str(ctx.role),+ actor_peer_id=ctx.actor_peer_id,+ bypass_acl=ctx.bypass_acl,+ telemetry_id=get_current_telemetry().telemetry_id or None,+ lock_handoff=handoff,)- )+ queue_manager = get_queue_manager()+ await queue_manager.enqueue(queue_manager.REINDEX, msg.to_dict())+ enqueued = True+ await service.viking_fs._async_agfs.pathlock_handoff(lease)+ lease = None+ await tracker.update_stage(+ task.task_id, "queued", account_id=ctx.account_id, user_id=ctx.user.user_id+ )+ except BaseException:+ if lease is not None and service is not None:+ await service.viking_fs._async_agfs.pathlock_release(lease)+ if not enqueued:+ await tracker.fail(+ task.task_id,+ "Failed to enqueue reindex processing",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ raisereturn {"task_id": task.task_id,"status": "accepted",@@ -293,7 +321,7 @@ class ReindexExecutor:tags: list[str] | None,tag_mode: str,) -> IngestOptions | None:- if mode == "prune_orphans" or (tags is None and tag_mode != "clear"):+ if tags is None and tag_mode != "clear":return Noneif tag_mode not in {"replace", "append", "clear"}:raise InvalidArgumentError(f"unsupported tag mode: {tag_mode}")@@ -438,6 +466,15 @@ class ReindexExecutor:if message:counters.warnings.append(f"Embedding queue failed during reindex: {message}")+ @staticmethod+ def _apply_semantic_tree_stats(counters: _ReindexCounters, stats: Any | None) -> None:+ if stats is None:+ return+ counters.rebuilt_records += int(getattr(stats, "indexed_records", 0) or 0)+ failures = list(getattr(stats, "failures", ()) or ())+ counters.failed_records += len(failures)+ counters.warnings.extend(failures)+def _is_resource_entry_for_namespace(self, uri: str, target_root: str) -> bool:if not uri.startswith(self._child_prefix(target_root)):return False@@ -455,9 +492,12 @@ class ReindexExecutor:uri: str,mode: str,run: _ReindexRunContext,+ recursive: bool = True,) -> None:counters = run.countersctx = run.ctx+ if not recursive:+ returnviking_fs = get_viking_fs()try:entries = await self._tree_all(viking_fs, uri, show_all_hidden=True, ctx=ctx)@@ -471,10 +511,19 @@ class ReindexExecutor:skill_roots.append(entry_uri)for skill_root in sorted(set(skill_roots)):- await self._reindex_skill(+ subtree = [+ entry+ for entry in entries+ if str(entry.get("uri") or "").startswith(skill_root.rstrip("/") + "/")+ ]+ await self._reindex_rfv(uri=skill_root,mode=mode,+ context_type="skill",+ recursive=True,run=run,+ root_is_dir=True,+ formal_entries=subtree,)if not skill_roots:@@ -500,15 +549,16 @@ class ReindexExecutor:uri: str,object_type: str,mode: str,- dry_run: bool = False,+ force: bool = False,recursive: bool = True,ingest_options: IngestOptions | None = None,ctx: RequestContext,+ existing_lease: dict | None = None,) -> dict[str, Any]:service = get_service()if service.viking_fs is None or service.vikingdb_manager is None:raise RuntimeError("OpenVikingService not initialized")- if mode != "prune_orphans" and not service.vikingdb_manager.has_queue_manager:+ if not service.vikingdb_manager.has_queue_manager:raise OpenVikingError("Reindex requires embedding queue",code="FAILED_PRECONDITION",@@ -523,19 +573,13 @@ class ReindexExecutor:if telemetry_id:wait_tracker.register_request(telemetry_id)- # prune_orphans only touches the vector store. A missing target has- # nothing on disk to protect, and acquiring a lock there would create- # the directory just to hold lock metadata. Refuse only when another- # owner is mid-write at that name (e.g. add_resource reserving it).- lease = None- if mode != "prune_orphans" or await service.viking_fs.exists(uri, ctx=ctx):- acquire_lock = service.viking_fs._async_agfs.pathlock_acquire_tree- stat = await service.viking_fs.stat(uri, ctx=ctx, skip_count=True)- if not stat.get("isDir", stat.get("is_dir")):- acquire_lock = service.viking_fs._async_agfs.pathlock_acquire_exact- lease = await acquire_lock(path)- elif await service.viking_fs._async_agfs.pathlock_is_locked(path):- raise ResourceBusyError(f"Resource is being processed: {uri}", uri=uri)+ acquire_lock = service.viking_fs._async_agfs.pathlock_acquire_tree+ stat = await service.viking_fs.stat(uri, ctx=ctx, skip_count=True)+ root_is_dir = bool(stat.get("isDir", stat.get("is_dir")))+ if not root_is_dir:+ acquire_lock = service.viking_fs._async_agfs.pathlock_acquire_exact+ lease = existing_lease or await acquire_lock(path)+ run: _ReindexRunContext | None = Nonetry:borrowed = (await service.viking_fs._async_agfs.pathlock_as_borrowed(lease)@@ -546,34 +590,27 @@ class ReindexExecutor:ctx=ctx,counters=counters,lock=borrowed,+ lease=lease,ingest_options=ingest_options,+ force=force,+ root_is_dir=root_is_dir,+ lock_root_uri=uri.rstrip("/"),)- if mode == "prune_orphans":- await self._prune_orphan_vectors(- uri=uri,- object_type=object_type,- dry_run=dry_run,- counters=counters,- ctx=ctx,- )- elif object_type == "global_namespace":- await self._reindex_global_namespace(- uri=uri,- mode=mode,- run=run,- )+ if object_type == "global_namespace":+ namespace_kwargs = {"uri": uri, "mode": mode, "run": run}+ if not recursive:+ namespace_kwargs["recursive"] = False+ await self._reindex_global_namespace(**namespace_kwargs)elif object_type == "user_namespace":- await self._reindex_user_namespace(- uri=uri,- mode=mode,- run=run,- )+ namespace_kwargs = {"uri": uri, "mode": mode, "run": run}+ if not recursive:+ namespace_kwargs["recursive"] = False+ await self._reindex_user_namespace(**namespace_kwargs)elif object_type == "skill_namespace":- await self._reindex_skill_namespace(- uri=uri,- mode=mode,- run=run,- )+ namespace_kwargs = {"uri": uri, "mode": mode, "run": run}+ if not recursive:+ namespace_kwargs["recursive"] = False+ await self._reindex_skill_namespace(**namespace_kwargs)elif object_type == "resource":resource_kwargs = {"uri": uri, "mode": mode, "run": run}if not recursive:@@ -596,15 +633,20 @@ class ReindexExecutor:details={"uri": uri},)- if telemetry_id and mode != "prune_orphans":+ if telemetry_id:await wait_tracker.wait_for_request(telemetry_id)+ self._apply_semantic_tree_stats(+ counters,+ SemanticProcessor.consume_tree_stats(telemetry_id=telemetry_id),+ )self._apply_embedding_wait_status(counters,wait_tracker.build_queue_status(telemetry_id),)finally:- if lease is not None:- await service.viking_fs._async_agfs.pathlock_release(lease)+ active_lease = run.lease if run is not None else lease+ if active_lease is not None and (run is None or not run.lease_handed_off):+ await service.viking_fs._async_agfs.pathlock_release(active_lease)if telemetry_id:wait_tracker.cleanup(telemetry_id)@@ -616,395 +658,263 @@ class ReindexExecutor:"scanned_records": counters.scanned_records,"rebuilt_records": counters.rebuilt_records,"deleted_records": counters.deleted_records,- "would_delete_records": counters.would_delete_records,"unsupported_records": counters.unsupported_records,"failed_records": counters.failed_records,"duration_ms": int((time.perf_counter() - started_at) * 1000),"warnings": counters.warnings,}- async def _run_tracked(+ async def _reindex_resource(self,- task_id: str,*,uri: str,- object_type: str,mode: str,- dry_run: bool = False,+ run: _ReindexRunContext,recursive: bool = True,- ingest_options: IngestOptions | None = None,- ctx: RequestContext,) -> None:- tracker = get_task_tracker()- tracker.register_running_task(task_id)- try:- await tracker.start(task_id, account_id=ctx.account_id, user_id=ctx.user.user_id)- with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):- result = await self._run(- uri=uri,- object_type=object_type,- mode=mode,- dry_run=dry_run,- recursive=recursive,- ingest_options=ingest_options,- ctx=ctx,- )- await tracker.complete(- task_id,- result,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- except asyncio.CancelledError:- # TaskWorkIndex finalizes after this active task and its queue work settle.- return- except Exception as exc:- await tracker.fail(- task_id,- str(exc),- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- finally:- await tracker.unregister_running_task(task_id)+ await self._reindex_rfv(+ uri=uri,+ mode=mode,+ context_type="resource",+ recursive=recursive,+ run=run,+ )- async def _prune_orphan_vectors(+ async def _reindex_rfv(self,*,uri: str,- object_type: str,- dry_run: bool,- counters: _ReindexCounters,- ctx: RequestContext,+ mode: str,+ context_type: str,+ recursive: bool,+ run: _ReindexRunContext,+ root_is_dir: bool | None = None,+ formal_entries: list[dict[str, Any]] | None = None,) -> None:- service = get_service()- vikingdb = service.vikingdb_manager- delete_groups: dict[tuple[str, str], tuple[RequestContext, list[str]]] = {}-- for context_type in self._prune_context_types(uri=uri, object_type=object_type):- offset = 0- while True:- filter_kwargs: dict[str, Any] = {}- if offset:- filter_kwargs["offset"] = offset- records = await vikingdb.filter(- filter=And(- [- Eq("account_id", ctx.account_id),- Eq("context_type", context_type),- Or(- [- PathScope("uri", uri, depth=0),- PathScope("uri", uri, depth=-1),- ]- ),- ]- ),- limit=PRUNE_ORPHAN_CANDIDATE_LIMIT,- output_fields=PRUNE_OUTPUT_FIELDS,- ctx=ctx,- **filter_kwargs,- )- if not records:- break-- for record in records:- counters.scanned_records += 1- if not self._is_supported_prune_record(record, counters):- continue- if not await self._is_orphan_vector_record(record, counters=counters, ctx=ctx):- continue-- if dry_run:- counters.would_delete_records += 1- continue-- delete_ctx = self._delete_ctx_for_prune_record(record, ctx)- key = (delete_ctx.account_id, delete_ctx.user.user_id)- if key not in delete_groups:- delete_groups[key] = (delete_ctx, [])- delete_groups[key][1].append(str(record["id"]))-- if len(records) < PRUNE_ORPHAN_CANDIDATE_LIMIT:- break- offset += len(records)-- if dry_run:- return+ from openviking.storage.context_update_plan import build_rfv_context_update_plan+ from openviking.storage.resource_rfv import build_rfv_snapshot+ from openviking.storage.resource_rnfv import RequestIntent+ from openviking.utils.resource_processor import ResourceProcessor- for delete_ctx, ids in delete_groups.values():- try:- deleted = await vikingdb.delete(ids, ctx=delete_ctx)- deleted_count = int(deleted if deleted is not None else len(ids))- counters.deleted_records += deleted_count- if deleted_count < len(ids):- failed_count = len(ids) - max(deleted_count, 0)- counters.failed_records += failed_count- counters.warnings.append(- f"Only deleted {deleted_count} of {len(ids)} orphan vectors for owner "- f"{delete_ctx.user.user_id}"- )- except Exception as exc:- counters.failed_records += len(ids)- counters.warnings.append(- f"Failed to delete {len(ids)} orphan vectors for owner "- f"{delete_ctx.user.user_id}: {exc}"- )-- def _prune_context_types(self, *, uri: str, object_type: str) -> list[str]:- if object_type == "resource":- return [ContextType.RESOURCE.value]- if object_type == "memory":- return [ContextType.MEMORY.value]- if object_type in {"skill", "skill_namespace"}:- return [ContextType.SKILL.value]- if object_type in {"global_namespace", "user_namespace"}:- return [- ContextType.RESOURCE.value,- ContextType.MEMORY.value,- ContextType.SKILL.value,- ]- return [str(context_type_for_uri(uri))]-- def _is_supported_prune_record(- self,- record: dict[str, Any],- counters: _ReindexCounters,- ) -> bool:- record_id = record.get("id")- uri = record.get("uri")- context_type = str(record.get("context_type") or "")- level = record.get("level")- if not record_id or not uri:- counters.unsupported_records += 1- counters.warnings.append(f"Skipping unknown prune record without id/uri: {record!r}")- return False- if context_type not in {- ContextType.RESOURCE.value,- ContextType.MEMORY.value,- ContextType.SKILL.value,- }:- counters.unsupported_records += 1- counters.warnings.append(f"Skipping unknown context_type for prune record {uri}")- return False- try:- normalized_level = int(level)- except (TypeError, ValueError):- counters.unsupported_records += 1- counters.warnings.append(f"Skipping unknown level for prune record {uri}: {level}")- return False- if normalized_level not in {- int(ContextLevel.ABSTRACT),- int(ContextLevel.OVERVIEW),- int(ContextLevel.DETAIL),- }:- counters.unsupported_records += 1- counters.warnings.append(f"Skipping unknown level for prune record {uri}: {level}")- return False- record["_prune_level"] = normalized_level- return True-- async def _is_orphan_vector_record(- self,- record: dict[str, Any],- *,- counters: _ReindexCounters,- ctx: RequestContext,- ) -> bool:- uri = str(record["uri"])- level = int(record.get("_prune_level", record["level"]))- context_type = str(record["context_type"])- owner_ctx = self._delete_ctx_for_prune_record(record, ctx)-- if level == int(ContextLevel.ABSTRACT):- abstract = await self._read_prune_source(- f"{uri}/.abstract.md",- ctx=owner_ctx,+ service = get_service()+ request = RequestIntent.from_ingest_options(+ target_uri=uri,+ processing_mode=mode,+ ingest_options=run.ingest_options,+ force=run.force,+ scalar_target_levels=frozenset({0, 1, 2}),+ )+ snapshot = await build_rfv_snapshot(+ viking_fs=service.viking_fs,+ vikingdb=service.vikingdb_manager,+ target_uri=uri,+ ctx=run.ctx,+ request_intent=request,+ recursive=recursive,+ source_read_concurrency=self._effective_file_vectorization_concurrency(),+ root_is_dir=run.root_is_dir if root_is_dir is None else root_is_dir,+ formal_inventory=(+ (True if root_is_dir is None else root_is_dir, formal_entries, True)+ if formal_entries is not None+ else None+ ),+ )+ diff, plan = build_rfv_context_update_plan(+ snapshot=snapshot,+ context_type=context_type,+ account_id=run.ctx.account_id,+ source_metadata=snapshot.source_metadata,+ )+ if (+ mode == "vectors_only"+ and not snapshot.formal.entries[""].is_dir+ and any(+ action.level == int(ContextLevel.DETAIL)+ and action.action.value in {"merge", "upsert"}+ for action in plan.direct_index_actions+ )+ ):+ enriched_snapshot = await self._with_single_file_parent_overview(+ snapshot,+ ctx=run.ctx,+ viking_fs=service.viking_fs,)- if abstract.error:- self._record_prune_source_error(- counters=counters,- uri=uri,- source_uri=f"{uri}/.abstract.md",- error=abstract.error,+ if enriched_snapshot is not snapshot:+ snapshot = enriched_snapshot+ diff, plan = build_rfv_context_update_plan(+ snapshot=snapshot,+ context_type=context_type,+ account_id=run.ctx.account_id,+ source_metadata=snapshot.source_metadata,)- return False- return (- not abstract.exists- or not abstract.text- or _is_not_ready_sentinel(abstract.text, _ABSTRACT_NOT_READY_SUFFIX)+ run.counters.scanned_records += len(snapshot.formal.entries)+ if plan.direct_index_actions:+ owner_ctx = self._content_owner_ctx(uri, run.ctx)+ delete_actions = tuple(+ action for action in plan.direct_index_actions if action.action.value == "delete")-- if level == int(ContextLevel.OVERVIEW):- overview = await self._read_prune_source(- f"{uri}/.overview.md",- ctx=owner_ctx,+ enqueue_actions = tuple(+ action for action in plan.direct_index_actions if action.action.value != "delete")- if overview.error:- self._record_prune_source_error(- counters=counters,- uri=uri,- source_uri=f"{uri}/.overview.md",- error=overview.error,- )- return False- if (- overview.exists- and overview.text- and not _is_not_ready_sentinel(overview.text, _OVERVIEW_NOT_READY_SUFFIX)- ):- return False- if context_type in {ContextType.RESOURCE.value, ContextType.SKILL.value}:- abstract = await self._read_prune_source(- f"{uri}/.abstract.md",+ if delete_actions:+ run.counters.deleted_records += await service.vikingdb_manager.strict_delete(+ [action.record_id for action in delete_actions],ctx=owner_ctx,)- if abstract.error:- self._record_prune_source_error(- counters=counters,- uri=uri,- source_uri=f"{uri}/.abstract.md",- error=abstract.error,- )- return False- return (- not abstract.exists- or not abstract.text- or _is_not_ready_sentinel(abstract.text, _ABSTRACT_NOT_READY_SUFFIX)+ if enqueue_actions:+ processor = getattr(service, "_resource_processor", None)+ if processor is None:+ processor = ResourceProcessor(service.vikingdb_manager)+ run.counters.rebuilt_records += await processor._enqueue_index_actions(+ enqueue_actions,+ ctx=owner_ctx,+ source_contents=dict(snapshot.source_contents),+ source_metadata=snapshot.source_metadata,)- return True-- if self._is_hidden_meta_file(uri):- return False- exists = await self._prune_source_exists(uri, ctx=owner_ctx)- if exists.error:- self._record_prune_source_error(- counters=counters,- uri=uri,- source_uri=uri,- error=exists.error,+ if plan.semantic_plan is not None:+ await self._enqueue_reindex_semantic_plan(+ semantic_plan=plan.semantic_plan,+ context_type=context_type,+ recursive=recursive,+ run=run,+ )+ elif plan.file_refresh is not None:+ service_processor = getattr(service, "_resource_processor", None)+ if service_processor is None:+ service_processor = ResourceProcessor(service.vikingdb_manager)+ await service_processor._get_summarizer().refresh_file_parent(+ file_uri=plan.file_refresh.file_uri,+ ctx=run.ctx,+ ingest_options=run.ingest_options,+ file_md5=plan.file_refresh.md5,)- return False- # A real filename can have the same spelling as a virtual chunk URI.- if exists.exists:- return False-- if "#" in uri:- # Generated chunks append a zero-padded index to the full base URI.- chunk_match = re.fullmatch(r"(.+)#chunk_[0-9]{4,}", uri)- if context_type == ContextType.MEMORY.value and chunk_match:- base_uri = chunk_match.group(1)- base = await self._read_prune_source(base_uri, ctx=owner_ctx)- if base.error:- self._record_prune_source_error(- counters=counters,- uri=uri,- source_uri=base_uri,- error=base.error,- )- return False- if not base.exists:- return True- expected = {- chunk_uri for chunk_uri, _chunk in self._chunk_memory_body(base_uri, base.text)- }- return uri not in expected- return False-- return True-- async def _read_prune_source(self, uri: str, *, ctx: RequestContext) -> _PruneSourceRead:- viking_fs = get_viking_fs()- try:- exists = await viking_fs.exists(uri, ctx=ctx)- except Exception as exc:- return _PruneSourceRead(exists=False, error=exc)- if not exists:- return _PruneSourceRead(exists=False)- try:- content = await viking_fs.read_file(uri, ctx=ctx)- except Exception as exc:- return _PruneSourceRead(exists=True, error=exc)- if isinstance(content, bytes):- text = content.decode("utf-8", errors="replace")- else:- text = str(content or "")- return _PruneSourceRead(exists=True, text=text)-- async def _prune_source_exists(self, uri: str, *, ctx: RequestContext) -> _PruneSourceRead:- try:- exists = await get_viking_fs().exists(uri, ctx=ctx)- except Exception as exc:- return _PruneSourceRead(exists=False, error=exc)- return _PruneSourceRead(exists=exists)- def _record_prune_source_error(- self,+ @staticmethod+ async def _with_single_file_parent_overview(+ snapshot: Any,*,- counters: _ReindexCounters | None,- uri: str,- source_uri: str,- error: Exception,- ) -> None:- if counters is None:- return- counters.failed_records += 1- counters.warnings.append(f"Skipped prune for {uri}: failed to read {source_uri}: {error}")-- def _delete_ctx_for_prune_record(- self,- record: dict[str, Any],ctx: RequestContext,- ) -> RequestContext:- uri = str(record.get("uri") or "")- owner = record.get("owner_user_id")- if not owner:- owner = owner_fields_for_uri(uri).get("owner_user_id")- if not owner or owner == ctx.user.user_id:- return ctx- return RequestContext(- user=UserIdentifier(ctx.account_id, str(owner)),- role=ctx.role,- actor_peer_id=ctx.actor_peer_id,- from_oauth=ctx.from_oauth,+ viking_fs: Any,+ ) -> Any:+ """Best-effort hydrate a file root's parent overview for L2 summary reuse."""+ parent = VikingURI(snapshot.request.target_uri).parent+ if parent is None:+ return snapshot+ try:+ raw = await viking_fs.read_file_bytes(f"{parent.uri}/.overview.md", ctx=ctx)+ overview = body_for_preview(raw)+ except Exception:+ return snapshot+ if not overview.strip():+ return snapshot+ return replace(+ snapshot,+ source_contents={+ **snapshot.source_contents,+ (parent.uri, int(ContextLevel.OVERVIEW)): overview,+ },)- async def _reindex_resource(+ async def _enqueue_reindex_semantic_plan(self,*,- uri: str,- mode: str,+ semantic_plan: Any,+ context_type: str,+ recursive: bool,run: _ReindexRunContext,- recursive: bool = True,) -> None:- counters = run.counters- ctx = run.ctx- if mode == "semantic_and_vectors":- semantic_kwargs = {- "uri": uri,- "context_type": "resource",- "ctx": ctx,- "lock": run.lock,- }- if not recursive:- semantic_kwargs["recursive"] = False- await self._run_semantic_processor(**semantic_kwargs)- vector_kwargs = {"uri": uri, "counters": counters, "ctx": ctx}- if not recursive:- vector_kwargs["recursive"] = False- await self._reindex_resource_vectors(- **self._with_ingest_options(- vector_kwargs,- run.ingest_options,- )- )+ """Persist one fully planned reindex DAG without persisting its source bytes."""+ from openviking.storage.queuefs import get_queue_manager++ owner_ctx = self._content_owner_ctx(semantic_plan.root_uri, run.ctx)+ telemetry_id = get_current_telemetry().telemetry_id+ msg = SemanticMsg(+ uri=semantic_plan.root_uri,+ context_type=context_type,+ recursive=recursive,+ account_id=owner_ctx.account_id,+ user_id=owner_ctx.user.user_id,+ group_ids=list(owner_ctx.group_ids),+ peer_id=owner_ctx.user.user_id,+ role=str(owner_ctx.role),+ telemetry_id=telemetry_id,+ generation_trigger="reindex",+ propagate_to_parent=recursive,+ plan=semantic_plan,+ )+ tracker = get_request_wait_tracker()+ if telemetry_id:+ tracker.register_semantic_root(telemetry_id, msg.id)++ queue_manager = get_queue_manager()+ semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)+ if run.lease is None:+ try:+ enqueue_id = await semantic_queue.enqueue(msg)+ except Exception:+ if telemetry_id:+ tracker.mark_semantic_failed(+ telemetry_id, msg.id, "semantic plan enqueue failed"+ )+ raise+ if enqueue_id == "deduplicated" and telemetry_id:+ tracker.mark_semantic_done(telemetry_id, msg.id, processed_delta=0)return- await self._reindex_resource_vectors(- **self._with_ingest_options(- {"uri": uri, "counters": counters, "ctx": ctx},- run.ingest_options,++ viking_fs = get_service().viking_fs+ agfs = viking_fs._async_agfs+ use_outer_lease = run.lock_root_uri in {None, semantic_plan.root_uri.rstrip("/")}+ plan_lease = run.lease+ acquired_plan_lease = False+ if not use_outer_lease:+ root_entry = next(+ entry for entry in semantic_plan.tree.entries if not entry.relative_path)- )+ acquire = (+ agfs.pathlock_acquire_tree+ if root_entry.kind == "directory"+ else agfs.pathlock_acquire_exact+ )+ kwargs = {"owner_lease_ref": run.lease} if run.lease is not None else {}+ plan_lease = await acquire(+ viking_fs._uri_to_path(semantic_plan.root_uri, ctx=owner_ctx), **kwargs+ )+ acquired_plan_lease = True+ assert plan_lease is not None+ handoff = await agfs.pathlock_to_handoff(plan_lease)+ handed_off = False+ try:+ await agfs.pathlock_handoff(plan_lease)+ handed_off = True+ msg.lock_handoff = handoff+ enqueue_id = await semantic_queue.enqueue(msg)+ if enqueue_id == "deduplicated":+ lock = await agfs.pathlock_adopt(handoff)+ await agfs.pathlock_release(lock)+ if not acquired_plan_lease:+ run.lease_handed_off = True+ run.lease = None+ run.lock = None+ if telemetry_id:+ tracker.mark_semantic_done(telemetry_id, msg.id, processed_delta=0)+ return+ if not acquired_plan_lease:+ run.lease_handed_off = True+ run.lease = None+ run.lock = None+ except BaseException:+ if handed_off:+ recovered_lease = await agfs.pathlock_adopt(handoff)+ if acquired_plan_lease:+ await agfs.pathlock_release(recovered_lease)+ else:+ run.lease = recovered_lease+ run.lock = await agfs.pathlock_as_borrowed(run.lease)+ elif acquired_plan_lease:+ await agfs.pathlock_release(plan_lease)+ if telemetry_id:+ tracker.mark_semantic_failed(telemetry_id, msg.id, "semantic plan enqueue failed")+ raiseasync def _reindex_skill(self,@@ -1013,56 +923,19 @@ class ReindexExecutor:mode: str,run: _ReindexRunContext,recursive: bool = True,+ formal_entries: list[dict[str, Any]] | None = None,) -> None:- counters = run.counters- ctx = run.ctx- if mode == "semantic_and_vectors":- if not recursive:- await self._regenerate_skill_semantics(uri=uri, ctx=ctx, lock=run.lock)- await self._reindex_skill_vectors(- uri=uri,- counters=counters,- ctx=ctx,- recursive=False,- ingest_options=run.ingest_options,- )- return- owner_ctx = self._content_owner_ctx(uri, ctx)- processor = await self._semantic_processor_for(owner_ctx)- executor = SemanticTreeExecutor(- processor=processor,- context_type="skill",- max_concurrent_llm=processor.max_concurrent_llm,- ctx=owner_ctx,- lock=run.lock,- generation_trigger="reindex",- ingest_options=run.ingest_options,- source={- "path": (await self._skill_meta(uri=uri, abstract="", ctx=ctx)).get(- "source_path", ""- )- },- )- try:- await executor.run(uri)- finally:- # Even a semantic error must settle already submitted Skill- # vectors before the caller releases its package lease.- await get_request_wait_tracker().wait_for_embeddings(- get_current_telemetry().telemetry_id- )- stats = executor.get_stats()- counters.scanned_records += stats.total_nodes- counters.rebuilt_records += stats.indexed_records- counters.failed_records += len(stats.failures)- counters.warnings.extend(stats.failures)- return- await self._reindex_skill_vectors(- **self._with_ingest_options(- {"uri": uri, "counters": counters, "ctx": ctx},- run.ingest_options,- )- )+ kwargs: dict[str, Any] = {+ "uri": uri,+ "mode": mode,+ "context_type": "skill",+ "recursive": recursive,+ "run": run,+ }+ if formal_entries is not None:+ kwargs["root_is_dir"] = True+ kwargs["formal_entries"] = formal_entries+ await self._reindex_rfv(**kwargs)async def _reindex_memory(self,@@ -1075,7 +948,7 @@ class ReindexExecutor:counters = run.countersctx = run.ctxif mode == "semantic_and_vectors":- stat = await get_viking_fs().stat(uri, ctx=ctx, skip_count=True)+ stat = await get_viking_fs().stat(uri, ctx=ctx)if stat.get("isDir", stat.get("is_dir")):semantic_kwargs = {"uri": uri,@@ -1098,7 +971,12 @@ class ReindexExecutor:returnawait self._reindex_memory_vectors(**self._with_ingest_options(- {"uri": uri, "counters": counters, "ctx": ctx},+ {+ "uri": uri,+ "counters": counters,+ "ctx": ctx,+ "recursive": recursive,+ },run.ingest_options,))@@ -1129,184 +1007,6 @@ class ReindexExecutor:)await processor.on_dequeue({"data": msg.to_json()}, lock=lock)- async def _reindex_resource_vectors(- self,- *,- uri: str,- counters: _ReindexCounters,- ctx: RequestContext,- recursive: bool = True,- ingest_options: IngestOptions | None = None,- ) -> None:- viking_fs = get_viking_fs()- try:- if not await viking_fs.exists(uri, ctx=ctx):- raise NotFoundError(uri, "resource")- stat = await viking_fs.stat(uri, ctx=ctx, skip_count=True)- is_dir = stat.get("isDir", stat.get("is_dir")) if isinstance(stat, dict) else False- if is_dir and recursive:- entries = await self._tree_all(viking_fs, uri, show_all_hidden=True, ctx=ctx)- else:- entries = []- except Exception as exc:- raise NotFoundError(uri, "resource") from exc-- if is_dir:- directories = [uri]- files: list[str] = []- for entry in entries:- entry_uri = entry.get("uri")- if not entry_uri:- continue- if entry.get("isDir"):- directories.append(entry_uri)- elif not self._is_hidden_meta_file(entry_uri):- files.append(entry_uri)- else:- directories = []- files = [uri]-- await self._reindex_resource_vectors_from_entries(- **self._with_ingest_options(- {- "root_uri": uri,- "directories": directories,- "files": files,- "counters": counters,- "ctx": ctx,- },- ingest_options,- )- )-- async def _reindex_resource_vectors_from_entries(- self,- *,- root_uri: str,- directories: Iterable[str],- files: Iterable[str],- counters: _ReindexCounters,- ctx: RequestContext,- ingest_options: IngestOptions | None = None,- ) -> None:- deduped_directories = []- seen_directories = set()- for directory_uri in directories:- if directory_uri and directory_uri not in seen_directories:- deduped_directories.append(directory_uri)- seen_directories.add(directory_uri)-- deduped_files = []- seen_files = set()- for file_uri in files:- if file_uri and file_uri not in seen_files:- deduped_files.append(file_uri)- seen_files.add(file_uri)-- for directory_uri in deduped_directories:- if directory_uri == "viking://":- continue- counters.scanned_records += 1- abstract = await self._read_directory_abstract(directory_uri, ctx=ctx)- overview = await self._read_directory_overview(directory_uri, ctx=ctx)- if not overview:- overview = abstract- if not abstract and not overview:- counters.unsupported_records += 1- counters.warnings.append(f"No semantic source found for {directory_uri}")- continue- if abstract:- try:- await self._upsert_context(- uri=directory_uri,- parent_uri=VikingURI(directory_uri).parent.uri,- abstract=abstract,- vector_text=embedding_text_for_body(- ContextLevel.ABSTRACT, directory_uri, abstract- ),- is_leaf=False,- context_type=context_type_for_uri(directory_uri),- level=ContextLevel.ABSTRACT,- ctx=ctx,- ingest_options=ingest_options,- )- counters.rebuilt_records += 1- except Exception as exc:- counters.failed_records += 1- counters.warnings.append(f"Failed to reindex {directory_uri} L0 vector: {exc}")- if overview:- try:- await self._upsert_context(- uri=directory_uri,- parent_uri=VikingURI(directory_uri).parent.uri,- # L1 abstract scalar carries the overview for Rerank.- abstract=_truncate_abstract_bytes(overview),- vector_text=embedding_text_for_body(- ContextLevel.OVERVIEW, directory_uri, overview- ),- is_leaf=False,- context_type=context_type_for_uri(directory_uri),- level=ContextLevel.OVERVIEW,- ctx=ctx,- ingest_options=ingest_options,- )- counters.rebuilt_records += 1- except Exception as exc:- counters.failed_records += 1- counters.warnings.append(f"Failed to reindex {directory_uri} L1 vector: {exc}")-- async def process_file(file_uri: str) -> _ReindexCounters:- file_counters = _ReindexCounters(scanned_records=1)- parent_uri = VikingURI(file_uri).parent.uri- try:- file_bytes = await get_viking_fs().read_file_bytes(file_uri, ctx=ctx)- except Exception as exc:- file_counters.failed_records += 1- file_counters.warnings.append(f"Failed to read {file_uri} for reindex: {exc}")- return file_counters- summary = await self._best_file_summary(file_uri, ctx=ctx)- vector_text = await self._best_resource_file_vector_text(- file_uri, summary, ctx=ctx, file_content=file_bytes- )- if not vector_text:- file_counters.unsupported_records += 1- file_counters.warnings.append(f"No vector source found for {file_uri}")- return file_counters- abstract = self._prefer_non_empty(summary, vector_text)- try:- await self._upsert_context(- uri=file_uri,- parent_uri=parent_uri,- abstract=abstract,- vector_text=vector_text,- is_leaf=True,- context_type=context_type_for_uri(file_uri),- level=ContextLevel.DETAIL,- ctx=ctx,- ingest_options=ingest_options,- md5=content_md5(file_bytes),- )- file_counters.rebuilt_records += 1- except Exception as exc:- file_counters.failed_records += 1- file_counters.warnings.append(f"Failed to reindex {file_uri} vector: {exc}")- return file_counters-- concurrency = self._effective_file_vectorization_concurrency()- if deduped_files:- logger.info(- "Reindex resource file vectorization: root=%s files=%d concurrency=%d",- root_uri,- len(deduped_files),- concurrency,- )- await self._run_ordered_counter_batches(- deduped_files,- concurrency=concurrency,- processor=process_file,- counters=counters,- )-async def reindex_directory_marker(self, *, dir_uri: str, level: ContextLevel, ctx: RequestContext) -> None:@@ -1380,11 +1080,13 @@ class ReindexExecutor:uri: str,mode: str,run: _ReindexRunContext,+ recursive: bool = True,) -> None:- counters = run.countersctx = run.ctxnormalized_uri = uri.rstrip("/")target_root = normalized_uri if normalized_uri else uri+ if not recursive:+ returnviking_fs = get_viking_fs()try:entries = await self._tree_all(viking_fs, target_root, show_all_hidden=True, ctx=ctx)@@ -1403,6 +1105,7 @@ class ReindexExecutor:uri=user_root,mode=mode,run=run,+ recursive=True,)return@@ -1451,31 +1154,53 @@ class ReindexExecutor:uri=skill_root,mode=skill_mode,run=run,+ recursive=True,+ formal_entries=[+ entry+ for entry in entries+ if str(entry.get("uri") or "").startswith(skill_root.rstrip("/") + "/")+ ],)- if mode == "semantic_and_vectors":- (- resource_directories,- resource_files,- ) = await self._refresh_namespace_resource_semantics(- target_root=target_root,- directories=resource_directories,- files=resource_files,+ resource_roots = [+ directory_uri+ for directory_uri in resource_directories+ if directory_uri.rstrip("/").endswith("/resources")+ ]+ covered_prefixes = tuple(root.rstrip("/") + "/" for root in resource_roots)+ for resource_root in resource_roots:+ await self._reindex_rfv(+ uri=resource_root,+ mode=mode,+ context_type="resource",+ recursive=True,run=run,+ root_is_dir=True,+ formal_entries=[+ entry+ for entry in entries+ if str(entry.get("uri") or "").startswith(resource_root.rstrip("/") + "/")+ ],)-- await self._reindex_resource_vectors_from_entries(- **self._with_ingest_options(- {- "root_uri": target_root,- "directories": resource_directories,- "files": resource_files,- "counters": counters,- "ctx": ctx,- },- run.ingest_options,+ for file_uri in resource_files:+ if any(file_uri.startswith(prefix) for prefix in covered_prefixes):+ continue+ if mode == "semantic_and_vectors":+ run.counters.unsupported_records += 1+ run.counters.warnings.append(+ f"Skipped {file_uri}: namespace semantic_and_vectors only refreshes "+ "resource directories"+ )+ continue+ await self._reindex_rfv(+ uri=file_uri,+ mode=mode,+ context_type="resource",+ recursive=False,+ run=run,+ root_is_dir=False,+ formal_entries=[],)- )async def _reindex_global_namespace(self,@@ -1483,10 +1208,12 @@ class ReindexExecutor:uri: str,mode: str,run: _ReindexRunContext,+ recursive: bool = True,) -> None:- counters = run.countersctx = run.ctxtarget_root = "viking://"+ if not recursive:+ returnviking_fs = get_viking_fs()try:entries = await self._tree_all(viking_fs, target_root, show_all_hidden=True, ctx=ctx)@@ -1524,126 +1251,39 @@ class ReindexExecutor:run=run,)- if mode == "semantic_and_vectors":- (- resource_directories,- resource_files,- ) = await self._refresh_namespace_resource_semantics(- target_root=target_root,- directories=resource_directories,- files=resource_files,+ if "viking://resources" in resource_directories:+ await self._reindex_rfv(+ uri="viking://resources",+ mode=mode,+ context_type="resource",+ recursive=True,run=run,+ root_is_dir=True,+ formal_entries=[+ entry+ for entry in entries+ if str(entry.get("uri") or "").startswith("viking://resources/")+ ],)-- await self._reindex_resource_vectors_from_entries(- **self._with_ingest_options(- {- "root_uri": target_root,- "directories": resource_directories,- "files": resource_files,- "counters": counters,- "ctx": ctx,- },- run.ingest_options,- )- )-- async def _reindex_skill_vectors(- self,- *,- uri: str,- counters: _ReindexCounters,- ctx: RequestContext,- recursive: bool = True,- ingest_options: IngestOptions | None = None,- ) -> None:- viking_fs = get_viking_fs()- owner_ctx = self._content_owner_ctx(uri, ctx)- pending = [uri]- while pending:- directory_uri = pending.pop()- counters.scanned_records += 1- abstract = await self._read_directory_abstract(directory_uri, ctx=ctx)- overview = await self._read_directory_overview(directory_uri, ctx=ctx)- for level in (0, 1):- if abstract if level == 0 else overview:- continue- record = await self._fetch_existing_record(- uri=directory_uri, level=level, ctx=owner_ctx+ for file_uri in resource_files:+ if file_uri.startswith("viking://resources/"):+ continue+ if mode == "semantic_and_vectors":+ run.counters.unsupported_records += 1+ run.counters.warnings.append(+ f"Skipped {file_uri}: namespace semantic_and_vectors only refreshes "+ "resource directories")- if level == 0:- abstract = self._record_abstract(record)- else:- overview = self._record_abstract(record)- if abstract or overview:- try:- await vectorize_directory_meta(- directory_uri,- abstract,- overview,- context_type="skill",- ctx=owner_ctx,- include_abstract=bool(abstract),- include_overview=bool(overview),- ingest_options=ingest_options,- content_is_body=True,- **(- {- "meta": await self._skill_meta(- uri=directory_uri, abstract=abstract, ctx=ctx- )- }- if classify_uri(directory_uri).is_skill_root- else {}- ),- )- counters.rebuilt_records += int(bool(abstract)) + int(bool(overview))- except Exception as exc:- counters.failed_records += 1- counters.warnings.append(f"Failed to reindex {directory_uri}: {exc}")- else:- counters.unsupported_records += 1- counters.warnings.append(f"No semantic source found for {directory_uri}")-- if not recursive:continue- from openviking.storage.queuefs.semantic_executor import _SKIP_FILENAMES- from openviking.storage.viking_fs import LS_ALL_NODES-- entries = await viking_fs.ls(directory_uri, node_limit=LS_ALL_NODES, ctx=ctx)- for entry in entries:- name = entry.get("name", "")- if not name or name.startswith(".") or name in _SKIP_FILENAMES:- continue- child_uri = VikingURI(directory_uri).join(name).uri- if entry.get("isDir", False):- pending.append(child_uri)- continue- counters.scanned_records += 1- try:- record = await self._fetch_existing_record(- uri=child_uri, level=2, ctx=owner_ctx- )- summary = self._record_abstract(record)- if not summary:- summary = await self._best_file_summary(child_uri, ctx=ctx)- enqueued = await vectorize_file(- file_path=child_uri,- summary_dict={"name": name, "summary": summary},- parent_uri=directory_uri,- context_type="skill",- ctx=owner_ctx,- preserve_existing_created_at=True,- ingest_options=ingest_options,- )- if enqueued:- counters.rebuilt_records += 1- else:- counters.unsupported_records += 1- counters.warnings.append(f"No vector source found for {child_uri}")- except Exception as exc:- counters.failed_records += 1- counters.warnings.append(f"Failed to reindex {child_uri}: {exc}")+ await self._reindex_rfv(+ uri=file_uri,+ mode=mode,+ context_type="resource",+ recursive=False,+ run=run,+ root_is_dir=False,+ formal_entries=[],+ )async def _reindex_memory_vectors(self,@@ -1880,9 +1520,7 @@ class ReindexExecutor:if content_type == ResourceContentType.TEXT:if self.vector_config_resolver is None:raise RuntimeError("ReindexExecutor requires a vector config resolver")- embedding_config = (- await self.vector_config_resolver.resolve(ctx.account_id)- ).embedding+ embedding_config = (await self.vector_config_resolver.resolve(ctx.account_id)).embeddingtext_source = embedding_config.text_sourceif text_source in SUMMARY_TEXT_SOURCES and summary:return summary@@ -2033,8 +1671,23 @@ class ReindexExecutor:except Exception:return ""- async def _read_memory_body(self, uri: str, *, ctx: RequestContext) -> _PruneSourceRead:- return await self._read_prune_source(uri, ctx=ctx)+ async def _read_memory_body(self, uri: str, *, ctx: RequestContext) -> _SourceRead:+ viking_fs = get_viking_fs()+ try:+ exists = await viking_fs.exists(uri, ctx=ctx)+ except Exception as exc:+ return _SourceRead(exists=False, error=exc)+ if not exists:+ return _SourceRead(exists=False)+ try:+ content = await viking_fs.read_file(uri, ctx=ctx)+ except Exception as exc:+ return _SourceRead(exists=True, error=exc)+ if isinstance(content, bytes):+ text = content.decode("utf-8", errors="replace")+ else:+ text = str(content or "")+ return _SourceRead(exists=True, text=text)def _chunk_memory_body(self, uri: str, body: str) -> Iterable[tuple[str, str]]:semantic = get_openviking_config().semanticdiff --git a/openviking/storage/abstract_overview.py b/openviking/storage/abstract_overview.pyindex 07a166491..7fa62c6e8 100644--- a/openviking/storage/abstract_overview.py+++ b/openviking/storage/abstract_overview.py@@ -12,7 +12,7 @@ import hashlibimport refrom dataclasses import dataclassfrom typing import TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional, Sequence, TypeVar-from urllib.parse import quote, unquote+from urllib.parse import quote, unquote, urlsplitimport yaml@@ -397,6 +397,62 @@ def body_for_preview(raw: str | bytes) -> str:return document.body.rstrip("\r\n")+def parse_overview_file_summaries(overview_content: str | bytes) -> Dict[str, str]:+ """Extract direct-file summaries from an overview body."""+ overview = body_for_preview(overview_content)+ if not overview.strip():+ return {}++ summaries: Dict[str, str] = {}+ current_file = ""+ current_summary_lines: list[str] = []++ def save_current() -> None:+ if current_file and current_summary_lines:+ summaries[current_file] = " ".join(current_summary_lines).strip()++ for line in overview.split("\n"):+ header_match = re.match(r"^###\s+(.+?)\s*$", line)+ if header_match:+ save_current()+ heading = header_match.group(1).strip()+ file_name = _overview_heading_file_name(heading)+ parts = file_name.split()+ current_file = parts[0] if len(parts) >= 2 and parts[0] == parts[1] else file_name+ current_summary_lines = []+ continue++ numbered_match = re.match(r"^\[(\d+)\]\s+(.+?):\s*(.+)$", line)+ if numbered_match:+ save_current()+ current_file = numbered_match.group(2).strip()+ current_summary_lines = [numbered_match.group(3).strip()]+ continue++ if current_file:+ stripped = line.strip()+ if stripped and not stripped.startswith("#"):+ current_summary_lines.append(stripped)++ save_current()+ return summaries+++def _overview_heading_file_name(heading: str) -> str:+ """Return the direct entry name represented by an overview H3 heading."""+ if heading.startswith("[") and heading.endswith(")"):+ destination_start = heading.rfind("](")+ if destination_start > 0:+ target = heading[destination_start + 2 : -1].strip()+ if target.startswith("<") and target.endswith(">"):+ target = target[1:-1].strip()+ if target.startswith("viking://"):+ path = unquote(urlsplit(target).path).rstrip("/")+ if path:+ return path.rsplit("/", 1)[-1]+ return heading++def body_for_embedding(raw: str | bytes,whitelist: Sequence[str] = EMBEDDING_METADATA_FIELDS,@@ -486,6 +542,7 @@ async def write_abstract_overview(consume_pending: Optional[int] = None,lock: Optional[Dict[str, Any]] = None,log_prefix: str = "[Semantic]",+ existing_raw: Optional[Mapping[int, str | bytes]] = None,) -> AbstractOverviewWriteResult:"""Render and atomically write sidecars while preserving source metadata.@@ -512,8 +569,20 @@ async def write_abstract_overview(logger.info("%s Skipping stale semantic write for %s", log_prefix, dir_uri)return AbstractOverviewWriteResult(wrote=False)- existing_overview = await _read_existing_document(viking_fs, overview_uri, ctx)- existing_abstract = await _read_existing_document(viking_fs, abstract_uri, ctx)+ existing_overview = (+ parse_abstract_overview(existing_raw[1])+ if existing_raw is not None and 1 in existing_raw+ else await _read_existing_document(viking_fs, overview_uri, ctx)+ if existing_raw is None+ else None+ )+ existing_abstract = (+ parse_abstract_overview(existing_raw[0])+ if existing_raw is not None and 0 in existing_raw+ else await _read_existing_document(viking_fs, abstract_uri, ctx)+ if existing_raw is None+ else None+ )merged_metadata = dict(metadata or {})for existing in (existing_overview, existing_abstract):if existing and "source" in existing.metadata and "source" not in merged_metadata:@@ -549,8 +618,20 @@ async def write_abstract_overview(rendered_abstract = render_abstract_overview(ContextLevel.ABSTRACT, dir_uri, abstract, merged_metadata)- current_overview = await _raw_if_exists(viking_fs, overview_uri, ctx)- current_abstract = await _raw_if_exists(viking_fs, abstract_uri, ctx)+ current_overview = (+ existing_raw.get(1)+ if existing_raw is not None+ else await _raw_if_exists(viking_fs, overview_uri, ctx)+ )+ current_abstract = (+ existing_raw.get(0)+ if existing_raw is not None+ else await _raw_if_exists(viking_fs, abstract_uri, ctx)+ )+ if isinstance(current_overview, bytes):+ current_overview = current_overview.decode("utf-8")+ if isinstance(current_abstract, bytes):+ current_abstract = current_abstract.decode("utf-8")if current_overview != rendered_overview:await viking_fs.write_file(@@ -704,6 +785,7 @@ async def read_abstract_overview_pending_snapshot(dir_uri: str,ctx: Optional[RequestContext],lock: Optional[Dict[str, Any]] = None,+ existing_raw: Optional[Mapping[int, str | bytes]] = None,) -> int:"""Read the pending counter captured at the start of an aggregation."""@@ -720,6 +802,14 @@ async def read_abstract_overview_pending_snapshot(pending_values.append(int(freshness["pending_child_changes"]))return max(pending_values, default=0)+ if existing_raw is not None:+ return _pending_of(+ [+ parse_abstract_overview(existing_raw[level]) if level in existing_raw else None+ for level in (1, 0)+ ]+ )+owns_lease = lock is Nonesnapshot_lease = lockif owns_lease:diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex b440fe64a..647d8375a 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -588,6 +588,9 @@ class TextEmbeddingHandler(DequeueHandlerBase):ctx: RequestContext,) -> str:inserted_data = embedding_msg.context_data+ materialized = inserted_data.pop("_materialized_content", None)+ if isinstance(materialized, str):+ return materialized[:VIKINGDB_CONTENT_MAX_SIZE]if inserted_data.get("is_leaf") and inserted_data.get("context_type") in (ContextType.RESOURCE.value,ContextType.SKILL.value,@@ -894,6 +897,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):embedding_msg,ctx,)+ inserted_data.pop("_materialized_content", None)if embedding_msg.action is IndexAction.MERGE:field_patch = embedding_msg.field_patchmerge_fields = dict(field_patch.values) if field_patch is not None else {}diff --git a/openviking/storage/content_write.py b/openviking/storage/content_write.pyindex 7572af63d..f06ddbbf1 100644--- a/openviking/storage/content_write.py+++ b/openviking/storage/content_write.py@@ -720,24 +720,45 @@ class ContentWriteCoordinator:) -> Dict[str, Any]:self._validate_tag_mode(mode)normalized_uri = self._validate_uri_path(uri, field_name="uri")- normalized_tags = normalize_search_tags(tags, discard_invalid=True)+ requested_mode = mode+ normalized_tags = (+ [] if requested_mode == "clear" else normalize_search_tags(tags, discard_invalid=True)+ )await self._viking_fs._ensure_access(normalized_uri, ctx, action=AclAction.WRITE)stat = await self._safe_stat(normalized_uri, ctx=ctx)+ context_type = context_type_for_uri(normalized_uri)+ root_uri = await self._resolve_root_uri(normalized_uri, ctx=ctx)+ if requested_mode == "replace" and not normalized_tags:+ return self._build_tags_result(+ uri=normalized_uri,+ updated_uris=[],+ skipped_count=1,+ failed_count=0,+ root_uri=root_uri,+ context_type=context_type,+ tags=normalized_tags,+ mode=requested_mode,+ )+ storage_mode = "replace" if requested_mode == "clear" else requested_modeif stat.get("isDir"):- return await self._set_directory_tags(+ result = await self._set_directory_tags(uri=normalized_uri,tags=normalized_tags,- mode=mode,+ mode=storage_mode,recursive=recursive,ctx=ctx,)- return await self._set_single_uri_tags(- uri=normalized_uri,- tags=normalized_tags,- mode=mode,- recursive=recursive,- ctx=ctx,- )+ else:+ result = await self._set_single_uri_tags(+ uri=normalized_uri,+ tags=normalized_tags,+ mode=storage_mode,+ recursive=recursive,+ ctx=ctx,+ )+ result["mode"] = requested_mode+ result["tags"] = normalized_tags+ return resultdef _build_write_result(self,@@ -1083,7 +1104,7 @@ class ContentWriteCoordinator:raise InvalidArgumentError(f"unsupported batch-write mode: {mode}")def _validate_tag_mode(self, mode: str) -> None:- if mode not in {"replace", "append"}:+ if mode not in {"replace", "append", "clear"}:raise InvalidArgumentError(f"unsupported tag mode: {mode}")def _ensure_content_write_policy(self, uri: str) -> None:diff --git a/openviking/storage/context_update_plan.py b/openviking/storage/context_update_plan.pyindex 755959ec4..80a7a9126 100644--- a/openviking/storage/context_update_plan.py+++ b/openviking/storage/context_update_plan.py@@ -11,6 +11,7 @@ from pathlib import PurePosixPathfrom typing import Any, Mappingfrom openviking.concurrency import bounded_map+from openviking.storage.abstract_overview import parse_overview_file_summariesfrom openviking.storage.index_action import FieldPatch, IndexActionfrom openviking.storage.resource_diff import ContentState, IndexStatefrom openviking.storage.resource_rnfv import (@@ -52,11 +53,6 @@ class FileVectorSource(str, Enum):SUMMARY_WHEN_AVAILABLE = "summary_when_available"-@dataclass(frozen=True)-class ParentPropagation:- enabled: bool = True--@dataclass(frozen=True)class FileRefreshIntent:"""Refresh a flat file and its parent after synchronous content commit."""@@ -247,7 +243,6 @@ class SemanticPlan:context_type: strtree: SemanticTreeSnapshotvectorize: bool = True- propagation: ParentPropagation = field(default_factory=ParentPropagation)file_vector_source: FileVectorSource = FileVectorSource.CONTENTingest_options: IngestOptions = field(default_factory=IngestOptions)source_metadata: Mapping[str, str] | None = None@@ -301,7 +296,6 @@ class SemanticPlan:context_type=str(data["context_type"]),tree=SemanticTreeSnapshot.from_dict(data.get("tree", {})),vectorize=bool(data.get("vectorize", True)),- propagation=ParentPropagation(**dict(data.get("propagation", {}))),file_vector_source=FileVectorSource(data.get("file_vector_source", FileVectorSource.CONTENT.value)),@@ -388,6 +382,7 @@ class DirectIndexAction:upsert_fields: Mapping[str, Any] = field(default_factory=dict)field_patch: FieldPatch | None = Nonemd5: str | None = None+ summary: str = ""def __post_init__(self) -> None:object.__setattr__(self, "action", IndexAction(self.action))@@ -588,11 +583,37 @@ def _field_patch(request: RequestIntent, record: VectorRecordSnapshot | None) ->return FieldPatch(values, modes) if values else None+def _rfv_file_summary(+ snapshot: Any,+ relative_path: str,+ record: VectorRecordSnapshot | None,+ overview_summaries: dict[str, Mapping[str, str]],+) -> str:+ file_name = relative_path.rsplit("/", 1)[-1]+ if not relative_path and not snapshot.formal.entries[""].is_dir:+ file_uri = snapshot.request.target_uri.rstrip("/")+ parent_uri = file_uri.rsplit("/", 1)[0]+ file_name = file_uri.rsplit("/", 1)[-1]+ else:+ parent_uri = _uri(snapshot.request.target_uri, _parent(relative_path))+ overview = snapshot.source_contents.get((parent_uri, 1))+ if isinstance(overview, (str, bytes)):+ summaries = overview_summaries.get(parent_uri)+ if summaries is None:+ summaries = parse_overview_file_summaries(overview)+ overview_summaries[parent_uri] = summaries+ summary = summaries.get(file_name)+ if summary:+ return summary+ return str(record.fields.get("abstract") or "") if record else ""++def _semantic_closure(diff: Any,new_kinds: Mapping[str, str],*,repair_indexes: bool = True,+ input_only_paths: frozenset[str] = frozenset(),) -> tuple[set[str], set[str], set[str]]:"""Return active nodes, membership-changing parents, and retained closure.@@ -612,6 +633,8 @@ def _semantic_closure(parent = _parent(parent)for path, entry in diff.entries.items():+ if path in input_only_paths:+ continuecontent_state = ContentState(entry.content_state)index_state = IndexState(entry.index_state)content_requires_semantics = content_state in {@@ -624,7 +647,12 @@ def _semantic_closure(or (repair_indexesand index_state- in {IndexState.MISSING, IndexState.PARTIAL, IndexState.LEVEL_CONFLICT}+ in {+ IndexState.MISSING,+ IndexState.PARTIAL,+ IndexState.LEVEL_CONFLICT,+ }+ or (repair_indexes and index_state is IndexState.STALE and bool(entry.level_states)))) and entry.new_kind in {"file", "directory"}:active.add(path)@@ -645,6 +673,51 @@ def _semantic_closure(return active, membership_changed, retained+def _promote_missing_semantic_inputs(+ *,+ active: set[str],+ retained: set[str],+ new_kinds: Mapping[str, str],+ records_by_path: Mapping[str, Mapping[int, VectorRecordSnapshot]],+ input_only_paths: frozenset[str] = frozenset(),+) -> None:+ """Promote retained inputs that cannot provide an abstract to a parent.++ An L2 record can be vector-complete after a vectors-only ingest while still+ lacking the file abstract required by its parent's first aggregation. The+ same applies to a retained directory without an L0 abstract. Semantic+ closure therefore tracks reusable abstracts independently from index MD5.+ """+ while True:+ missing_inputs = {+ path+ for path in retained - active+ if new_kinds.get(path) in {"file", "directory"}+ and (+ (+ record := records_by_path.get(path, {}).get(+ 2 if new_kinds.get(path) == "file" else 0+ )+ )+ is None+ or not str(record.fields.get("abstract") or "").strip()+ )+ }+ blocked_inputs = missing_inputs & input_only_paths+ if blocked_inputs:+ raise ValueError(+ "non-recursive semantic plan lacks reusable abstract for "+ + ", ".join(sorted(blocked_inputs))+ )+ promoted = missing_inputs - input_only_paths+ if not promoted:+ return+ active.update(promoted)+ for path in new_kinds:+ if path and _parent(path) in promoted:+ retained.add(path)++async def hydrate_context_plan_records(*,diff: Any,@@ -761,6 +834,7 @@ def build_context_update_plan(ingest_options: IngestOptions | Mapping[str, Any] | None = None,source_metadata: Mapping[str, str] | None = None,closure: tuple[set[str], set[str], set[str]] | None = None,+ input_only_paths: frozenset[str] = frozenset(),) -> ContextUpdatePlan:"""Compile resolved RNFV facts into synchronous and asynchronous actions.@@ -779,6 +853,7 @@ def build_context_update_plan(diff,new_kinds,repair_indexes=request.vectorize and request.processing_mode != "vectors_only",+ input_only_paths=input_only_paths,)semantic_enabled = request.processing_mode != "vectors_only"if not semantic_enabled:@@ -788,8 +863,11 @@ def build_context_update_plan(direct_actions: list[DirectIndexAction] = [DirectIndexAction(IndexAction.DELETE, record.uri, record.level, record.record_id)for record in duplicate_records+ if record.relative_path not in input_only_paths]for path, entry in sorted(diff.entries.items()):+ if path in input_only_paths:+ continuestate = ContentState(entry.content_state)kind = entry.new_kindif state in {ContentState.ADDED, ContentState.RESTORE, ContentState.MODIFIED}:@@ -874,11 +952,14 @@ def build_context_update_plan(),field_patch=field_patch if index_action is IndexAction.MERGE else None,md5=entry.md5,+ summary=str(record.fields.get("abstract") or "") if record else "",))scheduled_direct_ids = {action.record_id for action in direct_actions}for path, levels in records_by_path.items():+ if path in input_only_paths:+ continuefor record in levels.values():field_patch = _field_patch(request, record) if request.vectorize else Nonepatch_values = dict(field_patch.values) if field_patch is not None else {}@@ -918,6 +999,15 @@ def build_context_update_plan()scheduled_direct_ids.add(record.record_id)+ if semantic_enabled:+ _promote_missing_semantic_inputs(+ active=active,+ retained=retained,+ new_kinds=new_kinds,+ records_by_path=records_by_path,+ input_only_paths=input_only_paths,+ )+semantic_entries: list[SemanticTreeEntry] = []for path in sorted(retained, key=lambda value: (value.count("/"), value)):kind = new_kinds.get(path)@@ -926,7 +1016,9 @@ def build_context_update_plan(diff_entry = diff.entries.get(path)state = ContentState(diff_entry.content_state) if diff_entry else ContentState.UNCHANGEDaction = (- SemanticAction.GENERATE+ SemanticAction.REUSE+ if path in input_only_paths+ else SemanticAction.GENERATEif kind == "file" and path in activeelse SemanticAction.AGGREGATEif kind == "directory" and path in active@@ -959,6 +1051,8 @@ def build_context_update_plan()else IndexAction.UPSERT)+ if diff_entry and diff_entry.level_states:+ index_action = IndexAction.MERGEslots.append(IndexSlot(level,@@ -992,7 +1086,17 @@ def build_context_update_plan(request.vectorizeand diff_entryand IndexState(diff_entry.index_state)- in {IndexState.MISSING, IndexState.PARTIAL, IndexState.LEVEL_CONFLICT}+ in {+ IndexState.MISSING,+ IndexState.PARTIAL,+ IndexState.LEVEL_CONFLICT,+ }+ or (+ request.vectorize+ and diff_entry+ and IndexState(diff_entry.index_state) is IndexState.STALE+ and bool(diff_entry.level_states)+ )),))@@ -1025,6 +1129,116 @@ def build_context_update_plan()+def build_rfv_context_update_plan(+ *,+ snapshot: Any,+ context_type: str,+ account_id: str,+ source_metadata: Mapping[str, str] | None = None,+) -> tuple[Any, ContextUpdatePlan]:+ """Compile an R/F/V maintenance snapshot without inventing content writes."""+ from openviking.storage.resource_rfv import resolve_rfv_state++ diff = resolve_rfv_state(snapshot)+ if snapshot.request.processing_mode != "vectors_only":+ plan = build_context_update_plan(+ root_uri=snapshot.request.target_uri,+ context_type=context_type,+ request=snapshot.request,+ diff=diff,+ new_kinds={+ path: "directory" if entry.is_dir else "file"+ for path, entry in snapshot.formal.entries.items()+ },+ artifact_paths={},+ records=snapshot.vectors.records_by_id,+ is_code_repo=False,+ account_id=account_id,+ source_metadata=source_metadata or snapshot.source_metadata,+ input_only_paths=snapshot.input_only_paths,+ )+ if plan.content_tree_actions:+ raise ValueError("RFV maintenance plan must not mutate formal content")+ return diff, plan+ records_by_path, duplicate_records = _records_by_path(snapshot.vectors.records_by_id)+ overview_summaries: dict[str, Mapping[str, str]] = {}+ direct_actions: list[DirectIndexAction] = [+ DirectIndexAction(IndexAction.DELETE, record.uri, record.level, record.record_id)+ for record in duplicate_records+ if record.relative_path not in snapshot.input_only_paths+ ]++ for path, entry in sorted(diff.entries.items()):+ if path in snapshot.input_only_paths:+ continue+ existing = records_by_path.get(path, {})+ if entry.new_kind is None and IndexState(entry.index_state) is IndexState.ORPHAN:+ direct_actions.extend(+ DirectIndexAction(IndexAction.DELETE, record.uri, level, record.record_id)+ for level, record in sorted(existing.items())+ )+ continue++ expected_levels = {2} if entry.new_kind == "file" else {0, 1}+ for level, record in sorted(existing.items()):+ if level not in expected_levels or (+ snapshot.request.processing_mode == "vectors_only"+ and IndexState(entry.level_states.get(level, IndexState.ABSENT))+ is IndexState.ORPHAN+ ):+ direct_actions.append(+ DirectIndexAction(IndexAction.DELETE, record.uri, level, record.record_id)+ )++ for level in sorted(expected_levels):+ record = existing.get(level)+ level_state = IndexState(entry.level_states.get(level, IndexState.MISSING))+ field_patch = _field_patch(snapshot.request, record)+ if level_state in {IndexState.MISSING, IndexState.STALE}:+ if level not in entry.level_md5s:+ continue+ uri = _uri(snapshot.request.target_uri, path)+ direct_actions.append(+ DirectIndexAction(+ IndexAction.MERGE,+ uri,+ level,+ (+ record.record_id+ if record is not None+ else vector_record_id(account_id, uri, level)+ ),+ field_patch=field_patch,+ md5=entry.level_md5s.get(level),+ summary=_rfv_file_summary(snapshot, path, record, overview_summaries),+ )+ )+ elif field_patch is not None and record is not None:+ direct_actions.append(+ DirectIndexAction(+ IndexAction.UPDATE_FIELDS,+ record.uri,+ level,+ record.record_id,+ field_patch=field_patch.with_seed(+ {+ "uri": record.uri,+ "account_id": account_id,+ "level": level,+ **(_portable_existing_fields(record) or {}),+ **field_patch.resolve(record.fields),+ }+ ),+ )+ )++ return diff, ContextUpdatePlan(+ snapshot.request.target_uri,+ context_type,+ direct_index_actions=tuple(direct_actions),+ )++def _with_directory_root(snapshot: RNFVSnapshot, *, root_preexisting: bool) -> RNFVSnapshot:"""Add the logical directory root omitted by artifact/tree walks."""new_entries = {"": NewEntry(is_dir=True), **snapshot.new.entries}@@ -1200,13 +1414,13 @@ __all__ = ["IndexAction","IndexSlot","IndexState",- "ParentPropagation","SemanticAction","SemanticPlan","SemanticTreeEntry","SemanticTreeSnapshot","build_context_update_plan","build_context_update_plan_from_snapshot",+ "build_rfv_context_update_plan","execute_content_tree_actions","hydrate_context_plan_records",]diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 43a78b537..e045527b1 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -39,6 +39,7 @@ def init_queue_manager(max_concurrent_semantic: int = 32,max_concurrent_external_parse: int = 4,max_concurrent_add_resource: int = 4,+ max_concurrent_reindex: int = 4,max_concurrent_session_commit: int = DEFAULT_MAX_CONCURRENT_SESSION_COMMIT,max_concurrent_external_task: int = 10,*,@@ -54,6 +55,7 @@ def init_queue_manager(max_concurrent_semantic: Max concurrent semantic node work.max_concurrent_external_parse: Max concurrent ExternalParse tasks.max_concurrent_add_resource: Max concurrent AddResource tasks.+ max_concurrent_reindex: Max concurrent Reindex tasks.max_concurrent_session_commit: Max concurrent SessionCommit tasks.middlewares: Additional middleware, fixed at construction for all queues."""@@ -66,6 +68,7 @@ def init_queue_manager(max_concurrent_semantic=max_concurrent_semantic,max_concurrent_external_parse=max_concurrent_external_parse,max_concurrent_add_resource=max_concurrent_add_resource,+ max_concurrent_reindex=max_concurrent_reindex,max_concurrent_session_commit=max_concurrent_session_commit,max_concurrent_external_task=max_concurrent_external_task,middlewares=middlewares,@@ -92,6 +95,7 @@ class QueueManager:# Keep the on-disk name stable so pre-upgrade jobs remain recoverable.EXTERNAL_PARSE = "ExternalParse"ADD_RESOURCE = "AddResource"+ REINDEX = "Reindex"SESSION_COMMIT = "SessionCommit"EXTERNAL_TASK = "ExternalTask"# Account and user cleanup share one consumer. Retain the persisted name@@ -109,6 +113,7 @@ class QueueManager:max_concurrent_semantic: int = 32,max_concurrent_external_parse: int = 4,max_concurrent_add_resource: int = 4,+ max_concurrent_reindex: int = 4,max_concurrent_session_commit: int = DEFAULT_MAX_CONCURRENT_SESSION_COMMIT,max_concurrent_external_task: int = 10,*,@@ -122,6 +127,7 @@ class QueueManager:self._max_concurrent_semantic = max_concurrent_semanticself._max_concurrent_external_parse = max_concurrent_external_parseself._max_concurrent_add_resource = max_concurrent_add_resource+ self._max_concurrent_reindex = max_concurrent_reindexself._max_concurrent_session_commit = max_concurrent_session_commitself._max_concurrent_external_task = max_concurrent_external_taskself._queues: Dict[str, NamedQueue] = {}@@ -150,9 +156,7 @@ class QueueManager:if self._started:returnif self.SEMANTIC in self._queues and self._vlm_resolver is None:- raise RuntimeError(- "QueueManager requires a VLM resolver before semantic workers start"- )+ raise RuntimeError("QueueManager requires a VLM resolver before semantic workers start")self._started = True@@ -262,6 +266,8 @@ class QueueManager:return self._max_concurrent_external_parseif queue_name == self.ADD_RESOURCE:return self._max_concurrent_add_resource+ if queue_name == self.REINDEX:+ return self._max_concurrent_reindexif queue_name == self.SESSION_COMMIT:return self._max_concurrent_session_commitif queue_name == self.EXTERNAL_TASK:diff --git a/openviking/storage/queuefs/reindex_msg.py b/openviking/storage/queuefs/reindex_msg.pynew file mode 100644index 000000000..0db0397db--- /dev/null+++ b/openviking/storage/queuefs/reindex_msg.py@@ -0,0 +1,66 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Durable root request for asynchronous reindex work."""++from dataclasses import asdict, dataclass, field+from typing import Any, Dict, Optional+++@dataclass(kw_only=True)+class ReindexMsg:+ task_id: str+ uri: str+ object_type: str+ mode: str+ account_id: str+ user_id: str+ role: str+ force: bool = False+ recursive: bool = True+ tags: Optional[list[str]] = None+ tag_mode: str = "replace"+ group_ids: list[str] = field(default_factory=list)+ actor_peer_id: Optional[str] = None+ bypass_acl: bool = False+ telemetry_id: Optional[str] = None+ lock_handoff: Optional[Dict[str, Any]] = None+ lock_handoff_retry: int = 0++ def to_dict(self) -> Dict[str, Any]:+ return asdict(self)++ @classmethod+ def from_dict(cls, data: Dict[str, Any]) -> "ReindexMsg":+ required = ("task_id", "uri", "object_type", "mode", "account_id", "user_id")+ missing = [name for name in required if not data.get(name)]+ if missing:+ raise ValueError(f"Missing required reindex fields: {missing}")+ try:+ lock_handoff_retry = max(0, int(data.get("lock_handoff_retry", 0) or 0))+ except (TypeError, ValueError):+ lock_handoff_retry = 0+ return cls(+ task_id=str(data["task_id"]),+ uri=str(data["uri"]),+ object_type=str(data["object_type"]),+ mode=str(data["mode"]),+ account_id=str(data["account_id"]),+ user_id=str(data["user_id"]),+ role=str(data.get("role") or "root"),+ force=bool(data.get("force", False)),+ recursive=bool(data.get("recursive", True)),+ tags=list(data["tags"]) if isinstance(data.get("tags"), list) else None,+ tag_mode=str(data.get("tag_mode") or "replace"),+ group_ids=[str(value) for value in data.get("group_ids", [])],+ actor_peer_id=data.get("actor_peer_id")+ if isinstance(data.get("actor_peer_id"), str)+ else None,+ bypass_acl=bool(data.get("bypass_acl", False)),+ telemetry_id=data.get("telemetry_id")+ if isinstance(data.get("telemetry_id"), str)+ else None,+ lock_handoff=data.get("lock_handoff")+ if isinstance(data.get("lock_handoff"), dict)+ else None,+ lock_handoff_retry=lock_handoff_retry,+ )diff --git a/openviking/storage/queuefs/reindex_processor.py b/openviking/storage/queuefs/reindex_processor.pynew file mode 100644index 000000000..a9c15ea03--- /dev/null+++ b/openviking/storage/queuefs/reindex_processor.py@@ -0,0 +1,186 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Queue consumer that owns one durable asynchronous reindex request."""++from __future__ import annotations++import json+from typing import Any, Dict, Optional++from openviking.server.identity import RequestContext, Role+from openviking.service.task_tracker import TaskStatus, get_task_tracker+from openviking.service.task_work_index import bind_task_context, extract_task_metadata+from openviking.storage.queuefs.named_queue import DequeueHandlerBase+from openviking.storage.queuefs.process_result import ProcessResult+from openviking.storage.queuefs.reindex_msg import ReindexMsg+from openviking.telemetry import (+ OperationTelemetry,+ bind_telemetry,+ register_telemetry,+ resolve_telemetry,+ unregister_telemetry,+)+from openviking_cli.session.user_id import UserIdentifier+from openviking_cli.utils import get_logger++logger = get_logger(__name__)+++class ReindexProcessor(DequeueHandlerBase):+ def __init__(self, viking_fs: Any) -> None:+ self._viking_fs = viking_fs++ @staticmethod+ def _ctx(msg: ReindexMsg) -> RequestContext:+ return RequestContext(+ user=UserIdentifier(account_id=msg.account_id, user_id=msg.user_id),+ role=Role(msg.role),+ group_ids=tuple(msg.group_ids),+ actor_peer_id=msg.actor_peer_id,+ bypass_acl=msg.bypass_acl,+ )++ async def _adopt_or_reacquire(self, msg: ReindexMsg, ctx: RequestContext) -> Dict[str, Any]:+ agfs = self._viking_fs._async_agfs+ if msg.lock_handoff is not None:+ try:+ return await agfs.pathlock_adopt(msg.lock_handoff)+ except Exception:+ pass+ stat = await self._viking_fs.stat(msg.uri, ctx=ctx, skip_count=True)+ acquire = (+ agfs.pathlock_acquire_tree+ if stat.get("isDir", stat.get("is_dir"))+ else agfs.pathlock_acquire_exact+ )+ return await acquire(self._viking_fs._uri_to_path(msg.uri, ctx=ctx))++ async def _requeue_lock_handoff(self, msg: ReindexMsg, exc: Exception) -> bool:+ if msg.lock_handoff_retry >= 2:+ return False++ from openviking.storage.queuefs import get_queue_manager++ payload = msg.to_dict()+ payload["lock_handoff_retry"] = msg.lock_handoff_retry + 1+ queue_manager = get_queue_manager()+ await queue_manager.enqueue(queue_manager.REINDEX, payload)+ logger.warning(+ "[Reindex] Requeued task %s after lock handoff failure: %s",+ msg.task_id,+ exc,+ )+ return True++ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> ProcessResult:+ if not data:+ return ProcessResult.success()+ payload = data.get("data", data)+ if isinstance(payload, str):+ payload = json.loads(payload)+ msg = ReindexMsg.from_dict(payload)+ ctx = self._ctx(msg)+ tracker = get_task_tracker()+ metadata = extract_task_metadata(data)+ task = await tracker.get(+ msg.task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ if task is not None and task.status in {+ TaskStatus.COMPLETED,+ TaskStatus.FAILED,+ TaskStatus.CANCELLED,+ }:+ unregister_telemetry(msg.telemetry_id or "")+ return ProcessResult.success()+ lease = None+ try:+ lease = await self._adopt_or_reacquire(msg, ctx)+ except Exception as exc:+ if await self._requeue_lock_handoff(msg, exc):+ return ProcessResult.requeued()+ await tracker.fail(+ msg.task_id,+ f"Invalid lock_handoff: {exc}",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ unregister_telemetry(msg.telemetry_id or "")+ return ProcessResult.failed(f"Invalid lock_handoff: {exc}")++ telemetry_id = msg.telemetry_id or ""+ telemetry = resolve_telemetry(telemetry_id) if telemetry_id else None+ if telemetry is None:+ telemetry = OperationTelemetry(operation="reindex_job", enabled=True)+ if telemetry_id:+ telemetry.telemetry_id = telemetry_id+ else:+ telemetry_id = telemetry.telemetry_id+ register_telemetry(telemetry)+ try:+ await tracker.start(+ msg.task_id, account_id=ctx.account_id, user_id=ctx.user.user_id, stage="queued"+ )+ from openviking.server.dependencies import get_service+ from openviking.service.reindex_executor import ReindexExecutor++ service = get_service()+ executor = ReindexExecutor(+ vlm_resolver=service._vlm_resolver,+ vector_config_resolver=service._vector_config_resolver,+ )+ with (+ bind_telemetry(telemetry),+ bind_task_context(msg.task_id, ctx.account_id, ctx.user.user_id),+ ):+ # _run owns an existing lease and releases or hands it off on+ # every exit path. Clear this consumer's reference before the+ # call so an exception cannot release the same lease twice.+ executor_lease = lease+ lease = None+ result = await executor._run(+ uri=msg.uri,+ object_type=msg.object_type,+ mode=msg.mode,+ force=msg.force,+ recursive=msg.recursive,+ ingest_options=executor._resolve_ingest_options(+ mode=msg.mode, tags=msg.tags, tag_mode=msg.tag_mode+ ),+ ctx=ctx,+ existing_lease=executor_lease,+ )+ if metadata is not None:+ await tracker.wait_for_descendants(msg.task_id, metadata.work_id)+ await tracker.complete(+ msg.task_id, result, account_id=ctx.account_id, user_id=ctx.user.user_id+ )+ return ProcessResult.success()+ except Exception as exc:+ await tracker.fail(+ msg.task_id, str(exc), account_id=ctx.account_id, user_id=ctx.user.user_id+ )+ return ProcessResult.failed(str(exc))+ finally:+ if lease is not None:+ await self._viking_fs._async_agfs.pathlock_release(lease)+ unregister_telemetry(telemetry_id)++ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:+ """Release the root lock handoff before QueueFS ACKs cancelled work."""+ try:+ payload = data.get("data", data) if isinstance(data, dict) else data+ if isinstance(payload, str):+ payload = json.loads(payload)+ msg = ReindexMsg.from_dict(payload)+ except (json.JSONDecodeError, TypeError, ValueError) as exc:+ return ProcessResult.failed(str(exc))++ if msg.lock_handoff is not None:+ try:+ lock = await self._viking_fs._async_agfs.pathlock_adopt(msg.lock_handoff)+ await self._viking_fs._async_agfs.pathlock_release(lock)+ except Exception as exc:+ logger.warning("[Reindex] Failed to release cancelled lock handoff: %s", exc)+ return ProcessResult.cancelled()diff --git a/openviking/storage/queuefs/semantic_executor.py b/openviking/storage/queuefs/semantic_executor.pyindex 77ac836cb..f41029520 100644--- a/openviking/storage/queuefs/semantic_executor.py+++ b/openviking/storage/queuefs/semantic_executor.py@@ -7,7 +7,7 @@ import reimport threadingfrom contextlib import nullcontextfrom dataclasses import dataclass, field-from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Set+from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Mapping, Optional, Setfrom weakref import WeakKeyDictionaryfrom openviking.core.namespace import classify_uri@@ -198,6 +198,9 @@ class SemanticTreeExecutor:file_abstracts: Optional[Dict[str, str]] = None,semantic_plan: Optional["SemanticPlan"] = None,telemetry_id: str | None = None,+ source_contents: Optional[Mapping[tuple[str, int], str | bytes]] = None,+ source_raw_contents: Optional[Mapping[tuple[str, int], str | bytes]] = None,+ materialize_content: bool = False,):self._processor = processorself._context_type = context_type@@ -216,6 +219,11 @@ class SemanticTreeExecutor:self._changes_provided = changes is not Noneself._changes = changes or {}self._semantic_plan = semantic_plan+ self._source_contents = dict(source_contents or {})+ self._source_raw_contents = (+ dict(source_raw_contents) if source_raw_contents is not None else None+ )+ self._materialize_content = materialize_contentself._semantic_resource_root = (semantic_plan.root_uri.rstrip("/") if semantic_plan is not None else None)@@ -361,6 +369,22 @@ class SemanticTreeExecutor:self._stats.total_nodes = 1self._stats.done_nodes = 1return+ if isinstance(entry, SemanticTreeEntry) and entry.kind == "file":+ self._stats.total_nodes = 1+ self._stats.in_progress_nodes = 1+ # A file-root maintenance plan has no directory node or+ # sidecars of its own. Execute its planned L2 work directly;+ # the caller's parent freshness policy handles its parent.+ previous_incremental = self._incremental_update+ self._incremental_update = False+ try:+ await self._file_summary_task(root_uri, root_uri)+ finally:+ self._incremental_update = previous_incremental+ self._root_write_result = AbstractOverviewWriteResult(+ wrote=True, abstract_body_changed=True+ )+ returnowner = processing_owner.get()if owner is not None and self._task_context is not None:@@ -580,6 +604,16 @@ class SemanticTreeExecutor:dir_uri=dir_uri,ctx=self._ctx,lock=self._lock,+ existing_raw=(+ {+ level: raw+ for level in (0, 1)+ if (raw := self._source_raw_contents.get((dir_uri, level)))+ is not None+ }+ if self._source_raw_contents is not None+ else None+ ),)except LockAcquisitionError:if self._generation_trigger != "content_write":@@ -1047,6 +1081,13 @@ class SemanticTreeExecutor:self, dir_uri: str) -> tuple[Optional[str], Optional[str]]:if self._semantic_plan is not None:+ snapshot_overview = self._source_contents.get((dir_uri.rstrip("/"), 1))+ snapshot_abstract = self._source_contents.get((dir_uri.rstrip("/"), 0))+ if snapshot_overview is not None or snapshot_abstract is not None:+ return (+ str(snapshot_overview) if snapshot_overview is not None else None,+ str(snapshot_abstract) if snapshot_abstract is not None else None,+ )entry = self._plan_entries_by_uri.get(dir_uri.rstrip("/"))if entry is None:return None, None@@ -1100,8 +1141,10 @@ class SemanticTreeExecutor:else:self._file_change_status[file_path] = Trueif summary_dict is None:- file_content = None- if hasattr(self._viking_fs, "read_file_bytes"):+ file_content = self._source_contents.get((file_path.rstrip("/"), 2))+ if file_content is not None and not isinstance(file_content, bytes):+ raise ValueError(f"file vector source must be bytes: {file_path}")+ if file_content is None and hasattr(self._viking_fs, "read_file_bytes"):file_content = await self._viking_fs.read_file_bytes(file_path, ctx=self._ctx)summary_kwargs: Dict[str, Any] = {"llm_sem": self._llm_sem,@@ -1149,6 +1192,8 @@ class SemanticTreeExecutor:vectorize_kwargs["file_content"] = file_contentif self._telemetry_id is not None:vectorize_kwargs["telemetry_id"] = self._telemetry_id+ if self._materialize_content:+ vectorize_kwargs["materialize_content"] = Truemanifest_md5 = self._file_md5s.get(file_path.rstrip("/")) or Noneif file_content is not None and not manifest_md5:file_md5 = content_md5(file_content)@@ -1175,6 +1220,7 @@ class SemanticTreeExecutor:)if enqueued and slot is not None:self._scheduled_vector_record_ids.add(slot.record_id)+ self._stats.indexed_records += 1except Exception as e:logger.error("Failed to schedule vectorization for %s: %s",@@ -1344,6 +1390,15 @@ class SemanticTreeExecutor:consume_pending=consume_pending,lock=self._lock,log_prefix="[SemanticTree]",+ existing_raw=(+ {+ level: raw+ for level in (0, 1)+ if (raw := self._source_raw_contents.get((dir_uri, level))) is not None+ }+ if self._source_raw_contents is not None+ else None+ ),))if not wrote.wrote:@@ -1386,15 +1441,20 @@ class SemanticTreeExecutor:# The package root describes the skill definition, independently# of the summaries produced for its attachments.definition_changed = f"{dir_uri}/SKILL.md" in self._changed_paths+ plan_entry = self._plan_entries_by_uri.get(dir_uri.rstrip("/"))+ repair = bool(plan_entry is not None and plan_entry.repair)overview, abstract = await self._processor._skill_root_semantics(dir_uri,ctx=self._ctx,- regenerate=self._generation_trigger == "reindex" or definition_changed,+ regenerate=self._generation_trigger == "reindex"+ or definition_changed+ or repair,lock=self._lock,+ skill_content=self._source_contents.get((f"{dir_uri}/SKILL.md", 2)),)should_write = False- need_vectorize = not self._incremental_update or definition_changed- children_changed = definition_changed+ need_vectorize = not self._incremental_update or definition_changed or repair+ children_changed = definition_changed or repairelif self._generation_trigger == "content_copy" and not node.transfer_inputs_ready:need_vectorize = Falseshould_write = False@@ -1406,7 +1466,18 @@ class SemanticTreeExecutor:if not children_changed:overview, abstract = await self._read_existing_overview_abstract(dir_uri)should_write = overview is None or abstract is None- # Rebuilt sidecars must also replace their stale vectors.+ plan_entry = self._plan_entries_by_uri.get(dir_uri.rstrip("/"))+ if (+ self._semantic_plan is not None+ and plan_entry is not None+ and plan_entry.repair+ ):+ # RFV maintenance uses ``repair`` for both stale semantic+ # input and force rebuilds. Regenerate through the normal+ # semantic action before emitting its planned index slots.+ overview = None+ abstract = None+ should_write = Trueneed_vectorize = should_writechildren_changed = should_writeif should_write and (overview is None or abstract is None):@@ -1479,6 +1550,7 @@ class SemanticTreeExecutor:entry = self._plan_entries_by_uri.get(dir_uri.rstrip("/"))slots = {slot.level: slot for slot in entry.index_slots} if entry else {}+ enqueued_levels: set[int] = set()if need_vectorize and not self._skip_vectorization:assert overview is not None and abstract is not Nonetry:@@ -1499,7 +1571,7 @@ class SemanticTreeExecutor:return Falseold_body = slot.abstractnew_body = abstract if level == 0 else overview- return not old_body or old_body != new_body+ return entry.repair or not old_body or old_body != new_bodyinclude_abstract = should_emit(0)include_overview = should_emit(1)@@ -1521,6 +1593,10 @@ class SemanticTreeExecutor:},"include_abstract": include_abstract,"include_overview": include_overview,+ "md5s": {+ 0: content_md5(abstract.encode("utf-8")),+ 1: content_md5(overview.encode("utf-8")),+ },}if self._telemetry_id is not None:directory_vector_kwargs["telemetry_id"] = self._telemetry_id@@ -1556,8 +1632,6 @@ class SemanticTreeExecutor:if self._context_type != "skill":raiseself._record_skill_failure(dir_uri, e)- else:- enqueued_levels = set()for level, slot in sorted(slots.items()):if (@@ -1592,6 +1666,7 @@ class SemanticTreeExecutor:self._scheduled_vector_record_ids.update(slot.record_id for slot in slots.values() if slot.level in enqueued_levels)+ self._stats.indexed_records += len(enqueued_levels)finally:self._stats.done_nodes += 1self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 60e358227..e28b42d9b 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -8,7 +8,6 @@ import threadingimport timefrom contextlib import nullcontextfrom typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple-from urllib.parse import unquote, urlsplitfrom openviking.core.namespace import classify_urifrom openviking.observability.context import (@@ -45,6 +44,7 @@ from openviking.storage.abstract_overview import (deterministic_sample,freshness_metadata,markdown_safe_viking_uri,+ parse_overview_file_summaries,plan_abstract_overview_refresh,write_abstract_overview,)@@ -82,6 +82,7 @@ if TYPE_CHECKING:logger = get_logger(__name__)+class RequestQueueStats:processed: int = 0requeue_count: int = 0@@ -152,15 +153,23 @@ class SemanticProcessor(DequeueHandlerBase):async def _get_vlm_config(self, ctx: RequestContext) -> "VLMHandle":if self._vlm_resolver is None:- raise RuntimeError(- "SemanticProcessor requires a VLM resolver for account-owned work"- )+ raise RuntimeError("SemanticProcessor requires a VLM resolver for account-owned work")return await self._vlm_resolver.get_vlm(ctx.account_id)@classmethoddef _cache_tree_stats(cls, telemetry_id: str, uri: str, stats: SemanticTreeStats) -> None:with cls._stats_lock:if telemetry_id:+ previous = cls._tree_stats_by_telemetry_id.get(telemetry_id)+ if previous is not None:+ stats = SemanticTreeStats(+ total_nodes=previous.total_nodes + stats.total_nodes,+ pending_nodes=previous.pending_nodes + stats.pending_nodes,+ in_progress_nodes=previous.in_progress_nodes + stats.in_progress_nodes,+ done_nodes=previous.done_nodes + stats.done_nodes,+ failures=[*previous.failures, *stats.failures],+ indexed_records=previous.indexed_records + stats.indexed_records,+ )cls._tree_stats_by_telemetry_id[telemetry_id] = statscls._tree_stats_by_uri[uri] = statscls._tree_stats_order.append((telemetry_id, uri))@@ -286,8 +295,10 @@ class SemanticProcessor(DequeueHandlerBase):else:logger.warning(f"No queue manager available, cannot re-enqueue: {msg.uri}")- async def _enqueue_skill_retry(self, queue, msg: SemanticMsg, scope: SemanticLockScope) -> None:- """Transfer the live package lease to a retry before releasing this worker.+ async def _enqueue_semantic_retry(+ self, queue, msg: SemanticMsg, scope: SemanticLockScope+ ) -> None:+ """Transfer the live semantic lease to a retry before releasing this worker.Reusing the consumed handoff would require acquiring an unrelated lock,which conflicts with an update request still waiting under its outer lease.@@ -307,6 +318,10 @@ class SemanticProcessor(DequeueHandlerBase):scope._owned = Trueraise+ async def _enqueue_skill_retry(self, queue, msg: SemanticMsg, scope: SemanticLockScope) -> None:+ """Compatibility wrapper for existing skill retry callers."""+ await self._enqueue_semantic_retry(queue, msg, scope)+async def _requeue_semantic_msg_after_error(self,msg: SemanticMsg,@@ -529,8 +544,7 @@ class SemanticProcessor(DequeueHandlerBase):if self._vlm_resolver is None:raise RuntimeError(- "SemanticProcessor requires a VLM resolver "- "for account-owned work"+ "SemanticProcessor requires a VLM resolver for account-owned work")if not await work.acquire_lock(current_ctx):get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)@@ -557,12 +571,13 @@ class SemanticProcessor(DequeueHandlerBase):source=msg.plan.source_metadata,semantic_plan=msg.plan,telemetry_id=msg.telemetry_id,+ generation_trigger=msg.generation_trigger,)await executor.run(run_uri)self._cache_tree_stats(msg.telemetry_id, run_uri, executor.get_stats())- if not executor.stale and msg.plan.propagation.enabled:+ if not executor.stale:write_result = getattr(executor,"root_write_result",@@ -1232,6 +1247,7 @@ class SemanticProcessor(DequeueHandlerBase):llm_sem: asyncio.Semaphore,ctx: Optional[RequestContext] = None,file_content: Optional[bytes] = None,+ materialize_content: bool = False,) -> Dict[str, Any]:"""Generate summary for a single text file (code, documentation, or other text)."""viking_fs = get_viking_fs()@@ -1463,73 +1479,8 @@ class SemanticProcessor(DequeueHandlerBase):@classmethoddef _parse_overview_md(cls, overview_content: str) -> Dict[str, str]:- """Parse overview.md and extract file summaries.-- Args:- overview_content: Content of the overview.md file-- Returns:- Dictionary mapping file names to their summaries- """- import re-- summaries: Dict[str, str] = {}-- overview_content = body_for_preview(overview_content)- if not overview_content or not overview_content.strip():- return summaries-- lines = overview_content.split("\n")- current_file = None- current_summary_lines: List[str] = []-- for line in lines:- header_match = re.match(r"^###\s+(.+?)\s*$", line)- if header_match:- if current_file and current_summary_lines:- summaries[current_file] = " ".join(current_summary_lines).strip()-- file_name = cls._overview_heading_cache_key(header_match.group(1).strip())- parts = file_name.split()- if len(parts) >= 2 and parts[0] == parts[1]:- file_name = parts[0]-- current_file = file_name- current_summary_lines = []- continue-- numbered_match = re.match(r"^\[(\d+)\]\s+(.+?):\s*(.+)$", line)- if numbered_match:- if current_file and current_summary_lines:- summaries[current_file] = " ".join(current_summary_lines).strip()- current_file = numbered_match.group(2).strip()- current_summary_lines = [numbered_match.group(3).strip()]- continue-- if current_file:- stripped = line.strip()- if stripped and not stripped.startswith("#"):- current_summary_lines.append(stripped)-- if current_file and current_summary_lines:- summaries[current_file] = " ".join(current_summary_lines).strip()-- return summaries-- @staticmethod- def _overview_heading_cache_key(heading: str) -> str:- """Return the entry name represented by a plain or linked H3 heading."""- if heading.startswith("[") and heading.endswith(")"):- destination_start = heading.rfind("](")- if destination_start > 0:- target = heading[destination_start + 2 : -1].strip()- if target.startswith("<") and target.endswith(">"):- target = target[1:-1].strip()- if target.startswith("viking://"):- path = unquote(urlsplit(target).path).rstrip("/")- if path:- return path.rsplit("/", 1)[-1]- return heading+ """Parse overview.md and extract direct-file summaries."""+ return parse_overview_file_summaries(overview_content)async def _generate_overview(self,@@ -1850,6 +1801,7 @@ class SemanticProcessor(DequeueHandlerBase):ctx: RequestContext,regenerate: bool = False,lock: Optional[Dict[str, Any]] = None,+ skill_content: str | bytes | None = None,) -> Tuple[str, str]:"""Keep the package root tied only to its SKILL.md definition."""viking_fs = get_viking_fs()@@ -1866,7 +1818,11 @@ class SemanticProcessor(DequeueHandlerBase):from openviking.core.skill_loader import SkillLoaderfrom openviking.utils.skill_processor import SkillProcessor- content = await viking_fs.read_file(f"{uri}/SKILL.md", ctx=ctx)+ content = (+ skill_content+ if skill_content is not None+ else await viking_fs.read_file(f"{uri}/SKILL.md", ctx=ctx)+ )if isinstance(content, bytes):content = content.decode("utf-8")definition = SkillLoader.parse(content)@@ -1907,6 +1863,7 @@ class SemanticProcessor(DequeueHandlerBase):include_abstract: bool = True,include_overview: bool = True,telemetry_id: str | None = None,+ md5s: Optional[Dict[int, str]] = None,) -> set[int]:"""Create directory Context and enqueue to EmbeddingQueue."""@@ -1939,6 +1896,7 @@ class SemanticProcessor(DequeueHandlerBase):include_abstract=include_abstract,include_overview=include_overview,telemetry_id=telemetry_id,+ md5s=md5s,)async def _load_transfer_file_summaries(@@ -2004,6 +1962,7 @@ class SemanticProcessor(DequeueHandlerBase):ingest_options: IngestOptions | None = None,file_md5: Optional[str] = None,file_content: Optional[bytes] = None,+ materialize_content: bool = False,scalar_override: Optional[Dict[str, Any]] = None,field_patch: FieldPatch | None = None,action: str = "merge",@@ -2024,6 +1983,7 @@ class SemanticProcessor(DequeueHandlerBase):ingest_options=ingest_options,file_md5=file_md5,file_content=file_content,+ materialize_content=materialize_content,scalar_override=scalar_override,field_patch=field_patch,action=action,diff --git a/openviking/storage/queuefs/semantic_work.py b/openviking/storage/queuefs/semantic_work.pyindex 553221af5..65daf3124 100644--- a/openviking/storage/queuefs/semantic_work.py+++ b/openviking/storage/queuefs/semantic_work.py@@ -41,6 +41,7 @@ class SemanticMessageWork:self.msg = msgself.caller_lock = caller_lockself.scope: SemanticLockScope | None = None+ self._lock_closed = Falsedef start(self) -> None:pass@@ -61,22 +62,38 @@ class SemanticMessageWork:async def finish_processing(self, succeeded: bool) -> None:assert self.scope is not None- await self.scope.close()+ if self.msg.plan is not None and self.msg.lock_handoff is not None:+ await get_request_wait_tracker().wait_for_embeddings(+ self.msg.telemetry_id,+ stop_waiting=getattr(self.processor, "_embedding_worker_stopped", None),+ )+ if succeeded:+ await self.scope.close()+ self._lock_closed = Truedef failure_result(self, stats: SemanticTreeStats | None) -> ProcessResult | None:return Noneasync def reenqueue(self) -> None:- await self.processor._reenqueue_semantic_msg(self.msg)+ async def enqueue(queue, msg):+ if self.scope is not None and self.scope.lock is not None and self.scope._owned:+ await self.processor._enqueue_semantic_retry(queue, msg, self.scope)+ else:+ await queue.enqueue(msg)++ await self.processor._reenqueue_semantic_msg(self.msg, enqueue=enqueue)async def reenqueue_after_error(self) -> None:await self.reenqueue()async def skip(self) -> None:- pass+ if self.msg.lock_handoff is not None:+ await self.processor._release_cancelled_semantic_lock(self.msg)async def cancel(self) -> None:- pass+ if self.scope is None:+ await self.skip()+ get_request_wait_tracker().mark_semantic_done(self.msg.telemetry_id, self.msg.id)async def cancel_queued(self) -> None:if self.msg.telemetry_id and self.msg.id:@@ -84,7 +101,9 @@ class SemanticMessageWork:await self.processor._release_cancelled_semantic_lock(self.msg)async def close(self) -> None:- pass+ if self.scope is not None and not self._lock_closed:+ await self.scope.close()+ self._lock_closed = Trueclass SkillSemanticMessageWork(SemanticMessageWork):diff --git a/openviking/storage/resource_diff.py b/openviking/storage/resource_diff.pyindex a60b5a23f..c6a1c9149 100644--- a/openviking/storage/resource_diff.py+++ b/openviking/storage/resource_diff.py@@ -27,7 +27,7 @@ import asyncioimport jsonimport loggingfrom collections import Counter-from dataclasses import dataclass+from dataclasses import dataclass, fieldfrom enum import Enumfrom typing import Any, Dict, Mapping, Tuple@@ -115,6 +115,8 @@ class ResourceDiffEntry:old_kind: str | None = Nonenew_kind: str | None = Nonemd5: str | None = None+ level_states: Mapping[int, IndexState] = field(default_factory=dict)+ level_md5s: Mapping[int, str] = field(default_factory=dict)@dataclass(frozen=True)diff --git a/openviking/storage/resource_rfv.py b/openviking/storage/resource_rfv.pynew file mode 100644index 000000000..8fcd60b08--- /dev/null+++ b/openviking/storage/resource_rfv.py@@ -0,0 +1,419 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Typed R/F/V inputs for planning maintenance reindex operations."""++from __future__ import annotations++import asyncio+import json+from dataclasses import dataclass, field+from typing import Any, Mapping++from openviking.concurrency import bounded_map+from openviking.core.namespace import context_type_for_uri+from openviking.server.error_mapping import is_not_found_error+from openviking.storage.abstract_overview import body_for_preview+from openviking.storage.internal_names import is_storage_internal_name+from openviking.storage.resource_diff import (+ ContentState,+ IndexState,+ ResourceDiffEntry,+ ResourceDiffResult,+)+from openviking.storage.resource_rnfv import (+ RequestIntent,+ VectorIndexSnapshot,+ VectorRecordSnapshot,+ canonical_vector_records_by_level,+)+from openviking.utils.content_hash import content_md5+from openviking_cli.exceptions import NotFoundError+++@dataclass(frozen=True)+class RFVEntry:+ """One current formal-tree node and its per-level source fingerprints."""++ uri: str+ relative_path: str+ is_dir: bool+ level_md5s: Mapping[int, str] = field(default_factory=dict)++ def __post_init__(self) -> None:+ expected = {0, 1} if self.is_dir else {2}+ invalid = set(self.level_md5s) - expected+ if invalid:+ raise ValueError(f"invalid RFV levels for {self.relative_path}: {sorted(invalid)}")+++@dataclass(frozen=True)+class RFVFormalSnapshot:+ entries: Mapping[str, RFVEntry]+ complete: bool = True+++@dataclass(frozen=True)+class RFVSnapshot:+ request: RequestIntent+ formal: RFVFormalSnapshot+ vectors: VectorIndexSnapshot+ source_contents: Mapping[tuple[str, int], str | bytes] = field(default_factory=dict)+ source_raw_contents: Mapping[tuple[str, int], str | bytes] = field(default_factory=dict)+ source_metadata: Mapping[str, Any] | None = None+ input_only_paths: frozenset[str] = frozenset()+++def _relative_uri(root: str, uri: str) -> str | None:+ root = root.rstrip("/")+ if uri == root:+ return ""+ prefix = root + "/"+ return uri[len(prefix) :] if uri.startswith(prefix) else None+++def _is_sidecar(relative_path: str) -> bool:+ return relative_path.rsplit("/", 1)[-1] in {".abstract.md", ".overview.md"}+++def _is_control_file(relative_path: str) -> bool:+ name = relative_path.rsplit("/", 1)[-1]+ return name in {".source.json", ".image_mappings.json", ".artifact_manifest.json"}+++async def build_rfv_snapshot(+ *,+ viking_fs: Any,+ vikingdb: Any,+ target_uri: str,+ ctx: Any,+ request_intent: RequestIntent,+ recursive: bool = True,+ source_read_concurrency: int = 8,+ root_is_dir: bool | None = None,+ formal_inventory: tuple[bool, list[Mapping[str, Any]], bool] | None = None,+) -> RFVSnapshot:+ """Read one complete F/V inventory and each selected embedding source once."""+ root = target_uri.rstrip("/")+ projection = request_intent.required_vector_fields() | {"abstract"}+ if target_uri.rstrip("/").startswith(("viking://user/", "viking://agent/skills/")):+ projection = projection | {"name", "description", "tags"}++ resolved_root_is_dir = root_is_dir+ if formal_inventory is not None:+ resolved_root_is_dir = bool(formal_inventory[0])+ if resolved_root_is_dir is None and not recursive:+ stat = await viking_fs.stat(root, ctx=ctx, skip_count=True)+ resolved_root_is_dir = bool(stat.get("isDir", stat.get("is_dir")))+ include_direct_children = bool(resolved_root_is_dir) and not recursive++ async def read_formal_inventory() -> tuple[bool, list[Mapping[str, Any]], bool]:+ if formal_inventory is not None:+ return formal_inventory+ is_dir = resolved_root_is_dir+ if is_dir is None:+ stat = await viking_fs.stat(root, ctx=ctx, skip_count=True)+ is_dir = bool(stat.get("isDir", stat.get("is_dir")))+ if not is_dir:+ return is_dir, [], True+ if not recursive:+ entries = await viking_fs.ls(+ root,+ node_limit=None,+ show_all_hidden=True,+ ctx=ctx,+ )+ return is_dir, list(entries), True+ raw_entries = await viking_fs.tree(+ root,+ output="original",+ show_all_hidden=True,+ node_limit=None,+ level_limit=None,+ ctx=ctx,+ )+ complete = not any(entry.get("access") == "denied" for entry in raw_entries)+ return is_dir, list(raw_entries), complete++ formal_task = asyncio.create_task(read_formal_inventory())+ vector_task = asyncio.create_task(+ vikingdb.get_incremental_inventory_under_uri(+ root,+ ctx=ctx,+ output_fields=sorted(projection),+ recursive=recursive,+ include_direct_children=include_direct_children,+ )+ )+ try:+ (root_is_dir, raw_entries, complete), inventory = await asyncio.gather(+ formal_task, vector_task+ )+ except BaseException:+ for task in (formal_task, vector_task):+ if not task.done():+ task.cancel()+ await asyncio.gather(formal_task, vector_task, return_exceptions=True)+ raise+ complete = complete and not any(entry.get("access") == "denied" for entry in raw_entries)+ if not complete:+ raise ValueError("cannot build RFV snapshot from an incomplete formal tree")++ nodes: dict[str, tuple[str, bool]] = {"": (root, root_is_dir)}+ sidecars: dict[tuple[str, int], str] = {}+ source_metadata_uri: str | None = None+ for raw in raw_entries:+ if raw.get("access") == "denied":+ continue+ uri = str(raw.get("uri") or "")+ relative = _relative_uri(root, uri)+ if relative is None:+ complete = False+ continue+ relative_path = relative.strip("/")+ if not uri or not relative_path:+ continue+ name = relative_path.rsplit("/", 1)[-1]+ if _is_sidecar(relative_path):+ parent = relative_path.rsplit("/", 1)[0] if "/" in relative_path else ""+ level = 0 if name == ".abstract.md" else 1+ sidecars[(parent, level)] = uri+ continue+ if relative_path == ".source.json":+ source_metadata_uri = uri+ continue+ if is_storage_internal_name(name) or _is_control_file(relative_path):+ continue+ nodes[relative_path] = (uri, bool(raw.get("isDir")))++ if root_is_dir and not recursive:+ for level, name in ((0, ".abstract.md"), (1, ".overview.md")):+ sidecars[("", level)] = f"{root}/{name}"++ records_by_uri_level: dict[tuple[str, int], Mapping[str, Any]] = {}+ for record in inventory.values():+ uri = str(record.get("uri") or "")+ try:+ level = int(record.get("level", -1))+ except (TypeError, ValueError):+ continue+ records_by_uri_level.setdefault((uri, level), record)++ source_contents: dict[tuple[str, int], str | bytes] = {}+ source_raw_contents: dict[tuple[str, int], str | bytes] = {}+ level_md5s: dict[str, dict[int, str]] = {path: {} for path in nodes}+ source_jobs: list[tuple[str, str, int, bool]] = []+ for path, (uri, is_dir) in nodes.items():+ if include_direct_children and path:+ continue+ if is_dir:+ for level in (0, 1):+ source_uri = sidecars.get((path, level))+ if source_uri:+ source_jobs.append((path, source_uri, level, True))+ else:+ source_jobs.append((path, uri, 2, False))++ async def read_source(job: tuple[str, str, int, bool]) -> None:+ path, source_uri, level, is_sidecar = job+ try:+ raw = await viking_fs.read_file_bytes(source_uri, ctx=ctx)+ except Exception as exc:+ if is_sidecar and (+ isinstance(exc, (FileNotFoundError, KeyError, NotFoundError))+ or is_not_found_error(exc)+ ):+ return+ raise+ node_uri = nodes[path][0]+ if is_sidecar:+ source_raw_contents[(node_uri, level)] = raw+ body = body_for_preview(raw)+ if not body.strip():+ return+ source_contents[(node_uri, level)] = body+ source_md5 = content_md5(body.encode("utf-8"))+ level_md5s[path][level] = source_md5+ source = body+ else:+ source_md5 = content_md5(raw)+ level_md5s[path][level] = source_md5+ source = raw+ record = records_by_uri_level.get((node_uri, level))+ keep_file_source = (+ not is_sidecar+ and path.rsplit("/", 1)[-1] == "SKILL.md"+ and context_type_for_uri(root) == "skill"+ )+ if keep_file_source or (+ not is_sidecar+ and (+ request_intent.processing_mode != "vectors_only"+ or request_intent.force+ or record is None+ or str(record.get("md5") or "") != source_md5+ )+ ):+ source_contents[(node_uri, level)] = source++ await bounded_map(+ source_jobs,+ read_source,+ concurrency=max(1, source_read_concurrency),+ )+ source_metadata = None+ if source_metadata_uri is not None:+ try:+ raw_metadata = await viking_fs.read_file_bytes(source_metadata_uri, ctx=ctx)+ parsed_metadata = json.loads(raw_metadata)+ except (UnicodeDecodeError, json.JSONDecodeError):+ parsed_metadata = None+ if isinstance(parsed_metadata, dict):+ kind = parsed_metadata.get("kind") or parsed_metadata.get("type")+ source_uri = parsed_metadata.get("uri") or parsed_metadata.get("source")+ if kind and source_uri:+ source_metadata = {+ "kind": str(kind),+ "uri": str(source_uri),+ "path": str(source_uri),+ }+ vector_records: dict[str, VectorRecordSnapshot] = {}+ for record_id, record in inventory.items():+ uri = str(record.get("uri") or "")+ relative_path = _relative_uri(root, uri)+ if relative_path is None:+ raise ValueError(f"RFV vector record is outside target: {uri}")+ level = int(record.get("level", -1))+ fields = {+ name: record[name] for name in projection - {"id", "uri", "level"} if name in record+ }+ vector_records[str(record_id)] = VectorRecordSnapshot(+ str(record_id), uri, relative_path, level, fields+ )++ formal_entries = {+ path: RFVEntry(uri, path, is_dir, level_md5s[path]) for path, (uri, is_dir) in nodes.items()+ }+ return RFVSnapshot(+ request=request_intent,+ formal=RFVFormalSnapshot(formal_entries, complete=complete),+ vectors=VectorIndexSnapshot(vector_records, projection, complete=True),+ source_contents=source_contents,+ source_raw_contents=source_raw_contents,+ source_metadata=source_metadata,+ input_only_paths=frozenset(+ path for path in nodes if include_direct_children and path and "/" not in path+ ),+ )+++def _summarize_index_state(+ *,+ expected_levels: set[int],+ level_states: Mapping[int, IndexState],+ invalid_levels: set[int],+) -> IndexState:+ if invalid_levels:+ return IndexState.LEVEL_CONFLICT+ values = set(level_states.values())+ if not expected_levels:+ return IndexState.ABSENT+ if not level_states:+ return IndexState.MISSING+ if values == {IndexState.ABSENT}:+ return IndexState.ABSENT+ if IndexState.ORPHAN in values:+ return IndexState.ORPHAN+ if IndexState.STALE in values:+ return IndexState.STALE+ if IndexState.MISSING in values:+ return IndexState.PARTIAL if len(level_states) > 1 else IndexState.MISSING+ return IndexState.COMPLETE+++def resolve_rfv_state(snapshot: RFVSnapshot) -> ResourceDiffResult:+ """Resolve current formal nodes and vectors without inventing an N snapshot."""++ if not snapshot.formal.complete or not snapshot.vectors.complete:+ raise ValueError("cannot resolve an incomplete RFV snapshot")+ required = snapshot.request.required_vector_fields()+ if not snapshot.vectors.projected_fields.issuperset(required):+ raise ValueError("RFV vector projection misses required fields")+ root = snapshot.request.target_uri.rstrip("/")+ for path, formal in snapshot.formal.entries.items():+ expected_uri = root if not path else f"{root}/{path}"+ if formal.relative_path != path or formal.uri.rstrip("/") != expected_uri:+ raise ValueError(f"RFV formal entry is outside target: {formal.uri}")+ for record in snapshot.vectors.records_by_id.values():+ expected_uri = root if not record.relative_path else f"{root}/{record.relative_path}"+ if record.uri.rstrip("/") != expected_uri:+ raise ValueError(f"RFV vector record is outside target: {record.uri}")++ records_by_path: dict[str, list[VectorRecordSnapshot]] = {}+ for record in snapshot.vectors.records_by_id.values():+ records_by_path.setdefault(record.relative_path, []).append(record)++ entries: dict[str, ResourceDiffEntry] = {}+ all_paths = set(snapshot.formal.entries) | set(records_by_path)+ for path in sorted(all_paths):+ formal = snapshot.formal.entries.get(path)+ records = records_by_path.get(path, [])+ canonical, _duplicates = canonical_vector_records_by_level(records)+ if formal is None:+ entries[path] = ResourceDiffEntry(+ relative_path=path,+ content_state=ContentState.ABSENT,+ index_state=IndexState.ORPHAN,+ old_kind=None,+ new_kind=None,+ )+ continue++ kind = "directory" if formal.is_dir else "file"+ expected_levels = {0, 1} if formal.is_dir else {2}+ invalid_levels = set(canonical) - expected_levels+ level_states: dict[int, IndexState] = {}+ for level in sorted(expected_levels):+ source_md5 = str(formal.level_md5s.get(level) or "")+ record = canonical.get(level)+ if formal.is_dir and not source_md5:+ level_states[level] = (+ IndexState.MISSING+ if snapshot.request.processing_mode != "vectors_only"+ else IndexState.ABSENT+ )+ continue+ if record is None:+ level_states[level] = IndexState.MISSING+ continue+ stored_md5 = str(record.fields.get("md5") or "")+ level_states[level] = (+ IndexState.STALE+ if snapshot.request.force or not source_md5 or stored_md5 != source_md5+ else IndexState.COMPLETE+ )++ index_state = _summarize_index_state(+ expected_levels=expected_levels,+ level_states=level_states,+ invalid_levels=invalid_levels,+ )+ entries[path] = ResourceDiffEntry(+ relative_path=path,+ content_state=ContentState.UNCHANGED,+ index_state=index_state,+ old_kind=kind,+ new_kind=kind,+ md5=(formal.level_md5s.get(2) if not formal.is_dir else None),+ level_states=level_states,+ level_md5s=dict(formal.level_md5s),+ )+ return ResourceDiffResult(entries=entries)+++__all__ = [+ "RFVEntry",+ "RFVFormalSnapshot",+ "RFVSnapshot",+ "build_rfv_snapshot",+ "resolve_rfv_state",+]diff --git a/openviking/storage/resource_rnfv.py b/openviking/storage/resource_rnfv.pyindex f71e95c33..4b6d60a24 100644--- a/openviking/storage/resource_rnfv.py+++ b/openviking/storage/resource_rnfv.py@@ -96,6 +96,7 @@ class RequestIntent:processing_mode: strvectorize: bool = Truescalar_intents: tuple[ScalarIntent, ...] = ()+ force: bool = False@classmethoddef from_ingest_options(@@ -105,6 +106,8 @@ class RequestIntent:processing_mode: Any,ingest_options: IngestOptions | Mapping[str, Any] | None,vectorize: bool = True,+ force: bool = False,+ scalar_target_levels: frozenset[int] | None = None,) -> "RequestIntent":options = IngestOptions.from_value(ingest_options)scalar_intents: tuple[ScalarIntent, ...] = ()@@ -115,9 +118,8 @@ class RequestIntent:field="search_tags",mode=IngestOptions.vector_search_tag_mode(options.search_tag_mode),value=tuple(options.search_tags),- target_levels=(- frozenset({2}) if str(mode) == "vectors_only" else frozenset({0, 1, 2})- ),+ target_levels=scalar_target_levels+ or (frozenset({2}) if str(mode) == "vectors_only" else frozenset({0, 1, 2})),),)else:@@ -127,6 +129,7 @@ class RequestIntent:processing_mode=str(mode),vectorize=bool(vectorize),scalar_intents=scalar_intents,+ force=force,)def required_vector_fields(self) -> frozenset[str]:diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex 0dc884add..166cca13b 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -2132,8 +2132,10 @@ class VikingVectorIndexBackend:ctx: RequestContext,batch_size: int = 100,output_fields: Optional[List[str]] = None,+ recursive: bool = True,+ include_direct_children: bool = False,) -> Dict[str, Dict[str, Any]]:- """Strictly load lightweight L0/L1/L2 metadata below a resource root."""+ """Strictly load lightweight L0/L1/L2 metadata in a URI scope."""projection = list(dict.fromkeys([@@ -2146,7 +2148,11 @@ class VikingVectorIndexBackend:scope = And([Eq("account_id", ctx.account_id),- PathScope("uri", canonical_uri, depth=-1),+ PathScope(+ "uri",+ canonical_uri,+ depth=-1 if recursive else 1 if include_direct_children else 0,+ ),In("level", [0, 1, 2]),])diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex 6c58194ff..cb89b045b 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -128,9 +128,7 @@ def _apply_ingest_options(embedding_msg.context_data.get("search_tags"), incoming_tags)embedding_msg.context_data["search_tags"] = incoming_tags- embedding_msg.context_data.setdefault("_upsert_options", {})["search_tag_mode"] = (- tag_mode- )+ embedding_msg.context_data.setdefault("_upsert_options", {})["search_tag_mode"] = tag_modeasync def _enqueue_embedding_message(@@ -308,6 +306,7 @@ async def _build_image_data_uri(file_name: str,viking_fs,ctx: Optional[RequestContext],+ content: Optional[bytes] = None,) -> Optional[str]:"""Read an image file and encode it as a base64 ``data:`` URI.@@ -316,7 +315,8 @@ async def _build_image_data_uri(Returns None if the image cannot be read."""try:- content = await viking_fs.read_file_bytes(file_path, ctx=ctx)+ if content is None:+ content = await viking_fs.read_file_bytes(file_path, ctx=ctx)image_config = getattr(get_openviking_config(), "image", None)return image_bytes_to_model_data_uri(content, file_name, config=image_config)except Exception as e:@@ -412,6 +412,7 @@ async def vectorize_directory_meta(actions: Optional[Dict[int, IndexAction | str]] = None,field_patches: Optional[Dict[int, FieldPatch]] = None,telemetry_id: str | None = None,+ md5s: Optional[Dict[int, str]] = None,) -> set[int]:"""Vectorize directory metadata (.abstract.md and .overview.md).@@ -461,6 +462,7 @@ async def vectorize_directory_meta(account_id=ctx.account_id,owner_space=owner_space,meta=meta,+ md5=(md5s or {}).get(int(ContextLevel.ABSTRACT.value)),)context_abstract.set_vectorize(Vectorize(text=embedding_text_for_body(ContextLevel.ABSTRACT, uri, abstract))@@ -522,6 +524,7 @@ async def vectorize_directory_meta(account_id=ctx.account_id,owner_space=owner_space,meta=meta,+ md5=(md5s or {}).get(int(ContextLevel.OVERVIEW.value)),)context_overview.set_vectorize(Vectorize(text=embedding_text_for_body(ContextLevel.OVERVIEW, uri, overview))@@ -590,6 +593,7 @@ async def vectorize_file(ingest_options: IngestOptions | None = None,file_md5: Optional[str] = None,file_content: Optional[bytes] = None,+ materialize_content: bool = False,action: str = "merge",telemetry_id: str | None = None,) -> bool:@@ -701,7 +705,9 @@ async def vectorize_file(context.set_vectorize(Vectorize(text=embedding_text))elif content_type == ResourceContentType.IMAGE:# Multimodal embedders consume both parts; text-only embedders fall back to summary.- image_uri = await _build_image_data_uri(file_path, file_name, viking_fs, ctx)+ image_uri = await _build_image_data_uri(+ file_path, file_name, viking_fs, ctx, content=file_content+ )if image_uri:context.set_vectorize(Vectorize(text=summary, images=[image_uri]))elif summary:@@ -739,6 +745,21 @@ async def vectorize_file(_apply_ingest_options(embedding_msg, ingest_options)_apply_scalar_overrides(embedding_msg, scalar_override)+ if (+ materialize_content+ and file_content is not None+ and (+ content_type is ResourceContentType.TEXT+ or (content_type is None and is_text_file(file_name))+ )+ ):+ from openviking.storage.viking_vector_index_backend import (+ VIKINGDB_CONTENT_MAX_SIZE,+ )++ embedding_msg.context_data["_materialized_content"] = _coerce_text_file_content(+ file_content+ )[:VIKINGDB_CONTENT_MAX_SIZE]_apply_planned_field_patch(embedding_msg, field_patch)enqueued = await _enqueue_embedding_message(embedding_queue,diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.pyindex 5ce437ca4..2b3512d4b 100644--- a/openviking/utils/resource_processor.py+++ b/openviking/utils/resource_processor.py@@ -38,7 +38,11 @@ from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fsfrom openviking.storage.vikingdb_manager import VikingDBManagerfrom openviking.telemetry import get_current_telemetryfrom openviking.utils import is_github_url-from openviking.utils.embedding_utils import index_resource, vectorize_file+from openviking.utils.embedding_utils import (+ index_resource,+ vectorize_directory_meta,+ vectorize_file,+)from openviking.utils.git_auth import is_git_https_urlfrom openviking.utils.ingest_options import IngestOptionsfrom openviking.utils.log_correlation import log_correlation@@ -182,26 +186,20 @@ class ResourceProcessor:if self._summarizer is not None:return self._summarizerif self.vlm_resolver is None:- raise RuntimeError(- "ResourceProcessor requires a VLM resolver for account-owned work"- )+ raise RuntimeError("ResourceProcessor requires a VLM resolver for account-owned work")return Summarizer(await self._vlm_processor_for(ctx))def _get_vlm_processor(self) -> "VLMProcessor":"""Return an explicitly configured standalone VLM processor."""if self._vlm_processor is None:- raise RuntimeError(- "ResourceProcessor requires an explicitly configured VLMProcessor"- )+ raise RuntimeError("ResourceProcessor requires an explicitly configured VLMProcessor")return self._vlm_processorasync def _vlm_processor_for(self, ctx: RequestContext) -> "VLMProcessor":from openviking.parse.vlm import VLMProcessorif self.vlm_resolver is None:- raise RuntimeError(- "ResourceProcessor requires a VLM resolver for account-owned work"- )+ raise RuntimeError("ResourceProcessor requires a VLM resolver for account-owned work")return VLMProcessor(vlm=await self.vlm_resolver.get_vlm(ctx.account_id))def _get_media_processor(self):@@ -1257,9 +1255,7 @@ class ResourceProcessor:if vectors_only and target_preexisting and not root_is_file:diff = await SemanticProcessor(vlm_resolver=self.vlm_resolver- )._sync_topdown_recursive(- temp_uri, root_uri, ctx=ctx, lock=resource_lock- )+ )._sync_topdown_recursive(temp_uri, root_uri, ctx=ctx, lock=resource_lock)sync_deleted_files = list(getattr(diff, "deleted_files", []))sync_deleted_dirs = list(getattr(diff, "deleted_dirs", []))else:@@ -1381,20 +1377,44 @@ class ResourceProcessor:return {"kind": kind, "uri": str(path)}async def _enqueue_index_actions(- self, actions: Any, *, ctx: RequestContext, ingest_options: IngestOptions | None = None- ) -> None:+ self,+ actions: Any,+ *,+ ctx: RequestContext,+ ingest_options: IngestOptions | None = None,+ source_contents: Optional[Dict[tuple[str, int], str | bytes]] = None,+ source_metadata: Optional[Dict[str, Any]] = None,+ ) -> int:from collections import Counter+ from openviking.core.skill_loader import SkillLoaderfrom openviking.storage.index_action import IndexActionfrom openviking.storage.queuefs import get_queue_managerfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsgfrom openviking.telemetry import get_current_telemetryfrom openviking.utils.embedding_utils import _enqueue_embedding_message+ if source_contents is not None:+ missing_sources = [+ (action.uri, action.level)+ for action in actions+ if action.action in {IndexAction.UPSERT, IndexAction.MERGE}+ and (action.uri, action.level) not in source_contents+ ]+ if missing_sources:+ uri, level = missing_sources[0]+ source_name = (+ f"directory L{level}"+ if level in {int(ContextLevel.ABSTRACT), int(ContextLevel.OVERVIEW)}+ else "file"+ )+ raise ValueError(f"{source_name} vector source is unavailable: {uri}")+queue_manager = get_queue_manager()embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING, allow_create=True)telemetry_id = get_current_telemetry().telemetry_idaction_counts = Counter(action.action.value for action in actions)+ completed_actions = 0delete_ids = [action.record_id for action in actions if action.action == IndexAction.DELETE]if delete_ids:message = EmbeddingMsg.for_delete(@@ -1411,22 +1431,37 @@ class ResourceProcessor:message,failure_message="Failed to enqueue planned vector deletes",)+ completed_actions += len(delete_ids)+ directory_actions: dict[str, dict[int, Any]] = {}for action in actions:if action.action in {IndexAction.UPSERT, IndexAction.MERGE}:- if action.level != int(ContextLevel.DETAIL):- raise ValueError("Direct index upsert only supports file detail records")- await self._vectorize_resource_file(- action.uri,- ctx=ctx,- file_md5=action.md5,- ingest_options=ingest_options,- scalar_override={+ if action.level in {+ int(ContextLevel.ABSTRACT),+ int(ContextLevel.OVERVIEW),+ }:+ directory_actions.setdefault(action.uri, {})[action.level] = action+ continue+ file_kwargs = {+ "ctx": ctx,+ "file_md5": action.md5,+ "ingest_options": ingest_options,+ "scalar_override": {**dict(action.upsert_fields),"_record_id": action.record_id,},- action=action.action.value,- field_patch=action.field_patch,- )+ "action": action.action.value,+ "field_patch": action.field_patch,+ "summary": action.summary,+ }+ source = (source_contents or {}).get((action.uri, action.level))+ if source is not None:+ if not isinstance(source, bytes):+ raise ValueError(f"file vector source must be bytes: {action.uri}")+ file_kwargs["file_content"] = source+ if getattr(self.vikingdb, "uses_content_field", False):+ file_kwargs["materialize_content"] = True+ if await self._vectorize_resource_file(action.uri, **file_kwargs):+ completed_actions += 1continueif action.action != IndexAction.UPDATE_FIELDS:continue@@ -1442,11 +1477,78 @@ class ResourceProcessor:},telemetry_id=telemetry_id,)- await _enqueue_embedding_message(+ if await _enqueue_embedding_message(embedding_queue,message,failure_message=f"Failed to enqueue scalar update for {action.uri}",+ ):+ completed_actions += 1+ for uri, levels in directory_actions.items():+ abstract = (source_contents or {}).get((uri, int(ContextLevel.ABSTRACT)), "")+ overview = (source_contents or {}).get((uri, int(ContextLevel.OVERVIEW)), "")+ if isinstance(abstract, bytes):+ abstract = abstract.decode("utf-8")+ if isinstance(overview, bytes):+ overview = overview.decode("utf-8")+ if int(ContextLevel.ABSTRACT) in levels and not isinstance(abstract, str):+ raise ValueError(f"directory L0 vector source is unavailable: {uri}")+ if int(ContextLevel.OVERVIEW) in levels and not isinstance(overview, str):+ raise ValueError(f"directory L1 vector source is unavailable: {uri}")+ meta = None+ if context_type_for_uri(uri) == "skill" and abstract:+ try:+ skill = SkillLoader.parse(str(abstract))+ except ValueError:+ skill = None+ if skill is not None:+ meta = {+ "name": skill["name"],+ "description": skill.get("description", ""),+ "tags": skill.get("tags", []),+ "allowed_tools": skill.get("allowed_tools", []),+ }+ else:+ try:+ import yaml++ parsed = yaml.safe_load(str(abstract))+ except Exception:+ parsed = None+ if isinstance(parsed, dict):+ meta = {+ key: parsed.get(key, [] if key in {"tags", "allowed_tools"} else "")+ for key in ("name", "description", "tags", "allowed_tools")+ }+ if meta is not None and source_metadata is not None:+ meta["source_path"] = str(source_metadata.get("uri") or "")+ enqueued_levels = await vectorize_directory_meta(+ uri,+ abstract,+ overview,+ context_type=context_type_for_uri(uri),+ ctx=ctx,+ include_abstract=int(ContextLevel.ABSTRACT) in levels,+ include_overview=int(ContextLevel.OVERVIEW) in levels,+ content_is_body=True,+ actions={level: action.action.value for level, action in levels.items()},+ scalar_overrides={+ level: {+ **dict(action.upsert_fields),+ "_record_id": action.record_id,+ }+ for level, action in levels.items()+ },+ field_patches={+ level: action.field_patch+ for level, action in levels.items()+ if action.field_patch is not None+ },+ md5s={+ level: action.md5 for level, action in levels.items() if action.md5 is not None+ },+ **({"meta": meta} if meta is not None else {}),)+ completed_actions += len(enqueued_levels)logger.debug("[DirectIndexActions] %s root=%s action_counts=%s action_count=%d",log_correlation(),@@ -1454,6 +1556,7 @@ class ResourceProcessor:dict(action_counts),len(actions),)+ return completed_actionsasync def _delete_removed_resource_vectors(self,@@ -1581,14 +1684,17 @@ class ResourceProcessor:scalar_override: Optional[Dict[str, Any]] = None,field_patch: FieldPatch | None = None,action: str = "merge",- ) -> None:+ file_content: bytes | None = None,+ materialize_content: bool = False,+ summary: str = "",+ ) -> bool:parent = VikingURI(file_uri).parentif parent is None:- return+ return Falsename = file_uri.rsplit("/", 1)[-1]- await vectorize_file(+ return await vectorize_file(file_path=file_uri,- summary_dict={"name": name, "summary": ""},+ summary_dict={"name": name, "summary": summary},parent_uri=parent.uri,context_type=context_type_for_uri(file_uri),ctx=ctx,@@ -1597,6 +1703,8 @@ class ResourceProcessor:scalar_override=scalar_override,field_patch=field_patch,action=action,+ file_content=file_content,+ materialize_content=materialize_content,)async def reserve_unique_candidate(diff --git a/openviking_cli/utils/config/queue_worker_config.py b/openviking_cli/utils/config/queue_worker_config.pyindex 3ebb7f01a..d7f7453e0 100644--- a/openviking_cli/utils/config/queue_worker_config.py+++ b/openviking_cli/utils/config/queue_worker_config.py@@ -36,6 +36,7 @@ class QueueWorkersConfig(BaseModel):external_parse: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)add_resource: AddResourceQueueWorkerConfig = Field(default_factory=AddResourceQueueWorkerConfig)+ reindex: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)session_commit: QueueWorkerConfig = Field(default_factory=lambda: QueueWorkerConfig(max_concurrent=8))diff --git a/sdk/go/client_test.go b/sdk/go/client_test.goindex 2e027a197..88ab56b92 100644--- a/sdk/go/client_test.go+++ b/sdk/go/client_test.go@@ -366,7 +366,7 @@ func TestFindSendsImageQuery(t *testing.T) {}}-func TestReindexSendsDryRun(t *testing.T) {+func TestReindexSendsForceAndRecursive(t *testing.T) {client, closeServer := testClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if r.URL.Path != "/api/v1/content/reindex" {t.Fatalf("path = %s", r.URL.Path)@@ -378,14 +378,14 @@ func TestReindexSendsDryRun(t *testing.T) {if got := body["uri"]; got != "viking://resources/demo" {t.Fatalf("uri = %#v", got)}- if got := body["mode"]; got != "prune_orphans" {+ if got := body["mode"]; got != "vectors_only" {t.Fatalf("mode = %#v", got)}if got := body["wait"]; got != false {t.Fatalf("wait = %#v", got)}- if got := body["dry_run"]; got != true {- t.Fatalf("dry_run = %#v", got)+ if got := body["force"]; got != true {+ t.Fatalf("force = %#v", got)}if got := body["recursive"]; got != false {t.Fatalf("recursive = %#v", got)@@ -395,9 +395,9 @@ func TestReindexSendsDryRun(t *testing.T) {defer closeServer()if _, err := client.Reindex(context.Background(), "resources/demo", &ReindexOptions{- Mode: "prune_orphans",+ Mode: "vectors_only",Wait: false,- DryRun: true,+ Force: true,Recursive: Bool(false),}); err != nil {t.Fatal(err)diff --git a/sdk/go/filesystem.go b/sdk/go/filesystem.goindex ed5ca9014..fe256b5fc 100644--- a/sdk/go/filesystem.go+++ b/sdk/go/filesystem.go@@ -295,8 +295,9 @@ func (c *Client) BatchWrite(}// SetTags sets explicit k=v retrieval tags metadata for a file or directory.-// Valid modes are "replace" (default) and "append"; Recursive applies the tags-// to every file under a directory URI.+// Valid modes are "replace" (default), "append", and "clear". An empty+// replace request is a no-op; clear removes existing tags. Recursive applies+// the update to every file under a directory URI.func (c *Client) SetTags(ctx context.Context, uri string, tags []string, opts *SetTagsOptions) (map[string]any, error) {if opts == nil {opts = &SetTagsOptions{Mode: "replace"}@@ -305,9 +306,8 @@ func (c *Client) SetTags(ctx context.Context, uri string, tags []string, opts *Sif mode == "" {mode = "replace"}- // The server contract is tags:list[str]; a nil slice would marshal to JSON- // null and fail validation, so normalize to an empty list. With mode- // "replace" an empty list clears all tags.+ // Normalize nil to an empty list. The server treats replace + [] as a no-op+ // and clear as the explicit request to remove existing tags.if tags == nil {tags = []string{}}@@ -339,9 +339,11 @@ func (c *Client) Reindex(ctx context.Context, uri string, opts *ReindexOptions)"uri": NormalizeURI(uri),"mode": mode,"wait": opts.Wait,- "dry_run": opts.DryRun,"recursive": boolValue(opts.Recursive, true),}+ if opts.Force {+ payload["force"] = true+ }if opts.Tags != nil || opts.TagMode == "clear" {if opts.Tags != nil {payload["tags"] = opts.Tags@@ -352,7 +354,7 @@ func (c *Client) Reindex(ctx context.Context, uri string, opts *ReindexOptions)}payload["tag_mode"] = tagMode}- if err := mergeExtraProtected(payload, opts.Extra, "tags", "tag_mode"); err != nil {+ if err := mergeExtraProtected(payload, opts.Extra, "force", "tags", "tag_mode"); err != nil {return nil, err}var result map[string]anydiff --git a/sdk/go/types.go b/sdk/go/types.goindex de26f5d93..f0847deed 100644--- a/sdk/go/types.go+++ b/sdk/go/types.go@@ -268,7 +268,7 @@ type SetTagsOptions struct {type ReindexOptions struct {Mode stringWait bool- DryRun bool+ Force boolRecursive *boolTags []stringTagMode stringdiff --git a/sdk/python/openviking_sdk/client.py b/sdk/python/openviking_sdk/client.pyindex 2e29ea2ef..82d1a09e6 100644--- a/sdk/python/openviking_sdk/client.py+++ b/sdk/python/openviking_sdk/client.py@@ -1303,20 +1303,22 @@ class AsyncHTTPClient:async def set_tags(self,uri: str,- tags: List[str],+ tags: Optional[List[str]] = None,mode: str = "replace",recursive: bool = False,options: Optional[SetTagsOptions] = None,) -> Dict[str, Any]:+ fixed: Dict[str, Any] = {+ "uri": VikingURI.normalize(uri),+ "mode": mode,+ "recursive": recursive,+ }+ if tags is not None:+ fixed["tags"] = tagspayload = self._build_options_payload(options,SetTagsOptions,- fixed={- "uri": VikingURI.normalize(uri),- "tags": tags,- "mode": mode,- "recursive": recursive,- },+ fixed=fixed,)response = await self._request("POST",@@ -1778,8 +1780,8 @@ class AsyncHTTPClient:uri: str,mode: str = "vectors_only",wait: bool = True,- dry_run: bool = False,recursive: bool = True,+ force: bool = False,options: Optional[ReindexOptions] = None,) -> Dict[str, Any]:payload = self._build_options_payload(@@ -1789,10 +1791,12 @@ class AsyncHTTPClient:"uri": VikingURI.normalize(uri),"mode": mode,"wait": wait,- "dry_run": dry_run,+ "force": force,"recursive": recursive,},)+ if not payload.get("force"):+ payload.pop("force", None)response = await self._request("POST","/api/v1/content/reindex",@@ -2077,24 +2081,16 @@ class AsyncHTTPClient:)return self._handle_response(response)- def queue_status(- self, format: Optional[Literal["table", "json"]] = None- ) -> Dict[str, Any]:+ def queue_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:return run_async(self._get_queue_status(format=format))- def vikingdb_status(- self, format: Optional[Literal["table", "json"]] = None- ) -> Dict[str, Any]:+ def vikingdb_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:return run_async(self._get_vikingdb_status(format=format))- def models_status(- self, format: Optional[Literal["table", "json"]] = None- ) -> Dict[str, Any]:+ def models_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:return run_async(self._get_models_status(format=format))- def get_status(- self, format: Optional[Literal["table", "json"]] = None- ) -> Dict[str, Any]:+ def get_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:return run_async(self._get_system_status(format=format))def is_healthy(self) -> bool:@@ -2589,7 +2585,7 @@ class SyncHTTPClient:def set_tags(self,uri: str,- tags: List[str],+ tags: Optional[List[str]] = None,mode: str = "replace",recursive: bool = False,options: Optional[SetTagsOptions] = None,@@ -2862,20 +2858,19 @@ class SyncHTTPClient:uri: str,mode: str = "vectors_only",wait: bool = True,- dry_run: bool = False,recursive: bool = True,+ force: bool = False,options: Optional[ReindexOptions] = None,) -> Dict[str, Any]:- return run_async(- self._async_client.reindex(- uri,- mode=mode,- wait=wait,- dry_run=dry_run,- recursive=recursive,- options=options,- )- )+ kwargs: Dict[str, Any] = {+ "mode": mode,+ "wait": wait,+ "recursive": recursive,+ "options": options,+ }+ if force:+ kwargs["force"] = True+ return run_async(self._async_client.reindex(uri, **kwargs))def admin_create_account(self,@@ -3019,24 +3014,16 @@ class SyncHTTPClient:) -> Dict[str, Any]:return run_async(self._async_client.preflight_openviking_asset(name, repo_url, options))- def queue_status(- self, format: Optional[Literal["table", "json"]] = None- ) -> Dict[str, Any]:+ def queue_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:return self._async_client.queue_status(format=format)- def vikingdb_status(- self, format: Optional[Literal["table", "json"]] = None- ) -> Dict[str, Any]:+ def vikingdb_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:return self._async_client.vikingdb_status(format=format)- def models_status(- self, format: Optional[Literal["table", "json"]] = None- ) -> Dict[str, Any]:+ def models_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:return self._async_client.models_status(format=format)- def get_status(- self, format: Optional[Literal["table", "json"]] = None- ) -> Dict[str, Any]:+ def get_status(self, format: Optional[Literal["table", "json"]] = None) -> Dict[str, Any]:return self._async_client.get_status(format=format)def is_healthy(self) -> bool:diff --git a/sdk/python/openviking_sdk/options.py b/sdk/python/openviking_sdk/options.pyindex 63841b525..0358bfd55 100644--- a/sdk/python/openviking_sdk/options.py+++ b/sdk/python/openviking_sdk/options.py@@ -112,6 +112,7 @@ class SetTagsOptions(_ExtraOptions, total=False):class ReindexOptions(_ExtraOptions, total=False):+ force: booltags: List[str]tag_mode: Literal["replace", "append", "clear"]diff --git a/sdk/python/tests/test_async_client_behaviors.py b/sdk/python/tests/test_async_client_behaviors.pyindex 2d11ef909..5c12856fa 100644--- a/sdk/python/tests/test_async_client_behaviors.py+++ b/sdk/python/tests/test_async_client_behaviors.py@@ -310,9 +310,9 @@ async def test_async_http_client_reindex_posts_content_reindex():result = await client.reindex("viking://resources/demo",- mode="prune_orphans",+ mode="vectors_only",wait=False,- dry_run=True,+ force=True,options=None,)@@ -321,9 +321,9 @@ async def test_async_http_client_reindex_posts_content_reindex():"/api/v1/content/reindex",json={"uri": "viking://resources/demo",- "mode": "prune_orphans",+ "mode": "vectors_only","wait": False,- "dry_run": True,+ "force": True,"recursive": True,},)@@ -474,18 +474,16 @@ def test_sync_http_client_reindex_forwards_to_async_client():) as mock_run:result = client.reindex("viking://resources/demo",- mode="prune_orphans",+ mode="vectors_only",wait=False,- dry_run=True,)assert result == {"status": "accepted"}assert mock_run.calledmock_reindex.assert_called_once_with("viking://resources/demo",- mode="prune_orphans",+ mode="vectors_only",wait=False,- dry_run=True,recursive=True,options=None,)@@ -1339,6 +1337,25 @@ async def test_grep_omits_unset_tags_and_forwards_explicit_tags():assert fake_http.post.await_args_list[1].kwargs["json"]["tags"] == ["env=prod"]+async def test_set_tags_clear_omits_tags_from_sdk_request():+ client = AsyncHTTPClient(url="http://localhost:1933")+ fake_http = SimpleNamespace(post=AsyncMock(return_value=object()))+ client._http = fake_http+ client._handle_response_data = lambda _response: {"result": {"tags": []}}++ await client.set_tags("viking://resources/demo.md", mode="clear")++ fake_http.post.assert_awaited_once_with(+ "/api/v1/fs/attrs/set_tags",+ json={+ "uri": "viking://resources/demo.md",+ "mode": "clear",+ "recursive": False,+ },+ )++@pytest.mark.asyncioasync def test_glob_normalizes_scope_uri():client = AsyncHTTPClient(url="http://localhost:1933")diff --git a/sdk/typescript/src/client.ts b/sdk/typescript/src/client.tsindex 181fa1e0b..ea7fb1666 100644--- a/sdk/typescript/src/client.ts+++ b/sdk/typescript/src/client.ts@@ -682,7 +682,7 @@ export class OpenVikingClient {/** Set retrieval tags. */setTags(uri: string,- tags: string[],+ tags?: string[],options: SetTagsOptions = {},): Promise<JsonObject> {const body = compact({@@ -708,7 +708,7 @@ export class OpenVikingClient {uri: normalizeURI(uri),mode: options.mode ?? "vectors_only",wait: options.wait ?? true,- dry_run: options.dryRun ?? false,+ force: options.force || undefined,recursive: options.recursive ?? true,tags: options.tags,tag_mode:@@ -717,7 +717,7 @@ export class OpenVikingClient {: undefined,});return this.request("POST", "/api/v1/content/reindex", {- body: mergeExtra(body, options.extra, ["tags", "tag_mode"]),+ body: mergeExtra(body, options.extra, ["force", "tags", "tag_mode"]),});}diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.tsindex 197d51bbe..77c5dbac2 100644--- a/sdk/typescript/src/types.ts+++ b/sdk/typescript/src/types.ts@@ -131,7 +131,7 @@ export interface CompileOptions {}/** Retrieval tag update options. */export interface SetTagsOptions {- mode?: "replace" | "append";+ mode?: "replace" | "append" | "clear";recursive?: boolean;telemetry?: unknown;extra?: JsonObject;@@ -140,7 +140,7 @@ export interface SetTagsOptions {export interface ReindexOptions {mode?: string;wait?: boolean;- dryRun?: boolean;+ force?: boolean;recursive?: boolean;tags?: string[];tagMode?: "replace" | "append" | "clear";diff --git a/sdk/typescript/tests/client.test.ts b/sdk/typescript/tests/client.test.tsindex 93814c1ea..e11dc8a25 100644--- a/sdk/typescript/tests/client.test.ts+++ b/sdk/typescript/tests/client.test.ts@@ -176,12 +176,7 @@ describe("OpenVikingClient", () => {[{ role: "user", content: "hello" }],telemetry,);- await client.commitSession(- "session-1",- 2,- telemetry,- ["team=platform"],- );+ await client.commitSession("session-1", 2, telemetry, ["team=platform"]);expect(JSON.parse(String(fetcher.mock.calls[0]![1]?.body))).toEqual({messages: [{ role: "user", content: "hello" }],@@ -323,7 +318,7 @@ describe("OpenVikingClient", () => {});});- it("sends dry_run for prune_orphans reindex requests", async () => {+ it("sends force and recursive for reindex requests", async () => {const fetcher = vi.fn<typeof fetch>().mockResolvedValue(ok({ status: "completed" }));@@ -333,9 +328,9 @@ describe("OpenVikingClient", () => {});await client.reindex("resources", {- mode: "prune_orphans",+ mode: "vectors_only",wait: true,- dryRun: true,+ force: true,recursive: false,});@@ -343,9 +338,9 @@ describe("OpenVikingClient", () => {expect(String(url)).toBe("https://example.com/api/v1/content/reindex");expect(JSON.parse(String(init?.body))).toEqual({uri: "viking://resources",- mode: "prune_orphans",+ mode: "vectors_only",wait: true,- dry_run: true,+ force: true,recursive: false,});});@@ -439,20 +434,49 @@ describe("OpenVikingClient", () => {});it("sends explicit tags for write, list, tree, and grep", async () => {- const fetcher = vi.fn<typeof fetch>().mockImplementation(async () => ok({}));- const client = new OpenVikingClient({ baseUrl: "https://example.com", fetch: fetcher });+ const fetcher = vi+ .fn<typeof fetch>()+ .mockImplementation(async () => ok({}));+ const client = new OpenVikingClient({+ baseUrl: "https://example.com",+ fetch: fetcher,+ });- await client.write("resources/demo.md", "updated", { tags: [], tagMode: "replace" });+ await client.write("resources/demo.md", "updated", {+ tags: [],+ tagMode: "replace",+ });await client.list("resources", { tags: ["env=prod"], includeTags: true });await client.tree("resources", { tags: ["env=prod"], includeTags: true });- await client.grep("resources", "needle", { tags: ["env=prod"], includeTags: true });-- expect(JSON.parse(String(fetcher.mock.calls[0]![1]?.body))).toMatchObject({ tags: [], tag_mode: "replace" });- expect(new URL(String(fetcher.mock.calls[1]![0])).searchParams.get("tags")).toBe("env=prod");- expect(new URL(String(fetcher.mock.calls[1]![0])).searchParams.get("include_tags")).toBe("true");- expect(new URL(String(fetcher.mock.calls[2]![0])).searchParams.get("tags")).toBe("env=prod");- expect(new URL(String(fetcher.mock.calls[2]![0])).searchParams.get("include_tags")).toBe("true");- expect(JSON.parse(String(fetcher.mock.calls[3]![1]?.body))).toMatchObject({ tags: ["env=prod"], include_tags: true });+ await client.grep("resources", "needle", {+ tags: ["env=prod"],+ includeTags: true,+ });++ expect(JSON.parse(String(fetcher.mock.calls[0]![1]?.body))).toMatchObject({+ tags: [],+ tag_mode: "replace",+ });+ expect(+ new URL(String(fetcher.mock.calls[1]![0])).searchParams.get("tags"),+ ).toBe("env=prod");+ expect(+ new URL(String(fetcher.mock.calls[1]![0])).searchParams.get(+ "include_tags",+ ),+ ).toBe("true");+ expect(+ new URL(String(fetcher.mock.calls[2]![0])).searchParams.get("tags"),+ ).toBe("env=prod");+ expect(+ new URL(String(fetcher.mock.calls[2]![0])).searchParams.get(+ "include_tags",+ ),+ ).toBe("true");+ expect(JSON.parse(String(fetcher.mock.calls[3]![1]?.body))).toMatchObject({+ tags: ["env=prod"],+ include_tags: true,+ });});it("sends clear tag mode without tags", async () => {@@ -672,6 +696,22 @@ describe("OpenVikingClient", () => {).toThrow("extra cannot override uri");});+ it("omits tags when clear is requested", async () => {+ const fetcher = vi.fn<typeof fetch>().mockResolvedValue(ok({ updated: 1 }));+ const client = new OpenVikingClient({+ baseUrl: "https://example.com",+ fetch: fetcher,+ });++ await client.setTags("resources/demo.md", undefined, { mode: "clear" });++ expect(JSON.parse(String(fetcher.mock.calls[0]![1]?.body))).toEqual({+ uri: "viking://resources/demo.md",+ mode: "clear",+ recursive: false,+ });+ });+it("converts an existing Node.js image path to a data URI", async () => {const directory = await mkdtemp(join(tmpdir(), "openviking-sdk-image-"));const path = join(directory, "photo.png");@@ -1264,7 +1304,9 @@ describe("OpenVikingClient", () => {});await expect(client.getStatus()).resolves.toEqual({ is_healthy: true });- await expect(client.queueStatus("json")).resolves.toEqual({ name: "queue" });+ await expect(client.queueStatus("json")).resolves.toEqual({+ name: "queue",+ });await expect(client.modelsStatus("table")).resolves.toEqual({name: "models",});diff --git a/tests/client/test_rebuild_clients.py b/tests/client/test_rebuild_clients.pyindex b2aa201a8..f0bba6969 100644--- a/tests/client/test_rebuild_clients.py+++ b/tests/client/test_rebuild_clients.py@@ -130,9 +130,9 @@ async def test_async_http_client_reindex_posts_content_reindex():) as mock_handle:result = await client.reindex("viking://resources/demo",- mode="prune_orphans",+ mode="vectors_only",wait=False,- dry_run=True,+ force=True,)assert result == {"status": "completed"}@@ -140,9 +140,10 @@ async def test_async_http_client_reindex_posts_content_reindex():"/api/v1/content/reindex",json={"uri": "viking://resources/demo",- "mode": "prune_orphans",+ "mode": "vectors_only","wait": False,- "dry_run": True,+ "force": True,+ "recursive": True,},)assert mock_handle.called@@ -161,18 +162,20 @@ def test_sync_http_client_reindex_forwards_to_async_client():) as mock_run:result = client.reindex("viking://resources/demo",- mode="prune_orphans",+ mode="vectors_only",wait=False,- dry_run=True,+ force=True,)assert result == {"status": "accepted"}assert mock_run.calledmock_reindex.assert_called_once_with(- uri="viking://resources/demo",- mode="prune_orphans",+ "viking://resources/demo",+ mode="vectors_only",wait=False,- dry_run=True,+ recursive=True,+ force=True,+ options=None,)@@ -193,7 +196,6 @@ async def test_async_http_client_reindex_sends_non_recursive_flag():"uri": "viking://resources/demo","mode": "semantic_and_vectors","wait": True,- "dry_run": False,"recursive": False,},)@@ -207,9 +209,7 @@ def test_sync_http_client_reindex_forwards_non_recursive_flag():new_callable=Mock,return_value={"status": "completed"},) as mock_reindex:- with patch(- "openviking_sdk.client.run_async", return_value={"status": "completed"}- ):+ with patch("openviking_sdk.client.run_async", return_value={"status": "completed"}):client.reindex("viking://resources/demo",mode="semantic_and_vectors",@@ -217,11 +217,11 @@ def test_sync_http_client_reindex_forwards_non_recursive_flag():)mock_reindex.assert_called_once_with(- uri="viking://resources/demo",+ "viking://resources/demo",mode="semantic_and_vectors",wait=True,- dry_run=False,recursive=False,+ options=None,)diff --git a/tests/server/test_admin_rebuild_api.py b/tests/server/test_admin_rebuild_api.pyindex ef0cc0dc7..57bece0d1 100644--- a/tests/server/test_admin_rebuild_api.py+++ b/tests/server/test_admin_rebuild_api.py@@ -1,6 +1,5 @@"""Tests for reindex admin endpoint and executor behavior."""-import asynciofrom types import SimpleNamespacefrom unittest.mock import AsyncMock@@ -65,8 +64,8 @@ async def test_reindex_user_can_only_target_own_user_scope(monkeypatch):seen = {}class FakeService:- async def reindex(self, *, uri, mode, wait, dry_run=False, ctx):- seen.update(uri=uri, mode=mode, wait=wait, dry_run=dry_run, ctx=ctx)+ async def reindex(self, *, uri, mode, wait, ctx):+ seen.update(uri=uri, mode=mode, wait=wait, ctx=ctx)return {"status": "completed", "uri": uri, "mode": mode}monkeypatch.setattr("openviking.server.routers.content.get_service", lambda: FakeService())@@ -148,17 +147,20 @@ async def test_reindex_rejects_session_uri(admin_client: httpx.AsyncClient, uri:assert body["status"] == "error"-async def test_reindex_rejects_reason_field(admin_client: httpx.AsyncClient):- resp = await admin_client.post(- "/api/v1/content/reindex",- json={+def test_reindex_ignores_unknown_request_fields():+ from openviking.server.routers.content import ReindexRequest++ request = ReindexRequest.model_validate(+ {"uri": "viking://resources/demo","mode": "vectors_only","reason": "unused",- },- headers=ROOT_ACCOUNT_HEADERS,+ })- assert resp.status_code == 403++ assert request.uri == "viking://resources/demo"+ assert request.mode == "vectors_only"+ assert "reason" not in request.model_dump()async def test_reindex_root_requires_explicit_account(admin_client: httpx.AsyncClient):@@ -186,7 +188,6 @@ async def test_reindex_resource_vectors_only_wait_true(monkeypatch):mode,wait,ctx,- dry_run=False,tags=None,tag_mode="replace",):@@ -282,7 +283,7 @@ async def test_reindex_resource_vectors_only_wait_false(monkeypatch):from openviking.server.routers.content import ReindexRequest, reindexclass FakeService:- async def reindex(self, *, uri, mode, wait, ctx, dry_run=False):+ async def reindex(self, *, uri, mode, wait, ctx):return {"task_id": "rbld_123","status": "accepted",@@ -337,81 +338,36 @@ async def test_reindex_passes_non_recursive_to_service(monkeypatch):@pytest.mark.asyncio-async def test_reindex_prune_orphans_passes_dry_run_to_service(monkeypatch):+async def test_reindex_passes_force_to_service(monkeypatch):from openviking.server.routers.content import ReindexRequest, reindexseen = {}class FakeService:- async def reindex(self, *, uri, mode, wait, ctx, dry_run=False):- seen["uri"] = uri- seen["mode"] = mode- seen["wait"] = wait- seen["dry_run"] = dry_run- return {- "status": "completed",- "uri": uri,- "object_type": "resource",- "mode": mode,- "scanned_records": 1,- "deleted_records": 0,- "would_delete_records": 1,- "unsupported_records": 0,- "failed_records": 0,- "duration_ms": 1,- "warnings": [],- }+ async def reindex(self, **kwargs):+ seen.update(kwargs)+ return {"status": "completed"}ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="alice"),role=Role.ROOT,)- request = ReindexRequest(- uri="viking://resources/demo",- mode="prune_orphans",- wait=True,- dry_run=True,- )-monkeypatch.setattr("openviking.server.routers.content.get_service", lambda: FakeService())- response = await reindex(body=request, ctx=ctx)-- assert response.status == "ok"- assert response.result["would_delete_records"] == 1- assert seen == {- "uri": "viking://resources/demo",- "mode": "prune_orphans",- "wait": True,- "dry_run": True,- }--async def test_reindex_rejects_dry_run_for_non_prune_mode(monkeypatch):- from openviking.server.routers.content import ReindexRequest, reindex-- class FakeService:- async def reindex(self, **kwargs):- raise AssertionError("service should not be called")-- ctx = RequestContext(- user=UserIdentifier(account_id="test", user_id="alice"),- role=Role.ROOT,- )- request = ReindexRequest(- uri="viking://resources/demo",- mode="vectors_only",- wait=True,- dry_run=True,+ await reindex(+ body=ReindexRequest(+ uri="viking://resources/demo",+ mode="vectors_only",+ force=True,+ ),+ ctx=ctx,)- monkeypatch.setattr("openviking.server.routers.content.get_service", lambda: FakeService())- with pytest.raises(OpenVikingError, match="dry_run"):- await reindex(body=request, ctx=ctx)+ assert seen["force"] is True@pytest.mark.asyncio-async def test_reindex_executor_rejects_dry_run_for_non_prune_mode_direct():+async def test_reindex_rejects_non_recursive_namespace_before_work_starts():from openviking.service.reindex_executor import ReindexExecutorctx = RequestContext(@@ -419,12 +375,12 @@ async def test_reindex_executor_rejects_dry_run_for_non_prune_mode_direct():role=Role.ROOT,)- with pytest.raises(OpenVikingError, match="dry_run"):+ with pytest.raises(OpenVikingError, match="recursive=false"):await ReindexExecutor().execute(- uri="viking://resources/demo",+ uri="viking://user/alice",mode="vectors_only",wait=True,- dry_run=True,+ recursive=False,ctx=ctx,)@@ -443,44 +399,6 @@ async def test_reindex_executor_discards_invalid_tags_before_creating_backgroundassert ingest_options.search_tags == ["team=search"]-async def test_reindex_executor_ignores_tags_for_prune_orphans(monkeypatch):- from openviking.service.reindex_executor import ReindexExecutor-- executor = ReindexExecutor()- seen = {}-- async def fake_run(**kwargs):- seen.update(kwargs)- return {"status": "completed"}-- class FakeTracker:- async def has_running(self, *args, **kwargs):- return False-- monkeypatch.setattr(executor, "_run", fake_run)- monkeypatch.setattr(- "openviking.service.reindex_executor.get_task_tracker",- lambda: FakeTracker(),- )- ctx = RequestContext(- user=UserIdentifier(account_id="test", user_id="alice"),- role=Role.ROOT,- )-- result = await executor.execute(- uri="viking://resources/demo",- mode="prune_orphans",- wait=True,- tags=["invalid"],- tag_mode="invalid",- ctx=ctx,- )-- assert result == {"status": "completed"}- assert seen["ingest_options"] is None--@pytest.mark.asyncioasync def test_reindex_file_target_uses_exact_lock(monkeypatch):from types import SimpleNamespace@@ -510,7 +428,7 @@ async def test_reindex_file_target_uses_exact_lock(monkeypatch):def _uri_to_path(self, uri, ctx):return "/local/default/resources/demo.md"- async def stat(self, uri, ctx, skip_count=None):+ async def stat(self, uri, ctx, skip_count=True):return {"isDir": False}executor = ReindexExecutor()@@ -546,188 +464,152 @@ async def test_reindex_file_target_uses_exact_lock(monkeypatch):@pytest.mark.asyncio-async def test_reindex_prune_existing_file_uses_exact_lock(monkeypatch):- from types import SimpleNamespace-+async def test_reindex_executor_passes_tags_to_background_run(monkeypatch):from openviking.service.reindex_executor import ReindexExecutor+ from openviking.storage.queuefs.reindex_msg import ReindexMsg- calls = []-- class FakeAGFS:- async def pathlock_acquire_exact(self, path):- calls.append(("exact", path))- return {"id": "lease-1"}-- async def pathlock_acquire_tree(self, path):- calls.append(("tree", path))- raise AssertionError("existing file target must not acquire a tree lock")-- async def pathlock_as_borrowed(self, lease):- return lease-- async def pathlock_release(self, lease):- calls.append(("release", lease["id"]))-- class FakeFS:- _async_agfs = FakeAGFS()-- def _uri_to_path(self, uri, ctx):- return "/local/default/resources/demo.md"-- async def exists(self, uri, ctx):- return True-- async def stat(self, uri, ctx, skip_count=None):- return {"isDir": False}-- executor = ReindexExecutor()-- async def fake_prune_orphan_vectors(**kwargs):- return None-- monkeypatch.setattr(executor, "_prune_orphan_vectors", fake_prune_orphan_vectors)- monkeypatch.setattr(- "openviking.service.reindex_executor.get_service",- lambda: SimpleNamespace(- viking_fs=FakeFS(),- vikingdb_manager=SimpleNamespace(has_queue_manager=True),- ),- )- ctx = RequestContext(- user=UserIdentifier(account_id="test", user_id="alice"),- role=Role.ROOT,- )+ seen = {"enqueued": []}- result = await executor._run(- uri="viking://resources/demo.md",- object_type="resource",- mode="prune_orphans",- dry_run=True,- ctx=ctx,- )-- assert result["status"] == "completed"- assert calls == [- ("exact", "/local/default/resources/demo.md"),- ("release", "lease-1"),- ]--[email protected]("locked", [False, True])-async def test_reindex_prune_missing_target_does_not_lock(monkeypatch, locked):- """A missing prune target must not acquire a lock: doing so creates the directory."""- from types import SimpleNamespace+ class FakeTask:+ task_id = "task-1"- from openviking.service.reindex_executor import ReindexExecutor- from openviking.storage.errors import ResourceBusyError+ class FakeTracker:+ async def create_if_no_running(self, *args, **kwargs):+ return FakeTask()- calls = []+ update_stage = AsyncMock()+ fail = AsyncMock()class FakeAGFS:- async def pathlock_acquire_exact(self, path):- raise AssertionError("missing target must not acquire a lock")-async def pathlock_acquire_tree(self, path):- raise AssertionError("missing target must not acquire a lock")+ seen["path"] = path+ return {"lease": "root"}- async def pathlock_is_locked(self, path):- calls.append(("is_locked", path))- return locked+ async def pathlock_to_handoff(self, lease):+ assert lease == {"lease": "root"}+ return {"handoff": "root"}- async def pathlock_as_borrowed(self, lease):- raise AssertionError("no lease to borrow")+ async def pathlock_handoff(self, lease):+ assert lease == {"lease": "root"}async def pathlock_release(self, lease):- raise AssertionError("no lease to release")+ raise AssertionError(f"unexpected release: {lease}")- class FakeFS:+ class FakeVikingFS:_async_agfs = FakeAGFS()- def _uri_to_path(self, uri, ctx):- return "/local/default/resources/missing"+ async def stat(self, uri, *, ctx, skip_count):+ assert uri == "viking://resources/demo"+ assert skip_count is True+ return {"isDir": True}- async def exists(self, uri, ctx):- return False+ def _uri_to_path(self, uri, *, ctx):+ return "/resources/demo"- executor = ReindexExecutor()+ class FakeQueueManager:+ REINDEX = "Reindex"- async def fake_prune_orphan_vectors(**kwargs):- return None+ async def enqueue(self, queue_name, data):+ seen["enqueued"].append((queue_name, data))+ return "message-1"- monkeypatch.setattr(executor, "_prune_orphan_vectors", fake_prune_orphan_vectors)+ executor = ReindexExecutor()+ monkeypatch.setattr(+ "openviking.service.reindex_executor.get_task_tracker",+ lambda: FakeTracker(),+ )monkeypatch.setattr("openviking.service.reindex_executor.get_service",- lambda: SimpleNamespace(- viking_fs=FakeFS(),- vikingdb_manager=SimpleNamespace(has_queue_manager=True),- ),+ lambda: SimpleNamespace(viking_fs=FakeVikingFS()),)+ monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: FakeQueueManager())ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="alice"),role=Role.ROOT,)- run = executor._run(- uri="viking://resources/missing",- object_type="resource",- mode="prune_orphans",- dry_run=True,+ result = await executor.execute(+ uri="viking://resources/demo",+ mode="vectors_only",+ wait=False,+ tags=["Team=Search"],+ tag_mode="append",ctx=ctx,)- if locked:- with pytest.raises(ResourceBusyError):- await run- else:- assert (await run)["status"] == "completed"- assert calls == [("is_locked", "/local/default/resources/missing")]---async def test_reindex_executor_passes_tags_to_background_run(monkeypatch):- import asyncio+ assert result["status"] == "accepted"+ assert seen["path"] == "/resources/demo"+ assert len(seen["enqueued"]) == 1+ queue_name, payload = seen["enqueued"][0]+ assert queue_name == "Reindex"+ message = ReindexMsg.from_dict(payload)+ assert message.task_id == "task-1"+ assert message.tags == ["Team=Search"]+ assert message.tag_mode == "append"+ assert message.lock_handoff == {"handoff": "root"}+ assert message.actor_peer_id == ctx.actor_peer_id+ assert message.bypass_acl is ctx.bypass_acl++[email protected]("failure", ["stat", "lock"])+async def test_reindex_wait_false_marks_task_failed_when_submit_preparation_fails(+ monkeypatch, failure+):from openviking.service.reindex_executor import ReindexExecutor- seen = {}-class FakeTask:task_id = "task-1"- class FakeTracker:- async def create_if_no_running(self, *args, **kwargs):- return FakeTask()+ tracker = SimpleNamespace(+ create_if_no_running=AsyncMock(return_value=FakeTask()),+ fail=AsyncMock(),+ )- executor = ReindexExecutor()+ class FakeAGFS:+ async def pathlock_acquire_tree(self, _path):+ if failure == "lock":+ raise RuntimeError("root lock is busy")+ raise AssertionError("lock must not be acquired after stat failure")- async def fake_run_tracked(task_id, **kwargs):- seen["task_id"] = task_id- seen.update(kwargs)+ class FakeVikingFS:+ _async_agfs = FakeAGFS()++ async def stat(self, _uri, *, ctx, skip_count):+ del ctx, skip_count+ if failure == "stat":+ raise RuntimeError("root stat is unavailable")+ return {"isDir": True}++ def _uri_to_path(self, _uri, *, ctx):+ del ctx+ return "/resources/demo"- monkeypatch.setattr(executor, "_run_tracked", fake_run_tracked)+ monkeypatch.setattr("openviking.service.reindex_executor.get_task_tracker", lambda: tracker)monkeypatch.setattr(- "openviking.service.reindex_executor.get_task_tracker",- lambda: FakeTracker(),+ "openviking.service.reindex_executor.get_service",+ lambda: SimpleNamespace(viking_fs=FakeVikingFS()),)ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="alice"),role=Role.ROOT,)- result = await executor.execute(- uri="viking://resources/demo",- mode="vectors_only",- wait=False,- tags=["Team=Search"],- tag_mode="append",- ctx=ctx,- )- await asyncio.sleep(0)+ with pytest.raises(RuntimeError, match=f"root {failure}"):+ await ReindexExecutor().execute(+ uri="viking://resources/demo",+ mode="vectors_only",+ wait=False,+ ctx=ctx,+ )- assert result["status"] == "accepted"- assert seen["task_id"] == "task-1"- assert seen["ingest_options"].search_tags == ["team=search"]- assert seen["ingest_options"].search_tag_mode == "append"+ tracker.fail.assert_awaited_once_with(+ "task-1",+ "Failed to enqueue reindex processing",+ account_id="test",+ user_id="alice",+ )@pytest.mark.asyncio@@ -741,13 +623,13 @@ async def test_reindex_resource_passes_run_tags_to_vector_rebuild(monkeypatch):seen = {}- async def fake_reindex_resource_vectors(self, **kwargs):+ async def fake_reindex_rfv(self, **kwargs):seen.update(kwargs)monkeypatch.setattr(ReindexExecutor,- "_reindex_resource_vectors",- fake_reindex_resource_vectors,+ "_reindex_rfv",+ fake_reindex_rfv,)ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="alice"),@@ -765,641 +647,466 @@ async def test_reindex_resource_passes_run_tags_to_vector_rebuild(monkeypatch):),)- assert seen["ingest_options"] is ingest_options---async def test_reindex_upsert_uses_uri_owner_for_user_scoped_records(monkeypatch):- from openviking.service.reindex_executor import ReindexExecutor-- captured = {}-- class FakeVikingDB:- async def enqueue_embedding_msg(self, msg):- captured["msg"] = msg- return True-- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)-- service = ReindexExecutor()- ctx = RequestContext(- user=UserIdentifier(account_id="acct", user_id="admin"),- role=Role.ROOT,- )-- await service._upsert_context(- uri="viking://user/bob/memories/preferences/theme.md",- parent_uri="viking://user/bob/memories/preferences",- abstract="theme",- vector_text="theme",- is_leaf=True,- context_type="memory",- level=ContextLevel.DETAIL,- ctx=ctx,- )-- data = captured["msg"].context_data- assert data["user"]["user_id"] == "bob"- assert data["owner_user_id"] == "bob"- assert data["owner_space"] == "bob"---async def test_reindex_semantic_processor_uses_uri_owner_for_user_scoped_records(- monkeypatch, semantic_config-):- from openviking.service.reindex_executor import ReindexExecutor-- captured = {}-- class FakeSemanticProcessor:- def __init__(self, **_kwargs):- pass-- async def on_dequeue(self, payload, lock=None):- captured["payload"] = payload- captured["lock"] = lock- return ProcessResult.success()-- monkeypatch.setattr(- "openviking.service.reindex_executor.SemanticProcessor",- FakeSemanticProcessor,- )-- service = ReindexExecutor(vlm_resolver=semantic_config)- ctx = RequestContext(- user=UserIdentifier(account_id="acct", user_id="admin"),- role=Role.ROOT,- )-- await service._run_semantic_processor(- uri="viking://user/bob/memories/preferences",- context_type="memory",- ctx=ctx,- )-- data = captured["payload"]["data"]- assert '"user_id": "bob"' in data- assert '"peer_id": "bob"' in data- assert '"account_id": "acct"' in data---async def test_prune_orphans_candidate_filter_includes_target_uri(monkeypatch):- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters- from openviking.storage.expr import And, Eq, Or, PathScope-- target_uri = "viking://resources/demo/missing.txt"-- def matches_target_self(filter_expr):- if isinstance(filter_expr, And):- return all(matches_target_self(cond) for cond in filter_expr.conds)- if isinstance(filter_expr, Or):- return any(matches_target_self(cond) for cond in filter_expr.conds)- if isinstance(filter_expr, Eq):- return True- if isinstance(filter_expr, PathScope):- return filter_expr.path == target_uri and filter_expr.depth == 0- return False-- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return False-- class FakeVikingDB:- async def filter(self, *, filter, limit, output_fields, ctx):- if not matches_target_self(filter):- return []- return [- {- "id": "target",- "uri": target_uri,- "level": 2,- "context_type": "resource",- "account_id": "test",- }- ]-- async def delete(self, ids, *, ctx):- raise AssertionError("no records should be deleted")-- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())-- counters = _ReindexCounters()- ctx = RequestContext(- user=UserIdentifier(account_id="test", user_id="admin"),- role=Role.ROOT,- )-- await ReindexExecutor()._prune_orphan_vectors(- uri=target_uri,- object_type="resource",- dry_run=True,- counters=counters,- ctx=ctx,- )-- assert counters.scanned_records == 1- assert counters.would_delete_records == 1+ assert seen["run"].ingest_options is ingest_options@pytest.mark.asyncio-async def test_prune_orphans_dry_run_preserves_resource_l1_fallback_and_skips_unknown(- monkeypatch,-):[email protected]("mode", ["vectors_only", "semantic_and_vectors"])+async def test_reindex_resource_routes_both_modes_through_rfv_plan(monkeypatch, mode):from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters- class FakeVikingFS:- async def exists(self, uri, ctx=None):- if uri == "viking://resources/demo/.abstract.md":- return True- if uri == "viking://resources/demo/.overview.md":- return False- return False-- async def read_file(self, uri, ctx=None):- if uri == "viking://resources/demo/.abstract.md":- return "resource abstract"- raise FileNotFoundError(uri)-- class FakeVikingDB:- async def filter(self, *, filter, limit, output_fields, ctx):- assert limit == 100000- assert ctx.account_id == "test"- return [- {- "id": "keep_l1",- "uri": "viking://resources/demo",- "level": 1,- "context_type": "resource",- "account_id": "test",- },- {- "id": "delete_l2",- "uri": "viking://resources/demo/missing.txt",- "level": 2,- "context_type": "resource",- "account_id": "test",- },- {- "id": "skip_unknown",- "uri": "viking://resources/demo/unknown",- "level": 99,- "context_type": "resource",- "account_id": "test",- },- ]-- async def delete(self, ids, *, ctx):- raise AssertionError("dry_run should not delete")+ seen = {}- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())+ async def fake_reindex_rfv(self, **kwargs):+ seen.update(kwargs)- service = ReindexExecutor()- counters = _ReindexCounters()+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)ctx = RequestContext(- user=UserIdentifier(account_id="test", user_id="admin"),+ user=UserIdentifier(account_id="test", user_id="alice"),role=Role.ROOT,)+ run = _make_reindex_run(ctx, _ReindexCounters())+ run.force = True- await service._prune_orphan_vectors(+ await ReindexExecutor()._reindex_resource(uri="viking://resources/demo",- object_type="resource",- dry_run=True,- counters=counters,- ctx=ctx,+ mode=mode,+ recursive=False,+ run=run,)- assert counters.scanned_records == 3- assert counters.deleted_records == 0- assert counters.would_delete_records == 1- assert counters.unsupported_records == 1- assert any("unknown" in warning for warning in counters.warnings)+ assert seen == {+ "uri": "viking://resources/demo",+ "mode": mode,+ "context_type": "resource",+ "recursive": False,+ "run": run,+ }@pytest.mark.asyncio[email protected]("filename", ["theme.md", "report#chunk_notes.md", "report#chunk_0000"])-async def test_prune_orphans_deletes_stale_memory_chunk_with_uri_owner_ctx(monkeypatch, filename):[email protected]("mode", ["vectors_only", "semantic_and_vectors"])+async def test_reindex_skill_routes_both_modes_through_rfv_plan(monkeypatch, mode):from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters- base_uri = f"viking://user/bob/memories/preferences/{filename}"- deleted = {}-- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return uri == base_uri-- async def read_file(self, uri, ctx=None):- assert uri == base_uri- return "memory body"-- class FakeVikingDB:- async def filter(self, *, filter, limit, output_fields, ctx):- assert limit == 100000- assert ctx.account_id == "acct"- return [- {- "id": "keep_base",- "uri": base_uri,- "level": 2,- "context_type": "memory",- "account_id": "acct",- "owner_user_id": "bob",- },- {- "id": "keep_chunk",- "uri": f"{base_uri}#chunk_0000",- "level": 2,- "context_type": "memory",- "account_id": "acct",- "owner_user_id": "bob",- },- {- "id": "delete_chunk",- "uri": f"{base_uri}#chunk_0001",- "level": 2,- "context_type": "memory",- "account_id": "acct",- "owner_user_id": "bob",- },- ]-- async def delete(self, ids, *, ctx):- deleted["ids"] = ids- deleted["ctx"] = ctx- return len(ids)+ seen = {}- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())- monkeypatch.setattr(- ReindexExecutor,- "_chunk_memory_body",- lambda self, uri, body: [(f"{uri}#chunk_0000", body)],- )+ async def fake_reindex_rfv(self, **kwargs):+ seen.update(kwargs)- service = ReindexExecutor()- counters = _ReindexCounters()+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)ctx = RequestContext(- user=UserIdentifier(account_id="acct", user_id="admin"),+ user=UserIdentifier(account_id="test", user_id="alice"),role=Role.ROOT,)+ run = _make_reindex_run(ctx, _ReindexCounters())- await service._prune_orphan_vectors(- uri="viking://user/bob/memories",- object_type="memory",- dry_run=False,- counters=counters,- ctx=ctx,+ await ReindexExecutor()._reindex_skill(+ uri="viking://user/alice/skills/demo",+ mode=mode,+ recursive=False,+ run=run,)- assert deleted["ids"] == ["delete_chunk"]- assert deleted["ctx"].user.user_id == "bob"- assert counters.deleted_records == 1- assert counters.would_delete_records == 0+ assert seen == {+ "uri": "viking://user/alice/skills/demo",+ "mode": mode,+ "context_type": "skill",+ "recursive": False,+ "run": run,+ }@pytest.mark.asyncio-async def test_prune_orphans_skips_resource_l1_when_sidecar_read_fails(monkeypatch):- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters-- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return uri in {- "viking://resources/demo/.overview.md",- "viking://resources/demo/.abstract.md",- }-- async def read_file(self, uri, ctx=None):- raise OSError("temporary storage failure")-- class FakeVikingDB:- async def filter(self, *, filter, limit, output_fields, ctx):- return [- {- "id": "keep_l1",- "uri": "viking://resources/demo",- "level": 1,- "context_type": "resource",- "account_id": "test",- }- ]-- async def delete(self, ids, *, ctx):- raise AssertionError("read failures must not delete vectors")-- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())-- counters = _ReindexCounters()- ctx = RequestContext(- user=UserIdentifier(account_id="test", user_id="admin"),- role=Role.ROOT,- )-- await ReindexExecutor()._prune_orphan_vectors(- uri="viking://resources/demo",- object_type="resource",- dry_run=False,- counters=counters,- ctx=ctx,- )-- assert counters.deleted_records == 0- assert counters.failed_records == 1- assert any("failed to read" in warning for warning in counters.warnings)+async def test_reindex_rfv_skips_complete_file_without_enqueue_or_second_read(monkeypatch):+ from unittest.mock import AsyncMock--async def test_prune_orphans_skips_memory_chunks_when_base_read_fails(monkeypatch):from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters+ from openviking.utils.content_hash import content_md5- base_uri = "viking://user/bob/memories/preferences/theme.md"-- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return uri == base_uri+ uri = "viking://resources/demo.md"+ body = b"same body"- async def read_file(self, uri, ctx=None):- raise OSError("temporary storage failure")+ class FakeFS:+ stat = AsyncMock(return_value={"isDir": False})+ read_file_bytes = AsyncMock(return_value=body)- class FakeVikingDB:- async def filter(self, *, filter, limit, output_fields, ctx):- return [- {- "id": "keep_chunk",- "uri": f"{base_uri}#chunk_0001",+ class FakeDB:+ async def get_incremental_inventory_under_uri(+ self, target, *, ctx, output_fields, recursive=True, include_direct_children=False+ ):+ assert target == uri+ assert recursive is False+ assert include_direct_children is False+ return {+ "file-l2": {+ "id": "file-l2",+ "uri": uri,"level": 2,- "context_type": "memory",- "account_id": "acct",- "owner_user_id": "bob",+ "md5": content_md5(body),}- ]-- async def delete(self, ids, *, ctx):- raise AssertionError("base read failures must not delete chunks")-- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())+ }- counters = _ReindexCounters()+ resource_processor = SimpleNamespace(_enqueue_index_actions=AsyncMock())+ service = SimpleNamespace(+ viking_fs=FakeFS(),+ vikingdb_manager=FakeDB(),+ _resource_processor=resource_processor,+ )+ monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: service)ctx = RequestContext(- user=UserIdentifier(account_id="acct", user_id="admin"),+ user=UserIdentifier(account_id="test", user_id="alice"),role=Role.ROOT,)+ counters = _ReindexCounters()- await ReindexExecutor()._prune_orphan_vectors(- uri="viking://user/bob/memories",- object_type="memory",- dry_run=False,- counters=counters,- ctx=ctx,+ await ReindexExecutor()._reindex_rfv(+ uri=uri,+ mode="vectors_only",+ context_type="resource",+ recursive=False,+ run=_make_reindex_run(ctx, counters),)- assert counters.deleted_records == 0- assert counters.failed_records == 1- assert any("failed to read" in warning for warning in counters.warnings)+ resource_processor._enqueue_index_actions.assert_not_awaited()+ service.viking_fs.read_file_bytes.assert_awaited_once_with(uri, ctx=ctx)+ assert counters.scanned_records == 1+ assert counters.rebuilt_records == 0@pytest.mark.asyncio-async def test_prune_orphans_deletes_missing_file_with_string_level(monkeypatch):- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters+async def test_reindex_rfv_deletes_orphans_synchronously_with_owner_context(monkeypatch):+ from unittest.mock import AsyncMock- deleted = {}+ from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return False+ root = "viking://user/bob/resources/demo"- class FakeVikingDB:- async def filter(self, *, filter, limit, output_fields, ctx):- return [- {- "id": "delete_l2",- "uri": "viking://resources/demo/missing.txt",- "level": "2",- "context_type": "resource",- "account_id": "test",- }- ]+ class FakeFS:+ stat = AsyncMock(return_value={"isDir": True})+ tree = AsyncMock(return_value=[])- async def delete(self, ids, *, ctx):- deleted["ids"] = ids- return len(ids)+ class FakeDB:+ strict_delete = AsyncMock(return_value=1)- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())+ async def get_incremental_inventory_under_uri(+ self, target, *, ctx, output_fields, recursive=True, include_direct_children=False+ ):+ assert include_direct_children is False+ return {+ "orphan-l2": {+ "id": "orphan-l2",+ "uri": f"{root}/old.md",+ "level": 2,+ "md5": "old",+ }+ }- counters = _ReindexCounters()- ctx = RequestContext(+ db = FakeDB()+ service = SimpleNamespace(viking_fs=FakeFS(), vikingdb_manager=db)+ monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: service)+ admin_ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="admin"),role=Role.ROOT,)+ counters = _ReindexCounters()- await ReindexExecutor()._prune_orphan_vectors(- uri="viking://resources/demo",- object_type="resource",- dry_run=False,- counters=counters,- ctx=ctx,+ await ReindexExecutor()._reindex_rfv(+ uri=root,+ mode="vectors_only",+ context_type="resource",+ recursive=True,+ run=_make_reindex_run(admin_ctx, counters),)- assert deleted["ids"] == ["delete_l2"]+ db.strict_delete.assert_awaited_once()+ assert db.strict_delete.await_args.kwargs["ctx"].user.user_id == "bob"assert counters.deleted_records == 1- assert counters.unsupported_records == 0@pytest.mark.asyncio-async def test_prune_orphans_continues_after_delete_group_failure(monkeypatch):+async def test_reindex_rfv_enqueues_semantic_plan_without_snapshot_source_bytes(monkeypatch):+ import json++ import openviking.service.reindex_executor as reindex_modfrom openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters+ from openviking.storage.context_update_plan import (+ ContextUpdatePlan,+ SemanticPlan,+ SemanticTreeEntry,+ SemanticTreeSnapshot,+ )++ root = "viking://resources/demo"+ lease = {"id": "reindex-lease"}+ plan = SemanticPlan(+ root,+ "resource",+ SemanticTreeSnapshot((SemanticTreeEntry("", "directory", "unchanged", "aggregate"),)),+ )+ snapshot = SimpleNamespace(+ formal=SimpleNamespace(entries={"": object()}),+ source_contents={(root, 2): b"reindex source must stay in memory"},+ source_raw_contents={(root, 2): b"raw source must stay in memory"},+ source_metadata={"uri": "s3://source"},+ )+ handoff = {"owner_id": "semantic-worker", "covered_paths": []}+ calls = []- deleted = {}- filter_calls = 0+ class PathLock:+ async def pathlock_to_handoff(self, current):+ assert current == lease+ calls.append("to_handoff")+ return handoff++ async def pathlock_handoff(self, current):+ assert current == lease+ calls.append("handoff")++ class Queue:+ async def enqueue(self, msg):+ calls.append("enqueue")+ self.message = msg+ return "queued"++ queue = Queue()+ queue_manager = SimpleNamespace(SEMANTIC="semantic", get_queue=lambda *args, **kwargs: queue)+ service = SimpleNamespace(+ viking_fs=SimpleNamespace(_async_agfs=PathLock()),+ vikingdb_manager=SimpleNamespace(uses_content_field=False),+ )+ monkeypatch.setattr(reindex_mod, "get_service", lambda: service)+ monkeypatch.setattr(+ "openviking.storage.resource_rfv.build_rfv_snapshot",+ AsyncMock(return_value=snapshot),+ )+ monkeypatch.setattr(+ "openviking.storage.context_update_plan.build_rfv_context_update_plan",+ lambda **kwargs: (None, ContextUpdatePlan(root, "resource", semantic_plan=plan)),+ )+ monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: queue_manager)- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return False+ ctx = RequestContext(+ user=UserIdentifier(account_id="test", user_id="alice"),+ role=Role.ROOT,+ )+ run = _make_reindex_run(ctx, _ReindexCounters())+ run.lease = lease- class FakeVikingDB:- async def filter(self, *, filter, limit, output_fields, ctx):- nonlocal filter_calls- filter_calls += 1- if filter_calls > 1:- return []- return [- {- "id": "fail_delete",- "uri": "viking://user/alice/docs/missing.txt",- "level": 2,- "context_type": "resource",- "account_id": "acct",- "owner_user_id": "alice",- },- {- "id": "ok_delete",- "uri": "viking://user/bob/docs/missing.txt",- "level": 2,- "context_type": "resource",- "account_id": "acct",- "owner_user_id": "bob",- },- ]+ await ReindexExecutor()._reindex_rfv(+ uri=root,+ mode="semantic_and_vectors",+ context_type="resource",+ recursive=False,+ run=run,+ )- async def delete(self, ids, *, ctx):- if ctx.user.user_id == "alice":- raise RuntimeError("delete backend unavailable")- deleted["ids"] = ids- deleted["ctx"] = ctx- return len(ids)+ assert calls == ["to_handoff", "handoff", "enqueue"]+ assert run.lease_handed_off is True+ assert queue.message.plan == plan+ assert queue.message.propagate_to_parent is False+ assert queue.message.generation_trigger == "reindex"+ assert queue.message.lock_handoff == handoff+ assert b"reindex source must stay in memory" not in json.dumps(queue.message.to_dict()).encode()- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())- counters = _ReindexCounters()+async def test_single_file_vectors_only_reindex_reads_parent_overview_only_for_rebuild(monkeypatch):+ import openviking.service.reindex_executor as reindex_mod+ from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot+ from openviking.storage.resource_rnfv import RequestIntent, VectorIndexSnapshot++ file_uri = "viking://resources/demo/a.md"+ parent_uri = "viking://resources/demo"+ snapshot = RFVSnapshot(+ request=RequestIntent(file_uri, "vectors_only", force=True),+ formal=RFVFormalSnapshot({"": RFVEntry(file_uri, "", False, {2: "a-md5"})}),+ vectors=VectorIndexSnapshot({}, frozenset({"id", "uri", "level", "md5", "abstract"})),+ source_contents={(file_uri, 2): b"current body"},+ )+ fs = SimpleNamespace(+ read_file_bytes=AsyncMock(return_value=b"### a.md\nOverview summary."),+ )+ service = SimpleNamespace(+ viking_fs=fs,+ vikingdb_manager=SimpleNamespace(uses_content_field=False),+ _resource_processor=None,+ )+ enqueued = AsyncMock(return_value=1)+ monkeypatch.setattr(reindex_mod, "get_service", lambda: service)+ monkeypatch.setattr(+ "openviking.storage.resource_rfv.build_rfv_snapshot",+ AsyncMock(return_value=snapshot),+ )+ monkeypatch.setattr(+ "openviking.utils.resource_processor.ResourceProcessor._enqueue_index_actions",+ enqueued,+ )ctx = RequestContext(- user=UserIdentifier(account_id="acct", user_id="admin"),+ user=UserIdentifier(account_id="test", user_id="alice"),role=Role.ROOT,)+ run = _make_reindex_run(ctx, _ReindexCounters())+ run.root_is_dir = False- await ReindexExecutor()._prune_orphan_vectors(- uri="viking://user",- object_type="user_namespace",- dry_run=False,- counters=counters,- ctx=ctx,+ await ReindexExecutor()._reindex_rfv(+ uri=file_uri,+ mode="vectors_only",+ context_type="resource",+ recursive=False,+ run=run,)- assert deleted["ids"] == ["ok_delete"]- assert deleted["ctx"].user.user_id == "bob"- assert counters.deleted_records == 1- assert counters.failed_records == 1- assert any("Failed to delete" in warning for warning in counters.warnings)+ fs.read_file_bytes.assert_awaited_once_with(f"{parent_uri}/.overview.md", ctx=ctx)+ actions = enqueued.await_args.args[0]+ assert len(actions) == 1+ assert actions[0].summary == "Overview summary."@pytest.mark.asyncio-async def test_prune_orphans_pages_past_candidate_limit(monkeypatch):- from openviking.service import reindex_executor+async def test_single_file_vectors_only_noop_does_not_read_parent_overview(monkeypatch):+ import openviking.service.reindex_executor as reindex_modfrom openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot+ from openviking.storage.resource_rnfv import (+ RequestIntent,+ VectorIndexSnapshot,+ VectorRecordSnapshot,+ )++ file_uri = "viking://resources/demo/a.md"+ snapshot = RFVSnapshot(+ request=RequestIntent(file_uri, "vectors_only"),+ formal=RFVFormalSnapshot({"": RFVEntry(file_uri, "", False, {2: "a-md5"})}),+ vectors=VectorIndexSnapshot(+ {+ "a-l2": VectorRecordSnapshot(+ "a-l2", file_uri, "", 2, {"md5": "a-md5", "abstract": "summary"}+ )+ },+ frozenset({"id", "uri", "level", "md5", "abstract"}),+ ),+ )+ fs = SimpleNamespace(read_file_bytes=AsyncMock())+ service = SimpleNamespace(+ viking_fs=fs,+ vikingdb_manager=SimpleNamespace(uses_content_field=False),+ _resource_processor=None,+ )+ monkeypatch.setattr(reindex_mod, "get_service", lambda: service)+ monkeypatch.setattr(+ "openviking.storage.resource_rfv.build_rfv_snapshot",+ AsyncMock(return_value=snapshot),+ )+ ctx = RequestContext(+ user=UserIdentifier(account_id="test", user_id="alice"),+ role=Role.ROOT,+ )+ run = _make_reindex_run(ctx, _ReindexCounters())+ run.root_is_dir = False- filter_offsets = []- deleted = {}+ await ReindexExecutor()._reindex_rfv(+ uri=file_uri,+ mode="vectors_only",+ context_type="resource",+ recursive=False,+ run=run,+ )- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return not uri.endswith("missing.txt")+ fs.read_file_bytes.assert_not_awaited()- class FakeVikingDB:- async def filter(self, *, filter, limit, output_fields, ctx, offset=0):- filter_offsets.append(offset)- records = [- {- "id": "keep_1",- "uri": "viking://resources/demo/keep-1.txt",- "level": 2,- "context_type": "resource",- "account_id": "acct",- },- {- "id": "keep_2",- "uri": "viking://resources/demo/keep-2.txt",- "level": 2,- "context_type": "resource",- "account_id": "acct",- },- {- "id": "delete_3",- "uri": "viking://resources/demo/missing.txt",- "level": 2,- "context_type": "resource",- "account_id": "acct",- },- ]- return records[offset : offset + limit]- async def delete(self, ids, *, ctx):- deleted["ids"] = ids- return len(ids)+def test_reindex_merges_persisted_semantic_plan_stats_into_response_counters():+ from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters+ from openviking.storage.queuefs.semantic_executor import SemanticTreeStats++ counters = _ReindexCounters(rebuilt_records=2)+ ReindexExecutor._apply_semantic_tree_stats(+ counters,+ SemanticTreeStats(indexed_records=3, failures=["summary failed"]),+ )++ assert counters.rebuilt_records == 5+ assert counters.failed_records == 1+ assert counters.warnings == ["summary failed"]+++async def test_reindex_upsert_uses_uri_owner_for_user_scoped_records(monkeypatch):+ from openviking.service.reindex_executor import ReindexExecutor++ captured = {}++ class FakeVikingDB:+ async def enqueue_embedding_msg(self, msg):+ captured["msg"] = msg+ return Truefake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())- monkeypatch.setattr(reindex_executor, "PRUNE_ORPHAN_CANDIDATE_LIMIT", 2)- counters = _ReindexCounters()+ service = ReindexExecutor()ctx = RequestContext(user=UserIdentifier(account_id="acct", user_id="admin"),role=Role.ROOT,)- await ReindexExecutor()._prune_orphan_vectors(- uri="viking://resources/demo",- object_type="resource",- dry_run=False,- counters=counters,+ await service._upsert_context(+ uri="viking://user/bob/memories/preferences/theme.md",+ parent_uri="viking://user/bob/memories/preferences",+ abstract="theme",+ vector_text="theme",+ is_leaf=True,+ context_type="memory",+ level=ContextLevel.DETAIL,ctx=ctx,)- assert filter_offsets == [0, 2]- assert counters.scanned_records == 3- assert deleted["ids"] == ["delete_3"]- assert counters.deleted_records == 1+ data = captured["msg"].context_data+ assert data["user"]["user_id"] == "bob"+ assert data["owner_user_id"] == "bob"+ assert data["owner_space"] == "bob"@pytest.mark.asyncio-async def test_prune_orphans_reports_zero_delete_count_as_failure(monkeypatch):- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters+async def test_reindex_semantic_processor_uses_uri_owner_for_user_scoped_records(+ monkeypatch, semantic_config+):+ from openviking.service.reindex_executor import ReindexExecutor- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return False+ captured = {}- class FakeVikingDB:- async def filter(self, *, filter, limit, output_fields, ctx, offset=0):- if offset:- return []- return [- {- "id": "stale",- "uri": "viking://resources/demo/missing.txt",- "level": 2,- "context_type": "resource",- "account_id": "acct",- }- ]+ class FakeSemanticProcessor:+ def __init__(self, **_kwargs):+ pass- async def delete(self, ids, *, ctx):- return 0+ async def on_dequeue(self, payload, lock=None):+ captured["payload"] = payload+ captured["lock"] = lock+ return ProcessResult.success()- fake_service = type("Svc", (), {"vikingdb_manager": FakeVikingDB()})()- monkeypatch.setattr("openviking.service.reindex_executor.get_service", lambda: fake_service)- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())+ monkeypatch.setattr(+ "openviking.service.reindex_executor.SemanticProcessor",+ FakeSemanticProcessor,+ )- counters = _ReindexCounters()+ service = ReindexExecutor(vlm_resolver=semantic_config)ctx = RequestContext(user=UserIdentifier(account_id="acct", user_id="admin"),role=Role.ROOT,)- await ReindexExecutor()._prune_orphan_vectors(- uri="viking://resources/demo",- object_type="resource",- dry_run=False,- counters=counters,+ await service._run_semantic_processor(+ uri="viking://user/bob/memories/preferences",+ context_type="memory",ctx=ctx,)- assert counters.deleted_records == 0- assert counters.failed_records == 1- assert any("Only deleted 0 of 1" in warning for warning in counters.warnings)+ data = captured["payload"]["data"]+ assert '"user_id": "bob"' in data+ assert '"peer_id": "bob"' in data+ assert '"account_id": "acct"' in data@pytest.mark.asyncio@@ -1433,9 +1140,7 @@ async def test_reindex_semantic_processor_uses_configured_vlm_concurrency(user=UserIdentifier(account_id="test", user_id="alice"),role=Role.ROOT,)- await reindex_mod.ReindexExecutor(- vlm_resolver=semantic_config- )._run_semantic_processor(+ await reindex_mod.ReindexExecutor(vlm_resolver=semantic_config)._run_semantic_processor(uri="viking://resources/demo",context_type="resource",ctx=ctx,@@ -1457,7 +1162,7 @@ async def test_reindex_memory_semantic_and_vectors_rebuilds_full_subtree(monkeypseen["vectors"].append(uri)class FakeVikingFS:- async def stat(self, uri, ctx=None, skip_count=None):+ async def stat(self, uri, ctx=None, skip_count=True):return {"isDir": True}monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)@@ -1521,14 +1226,10 @@ async def test_reindex_resource_non_recursive_limits_semantics_and_vectors(monkeseen = {}- async def fake_run_semantic_processor(self, **kwargs):- seen["semantic"] = kwargs-- async def fake_reindex_resource_vectors(self, **kwargs):- seen["vectors"] = kwargs+ async def fake_reindex_rfv(self, **kwargs):+ seen.update(kwargs)- monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)- monkeypatch.setattr(ReindexExecutor, "_reindex_resource_vectors", fake_reindex_resource_vectors)+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="alice"),@@ -1541,8 +1242,8 @@ async def test_reindex_resource_non_recursive_limits_semantics_and_vectors(monkerun=_make_reindex_run(ctx, _ReindexCounters()),)- assert seen["semantic"]["recursive"] is False- assert seen["vectors"]["recursive"] is False+ assert seen["recursive"] is False+ assert seen["mode"] == "semantic_and_vectors"@pytest.mark.asyncio@@ -1558,7 +1259,7 @@ async def test_reindex_memory_non_recursive_limits_semantics_and_vectors(monkeypseen["vectors"] = kwargsclass FakeVikingFS:- async def stat(self, uri, ctx=None, skip_count=None):+ async def stat(self, uri, ctx=None, skip_count=True):return {"isDir": True}monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)@@ -1586,16 +1287,10 @@ async def test_reindex_skill_non_recursive_limits_vectors_to_directory(monkeypatseen = {}- async def fake_regenerate_skill_semantics(self, **kwargs):- seen["semantic"] = kwargs-- async def fake_reindex_skill_vectors(self, **kwargs):- seen["vectors"] = kwargs+ async def fake_reindex_rfv(self, **kwargs):+ seen.update(kwargs)- monkeypatch.setattr(- ReindexExecutor, "_regenerate_skill_semantics", fake_regenerate_skill_semantics- )- monkeypatch.setattr(ReindexExecutor, "_reindex_skill_vectors", fake_reindex_skill_vectors)+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="alice"),@@ -1608,20 +1303,20 @@ async def test_reindex_skill_non_recursive_limits_vectors_to_directory(monkeypatrun=_make_reindex_run(ctx, _ReindexCounters()),)- assert seen["semantic"]["uri"] == "viking://user/alice/skills/demo"- assert seen["vectors"]["recursive"] is False+ assert seen["uri"] == "viking://user/alice/skills/demo"+ assert seen["recursive"] is False@pytest.mark.asyncio-async def test_reindex_skill_vectors_only_ignores_non_recursive_flag(monkeypatch):+async def test_reindex_skill_vectors_only_honors_non_recursive_flag(monkeypatch):from openviking.service.reindex_executor import ReindexExecutor, _ReindexCountersseen = {}- async def fake_reindex_skill_vectors(self, **kwargs):+ async def fake_reindex_rfv(self, **kwargs):seen.update(kwargs)- monkeypatch.setattr(ReindexExecutor, "_reindex_skill_vectors", fake_reindex_skill_vectors)+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="alice"),@@ -1634,49 +1329,37 @@ async def test_reindex_skill_vectors_only_ignores_non_recursive_flag(monkeypatchrun=_make_reindex_run(ctx, _ReindexCounters()),)- assert "recursive" not in seen+ assert seen["recursive"] is False@pytest.mark.asyncio-async def test_reindex_resource_vectors_non_recursive_skips_tree(monkeypatch):+async def test_reindex_resource_vectors_only_honors_non_recursive(monkeypatch):from openviking.service.reindex_executor import ReindexExecutor, _ReindexCountersseen = {}- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return True-- async def stat(self, uri, ctx=None, skip_count=True):- return {"isDir": True}-- async def fail_tree_all(*args, **kwargs):- raise AssertionError("non-recursive vector rebuild must not walk descendants")-- async def fake_reindex_from_entries(self, **kwargs):+ async def fake_reindex_rfv(self, **kwargs):seen.update(kwargs)- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())- monkeypatch.setattr(ReindexExecutor, "_tree_all", fail_tree_all)monkeypatch.setattr(ReindexExecutor,- "_reindex_resource_vectors_from_entries",- fake_reindex_from_entries,+ "_reindex_rfv",+ fake_reindex_rfv,)-ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="alice"),role=Role.ROOT,)- await ReindexExecutor()._reindex_resource_vectors(++ await ReindexExecutor()._reindex_resource(uri="viking://resources/demo",- counters=_ReindexCounters(),- ctx=ctx,+ mode="vectors_only",recursive=False,+ run=_make_reindex_run(ctx, _ReindexCounters()),)- assert seen["directories"] == ["viking://resources/demo"]- assert seen["files"] == []+ assert seen["recursive"] is False@pytest.mark.asyncio@@ -1777,27 +1460,17 @@ async def test_reindex_user_namespace_semantic_and_vectors_promotes_memory_mode({"uri": "viking://user/default/resources", "isDir": True},]- seen = {"memory_modes": [], "semantic_calls": [], "resource_calls": []}+ seen = {"memory_modes": [], "rfv_calls": []}async def fake_reindex_memory(self, *, uri, mode, run):seen["memory_modes"].append((uri, mode))- async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):- seen["semantic_calls"].append((uri, context_type))-- async def fake_reindex_resource_vectors_from_entries(- self, *, root_uri, directories, files, counters, ctx- ):- seen["resource_calls"].append((root_uri, list(directories), list(files)))+ async def fake_reindex_rfv(self, **kwargs):+ seen["rfv_calls"].append(kwargs)monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())monkeypatch.setattr(ReindexExecutor, "_reindex_memory", fake_reindex_memory)- monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)- monkeypatch.setattr(- ReindexExecutor,- "_reindex_resource_vectors_from_entries",- fake_reindex_resource_vectors_from_entries,- )+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)service = ReindexExecutor()counters = _ReindexCounters()@@ -1813,8 +1486,7 @@ async def test_reindex_user_namespace_semantic_and_vectors_promotes_memory_mode()assert seen["memory_modes"] == [("viking://user/default/memories", "semantic_and_vectors")]- assert seen["semantic_calls"] == [("viking://user/default/resources", "resource")]- assert seen["resource_calls"]+ assert [call["uri"] for call in seen["rfv_calls"]] == ["viking://user/default/resources"]@pytest.mark.asyncio@@ -1839,27 +1511,17 @@ async def test_reindex_user_namespace_semantic_and_vectors_does_not_reprocess_me{"uri": "viking://user/default/resources", "isDir": True},]- seen = {"memory_modes": [], "semantic_calls": []}+ seen = {"memory_modes": [], "rfv_calls": []}async def fake_reindex_memory(self, *, uri, mode, run):seen["memory_modes"].append((uri, mode))- async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):- seen["semantic_calls"].append((uri, context_type))-- async def fake_reindex_resource_vectors_from_entries(- self, *, root_uri, directories, files, counters, ctx- ):- return None+ async def fake_reindex_rfv(self, **kwargs):+ seen["rfv_calls"].append(kwargs)monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())monkeypatch.setattr(ReindexExecutor, "_reindex_memory", fake_reindex_memory)- monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)- monkeypatch.setattr(- ReindexExecutor,- "_reindex_resource_vectors_from_entries",- fake_reindex_resource_vectors_from_entries,- )+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)service = ReindexExecutor()counters = _ReindexCounters()@@ -1875,7 +1537,7 @@ async def test_reindex_user_namespace_semantic_and_vectors_does_not_reprocess_me)assert seen["memory_modes"] == [("viking://user/default/memories", "semantic_and_vectors")]- assert seen["semantic_calls"] == [("viking://user/default/resources", "resource")]+ assert [call["uri"] for call in seen["rfv_calls"]] == ["viking://user/default/resources"]@pytest.mark.asyncio@@ -1900,23 +1562,13 @@ async def test_reindex_user_namespace_semantic_and_vectors_skips_uncovered_root_{"uri": "viking://user/default/profile.md", "isDir": False},]- seen = {"semantic_calls": [], "resource_files": []}-- async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):- seen["semantic_calls"].append((uri, context_type))+ seen = {"rfv_calls": []}- async def fake_reindex_resource_vectors_from_entries(- self, *, root_uri, directories, files, counters, ctx- ):- seen["resource_files"] = list(files)+ async def fake_reindex_rfv(self, **kwargs):+ seen["rfv_calls"].append(kwargs)monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())- monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)- monkeypatch.setattr(- ReindexExecutor,- "_reindex_resource_vectors_from_entries",- fake_reindex_resource_vectors_from_entries,- )+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)service = ReindexExecutor()counters = _ReindexCounters()@@ -1931,8 +1583,7 @@ async def test_reindex_user_namespace_semantic_and_vectors_skips_uncovered_root_run=_make_reindex_run(ctx, counters),)- assert seen["semantic_calls"] == [("viking://user/default/resources", "resource")]- assert seen["resource_files"] == ["viking://user/default/resources/doc.md"]+ assert [call["uri"] for call in seen["rfv_calls"]] == ["viking://user/default/resources"]assert counters.unsupported_records == 1assert "viking://user/default/profile.md" in counters.warnings[0]@@ -1961,11 +1612,11 @@ async def test_reindex_skill_namespace_reindexes_only_skill_roots(monkeypatch):seen = []- async def fake_reindex_skill(self, *, uri, mode, run):- seen.append((uri, mode))+ async def fake_reindex_rfv(self, **kwargs):+ seen.append(kwargs)monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())- monkeypatch.setattr(ReindexExecutor, "_reindex_skill", fake_reindex_skill)+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)service = ReindexExecutor()counters = _ReindexCounters()@@ -1980,7 +1631,13 @@ async def test_reindex_skill_namespace_reindexes_only_skill_roots(monkeypatch):run=_make_reindex_run(ctx, counters),)- assert seen == [("viking://user/default/skills/my_skill", "semantic_and_vectors")]+ assert [(call["uri"], call["context_type"]) for call in seen] == [+ ("viking://user/default/skills/my_skill", "skill")+ ]+ assert {entry["uri"] for entry in seen[0]["formal_entries"]} == {+ "viking://user/default/skills/my_skill/assets",+ "viking://user/default/skills/my_skill/SKILL.md",+ }@pytest.mark.asyncio@@ -2005,27 +1662,17 @@ async def test_reindex_global_namespace_semantic_and_vectors_propagates_to_child{"uri": "viking://resources", "isDir": True},]- seen = {"user_modes": [], "semantic_calls": [], "resource_calls": []}+ seen = {"user_modes": [], "rfv_calls": []}async def fake_reindex_user_namespace(self, *, uri, mode, run):seen["user_modes"].append((uri, mode))- async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):- seen["semantic_calls"].append((uri, context_type))-- async def fake_reindex_resource_vectors_from_entries(- self, *, root_uri, directories, files, counters, ctx- ):- seen["resource_calls"].append((root_uri, list(directories), list(files)))+ async def fake_reindex_rfv(self, **kwargs):+ seen["rfv_calls"].append(kwargs)- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())- monkeypatch.setattr(ReindexExecutor, "_reindex_user_namespace", fake_reindex_user_namespace)- monkeypatch.setattr(ReindexExecutor, "_run_semantic_processor", fake_run_semantic_processor)- monkeypatch.setattr(- ReindexExecutor,- "_reindex_resource_vectors_from_entries",- fake_reindex_resource_vectors_from_entries,- )+ monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())+ monkeypatch.setattr(ReindexExecutor, "_reindex_user_namespace", fake_reindex_user_namespace)+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)service = ReindexExecutor()counters = _ReindexCounters()@@ -2041,8 +1688,7 @@ async def test_reindex_global_namespace_semantic_and_vectors_propagates_to_child)assert seen["user_modes"] == [("viking://user/default", "semantic_and_vectors")]- assert seen["semantic_calls"] == [("viking://resources", "resource")]- assert seen["resource_calls"]+ assert [call["uri"] for call in seen["rfv_calls"]] == ["viking://resources"]@pytest.mark.asyncio@@ -2137,103 +1783,6 @@ async def test_reindex_upsert_context_omits_search_tags_without_ingest_options(m@pytest.mark.asyncio-async def test_reindex_resource_vectors_parallelize_files_and_isolate_failures(monkeypatch):- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters- from openviking_cli.exceptions import OpenVikingError-- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return True-- async def stat(self, uri, ctx=None, skip_count=True):- return {"isDir": True}-- async def read_file_bytes(self, uri, ctx=None):- return uri.encode()-- async def tree(- self,- uri,- output="original",- show_all_hidden=True,- node_limit=1000,- level_limit=3,- ctx=None,- ):- assert node_limit is None- assert level_limit is None- return [- {"uri": "viking://resources/demo/bad.txt", "isDir": False},- {"uri": "viking://resources/demo/good.txt", "isDir": False},- ]-- async def fake_read_directory_abstract(self, uri, *, ctx):- return ""-- async def fake_read_directory_overview(self, uri, *, ctx):- return ""-- async def fake_best_file_summary(self, uri, *, ctx):- return f"summary:{uri.rsplit('/', 1)[-1]}"-- async def fake_best_resource_file_vector_text(self, uri, summary, ctx, file_content=None):- return summary-- seen = []- both_entered = asyncio.Event()- release = asyncio.Event()-- async def fake_upsert_context(self, **kwargs):- seen.append(kwargs["uri"])- if len(seen) == 2:- both_entered.set()- await release.wait()- if kwargs["uri"].endswith("bad.txt"):- raise OpenVikingError("boom", code="PROCESSING_ERROR")-- monkeypatch.setattr(- "openviking.service.reindex_executor.get_openviking_config",- lambda: SimpleNamespace(reindex=SimpleNamespace(file_vectorization_concurrency=8)),- )- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())- monkeypatch.setattr(ReindexExecutor, "_read_directory_abstract", fake_read_directory_abstract)- monkeypatch.setattr(ReindexExecutor, "_read_directory_overview", fake_read_directory_overview)- monkeypatch.setattr(ReindexExecutor, "_best_file_summary", fake_best_file_summary)- monkeypatch.setattr(- ReindexExecutor,- "_best_resource_file_vector_text",- fake_best_resource_file_vector_text,- )- monkeypatch.setattr(ReindexExecutor, "_upsert_context", fake_upsert_context)-- service = ReindexExecutor()- counters = _ReindexCounters()- ctx = RequestContext(- user=UserIdentifier(account_id="test", user_id="alice"),- role=Role.ROOT,- )-- task = asyncio.create_task(- service._reindex_resource_vectors(- uri="viking://resources/demo",- counters=counters,- ctx=ctx,- )- )- try:- await asyncio.wait_for(both_entered.wait(), timeout=1)- finally:- release.set()- await task-- assert set(seen) == {- "viking://resources/demo/bad.txt",- "viking://resources/demo/good.txt",- }- assert counters.failed_records == 1- assert counters.rebuilt_records == 1--@pytest.mark.asyncioasync def test_reindex_semantic_processor_runs_with_skip_vectorization(monkeypatch, semantic_config@@ -2356,122 +1905,7 @@ async def test_reindex_semantic_processor_sets_memory_aggregation_policy(@pytest.mark.asyncio-async def test_reindex_resource_l2_falls_back_to_vector_text_when_summary_missing(monkeypatch):- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters-- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return True-- async def stat(self, uri, ctx=None, skip_count=True):- return {"isDir": True}-- async def read_file_bytes(self, uri, ctx=None):- return b"raw file body"-- async def tree(- self,- uri,- output="original",- show_all_hidden=True,- node_limit=1000,- level_limit=3,- ctx=None,- ):- assert node_limit is None- assert level_limit is None- return [{"uri": "viking://resources/demo/file.txt", "isDir": False}]-- seen = {}-- async def fake_read_directory_abstract(self, uri, *, ctx):- return "skill abstract"-- async def fake_read_directory_overview(self, uri, *, ctx):- return ""-- async def fake_best_file_summary(self, uri, *, ctx):- return ""-- async def fake_best_resource_file_vector_text(self, uri, summary, ctx, file_content=None):- return "raw file body"-- async def fake_upsert_context(self, **kwargs):- seen[kwargs["uri"]] = kwargs-- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())- monkeypatch.setattr(ReindexExecutor, "_read_directory_abstract", fake_read_directory_abstract)- monkeypatch.setattr(ReindexExecutor, "_read_directory_overview", fake_read_directory_overview)- monkeypatch.setattr(ReindexExecutor, "_best_file_summary", fake_best_file_summary)- monkeypatch.setattr(- ReindexExecutor,- "_best_resource_file_vector_text",- fake_best_resource_file_vector_text,- )- monkeypatch.setattr(ReindexExecutor, "_upsert_context", fake_upsert_context)-- service = ReindexExecutor()- counters = _ReindexCounters()- ctx = RequestContext(- user=UserIdentifier(account_id="test", user_id="alice"),- role=Role.ROOT,- )-- await service._reindex_resource_vectors(- uri="viking://resources/demo",- counters=counters,- ctx=ctx,- )-- assert seen["viking://resources/demo/file.txt"]["abstract"] == "raw file body"--@pytest.mark.asyncio-async def test_reindex_resource_file_writes_md5_from_same_bytes(monkeypatch):- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters- from openviking.utils.content_hash import content_md5-- file_uri = "viking://resources/demo/file.txt"- raw = b"current body"-- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return True-- async def stat(self, uri, ctx=None, skip_count=True):- return {"isDir": False}-- async def read_file_bytes(self, uri, ctx=None):- assert uri == file_uri- return raw-- seen = {}-- async def fake_best_file_summary(self, uri, *, ctx):- return "summary"-- async def fake_best_resource_file_vector_text(self, uri, summary, ctx, file_content=None):- return "current body"-- async def fake_upsert_context(self, **kwargs):- seen.update(kwargs)-- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())- monkeypatch.setattr(ReindexExecutor, "_best_file_summary", fake_best_file_summary)- monkeypatch.setattr(- ReindexExecutor,- "_best_resource_file_vector_text",- fake_best_resource_file_vector_text,- )- monkeypatch.setattr(ReindexExecutor, "_upsert_context", fake_upsert_context)-- counters = _ReindexCounters()- ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="alice"), role=Role.ROOT)- await ReindexExecutor()._reindex_resource_vectors(uri=file_uri, counters=counters, ctx=ctx)-- assert seen["md5"] == content_md5(raw)--@pytest.mark.asyncioasync def test_reindex_resource_vector_text_uses_existing_record_for_non_text(monkeypatch):from openviking.service.reindex_executor import ReindexExecutor@@ -2512,9 +1946,13 @@ async def test_reindex_resource_vector_text_summary_first_skips_content_read(monmonkeypatch.setattr(ReindexExecutor, "_safe_read_text", fail_if_content_read)monkeypatch.setattr(ReindexExecutor, "_fetch_existing_record", fake_fetch_existing_record)- resolver = SimpleNamespace(resolve=AsyncMock(return_value=SimpleNamespace(- embedding=SimpleNamespace(text_source="summary_first"),- )))+ resolver = SimpleNamespace(+ resolve=AsyncMock(+ return_value=SimpleNamespace(+ embedding=SimpleNamespace(text_source="summary_first"),+ )+ )+ )service = ReindexExecutor(vector_config_resolver=resolver)ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="alice"),@@ -2602,8 +2040,8 @@ async def test_reindex_file_summary_reads_existing_record_as_uri_owner(monkeypatasync def test_reindex_memory_fallback_reads_existing_record_as_uri_owner(monkeypatch):from openviking.service.reindex_executor import (ReindexExecutor,- _PruneSourceRead,_ReindexCounters,+ _SourceRead,)captured = {}@@ -2613,11 +2051,11 @@ async def test_reindex_memory_fallback_reads_existing_record_as_uri_owner(monkeyasync def exists(self, uri, ctx=None):return True- async def stat(self, uri, ctx=None, skip_count=None):+ async def stat(self, uri, ctx=None, skip_count=True):return {"isDir": False}async def fake_read_memory_body(self, uri, *, ctx):- return _PruneSourceRead(exists=True, text="")+ return _SourceRead(exists=True, text="")async def fake_fetch_existing_record(self, *, uri, level, ctx):captured["ctx"] = ctx@@ -2661,7 +2099,7 @@ async def test_reindex_memory_skips_fallback_when_body_read_fails(monkeypatch):async def exists(self, uri, ctx=None):return True- async def stat(self, uri, ctx=None, skip_count=None):+ async def stat(self, uri, ctx=None, skip_count=True):return {"isDir": False}async def read_file(self, uri, ctx=None):@@ -2728,79 +2166,25 @@ async def test_reindex_resource_vector_text_skips_non_text_body_without_summary(@pytest.mark.asyncio-async def test_reindex_resource_vectors_accepts_single_file_uri(monkeypatch):- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters-- class FakeVikingFS:- async def exists(self, uri, ctx=None):- return True-- async def stat(self, uri, ctx=None, skip_count=True):- return {"isDir": False}-- async def read_file_bytes(self, uri, ctx=None):- return b"file summary"-- async def tree(self, *args, **kwargs):- raise AssertionError("single-file reindex should not call tree")-- seen = {}-- async def fake_best_file_summary(self, uri, *, ctx):- return "file summary"-- async def fake_best_resource_file_vector_text(self, uri, summary, ctx, file_content=None):- return summary-- async def fake_upsert_context(self, **kwargs):- seen[kwargs["uri"]] = kwargs-- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())- monkeypatch.setattr(ReindexExecutor, "_best_file_summary", fake_best_file_summary)- monkeypatch.setattr(- ReindexExecutor,- "_best_resource_file_vector_text",- fake_best_resource_file_vector_text,- )- monkeypatch.setattr(ReindexExecutor, "_upsert_context", fake_upsert_context)-- service = ReindexExecutor()- counters = _ReindexCounters()- ctx = RequestContext(- user=UserIdentifier(account_id="test", user_id="alice"),- role=Role.ROOT,- )-- await service._reindex_resource_vectors(- uri="viking://resources/demo/file.txt",- counters=counters,- ctx=ctx,- )-- assert list(seen) == ["viking://resources/demo/file.txt"]- assert counters.scanned_records == 1- assert counters.rebuilt_records == 1--@pytest.mark.asyncioasync def test_reindex_memory_l2_falls_back_to_body_when_abstract_missing(monkeypatch):from openviking.service.reindex_executor import (ReindexExecutor,- _PruneSourceRead,_ReindexCounters,+ _SourceRead,)class FakeVikingFS:async def exists(self, uri, ctx=None):return True- async def stat(self, uri, ctx=None, skip_count=None):+ async def stat(self, uri, ctx=None, skip_count=True):return {"isDir": False}seen = {}async def fake_read_memory_body(self, uri, *, ctx):- return _PruneSourceRead(exists=True, text="memory body text")+ return _SourceRead(exists=True, text="memory body text")async def fake_fetch_existing_record(self, *, uri, level, ctx):return None@@ -2837,15 +2221,15 @@ async def test_reindex_memory_l2_falls_back_to_body_when_abstract_missing(monkeyasync def test_reindex_memory_l2_strips_memory_fields_from_abstract(monkeypatch):from openviking.service.reindex_executor import (ReindexExecutor,- _PruneSourceRead,_ReindexCounters,+ _SourceRead,)class FakeVikingFS:async def exists(self, uri, ctx=None):return True- async def stat(self, uri, ctx=None, skip_count=None):+ async def stat(self, uri, ctx=None, skip_count=True):return {"isDir": False}seen = {}@@ -2855,7 +2239,7 @@ async def test_reindex_memory_l2_strips_memory_fields_from_abstract(monkeypatch))async def fake_read_memory_body(self, uri, *, ctx):- return _PruneSourceRead(exists=True, text=raw_body)+ return _SourceRead(exists=True, text=raw_body)async def fake_fetch_existing_record(self, *, uri, level, ctx):return None@@ -2899,15 +2283,15 @@ async def test_reindex_memory_l2_strips_memory_fields_from_abstract(monkeypatch)async def test_reindex_memory_vectors_walks_deep_subtree(monkeypatch):from openviking.service.reindex_executor import (ReindexExecutor,- _PruneSourceRead,_ReindexCounters,+ _SourceRead,)class FakeVikingFS:async def exists(self, uri, ctx=None):return True- async def stat(self, uri, ctx=None, skip_count=None):+ async def stat(self, uri, ctx=None, skip_count=True):return {"isDir": True}async def tree(@@ -2934,7 +2318,7 @@ async def test_reindex_memory_vectors_walks_deep_subtree(monkeypatch):seen = {}async def fake_read_memory_body(self, uri, *, ctx):- return _PruneSourceRead(exists=True, text="likes spicy food")+ return _SourceRead(exists=True, text="likes spicy food")async def fake_fetch_existing_record(self, *, uri, level, ctx):return None@@ -2974,15 +2358,15 @@ async def test_reindex_memory_vectors_rebuilds_directory_levels_without_regenerafrom openviking.core.context import ContextLevelfrom openviking.service.reindex_executor import (ReindexExecutor,- _PruneSourceRead,_ReindexCounters,+ _SourceRead,)class FakeVikingFS:async def exists(self, uri, ctx=None):return True- async def stat(self, uri, ctx=None, skip_count=None):+ async def stat(self, uri, ctx=None, skip_count=True):return {"isDir": True}async def tree(@@ -3017,7 +2401,7 @@ async def test_reindex_memory_vectors_rebuilds_directory_levels_without_regenerareturn ""async def fake_read_memory_body(self, uri, *, ctx):- return _PruneSourceRead(exists=True, text="likes spicy food")+ return _SourceRead(exists=True, text="likes spicy food")async def fake_fetch_existing_record(self, *, uri, level, ctx):return None@@ -3095,34 +2479,17 @@ async def test_reindex_user_namespace_partitions_memory_skill_and_resource(monke{"uri": "viking://user/default/skills/my_skill/SKILL.md", "isDir": False},]- seen = {"memory": [], "skill": [], "resource_dirs": [], "resource_files": []}+ seen = {"memory": [], "rfv": []}async def fake_reindex_memory(self, *, uri, mode, run):seen["memory"].append((uri, mode))- async def fake_reindex_skill(self, *, uri, mode, run):- seen["skill"].append((uri, mode))-- async def fake_reindex_resource_vectors_from_entries(- self,- *,- root_uri,- directories,- files,- counters,- ctx,- ):- seen["resource_dirs"] = list(directories)- seen["resource_files"] = list(files)+ async def fake_reindex_rfv(self, **kwargs):+ seen["rfv"].append(kwargs)monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())monkeypatch.setattr(ReindexExecutor, "_reindex_memory", fake_reindex_memory)- monkeypatch.setattr(ReindexExecutor, "_reindex_skill", fake_reindex_skill)- monkeypatch.setattr(- ReindexExecutor,- "_reindex_resource_vectors_from_entries",- fake_reindex_resource_vectors_from_entries,- )+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)service = ReindexExecutor()counters = _ReindexCounters()@@ -3138,14 +2505,10 @@ async def test_reindex_user_namespace_partitions_memory_skill_and_resource(monke)assert seen["memory"] == [("viking://user/default/memories", "vectors_only")]- assert seen["skill"] == [("viking://user/default/skills/my_skill", "vectors_only")]- assert "viking://user/default/resources" in seen["resource_dirs"]- assert "viking://user/default/memories" not in seen["resource_dirs"]- assert "viking://user/default/skills" not in seen["resource_dirs"]- assert "viking://user/default/sessions" not in seen["resource_dirs"]- assert seen["resource_files"] == [- "viking://user/default/resources/doc.md",- "viking://user/default/profile.md",+ assert [(call["uri"], call["context_type"]) for call in seen["rfv"]] == [+ ("viking://user/default/skills/my_skill", "skill"),+ ("viking://user/default/resources", "resource"),+ ("viking://user/default/profile.md", "resource"),]@@ -3177,30 +2540,17 @@ async def test_reindex_global_namespace_partitions_user_and_resources(monkeypatc{"uri": "viking://README.md", "isDir": False},]- seen = {"user": [], "resource_dirs": [], "resource_files": []}+ seen = {"user": [], "rfv": []}async def fake_reindex_user_namespace(self, *, uri, mode, run):seen["user"].append((uri, mode))- async def fake_reindex_resource_vectors_from_entries(- self,- *,- root_uri,- directories,- files,- counters,- ctx,- ):- seen["resource_dirs"] = list(directories)- seen["resource_files"] = list(files)+ async def fake_reindex_rfv(self, **kwargs):+ seen["rfv"].append(kwargs)monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())monkeypatch.setattr(ReindexExecutor, "_reindex_user_namespace", fake_reindex_user_namespace)- monkeypatch.setattr(- ReindexExecutor,- "_reindex_resource_vectors_from_entries",- fake_reindex_resource_vectors_from_entries,- )+ monkeypatch.setattr(ReindexExecutor, "_reindex_rfv", fake_reindex_rfv)service = ReindexExecutor()counters = _ReindexCounters()@@ -3216,70 +2566,12 @@ async def test_reindex_global_namespace_partitions_user_and_resources(monkeypatc)assert seen["user"] == [("viking://user/default", "vectors_only")]- assert "viking://resources" in seen["resource_dirs"]- assert "viking://" not in seen["resource_dirs"]- assert "viking://user" not in seen["resource_dirs"]- assert "viking://session" not in seen["resource_dirs"]- assert "viking://session/default" not in seen["resource_dirs"]- assert seen["resource_files"] == ["viking://resources/demo.txt"]---async def test_reindex_skill_vectors_non_recursive_skips_skill_detail(monkeypatch):- from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters-- class FakeVikingFS:- async def exists(self, uri, ctx=None):- raise AssertionError(f"must not inspect skill detail: {uri}")-- async def fake_read_directory_abstract(self, uri, *, ctx):- return "skill abstract"-- async def fake_read_directory_overview(self, uri, *, ctx):- return "skill overview"-- async def fake_safe_read_text(self, uri, *, ctx):- raise AssertionError(f"must not read skill detail: {uri}")-- async def fake_skill_meta(self, *, uri, abstract, ctx):- return {"name": "my_skill", "description": abstract}-- upserts = []-- async def fake_vectorize_directory(uri, abstract, overview, **kwargs):- upserts.extend([(uri, ContextLevel.ABSTRACT), (uri, ContextLevel.OVERVIEW)])-- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())- monkeypatch.setattr(ReindexExecutor, "_read_directory_abstract", fake_read_directory_abstract)- monkeypatch.setattr(ReindexExecutor, "_read_directory_overview", fake_read_directory_overview)- monkeypatch.setattr(ReindexExecutor, "_safe_read_text", fake_safe_read_text)- monkeypatch.setattr(ReindexExecutor, "_skill_meta", fake_skill_meta)- monkeypatch.setattr(- "openviking.service.reindex_executor.vectorize_directory_meta", fake_vectorize_directory- )-- counters = _ReindexCounters()- ctx = RequestContext(- user=UserIdentifier(account_id="test", user_id="alice"),- role=Role.ROOT,- )- uri = "viking://user/skills/my_skill"-- await ReindexExecutor()._reindex_skill_vectors(- uri=uri,- counters=counters,- ctx=ctx,- recursive=False,- )-- assert upserts == [- (uri, ContextLevel.ABSTRACT),- (uri, ContextLevel.OVERVIEW),+ assert [(call["uri"], call["context_type"]) for call in seen["rfv"]] == [+ ("viking://resources", "resource")]- assert counters.scanned_records == 1- assert counters.rebuilt_records == 2@pytest.mark.asyncioasync def test_reindex_upsert_ignores_empty_replace_tags(monkeypatch):from types import SimpleNamespace@@ -3377,7 +2669,7 @@ async def test_openviking_service_reindex_uses_default_root_context(monkeypatch)seen["vlm_resolver"] = vlm_resolverseen["vector_config_resolver"] = vector_config_resolver- async def execute(self, *, uri, mode, wait, ctx, dry_run=False):+ async def execute(self, *, uri, mode, wait, ctx):seen["uri"] = uriseen["mode"] = modeseen["wait"] = wait@@ -3424,7 +2716,7 @@ async def test_openviking_service_reindex_keeps_canonical_user_id(monkeypatch):seen["vlm_resolver"] = vlm_resolverseen["vector_config_resolver"] = vector_config_resolver- async def execute(self, *, uri, mode, wait, ctx, dry_run=False):+ async def execute(self, *, uri, mode, wait, ctx):seen["uri"] = uriseen["ctx"] = ctxreturn {"status": "completed", "uri": uri}@@ -3450,9 +2742,8 @@ async def test_openviking_service_reindex_keeps_canonical_user_id(monkeypatch):result = await OpenVikingService.reindex(service,uri="viking://user/resources/memories",- mode="prune_orphans",+ mode="vectors_only",wait=True,- dry_run=True,ctx=ctx,)diff --git a/tests/server/test_api_content_write.py b/tests/server/test_api_content_write.pyindex d1834806a..9fba532cc 100644--- a/tests/server/test_api_content_write.py+++ b/tests/server/test_api_content_write.py@@ -302,11 +302,72 @@ async def test_api_create_mode_regression_replace_unchanged(client_with_resourceassert read_resp.json()["result"] == "# Updated\n\nFresh content."-async def test_set_tags_requires_tags_field(client_with_resource):- client, uri = client_with_resource- file_uri = await _first_file_uri(client, uri)- resp = await client.post("/api/v1/fs/attrs/set_tags", json={"uri": file_uri})- assert resp.status_code == 400+async def test_set_tags_omitted_tags_is_noop(client, service, monkeypatch):+ captured = {}++ async def fake_set_tags(+ *,+ uri,+ tags,+ ctx,+ mode="replace",+ recursive=False,+ ):+ del ctx+ captured["uri"] = uri+ captured["tags"] = tags+ captured["mode"] = mode+ captured["recursive"] = recursive+ return {"uri": uri, "tags": tags, "mode": mode}++ monkeypatch.setattr(service.fs, "set_tags", fake_set_tags)++ resp = await client.post(+ "/api/v1/fs/attrs/set_tags",+ json={"uri": "viking://resources/demo/file.md"},+ )++ assert resp.status_code == 200+ assert captured == {+ "uri": "viking://resources/demo/file.md",+ "tags": [],+ "mode": "replace",+ "recursive": False,+ }+++async def test_set_tags_clear_allows_omitted_tags(client, service, monkeypatch):+ captured = {}++ async def fake_set_tags(+ *,+ uri,+ tags,+ ctx,+ mode="replace",+ recursive=False,+ ):+ del ctx+ captured["uri"] = uri+ captured["tags"] = tags+ captured["mode"] = mode+ captured["recursive"] = recursive+ return {"uri": uri, "tags": tags, "mode": mode}++ monkeypatch.setattr(service.fs, "set_tags", fake_set_tags)++ resp = await client.post(+ "/api/v1/fs/attrs/set_tags",+ json={"uri": "viking://resources/demo/file.md", "mode": "clear"},+ )++ assert resp.status_code == 200+ assert captured == {+ "uri": "viking://resources/demo/file.md",+ "tags": [],+ "mode": "clear",+ "recursive": False,+ }async def test_set_tags_passes_tags_to_service(client, service, monkeypatch):diff --git a/tests/server/test_content_write_service.py b/tests/server/test_content_write_service.pyindex da44b0ddc..32fc479ff 100644--- a/tests/server/test_content_write_service.py+++ b/tests/server/test_content_write_service.py@@ -1551,6 +1551,68 @@ async def test_set_tags_append_merges_existing_tags(monkeypatch):assert fake_store.update_calls == [(file_uri, ["env=prod", "team=search"], "append")]+async def test_set_tags_clear_ignores_values_and_clears_existing_tags(monkeypatch):+ file_uri = "viking://resources/demo/doc.md"+ root_uri = "viking://resources/demo"+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)+ fake_vfs = _FakeVikingFS(file_uri=file_uri, root_uri=root_uri)+ coordinator = ContentWriteCoordinator(viking_fs=fake_vfs)++ class _FakeVectorStore:+ def __init__(self):+ self.update_calls = []++ async def update_search_tags(self, uri: str, tags, *, mode: str, levels=None, ctx=None):+ del ctx+ assert levels is None+ self.update_calls.append((uri, list(tags), mode))+ return [{"uri": uri}]++ fake_store = _FakeVectorStore()+ fake_vfs.vector_store = fake_store++ result = await coordinator.set_tags(+ uri=file_uri,+ tags=["team=ignored"],+ mode="clear",+ ctx=ctx,+ )++ assert result["mode"] == "clear"+ assert result["tags"] == []+ assert result["tags_updated"] is True+ assert fake_store.update_calls == [(file_uri, [], "replace")]+++async def test_set_tags_empty_replace_is_noop(monkeypatch):+ file_uri = "viking://resources/demo/doc.md"+ root_uri = "viking://resources/demo"+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)+ fake_vfs = _FakeVikingFS(file_uri=file_uri, root_uri=root_uri)+ coordinator = ContentWriteCoordinator(viking_fs=fake_vfs)++ class _FakeVectorStore:+ async def update_search_tags(self, *args, **kwargs):+ raise AssertionError("replace with [] must not update stored tags")++ fake_vfs.vector_store = _FakeVectorStore()++ result = await coordinator.set_tags(+ uri=file_uri,+ tags=[],+ mode="replace",+ ctx=ctx,+ )++ assert result["mode"] == "replace"+ assert result["tags"] == []+ assert result["tags_updated"] is False+ assert result["success_count"] == 0+ assert result["skipped_count"] == 1++@pytest.mark.asyncioasync def test_set_tags_discards_non_kv_tags(monkeypatch):file_uri = "viking://resources/demo/doc.md"diff --git a/tests/service/test_skill_package_reindex.py b/tests/service/test_skill_package_reindex.pydeleted file mode 100644index 5d983dc68..000000000--- a/tests/service/test_skill_package_reindex.py+++ /dev/null@@ -1,97 +0,0 @@-from types import SimpleNamespace-from unittest.mock import AsyncMock--import pytest--from openviking.server.identity import RequestContext, Role-from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters-from openviking_cli.session.user_id import UserIdentifier---async def test_vectors_only_reindexes_entire_skill_with_common_media_inputs(monkeypatch):- root = "viking://agent/skills/demo"- tree = {- root: [- {"name": "SKILL.md"},- {"name": "reference", "isDir": True},- {"name": ".source.json"},- ],- root + "/reference": [- {"name": "image.png"},- {"name": "video.mp4"},- {"name": "nested", "isDir": True},- ],- root + "/reference/nested": [- {"name": "api.md"},- {"name": ".abstract.md"},- {"name": "messages.jsonl"},- ],- }- fs = SimpleNamespace(ls=AsyncMock(side_effect=lambda uri, **kw: tree[uri]))- monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: fs)- executor = ReindexExecutor()- executor._read_directory_abstract = AsyncMock(return_value="name: demo\ndescription: Demo")- executor._read_directory_overview = AsyncMock(return_value="overview")- executor._fetch_existing_record = AsyncMock(- side_effect=lambda **kw: {"abstract": "media summary", "meta": {"source_path": "/tmp/demo"}}- )- directories = AsyncMock()- files = AsyncMock(return_value=True)- monkeypatch.setattr("openviking.service.reindex_executor.vectorize_directory_meta", directories)- monkeypatch.setattr("openviking.service.reindex_executor.vectorize_file", files)- monkeypatch.setattr(- "openviking.service.reindex_executor.SemanticProcessor",- lambda **kw: pytest.fail("vectors_only must not generate summaries"),- )- counters = _ReindexCounters()- ctx = RequestContext(user=UserIdentifier("acc", "alice"), role=Role.USER)- await executor._reindex_skill_vectors(uri=root, ctx=ctx, counters=counters)- assert {call.args[0] for call in directories.await_args_list} == set(tree)- expected = {- root + "/SKILL.md",- root + "/reference/image.png",- root + "/reference/video.mp4",- root + "/reference/nested/api.md",- }- assert {call.kwargs["file_path"] for call in files.await_args_list} == expected- assert all(call.kwargs["context_type"] == "skill" for call in files.await_args_list)- assert all(- call.kwargs["summary_dict"]["summary"] == "media summary" for call in files.await_args_list- )- assert directories.await_args_list[0].kwargs["meta"]["source_path"] == "/tmp/demo"- assert counters.scanned_records == 7 and counters.rebuilt_records == 10--- "abstract",- [- "Manage backups.\nUse when: a backup needs recovery.",- "Use when: recovery is needed",- "A plain description",- ],-)-async def test_vectors_only_preserves_legacy_skill_descriptions(monkeypatch, abstract):- executor = ReindexExecutor()- executor._read_directory_abstract = AsyncMock(return_value=abstract)- executor._read_directory_overview = AsyncMock(return_value="Overview")- previous_meta = {- "name": "demo",- "tags": ["backup"],- "allowed_tools": ["Read"],- "source_path": "/demo",- }- executor._fetch_existing_record = AsyncMock(return_value={"meta": previous_meta})- monkeypatch.setattr(- "openviking.service.reindex_executor.get_viking_fs", lambda: SimpleNamespace()- )- vectorize = AsyncMock()- monkeypatch.setattr("openviking.service.reindex_executor.vectorize_directory_meta", vectorize)- counters = _ReindexCounters()- ctx = RequestContext(user=UserIdentifier("acc", "alice"), role=Role.USER)- await executor._reindex_skill_vectors(- uri="viking://agent/skills/demo", counters=counters, ctx=ctx, recursive=False- )- assert counters.failed_records == 0 and counters.rebuilt_records == 2- assert vectorize.await_args.kwargs["meta"] == {**previous_meta, "description": abstract}diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.pyindex 805262634..19c8f8f35 100644--- a/tests/storage/test_collection_schemas.py+++ b/tests/storage/test_collection_schemas.py@@ -86,7 +86,8 @@ class TextEmbeddingHandler(ProductionTextEmbeddingHandler):provider = AccountEmbeddingProvider(AccountVectorConfigResolver(manager), manager)vikingdb.account_uses_content_field = AsyncMock(return_value=getattr(- vikingdb, "uses_content_field",+ vikingdb,+ "uses_content_field",config.storage.vectordb.backend in {"volcengine", "vikingdb"},))@@ -836,9 +837,10 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_wcollection_schemas.logger.removeHandler(caplog.handler)warnings = [record.message for record in caplog.records if record.levelno == logging.WARNING]- assert warnings.count(- "Embedding circuit breaker is open; re-enqueueing messages account=default"- ) == 1+ assert (+ warnings.count("Embedding circuit breaker is open; re-enqueueing messages account=default")+ == 1+ )for result in (first_result, second_result):assert result.outcome is ProcessOutcome.REQUEUEDassert result.value is None@@ -999,6 +1001,33 @@ async def test_embedding_handler_materialize_content_keeps_inline(monkeypatch):assert content == "already inline"+async def test_embedding_handler_materialize_content_uses_preloaded_source(monkeypatch):+ class _DummyVikingDB:+ is_closing = False++ class _BrokenFS:+ async def read_file(self, uri, *, ctx):+ raise AssertionError(f"must not reread {uri}")++ monkeypatch.setattr("openviking.storage.viking_fs.get_viking_fs", lambda: _BrokenFS())+ handler = TextEmbeddingHandler(_DummyVikingDB())+ msg = EmbeddingMsg(+ "embedding text",+ {+ "uri": "viking://resources/current.txt",+ "abstract": "abstract",+ "is_leaf": True,+ "context_type": "resource",+ "account_id": "default",+ "_materialized_content": "current body",+ },+ )+ ctx = RequestContext(user=UserIdentifier("default", "default"), role=Role.ROOT)++ assert await handler._materialize_content(msg, ctx) == "current body"++@pytest.mark.asyncioasync def test_embedding_handler_propagates_account_id_on_error(monkeypatch):class _DummyVikingDB:diff --git a/tests/storage/test_context_update_plan.py b/tests/storage/test_context_update_plan.pyindex 2d2869550..3e4cdcacd 100644--- a/tests/storage/test_context_update_plan.py+++ b/tests/storage/test_context_update_plan.py@@ -306,6 +306,25 @@ def test_semantic_plan_validates_ancestors_without_rescanning_entries():assert entries.iterations <= 4+def test_semantic_plan_serialization_ignores_legacy_parent_propagation():+ from openviking.storage.context_update_plan import (+ SemanticPlan,+ SemanticTreeEntry,+ SemanticTreeSnapshot,+ )++ plan = SemanticPlan(+ "viking://resources/repo",+ "resource",+ SemanticTreeSnapshot((SemanticTreeEntry("", "directory", "unchanged", "aggregate"),)),+ )++ serialized = plan.to_dict()++ assert "propagation" not in serialized+ assert SemanticPlan.from_dict({**serialized, "propagation": {"enabled": False}}) == plan++def test_semantic_plan_rejects_missing_higher_ancestor_independent_of_entry_order():from openviking.storage.context_update_plan import (IndexSlot,@@ -971,6 +990,53 @@ def test_healthy_noop_produces_no_actions_or_semantic_plan():assert plan.is_noop()assert plan.content_tree_actions == ()assert plan.semantic_plan is None+++def test_shared_semantic_closure_generates_complete_file_without_abstract():+ """RNFV add_resources and RFV reindex share this closure rule."""+ from openviking.storage.context_update_plan import (+ ContentState,+ IndexState,+ SemanticAction,+ build_context_update_plan,+ )+ from openviking.storage.resource_diff import ResourceDiffEntry, ResourceDiffResult+ from openviking.storage.resource_rnfv import RequestIntent, VectorRecordSnapshot++ root = "viking://resources/repo"+ plan = build_context_update_plan(+ root_uri=root,+ context_type="resource",+ request=RequestIntent(root, "semantic_and_vectors"),+ diff=ResourceDiffResult(+ {+ "": ResourceDiffEntry(+ "",+ ContentState.UNCHANGED,+ IndexState.MISSING,+ old_kind="directory",+ new_kind="directory",+ ),+ "a.md": ResourceDiffEntry(+ "a.md",+ ContentState.UNCHANGED,+ IndexState.COMPLETE,+ old_kind="file",+ new_kind="file",+ md5="same",+ ),+ }+ ),+ new_kinds={"": "directory", "a.md": "file"},+ artifact_paths={"a.md": "repository/a.md"},+ records={"a-l2": VectorRecordSnapshot("a-l2", f"{root}/a.md", "a.md", 2, {"md5": "same"})},+ is_code_repo=False,+ account_id="acc",+ )++ entries = {entry.relative_path: entry for entry in plan.semantic_plan.tree.entries}+ assert entries[""].semantic_action is SemanticAction.AGGREGATE+ assert entries["a.md"].semantic_action is SemanticAction.GENERATEassert plan.direct_index_actions == ()@@ -1137,12 +1203,8 @@ def test_healthy_directory_rnfv_clear_updates_all_vector_levels():ingest_options=IngestOptions.from_search_tags(None, mode="clear"),)records = {- "root-l0": VectorRecordSnapshot(- "root-l0", root, "", 0, {"search_tags": ["scope=old"]}- ),- "root-l1": VectorRecordSnapshot(- "root-l1", root, "", 1, {"search_tags": ["scope=old"]}- ),+ "root-l0": VectorRecordSnapshot("root-l0", root, "", 0, {"search_tags": ["scope=old"]}),+ "root-l1": VectorRecordSnapshot("root-l1", root, "", 1, {"search_tags": ["scope=old"]}),"a-l2": VectorRecordSnapshot("a-l2",root + "/a.py",@@ -1237,11 +1299,7 @@ def test_healthy_rnfv_clear_is_noop_when_vector_has_no_tags(existing_tags):),new_kinds={"a.py": "file"},artifact_paths={"a.py": "repository/a.py"},- records={- "a-l2": VectorRecordSnapshot(- "a-l2", root + "/a.py", "a.py", 2, fields- )- },+ records={"a-l2": VectorRecordSnapshot("a-l2", root + "/a.py", "a.py", 2, fields)},is_code_repo=False,account_id="acc",)@@ -1297,11 +1355,7 @@ def test_healthy_rnfv_non_empty_replace_compares_against_vector_tags(),new_kinds={"a.py": "file"},artifact_paths={"a.py": "repository/a.py"},- records={- "a-l2": VectorRecordSnapshot(- "a-l2", root + "/a.py", "a.py", 2, fields- )- },+ records={"a-l2": VectorRecordSnapshot("a-l2", root + "/a.py", "a.py", 2, fields)},is_code_repo=False,account_id="acc",)@@ -2583,6 +2637,11 @@ async def test_resource_processor_dispatches_direct_index_actions_without_semant)monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: queue_manager)monkeypatch.setattr("openviking.utils.embedding_utils._enqueue_embedding_message", enqueue)+ vectorize_directory = AsyncMock(return_value={0, 1})+ monkeypatch.setattr(+ "openviking.utils.resource_processor.vectorize_directory_meta",+ vectorize_directory,+ )processor = ResourceProcessor(SimpleNamespace(get_embedder=lambda: None))processor._vectorize_resource_file = AsyncMock()ctx = RequestContext(UserIdentifier("acc", "user"), Role.USER)@@ -2619,9 +2678,33 @@ async def test_resource_processor_dispatches_direct_index_actions_without_semant{"search_tags": "append"},),md5="new-md5",+ summary="existing summary",+ ),+ DirectIndexAction(+ "merge",+ "viking://resources/repo",+ 0,+ "repo-l0",+ field_patch=FieldPatch(+ {"search_tags": ["scope=new"]},+ {"search_tags": "replace"},+ ),+ md5="abstract-md5",+ ),+ DirectIndexAction(+ "upsert",+ "viking://resources/repo",+ 1,+ "repo-l1",+ md5="overview-md5",),),ctx=ctx,+ source_contents={+ ("viking://resources/repo/c.py", 2): b"print('current')",+ ("viking://resources/repo", 0): "current abstract",+ ("viking://resources/repo", 1): "current overview",+ },)assert [msg.action.value for msg in enqueued] == ["delete",@@ -2644,7 +2727,93 @@ async def test_resource_processor_dispatches_direct_index_actions_without_semant{"search_tags": ["scope=new"]},{"search_tags": "append"},),+ summary="existing summary",+ file_content=b"print('current')",)+ vectorize_directory.assert_awaited_once_with(+ "viking://resources/repo",+ "current abstract",+ "current overview",+ context_type="resource",+ ctx=ctx,+ include_abstract=True,+ include_overview=True,+ content_is_body=True,+ actions={0: "merge", 1: "upsert"},+ scalar_overrides={+ 0: {"_record_id": "repo-l0"},+ 1: {"_record_id": "repo-l1"},+ },+ field_patches={+ 0: FieldPatch(+ {"search_tags": ["scope=new"]},+ {"search_tags": "replace"},+ )+ },+ md5s={0: "abstract-md5", 1: "overview-md5"},+ )+++async def test_resource_processor_rejects_direct_upsert_without_snapshot_source():+ from openviking.server.identity import RequestContext, Role+ from openviking.storage.context_update_plan import DirectIndexAction+ from openviking.utils.resource_processor import ResourceProcessor+ from openviking_cli.session.user_id import UserIdentifier++ processor = ResourceProcessor(SimpleNamespace(get_embedder=lambda: None))+ ctx = RequestContext(UserIdentifier("acc", "user"), Role.USER)++ with pytest.raises(ValueError, match="directory L0 vector source is unavailable"):+ await processor._enqueue_index_actions(+ (+ DirectIndexAction(+ "upsert",+ "viking://resources/repo",+ 0,+ "repo-l0",+ md5="abstract-md5",+ ),+ ),+ ctx=ctx,+ source_contents={},+ )+++async def test_resource_processor_preserves_skill_metadata_in_direct_directory_merge(monkeypatch):+ from openviking.server.identity import RequestContext, Role+ from openviking.storage.context_update_plan import DirectIndexAction+ from openviking.utils.resource_processor import ResourceProcessor+ from openviking_cli.session.user_id import UserIdentifier++ vectorize_directory = AsyncMock(return_value={0})+ monkeypatch.setattr(+ "openviking.utils.resource_processor.vectorize_directory_meta", vectorize_directory+ )+ monkeypatch.setattr(+ "openviking.storage.queuefs.get_queue_manager",+ lambda: SimpleNamespace(EMBEDDING="embedding", get_queue=lambda *a, **k: object()),+ )+ processor = ResourceProcessor(SimpleNamespace(get_embedder=lambda: None))+ ctx = RequestContext(UserIdentifier("acc", "owner"), Role.USER)+ uri = "viking://user/owner/skills/demo"++ await processor._enqueue_index_actions(+ (DirectIndexAction("merge", uri, 0, "skill-l0", md5="new"),),+ ctx=ctx,+ source_contents={+ (uri, 0): "name: demo\ndescription: Demo\ntags: [tool]\nallowed_tools: [Read]",+ },+ )++ meta = vectorize_directory.await_args.kwargs["meta"]+ assert meta == {+ "name": "demo",+ "description": "Demo",+ "tags": ["tool"],+ "allowed_tools": ["Read"],+ }def test_semantic_message_roundtrip_uses_explicit_plan():@@ -3377,7 +3546,7 @@ async def test_semantic_processor_runs_only_plan_execution_roots(monkeypatch):root_write_result = Nonedef __init__(self, **kwargs):- calls.append(("init", kwargs["semantic_plan"]))+ calls.append(("init", kwargs))async def run(self, uri):calls.append(("run", uri))@@ -3464,11 +3633,13 @@ async def test_semantic_processor_runs_only_plan_execution_roots(monkeypatch):user_id="user",role="user",plan=plan,+ generation_trigger="reindex",)await processor.on_dequeue(msg.to_dict())assert [value for kind, value in calls if kind == "run"] == [root]+ assert calls[0][1]["generation_trigger"] == "reindex"@pytest.mark.asynciodiff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.pyindex 55d7e79ff..662569a10 100644--- a/tests/storage/test_queue_manager.py+++ b/tests/storage/test_queue_manager.py@@ -40,11 +40,13 @@ def test_queue_concurrency_uses_separate_configured_values() -> None:agfs=object(),max_concurrent_external_parse=9,max_concurrent_add_resource=7,+ max_concurrent_reindex=6,max_concurrent_session_commit=5,)assert manager._max_concurrent_for_queue(manager.EXTERNAL_PARSE) == 9assert manager._max_concurrent_for_queue(manager.ADD_RESOURCE) == 7+ assert manager._max_concurrent_for_queue(manager.REINDEX) == 6assert manager._max_concurrent_for_queue(manager.SESSION_COMMIT) == 5@@ -143,9 +145,7 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(object(), max_concurrent_semantic=concurrency, max_concurrent_embedding=concurrency)manager._poll_interval = 0.001- manager.set_vlm_resolver(- SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace()))- )+ manager.set_vlm_resolver(SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace())))manager.setup_standard_queues(object(), start=False)semantic = manager._queues[manager.SEMANTIC]._dequeue_handlermanager._queues = {diff --git a/tests/storage/test_reindex_msg.py b/tests/storage/test_reindex_msg.pynew file mode 100644index 000000000..578982af6--- /dev/null+++ b/tests/storage/test_reindex_msg.py@@ -0,0 +1,362 @@+from types import SimpleNamespace+from unittest.mock import AsyncMock, Mock++import pytest++from openviking.service.task_tracker import TaskStatus+from openviking.storage.errors import LockAcquisitionError+from openviking.storage.queuefs.reindex_msg import ReindexMsg+++def test_reindex_msg_round_trips_only_request_descriptor():+ message = ReindexMsg(+ task_id="task-1",+ uri="viking://resources/demo",+ object_type="resource",+ mode="semantic_and_vectors",+ force=True,+ recursive=False,+ tags=["team=search"],+ tag_mode="append",+ account_id="account-1",+ user_id="user-1",+ group_ids=["group-1"],+ role="admin",+ actor_peer_id="peer-1",+ bypass_acl=True,+ lock_handoff={"lease_ref": "lease-1"},+ )++ restored = ReindexMsg.from_dict(message.to_dict())++ assert restored == message+ assert "source_contents" not in restored.to_dict()+ assert "snapshot" not in restored.to_dict()+++def test_reindex_processor_restores_actor_peer_and_acl_bypass_context():+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor++ message = ReindexMsg(+ task_id="task-1",+ uri="viking://resources/demo",+ object_type="resource",+ mode="vectors_only",+ account_id="account-1",+ user_id="user-1",+ role="admin",+ actor_peer_id="peer-1",+ bypass_acl=True,+ )++ ctx = ReindexProcessor._ctx(message)++ assert ctx.actor_peer_id == "peer-1"+ assert ctx.bypass_acl is True+++def test_reindex_msg_rejects_missing_descriptor_fields():+ with pytest.raises(ValueError, match="task_id"):+ ReindexMsg.from_dict({"uri": "viking://resources/demo"})+++async def test_reindex_processor_terminalizes_root_after_descendants(monkeypatch):+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor++ tracker = SimpleNamespace(+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.RUNNING)),+ start=AsyncMock(),+ complete=AsyncMock(),+ fail=AsyncMock(),+ wait_for_descendants=AsyncMock(),+ )+ monkeypatch.setattr(+ "openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)+ )+ monkeypatch.setattr(+ "openviking.server.dependencies.get_service",+ lambda: SimpleNamespace(_vlm_resolver=None, _vector_config_resolver=None),+ )++ result = {"status": "completed", "rebuilt_records": 3}+ monkeypatch.setattr(+ "openviking.service.reindex_executor.ReindexExecutor._run", AsyncMock(return_value=result)+ )+ viking_fs = SimpleNamespace(_async_agfs=SimpleNamespace(pathlock_release=AsyncMock()))+ processor = ReindexProcessor(viking_fs)+ processor._adopt_or_reacquire = AsyncMock(return_value={"lease": "root"})+ message = ReindexMsg(+ task_id="task-1",+ uri="viking://resources/demo",+ object_type="resource",+ mode="vectors_only",+ account_id="account-1",+ user_id="user-1",+ role="admin",+ )++ outcome = await processor.on_dequeue(+ {"id": "queue-1", "data": {**message.to_dict(), "_task_work_id": "root-work"}}+ )++ assert outcome.outcome.value == "success"+ tracker.wait_for_descendants.assert_awaited_once_with("task-1", "root-work")+ tracker.complete.assert_awaited_once_with(+ "task-1", result, account_id="account-1", user_id="user-1"+ )+++async def test_reindex_processor_restores_root_telemetry_for_executor(monkeypatch):+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor+ from openviking.telemetry import (+ OperationTelemetry,+ get_current_telemetry,+ register_telemetry,+ unregister_telemetry,+ )++ tracker = SimpleNamespace(+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.RUNNING)),+ start=AsyncMock(),+ complete=AsyncMock(),+ fail=AsyncMock(),+ wait_for_descendants=AsyncMock(),+ )+ monkeypatch.setattr(+ "openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)+ )+ monkeypatch.setattr(+ "openviking.server.dependencies.get_service",+ lambda: SimpleNamespace(_vlm_resolver=None, _vector_config_resolver=None),+ )+ telemetry = OperationTelemetry(operation="content_reindex", enabled=True)+ telemetry_id = telemetry.telemetry_id+ register_telemetry(telemetry)+ seen = {}++ async def capture_telemetry(_self, **_kwargs):+ seen["telemetry_id"] = get_current_telemetry().telemetry_id+ return {"status": "completed"}++ monkeypatch.setattr(+ "openviking.service.reindex_executor.ReindexExecutor._run",+ capture_telemetry,+ )+ processor = ReindexProcessor(+ SimpleNamespace(_async_agfs=SimpleNamespace(pathlock_release=AsyncMock()))+ )+ processor._adopt_or_reacquire = AsyncMock(return_value={"lease": "root"})+ message = ReindexMsg(+ task_id="task-1",+ uri="viking://resources/demo",+ object_type="resource",+ mode="semantic_and_vectors",+ account_id="account-1",+ user_id="user-1",+ role="admin",+ telemetry_id=telemetry_id,+ )++ try:+ outcome = await processor.on_dequeue({"id": "queue-1", "data": message.to_dict()})+ finally:+ unregister_telemetry(telemetry_id)++ assert outcome.outcome.value == "success"+ assert seen["telemetry_id"] == telemetry_id+++async def test_reindex_processor_replay_of_completed_task_skips_force_rebuild(monkeypatch):+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor++ tracker = SimpleNamespace(+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.COMPLETED)),+ start=AsyncMock(),+ complete=AsyncMock(),+ fail=AsyncMock(),+ wait_for_descendants=AsyncMock(),+ )+ monkeypatch.setattr(+ "openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)+ )+ monkeypatch.setattr(+ "openviking.server.dependencies.get_service",+ lambda: SimpleNamespace(_vlm_resolver=None, _vector_config_resolver=None),+ )+ monkeypatch.setattr(+ "openviking.service.reindex_executor.ReindexExecutor._run",+ AsyncMock(return_value={"status": "completed"}),+ )+ viking_fs = SimpleNamespace(_async_agfs=SimpleNamespace(pathlock_release=AsyncMock()))+ processor = ReindexProcessor(viking_fs)+ processor._adopt_or_reacquire = AsyncMock(return_value={"lease": "must-not-adopt"})+ message = ReindexMsg(+ task_id="task-1",+ uri="viking://resources/demo",+ object_type="resource",+ mode="semantic_and_vectors",+ force=True,+ account_id="account-1",+ user_id="user-1",+ role="admin",+ lock_handoff={"lease_ref": "old"},+ )++ outcome = await processor.on_dequeue(+ {"id": "queue-1", "data": {**message.to_dict(), "_task_work_id": "root-work"}}+ )++ assert outcome.outcome.value == "success"+ processor._adopt_or_reacquire.assert_not_awaited()+ tracker.start.assert_not_awaited()+ tracker.complete.assert_not_awaited()+ tracker.fail.assert_not_awaited()+++async def test_reindex_processor_requeues_lock_handoff_failure_before_ack(monkeypatch):+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor++ tracker = SimpleNamespace(+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.RUNNING)),+ start=AsyncMock(),+ complete=AsyncMock(),+ fail=AsyncMock(),+ wait_for_descendants=AsyncMock(),+ )+ queue_manager = SimpleNamespace(REINDEX="Reindex", enqueue=AsyncMock(return_value="retry-1"))+ monkeypatch.setattr(+ "openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)+ )+ monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: queue_manager)+ processor = ReindexProcessor(SimpleNamespace(_async_agfs=SimpleNamespace()))+ processor._adopt_or_reacquire = AsyncMock(side_effect=LockAcquisitionError("busy"))+ message = ReindexMsg(+ task_id="task-1",+ uri="viking://resources/demo",+ object_type="resource",+ mode="vectors_only",+ account_id="account-1",+ user_id="user-1",+ role="admin",+ )++ outcome = await processor.on_dequeue(+ {"id": "queue-1", "data": {**message.to_dict(), "_task_work_id": "root-work"}}+ )++ assert outcome.outcome.value == "requeued"+ queue_manager.enqueue.assert_awaited_once_with(+ "Reindex", {**message.to_dict(), "lock_handoff_retry": 1}+ )+ tracker.fail.assert_not_awaited()+++async def test_reindex_processor_fails_after_lock_handoff_retry_limit(monkeypatch):+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor++ tracker = SimpleNamespace(+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.RUNNING)),+ start=AsyncMock(),+ complete=AsyncMock(),+ fail=AsyncMock(),+ wait_for_descendants=AsyncMock(),+ )+ monkeypatch.setattr(+ "openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)+ )+ processor = ReindexProcessor(SimpleNamespace(_async_agfs=SimpleNamespace()))+ processor._adopt_or_reacquire = AsyncMock(side_effect=LockAcquisitionError("busy"))+ message = ReindexMsg(+ task_id="task-1",+ uri="viking://resources/demo",+ object_type="resource",+ mode="vectors_only",+ account_id="account-1",+ user_id="user-1",+ role="admin",+ lock_handoff_retry=2,+ )++ outcome = await processor.on_dequeue({"id": "queue-1", "data": message.to_dict()})++ assert outcome.outcome.value == "failed"+ tracker.fail.assert_awaited_once()+++async def test_reindex_processor_cancellation_releases_root_lock_handoff():+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor++ agfs = SimpleNamespace(+ pathlock_adopt=AsyncMock(return_value={"lease": "root"}),+ pathlock_release=AsyncMock(),+ )+ processor = ReindexProcessor(SimpleNamespace(_async_agfs=agfs))+ message = ReindexMsg(+ task_id="task-1",+ uri="viking://resources/demo",+ object_type="resource",+ mode="vectors_only",+ account_id="account-1",+ user_id="user-1",+ role="admin",+ lock_handoff={"lease_ref": "root-handoff"},+ )++ outcome = await processor.on_cancelled({"data": message.to_dict()})++ assert outcome.outcome.value == "cancelled"+ agfs.pathlock_adopt.assert_awaited_once_with({"lease_ref": "root-handoff"})+ agfs.pathlock_release.assert_awaited_once_with({"lease": "root"})+++async def test_reindex_processor_does_not_release_lease_owned_by_failed_executor(monkeypatch):+ from openviking.storage.queuefs.reindex_processor import ReindexProcessor++ tracker = SimpleNamespace(+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.RUNNING)),+ start=AsyncMock(),+ complete=AsyncMock(),+ fail=AsyncMock(),+ wait_for_descendants=AsyncMock(),+ )+ monkeypatch.setattr(+ "openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)+ )+ monkeypatch.setattr(+ "openviking.server.dependencies.get_service",+ lambda: SimpleNamespace(_vlm_resolver=None, _vector_config_resolver=None),+ )+ agfs = SimpleNamespace(pathlock_release=AsyncMock())++ async def fail_after_releasing(_self, *, existing_lease, **_kwargs):+ await agfs.pathlock_release(existing_lease)+ raise RuntimeError("semantic plan enqueue failed")++ monkeypatch.setattr(+ "openviking.service.reindex_executor.ReindexExecutor._run",+ fail_after_releasing,+ )+ processor = ReindexProcessor(SimpleNamespace(_async_agfs=agfs))+ processor._adopt_or_reacquire = AsyncMock(return_value={"lease": "root"})+ message = ReindexMsg(+ task_id="task-1",+ uri="viking://resources/demo",+ object_type="resource",+ mode="semantic_and_vectors",+ account_id="account-1",+ user_id="user-1",+ role="admin",+ )++ outcome = await processor.on_dequeue({"id": "queue-1", "data": message.to_dict()})++ assert outcome.outcome.value == "failed"+ agfs.pathlock_release.assert_awaited_once_with({"lease": "root"})+ tracker.fail.assert_awaited_once()diff --git a/tests/storage/test_resource_rfv.py b/tests/storage/test_resource_rfv.pynew file mode 100644index 000000000..b27bfd328--- /dev/null+++ b/tests/storage/test_resource_rfv.py@@ -0,0 +1,1065 @@+from types import SimpleNamespace+from unittest.mock import ANY, AsyncMock++import pytest++from openviking.storage.resource_diff import IndexState+from openviking.storage.resource_rnfv import (+ RequestIntent,+ VectorIndexSnapshot,+ VectorRecordSnapshot,+)+++def _record(record_id: str, uri: str, relative_path: str, level: int, md5: str):+ return VectorRecordSnapshot(+ record_id=record_id,+ uri=uri,+ relative_path=relative_path,+ level=level,+ fields={"md5": md5},+ )+++def test_rfv_resolver_skips_complete_file_and_marks_changed_or_forced_file_stale():+ from openviking.storage.resource_rfv import (+ RFVEntry,+ RFVFormalSnapshot,+ RFVSnapshot,+ resolve_rfv_state,+ )++ root = "viking://resources/demo"+ record = _record("readme-l2", f"{root}/README.md", "README.md", 2, "same")+ formal = RFVFormalSnapshot(+ entries={+ "": RFVEntry(uri=root, relative_path="", is_dir=True),+ "README.md": RFVEntry(+ uri=f"{root}/README.md",+ relative_path="README.md",+ is_dir=False,+ level_md5s={2: "same"},+ ),+ }+ )+ vectors = VectorIndexSnapshot(+ records_by_id={record.record_id: record},+ projected_fields=frozenset({"id", "uri", "level", "md5"}),+ )++ normal = resolve_rfv_state(RFVSnapshot(RequestIntent(root, "vectors_only"), formal, vectors))+ assert normal.entries["README.md"].content_state.value == "unchanged"+ assert normal.entries["README.md"].index_state == IndexState.COMPLETE+ assert normal.entries["README.md"].level_states == {2: IndexState.COMPLETE}++ changed_formal = RFVFormalSnapshot(+ entries={+ **formal.entries,+ "README.md": RFVEntry(+ uri=f"{root}/README.md",+ relative_path="README.md",+ is_dir=False,+ level_md5s={2: "changed"},+ ),+ }+ )+ changed = resolve_rfv_state(+ RFVSnapshot(RequestIntent(root, "vectors_only"), changed_formal, vectors)+ )+ assert changed.entries["README.md"].index_state == IndexState.STALE++ forced = resolve_rfv_state(+ RFVSnapshot(RequestIntent(root, "vectors_only", force=True), formal, vectors)+ )+ assert forced.entries["README.md"].index_state == IndexState.STALE+++def test_rfv_resolver_tracks_directory_levels_independently():+ from openviking.storage.resource_rfv import (+ RFVEntry,+ RFVFormalSnapshot,+ RFVSnapshot,+ resolve_rfv_state,+ )++ root = "viking://resources/demo"+ records = {+ "root-l0": _record("root-l0", root, "", 0, "abstract-same"),+ "root-l1": _record("root-l1", root, "", 1, "overview-old"),+ }+ snapshot = RFVSnapshot(+ RequestIntent(root, "vectors_only"),+ RFVFormalSnapshot(+ entries={+ "": RFVEntry(+ uri=root,+ relative_path="",+ is_dir=True,+ level_md5s={0: "abstract-same", 1: "overview-new"},+ )+ }+ ),+ VectorIndexSnapshot(+ records_by_id=records,+ projected_fields=frozenset({"id", "uri", "level", "md5"}),+ ),+ )++ resolved = resolve_rfv_state(snapshot)+ assert resolved.entries[""].index_state == IndexState.STALE+ assert resolved.entries[""].level_states == {+ 0: IndexState.COMPLETE,+ 1: IndexState.STALE,+ }+++def test_rfv_resolver_marks_missing_and_orphan_records():+ from openviking.storage.resource_rfv import (+ RFVEntry,+ RFVFormalSnapshot,+ RFVSnapshot,+ resolve_rfv_state,+ )++ root = "viking://resources/demo"+ orphan = _record("old-l2", f"{root}/old.md", "old.md", 2, "old")+ snapshot = RFVSnapshot(+ RequestIntent(root, "vectors_only"),+ RFVFormalSnapshot(+ entries={+ "": RFVEntry(uri=root, relative_path="", is_dir=True),+ "new.md": RFVEntry(+ uri=f"{root}/new.md",+ relative_path="new.md",+ is_dir=False,+ level_md5s={2: "new"},+ ),+ }+ ),+ VectorIndexSnapshot(+ records_by_id={orphan.record_id: orphan},+ projected_fields=frozenset({"id", "uri", "level", "md5"}),+ ),+ )++ resolved = resolve_rfv_state(snapshot)+ assert resolved.entries["new.md"].index_state == IndexState.MISSING+ assert resolved.entries["old.md"].index_state == IndexState.ORPHAN+ assert resolved.entries["old.md"].content_state.value == "absent"+++def test_rfv_single_file_semantic_plan_uses_generate_instead_of_file_refresh():+ from openviking.storage.context_update_plan import SemanticAction, build_rfv_context_update_plan+ from openviking.storage.resource_rfv import (+ RFVEntry,+ RFVFormalSnapshot,+ RFVSnapshot,+ )++ uri = "viking://resources/report.md"+ record = _record("report-l2", uri, "", 2, "old")+ snapshot = RFVSnapshot(+ RequestIntent(uri, "semantic_and_vectors"),+ RFVFormalSnapshot({"": RFVEntry(uri, "", False, {2: "new"})}),+ VectorIndexSnapshot(+ {record.record_id: record},+ frozenset({"id", "uri", "level", "md5", "abstract"}),+ ),+ source_contents={(uri, 2): b"new"},+ )++ _, plan = build_rfv_context_update_plan(+ snapshot=snapshot,+ context_type="resource",+ account_id="acc",+ )++ assert plan.file_refresh is None+ assert plan.semantic_plan is not None+ entry = plan.semantic_plan.tree.entries[0]+ assert entry.kind == "file"+ assert entry.semantic_action is SemanticAction.GENERATE+ assert entry.slot(2).record_id == "report-l2"+++def test_rfv_missing_directory_sidecar_is_deleted_for_vectors_only_but_repaired_for_semantic():+ from openviking.storage.resource_rfv import (+ RFVEntry,+ RFVFormalSnapshot,+ RFVSnapshot,+ resolve_rfv_state,+ )++ root = "viking://resources/demo"+ l0 = _record("root-l0", root, "", 0, "old")+ formal = RFVFormalSnapshot({"": RFVEntry(root, "", True, {})})+ vectors = VectorIndexSnapshot({l0.record_id: l0}, frozenset({"id", "uri", "level", "md5"}))++ vectors_only = resolve_rfv_state(+ RFVSnapshot(RequestIntent(root, "vectors_only"), formal, vectors)+ )+ semantic = resolve_rfv_state(+ RFVSnapshot(RequestIntent(root, "semantic_and_vectors"), formal, vectors)+ )++ assert vectors_only.entries[""].level_states == {+ 0: IndexState.ABSENT,+ 1: IndexState.ABSENT,+ }+ assert semantic.entries[""].level_states == {+ 0: IndexState.MISSING,+ 1: IndexState.MISSING,+ }+++def test_rfv_plan_compiles_per_level_repairs_and_orphan_cleanup_without_content_actions():+ from openviking.storage.context_update_plan import build_rfv_context_update_plan+ from openviking.storage.index_action import IndexAction+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot++ root = "viking://resources/demo"+ records = {+ "root-l0": _record("root-l0", root, "", 0, "root-old"),+ "same-l2": _record("same-l2", f"{root}/same.md", "same.md", 2, "same"),+ "stale-l2": _record("stale-l2", f"{root}/stale.md", "stale.md", 2, "old"),+ "orphan-l2": _record("orphan-l2", f"{root}/orphan.md", "orphan.md", 2, "orphan"),+ }+ snapshot = RFVSnapshot(+ RequestIntent(root, "vectors_only"),+ RFVFormalSnapshot(+ entries={+ "": RFVEntry(+ uri=root,+ relative_path="",+ is_dir=True,+ level_md5s={0: "root-new", 1: "overview-new"},+ ),+ "same.md": RFVEntry(+ uri=f"{root}/same.md",+ relative_path="same.md",+ is_dir=False,+ level_md5s={2: "same"},+ ),+ "stale.md": RFVEntry(+ uri=f"{root}/stale.md",+ relative_path="stale.md",+ is_dir=False,+ level_md5s={2: "new"},+ ),+ }+ ),+ VectorIndexSnapshot(+ records_by_id=records,+ projected_fields=frozenset({"id", "uri", "level", "md5"}),+ ),+ )++ diff, plan = build_rfv_context_update_plan(+ snapshot=snapshot,+ context_type="resource",+ account_id="acc",+ )++ assert plan.content_tree_actions == ()+ assert plan.semantic_plan is None+ assert diff.entries["same.md"].level_states == {2: IndexState.COMPLETE}+ assert all(+ action.action is IndexAction.MERGE and action.upsert_fields == {}+ for action in plan.direct_index_actions+ if action.action is not IndexAction.DELETE+ )+ assert {+ (action.action, action.uri, action.level, action.record_id, action.md5)+ for action in plan.direct_index_actions+ } == {+ (IndexAction.MERGE, root, 0, "root-l0", "root-new"),+ (IndexAction.MERGE, root, 1, action_record_id(root, 1), "overview-new"),+ (IndexAction.MERGE, f"{root}/stale.md", 2, "stale-l2", "new"),+ (IndexAction.DELETE, f"{root}/orphan.md", 2, "orphan-l2", None),+ }+++def test_rfv_semantic_plan_reuses_existing_semantic_actions_for_stale_nodes():+ from openviking.storage.context_update_plan import (+ SemanticAction,+ build_rfv_context_update_plan,+ )+ from openviking.storage.index_action import IndexAction+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot++ root = "viking://resources/demo"+ records = {+ "root-l0": VectorRecordSnapshot(+ "root-l0", root, "", 0, {"md5": "old-l0", "abstract": "old root"}+ ),+ "root-l1": VectorRecordSnapshot(+ "root-l1", root, "", 1, {"md5": "old-l1", "abstract": "old overview"}+ ),+ "a-l2": VectorRecordSnapshot(+ "a-l2",+ f"{root}/a.md",+ "a.md",+ 2,+ {"md5": "old-file", "abstract": "old file"},+ ),+ }+ snapshot = RFVSnapshot(+ RequestIntent(root, "semantic_and_vectors"),+ RFVFormalSnapshot(+ {+ "": RFVEntry(root, "", True, {0: "new-l0", 1: "new-l1"}),+ "a.md": RFVEntry(f"{root}/a.md", "a.md", False, {2: "new-file"}),+ }+ ),+ VectorIndexSnapshot(+ records,+ frozenset({"id", "uri", "level", "md5", "abstract"}),+ ),+ )++ _diff, plan = build_rfv_context_update_plan(+ snapshot=snapshot, context_type="resource", account_id="acc"+ )++ assert plan.content_tree_actions == ()+ assert plan.direct_index_actions == ()+ entries = {entry.relative_path: entry for entry in plan.semantic_plan.tree.entries}+ assert entries[""].semantic_action is SemanticAction.AGGREGATE+ assert entries["a.md"].semantic_action is SemanticAction.GENERATE+ assert entries[""].repair is True+ assert entries["a.md"].repair is True+ assert [(slot.level, slot.action) for slot in entries[""].index_slots] == [+ (0, IndexAction.MERGE),+ (1, IndexAction.MERGE),+ ]+ assert [(slot.level, slot.action) for slot in entries["a.md"].index_slots] == [+ (2, IndexAction.MERGE)+ ]+++def test_rfv_semantic_plan_generates_complete_file_without_reusable_abstract():+ """A vectors-only L2 cannot satisfy a parent's first semantic aggregation."""+ from openviking.storage.context_update_plan import (+ SemanticAction,+ build_rfv_context_update_plan,+ )+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot++ root = "viking://resources/demo"+ snapshot = RFVSnapshot(+ RequestIntent(root, "semantic_and_vectors"),+ RFVFormalSnapshot(+ {+ "": RFVEntry(root, "", True, {}),+ "a.md": RFVEntry(f"{root}/a.md", "a.md", False, {2: "same"}),+ }+ ),+ VectorIndexSnapshot(+ {"a-l2": _record("a-l2", f"{root}/a.md", "a.md", 2, "same")},+ frozenset({"id", "uri", "level", "md5", "abstract"}),+ ),+ source_contents={(f"{root}/a.md", 2): b"current file body"},+ )++ _diff, plan = build_rfv_context_update_plan(+ snapshot=snapshot, context_type="resource", account_id="acc"+ )++ entries = {entry.relative_path: entry for entry in plan.semantic_plan.tree.entries}+ assert entries[""].semantic_action is SemanticAction.AGGREGATE+ assert entries["a.md"].semantic_action is SemanticAction.GENERATE+++def test_rfv_semantic_file_uses_minimal_plan_without_duplicate_direct_upsert():+ from openviking.storage.context_update_plan import build_rfv_context_update_plan+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot++ uri = "viking://resources/demo.md"+ snapshot = RFVSnapshot(+ RequestIntent(uri, "semantic_and_vectors", force=True),+ RFVFormalSnapshot({"": RFVEntry(uri, "", False, {2: "current"})}),+ VectorIndexSnapshot(+ {"file-l2": _record("file-l2", uri, "", 2, "current")},+ frozenset({"id", "uri", "level", "md5", "abstract"}),+ ),+ )++ _diff, plan = build_rfv_context_update_plan(+ snapshot=snapshot, context_type="resource", account_id="acc"+ )++ assert plan.semantic_plan is not None+ assert plan.direct_index_actions == ()+ assert plan.file_refresh is None+ entry = plan.semantic_plan.tree.entries[0]+ assert entry.semantic_action.value == "generate"+ assert entry.slot(2).record_id == "file-l2"+++def test_rfv_semantic_file_is_noop_when_fingerprint_and_scalars_are_complete():+ from openviking.storage.context_update_plan import build_rfv_context_update_plan+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot++ uri = "viking://resources/demo.md"+ snapshot = RFVSnapshot(+ RequestIntent(uri, "semantic_and_vectors"),+ RFVFormalSnapshot({"": RFVEntry(uri, "", False, {2: "same"})}),+ VectorIndexSnapshot(+ {"file-l2": _record("file-l2", uri, "", 2, "same")},+ frozenset({"id", "uri", "level", "md5", "abstract"}),+ ),+ )++ _diff, plan = build_rfv_context_update_plan(+ snapshot=snapshot, context_type="resource", account_id="acc"+ )++ assert plan.is_noop()+++def test_rfv_semantic_file_applies_scalar_only_update_without_refresh():+ from openviking.storage.context_update_plan import build_rfv_context_update_plan+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot+ from openviking.storage.resource_rnfv import ScalarIntent++ uri = "viking://resources/demo.md"+ snapshot = RFVSnapshot(+ RequestIntent(+ uri,+ "semantic_and_vectors",+ scalar_intents=(ScalarIntent("search_tags", "replace", ("team=search",)),),+ ),+ RFVFormalSnapshot({"": RFVEntry(uri, "", False, {2: "same"})}),+ VectorIndexSnapshot(+ {"file-l2": _record("file-l2", uri, "", 2, "same")},+ frozenset({"id", "uri", "level", "md5", "abstract", "search_tags"}),+ ),+ )++ _diff, plan = build_rfv_context_update_plan(+ snapshot=snapshot, context_type="resource", account_id="acc"+ )++ assert plan.file_refresh is None+ assert [action.action.value for action in plan.direct_index_actions] == ["update_fields"]+++def test_rfv_semantic_file_missing_l2_uses_minimal_plan_with_request_scalars():+ from openviking.storage.context_update_plan import build_rfv_context_update_plan+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot+ from openviking.storage.resource_rnfv import ScalarIntent++ uri = "viking://resources/demo.md"+ snapshot = RFVSnapshot(+ RequestIntent(+ uri,+ "semantic_and_vectors",+ scalar_intents=(ScalarIntent("search_tags", "replace", ("team=search",)),),+ ),+ RFVFormalSnapshot({"": RFVEntry(uri, "", False, {2: "same"})}),+ VectorIndexSnapshot(+ {},+ frozenset({"id", "uri", "level", "md5", "abstract", "search_tags"}),+ ),+ )++ _diff, plan = build_rfv_context_update_plan(+ snapshot=snapshot, context_type="resource", account_id="acc"+ )++ assert plan.file_refresh is None+ assert plan.semantic_plan is not None+ assert plan.direct_index_actions == ()+ entry = plan.semantic_plan.tree.entries[0]+ assert entry.semantic_action.value == "generate"+ assert entry.slot(2).field_patch.values == {"search_tags": ["team=search"]}+ assert entry.slot(2).field_patch.modes == {"search_tags": "replace"}+++def action_record_id(uri: str, level: int) -> str:+ from openviking.storage.vector_ids import vector_record_id++ return vector_record_id("acc", uri, level)+++class _SnapshotFS:+ def __init__(self, root: str):+ self.root = root+ self.tree = AsyncMock(+ return_value=[+ {"uri": f"{root}/docs", "rel_path": "docs", "isDir": True},+ {+ "uri": f"{root}/docs/.abstract.md",+ "rel_path": "docs/.abstract.md",+ "isDir": False,+ },+ {+ "uri": f"{root}/docs/.overview.md",+ "rel_path": "docs/.overview.md",+ "isDir": False,+ },+ {"uri": f"{root}/docs/a.md", "rel_path": "docs/a.md", "isDir": False},+ ]+ )+ self.stat = AsyncMock(return_value={"isDir": True})+ self.read_file_bytes = AsyncMock(+ side_effect=lambda uri, ctx=None: {+ f"{root}/docs/.abstract.md": b"docs abstract",+ f"{root}/docs/.overview.md": b"docs overview",+ f"{root}/docs/a.md": b"file body",+ }[uri]+ )+++class _SnapshotDB:+ def __init__(self):+ self.get_incremental_inventory_under_uri = AsyncMock(return_value={})+++async def test_build_rfv_snapshot_reads_one_f_inventory_one_v_inventory_and_each_source_once():+ from openviking.storage.resource_rfv import build_rfv_snapshot+ from openviking.utils.content_hash import content_md5++ root = "viking://resources/demo"+ fs = _SnapshotFS(root)+ db = _SnapshotDB()++ snapshot = await build_rfv_snapshot(+ viking_fs=fs,+ vikingdb=db,+ target_uri=root,+ ctx=object(),+ request_intent=RequestIntent(root, "vectors_only"),+ recursive=True,+ root_is_dir=True,+ )++ fs.tree.assert_awaited_once()+ fs.stat.assert_not_awaited()+ db.get_incremental_inventory_under_uri.assert_awaited_once()+ assert fs.read_file_bytes.await_count == 3+ assert snapshot.formal.entries["docs"].level_md5s == {+ 0: content_md5(b"docs abstract"),+ 1: content_md5(b"docs overview"),+ }+ assert snapshot.formal.entries["docs/a.md"].level_md5s == {2: content_md5(b"file body")}+ assert snapshot.source_contents == {+ (f"{root}/docs", 0): "docs abstract",+ (f"{root}/docs", 1): "docs overview",+ (f"{root}/docs/a.md", 2): b"file body",+ }+++async def test_build_rfv_snapshot_non_recursive_directory_reads_only_direct_children():+ from openviking.storage.resource_rfv import build_rfv_snapshot++ root = "viking://resources/demo"+ fs = _SnapshotFS(root)+ fs.ls = AsyncMock(+ return_value=[+ {"uri": f"{root}/docs", "name": "docs", "isDir": True},+ {"uri": f"{root}/a.md", "name": "a.md", "isDir": False},+ {"uri": f"{root}/.abstract.md", "name": ".abstract.md", "isDir": False},+ {"uri": f"{root}/.overview.md", "name": ".overview.md", "isDir": False},+ ]+ )+ db = _SnapshotDB()++ snapshot = await build_rfv_snapshot(+ viking_fs=fs,+ vikingdb=db,+ target_uri=root,+ ctx=object(),+ request_intent=RequestIntent(root, "vectors_only"),+ recursive=False,+ )++ fs.tree.assert_not_awaited()+ fs.ls.assert_awaited_once_with(root, node_limit=None, show_all_hidden=True, ctx=ANY)+ assert set(snapshot.formal.entries) == {"", "docs", "a.md"}+ assert snapshot.input_only_paths == frozenset({"docs", "a.md"})+ assert db.get_incremental_inventory_under_uri.await_args.kwargs["recursive"] is False+ assert (+ db.get_incremental_inventory_under_uri.await_args.kwargs["include_direct_children"] is True+ )+++async def test_vectors_only_force_plan_preserves_existing_file_summary():+ from openviking.storage.context_update_plan import (+ ContextUpdatePlan,+ IndexAction,+ build_rfv_context_update_plan,+ )+ from openviking.storage.resource_rfv import build_rfv_snapshot++ uri = "viking://resources/demo/a.md"+ fs = SimpleNamespace(+ stat=AsyncMock(return_value={"isDir": False}),+ read_file_bytes=AsyncMock(return_value=b"current body"),+ )+ db = SimpleNamespace(+ get_incremental_inventory_under_uri=AsyncMock(+ return_value={+ "a-l2": {+ "id": "a-l2",+ "uri": uri,+ "level": 2,+ "md5": "old-md5",+ "abstract": "existing summary",+ }+ }+ )+ )++ snapshot = await build_rfv_snapshot(+ viking_fs=fs,+ vikingdb=db,+ target_uri=uri,+ ctx=object(),+ request_intent=RequestIntent(uri, "vectors_only", force=True),+ recursive=False,+ )+ _, plan = build_rfv_context_update_plan(+ snapshot=snapshot,+ context_type="resource",+ account_id="acc",+ )++ assert db.get_incremental_inventory_under_uri.await_args.kwargs["output_fields"] == [+ "abstract",+ "id",+ "level",+ "md5",+ "uri",+ ]+ assert len(plan.direct_index_actions) == 1+ action = plan.direct_index_actions[0]+ assert action.action is IndexAction.MERGE+ assert action.record_id == "a-l2"+ assert action.summary == "existing summary"+ assert (+ ContextUpdatePlan.from_dict(plan.to_dict()).direct_index_actions[0].summary+ == "existing summary"+ )+++def test_vectors_only_missing_file_uses_parent_overview_summary():+ from openviking.storage.context_update_plan import IndexAction, build_rfv_context_update_plan+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot++ root = "viking://resources/demo"+ file_uri = f"{root}/a.md"+ snapshot = RFVSnapshot(+ request=RequestIntent(root, "vectors_only"),+ formal=RFVFormalSnapshot(+ {+ "": RFVEntry(root, "", True, {0: "root-l0", 1: "root-l1"}),+ "a.md": RFVEntry(file_uri, "a.md", False, {2: "a-md5"}),+ }+ ),+ vectors=VectorIndexSnapshot(+ {+ "root-l0": VectorRecordSnapshot(+ "root-l0", root, "", 0, {"md5": "root-l0", "abstract": "root abstract"}+ ),+ "root-l1": VectorRecordSnapshot(+ "root-l1", root, "", 1, {"md5": "root-l1", "abstract": "root overview"}+ ),+ },+ frozenset({"id", "uri", "level", "md5"}),+ ),+ source_contents={(root, 1): "### a.md\nCurrent overview summary."},+ )++ _, plan = build_rfv_context_update_plan(+ snapshot=snapshot,+ context_type="resource",+ account_id="acc",+ )++ assert len(plan.direct_index_actions) == 1+ action = plan.direct_index_actions[0]+ assert action.action is IndexAction.MERGE+ assert action.uri == file_uri+ assert action.summary == "Current overview summary."+++def test_vectors_only_reuses_one_overview_parse_per_parent(monkeypatch):+ from openviking.storage import context_update_plan+ from openviking.storage.context_update_plan import build_rfv_context_update_plan+ from openviking.storage.resource_rfv import RFVEntry, RFVFormalSnapshot, RFVSnapshot++ root = "viking://resources/demo"+ snapshot = RFVSnapshot(+ request=RequestIntent(root, "vectors_only"),+ formal=RFVFormalSnapshot(+ {+ "": RFVEntry(root, "", True, {0: "root-l0", 1: "root-l1"}),+ "a.md": RFVEntry(f"{root}/a.md", "a.md", False, {2: "a-md5"}),+ "b.md": RFVEntry(f"{root}/b.md", "b.md", False, {2: "b-md5"}),+ }+ ),+ vectors=VectorIndexSnapshot(+ {+ "root-l0": VectorRecordSnapshot("root-l0", root, "", 0, {"md5": "root-l0"}),+ "root-l1": VectorRecordSnapshot("root-l1", root, "", 1, {"md5": "root-l1"}),+ },+ frozenset({"id", "uri", "level", "md5"}),+ ),+ source_contents={(root, 1): "already loaded overview"},+ )+ calls = []++ def parse(overview):+ calls.append(overview)+ return {"a.md": "a summary", "b.md": "b summary"}++ monkeypatch.setattr(context_update_plan, "parse_overview_file_summaries", parse)++ _, plan = build_rfv_context_update_plan(+ snapshot=snapshot,+ context_type="resource",+ account_id="acc",+ )++ assert [action.summary for action in plan.direct_index_actions] == ["a summary", "b summary"]+ assert calls == ["already loaded overview"]+++async def test_non_recursive_semantic_plan_reuses_direct_child_summaries_without_rebuilding_them():+ from openviking.storage.context_update_plan import (+ SemanticAction,+ build_rfv_context_update_plan,+ )+ from openviking.storage.resource_rfv import build_rfv_snapshot++ root = "viking://resources/demo"++ class _OneLevelFS:+ stat = AsyncMock(return_value={"isDir": True})+ tree = AsyncMock(side_effect=AssertionError("non-recursive plan must not walk descendants"))+ ls = AsyncMock(+ return_value=[+ {"uri": f"{root}/a.md", "name": "a.md", "isDir": False},+ {"uri": f"{root}/child", "name": "child", "isDir": True},+ {"uri": f"{root}/.abstract.md", "name": ".abstract.md", "isDir": False},+ {"uri": f"{root}/.overview.md", "name": ".overview.md", "isDir": False},+ ]+ )+ read_file_bytes = AsyncMock(+ side_effect=lambda uri, ctx=None: {+ f"{root}/.abstract.md": b"root abstract",+ f"{root}/.overview.md": b"root overview",+ }[uri]+ )++ records = {+ "root-l0": {+ "id": "root-l0",+ "uri": root,+ "level": 0,+ "md5": "root-a",+ "abstract": "root abstract",+ },+ "root-l1": {+ "id": "root-l1",+ "uri": root,+ "level": 1,+ "md5": "root-o",+ "abstract": "root overview",+ },+ "a-l2": {+ "id": "a-l2",+ "uri": f"{root}/a.md",+ "level": 2,+ "md5": "a-md5",+ "abstract": "a summary",+ },+ "child-l0": {+ "id": "child-l0",+ "uri": f"{root}/child",+ "level": 0,+ "md5": "child-a",+ "abstract": "child abstract",+ },+ }+ db = SimpleNamespace(get_incremental_inventory_under_uri=AsyncMock(return_value=records))++ snapshot = await build_rfv_snapshot(+ viking_fs=_OneLevelFS(),+ vikingdb=db,+ target_uri=root,+ ctx=object(),+ request_intent=RequestIntent(root, "semantic_and_vectors", force=True),+ recursive=False,+ )+ _, plan = build_rfv_context_update_plan(+ snapshot=snapshot, context_type="resource", account_id="acc"+ )++ assert db.get_incremental_inventory_under_uri.await_args.kwargs == {+ "ctx": ANY,+ "output_fields": ["abstract", "id", "level", "md5", "uri"],+ "recursive": False,+ "include_direct_children": True,+ }+ entries = {entry.relative_path: entry for entry in plan.semantic_plan.tree.entries}+ assert entries[""].semantic_action is SemanticAction.AGGREGATE+ assert entries["a.md"].semantic_action is SemanticAction.REUSE+ assert entries["child"].semantic_action is SemanticAction.REUSE+ assert entries["a.md"].slot(2).record_id == "a-l2"+ assert entries["child"].slot(0).record_id == "child-l0"+++async def test_non_recursive_semantic_plan_rejects_direct_child_without_reusable_abstract():+ from openviking.storage.context_update_plan import build_rfv_context_update_plan+ from openviking.storage.resource_rfv import build_rfv_snapshot++ root = "viking://resources/demo"++ class _OneLevelFS:+ stat = AsyncMock(return_value={"isDir": True})+ ls = AsyncMock(+ return_value=[+ {"uri": f"{root}/child", "name": "child", "isDir": True},+ {"uri": f"{root}/.abstract.md", "name": ".abstract.md", "isDir": False},+ {"uri": f"{root}/.overview.md", "name": ".overview.md", "isDir": False},+ ]+ )+ read_file_bytes = AsyncMock(+ side_effect=lambda uri, ctx=None: {+ f"{root}/.abstract.md": b"root abstract",+ f"{root}/.overview.md": b"root overview",+ }[uri]+ )++ db = SimpleNamespace(+ get_incremental_inventory_under_uri=AsyncMock(+ return_value={+ "root-l0": {+ "id": "root-l0",+ "uri": root,+ "level": 0,+ "md5": "root-a",+ "abstract": "root abstract",+ },+ "root-l1": {+ "id": "root-l1",+ "uri": root,+ "level": 1,+ "md5": "root-o",+ "abstract": "root overview",+ },+ "child-l0": {+ "id": "child-l0",+ "uri": f"{root}/child",+ "level": 0,+ "md5": "child-a",+ "abstract": "",+ },+ }+ )+ )+ snapshot = await build_rfv_snapshot(+ viking_fs=_OneLevelFS(),+ vikingdb=db,+ target_uri=root,+ ctx=object(),+ request_intent=RequestIntent(root, "semantic_and_vectors", force=True),+ recursive=False,+ )++ with pytest.raises(+ ValueError, match="non-recursive semantic plan lacks reusable abstract for child"+ ):+ build_rfv_context_update_plan(snapshot=snapshot, context_type="resource", account_id="acc")+++async def test_non_recursive_plan_keeps_hidden_children_and_never_mutates_input_only_nodes():+ from openviking.storage.context_update_plan import build_rfv_context_update_plan+ from openviking.storage.resource_rfv import build_rfv_snapshot+ from openviking.utils.ingest_options import IngestOptions++ root = "viking://resources/demo"++ class _OneLevelFS:+ stat = AsyncMock(return_value={"isDir": True})+ ls = AsyncMock(+ return_value=[+ {"uri": f"{root}/.env", "name": ".env", "isDir": False},+ {"uri": f"{root}/a.md", "name": "a.md", "isDir": False},+ {"uri": f"{root}/.abstract.md", "name": ".abstract.md", "isDir": False},+ {"uri": f"{root}/.overview.md", "name": ".overview.md", "isDir": False},+ ]+ )+ read_file_bytes = AsyncMock(+ side_effect=lambda uri, ctx=None: {+ f"{root}/.abstract.md": b"root abstract",+ f"{root}/.overview.md": b"root overview",+ }[uri]+ )++ db = SimpleNamespace(+ get_incremental_inventory_under_uri=AsyncMock(+ return_value={+ "root-l0": {"id": "root-l0", "uri": root, "level": 0, "abstract": "old"},+ "root-l1": {"id": "root-l1", "uri": root, "level": 1, "abstract": "old"},+ "env-l2": {+ "id": "env-l2",+ "uri": f"{root}/.env",+ "level": 2,+ "md5": "old-env",+ "abstract": "env summary",+ "search_tags": ["team=old"],+ },+ "a-l2": {+ "id": "a-l2",+ "uri": f"{root}/a.md",+ "level": 2,+ "md5": "old-a",+ "abstract": "a summary",+ "search_tags": ["team=old"],+ },+ }+ )+ )+ request = RequestIntent.from_ingest_options(+ target_uri=root,+ processing_mode="semantic_and_vectors",+ force=True,+ ingest_options=IngestOptions.from_search_tags(["team=new"], mode="replace"),+ )++ snapshot = await build_rfv_snapshot(+ viking_fs=_OneLevelFS(),+ vikingdb=db,+ target_uri=root,+ ctx=object(),+ request_intent=request,+ recursive=False,+ )+ _, plan = build_rfv_context_update_plan(+ snapshot=snapshot,+ context_type="resource",+ account_id="acc",+ )++ assert {".env", "a.md"}.issubset(snapshot.formal.entries)+ assert snapshot.input_only_paths == frozenset({".env", "a.md"})+ assert {action.record_id for action in plan.direct_index_actions}.isdisjoint({"env-l2", "a-l2"})+++async def test_build_rfv_snapshot_keeps_dotfiles_and_excludes_control_files():+ from openviking.storage.resource_rfv import build_rfv_snapshot++ root = "viking://resources/demo"+ fs = _SnapshotFS(root)+ fs.tree.return_value = [+ {"uri": f"{root}/.env", "rel_path": ".env", "isDir": False},+ {"uri": f"{root}/.source.json", "rel_path": ".source.json", "isDir": False},+ {"uri": f"{root}/.path.ovlock", "rel_path": ".path.ovlock", "isDir": False},+ ]+ fs.read_file_bytes.side_effect = lambda uri, ctx=None: b"KEY=value"++ snapshot = await build_rfv_snapshot(+ viking_fs=fs,+ vikingdb=_SnapshotDB(),+ target_uri=root,+ ctx=object(),+ request_intent=RequestIntent(root, "vectors_only"),+ )++ assert set(snapshot.formal.entries) == {"", ".env"}+++async def test_build_rfv_snapshot_reads_skill_source_metadata_from_same_f_inventory():+ from openviking.storage.resource_rfv import build_rfv_snapshot++ root = "viking://user/alice/skills/demo"+ fs = _SnapshotFS(root)+ fs.tree.return_value = [+ {"uri": f"{root}/.source.json", "rel_path": ".source.json", "isDir": False}+ ]+ fs.read_file_bytes.side_effect = lambda uri, ctx=None: (+ b'{"type":"git","source":"https://example/repo"}'+ )++ snapshot = await build_rfv_snapshot(+ viking_fs=fs,+ vikingdb=_SnapshotDB(),+ target_uri=root,+ ctx=object(),+ request_intent=RequestIntent(root, "semantic_and_vectors"),+ )++ assert snapshot.source_metadata == {+ "kind": "git",+ "uri": "https://example/repo",+ "path": "https://example/repo",+ }+ assert ".source.json" not in snapshot.formal.entries+++def test_rfv_resolver_rejects_formal_entry_outside_request_scope():+ from openviking.storage.resource_rfv import (+ RFVEntry,+ RFVFormalSnapshot,+ RFVSnapshot,+ resolve_rfv_state,+ )++ root = "viking://resources/demo"+ snapshot = RFVSnapshot(+ RequestIntent(root, "vectors_only"),+ RFVFormalSnapshot(+ {+ "outside.md": RFVEntry(+ "viking://resources/other.md",+ "outside.md",+ False,+ {2: "fingerprint"},+ )+ }+ ),+ VectorIndexSnapshot({}, frozenset({"id", "uri", "level", "md5"})),+ )++ with pytest.raises(ValueError, match="outside target"):+ resolve_rfv_state(snapshot)+++async def test_build_rfv_snapshot_rejects_denied_reused_formal_inventory():+ from openviking.storage.resource_rfv import build_rfv_snapshot++ root = "viking://resources/demo"+ fs = _SnapshotFS(root)++ with pytest.raises(ValueError, match="incomplete formal tree"):+ await build_rfv_snapshot(+ viking_fs=fs,+ vikingdb=_SnapshotDB(),+ target_uri=root,+ ctx=object(),+ request_intent=RequestIntent(root, "vectors_only"),+ root_is_dir=True,+ formal_inventory=(+ True,+ [+ {+ "uri": f"{root}/private.md",+ "isDir": False,+ "access": "denied",+ }+ ],+ True,+ ),+ )diff --git a/tests/storage/test_semantic_executor_incremental.py b/tests/storage/test_semantic_executor_incremental.pyindex 84ddb057d..74213eb4b 100644--- a/tests/storage/test_semantic_executor_incremental.py+++ b/tests/storage/test_semantic_executor_incremental.py@@ -138,15 +138,19 @@ class _FakeProcessor:ingest_options=None,file_md5=None,file_content=None,+ materialize_content=False,scalar_override=None,action="merge",+ field_patch=None,):self.vectorized_files.append(file_path)self.file_ingest_options[file_path] = ingest_optionsself.file_md5s[file_path] = file_md5self.file_contents[("vector", file_path)] = file_content+ self.file_contents[("materialize_content", file_path)] = materialize_contentself.file_contents[("action", file_path)] = actionself.file_contents[("scalar_override", file_path)] = scalar_override+ self.file_contents[("field_patch", file_path)] = field_patchreturn Trueasync def _vectorize_directory(@@ -157,15 +161,20 @@ class _FakeProcessor:overview,ctx=None,ingest_options=None,+ skill_source_path="",scalar_overrides=None,actions=None,+ field_patches=None,include_abstract=True,include_overview=True,+ md5s=None,):self.directory_ingest_options[uri] = ingest_optionsself.vectorized_dirs.append(uri)self.file_contents[("actions", uri)] = actionsself.file_contents[("scalar_overrides", uri)] = scalar_overrides+ self.file_contents[("field_patches", uri)] = field_patches+ self.file_contents[("md5s", uri)] = md5sreturn {level for level, included in ((0, include_abstract), (1, include_overview)) if included}@@ -211,6 +220,172 @@ class _FakeProcessor:)+async def test_file_root_semantic_plan_runs_file_node_without_listing_or_sidecars(monkeypatch):+ from openviking.storage.context_update_plan import (+ IndexSlot,+ SemanticPlan,+ SemanticTreeEntry,+ SemanticTreeSnapshot,+ )++ file_uri = "viking://resources/report.md"+ fake_fs = _FakeVikingFS(tree={}, file_contents={file_uri: b"report body"})+ fake_fs.ls = AsyncMock(side_effect=AssertionError("semantic plan must not list file root"))+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs+ )+ processor = _FakeProcessor(fake_fs)+ plan = SemanticPlan(+ file_uri,+ "resource",+ SemanticTreeSnapshot(+ (+ SemanticTreeEntry(+ "",+ "file",+ "modified",+ "generate",+ md5="fresh-md5",+ index_slots=(IndexSlot(2, "report-l2", action="upsert"),),+ ),+ )+ ),+ )+ executor = SemanticTreeExecutor(+ processor=processor,+ context_type="resource",+ max_concurrent_llm=1,+ ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),+ semantic_plan=plan,+ )++ await executor.run(file_uri)++ assert processor.summarized_files == [file_uri]+ assert processor.vectorized_files == [file_uri]+ assert fake_fs.writes == []+ assert executor.get_stats().indexed_records == 1+++async def test_semantic_plan_repair_regenerates_and_reembeds_directory(monkeypatch):+ from openviking.storage.context_update_plan import (+ IndexSlot,+ SemanticPlan,+ SemanticTreeEntry,+ SemanticTreeSnapshot,+ )+ from openviking.utils.content_hash import content_md5++ root_uri = "viking://resources/root"+ fake_fs = _FakeVikingFS(tree={root_uri: []}, file_contents={})+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs+ )+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_executor.get_openviking_config",+ lambda: SimpleNamespace(semantic=SimpleNamespace(overview_sample_limit=32)),+ )+ processor = _FakeProcessor(fake_fs)+ plan = SemanticPlan(+ root_uri,+ "resource",+ SemanticTreeSnapshot(+ (+ SemanticTreeEntry(+ "",+ "directory",+ "unchanged",+ "aggregate",+ index_slots=(+ IndexSlot(0, "root-l0", {"abstract": "stale abstract"}, action="upsert"),+ IndexSlot(1, "root-l1", {"abstract": "stale overview"}, action="upsert"),+ ),+ repair=True,+ ),+ )+ ),+ )+ executor = SemanticTreeExecutor(+ processor=processor,+ context_type="resource",+ max_concurrent_llm=1,+ ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),+ semantic_plan=plan,+ generation_trigger="reindex",+ source_contents={+ (root_uri, 0): "current abstract",+ (root_uri, 1): "current overview",+ },+ )++ await executor.run(root_uri)++ assert processor.generated_overviews == [root_uri]+ assert processor.vectorized_dirs == [root_uri]+ assert processor.file_contents[("md5s", root_uri)] == {+ 0: content_md5(b"abstract"),+ 1: content_md5(b"FILES:"),+ }+++async def test_skill_plan_repair_regenerates_and_reembeds_skill_root(monkeypatch):+ from openviking.storage.context_update_plan import (+ IndexSlot,+ SemanticPlan,+ SemanticTreeEntry,+ SemanticTreeSnapshot,+ )++ root_uri = "viking://user/alice/skills/demo"+ fake_fs = _FakeVikingFS(tree={root_uri: []}, file_contents={})+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs+ )+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_executor.get_openviking_config",+ lambda: SimpleNamespace(semantic=SimpleNamespace(overview_sample_limit=32)),+ )+ processor = _FakeProcessor(fake_fs)+ processor._skill_root_semantics = AsyncMock(return_value=("skill overview", "skill abstract"))+ plan = SemanticPlan(+ root_uri,+ "skill",+ SemanticTreeSnapshot(+ (+ SemanticTreeEntry(+ "",+ "directory",+ "unchanged",+ "aggregate",+ index_slots=(+ IndexSlot(0, "skill-l0", {"abstract": "old"}, action="upsert"),+ IndexSlot(1, "skill-l1", {"abstract": "old"}, action="upsert"),+ ),+ repair=True,+ ),+ )+ ),+ )+ executor = SemanticTreeExecutor(+ processor=processor,+ context_type="skill",+ max_concurrent_llm=1,+ ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),+ semantic_plan=plan,+ generation_trigger="content_write",+ source_contents={(f"{root_uri}/SKILL.md", 2): b"name: demo"},+ )++ await executor.run(root_uri)++ processor._skill_root_semantics.assert_awaited_once()+ assert processor._skill_root_semantics.await_args.kwargs["regenerate"] is True+ assert processor.vectorized_dirs == [root_uri]++@pytest.mark.asyncio@pytest.mark.parametrize("sidecar_state", ["valid", "malformed_overview", "malformed_abstract"])async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypatch, sidecar_state):diff --git a/tests/storage/test_semantic_processor_lock_ownership.py b/tests/storage/test_semantic_processor_lock_ownership.pyindex ded8fd46f..322144086 100644--- a/tests/storage/test_semantic_processor_lock_ownership.py+++ b/tests/storage/test_semantic_processor_lock_ownership.py@@ -9,8 +9,11 @@ from unittest.mock import AsyncMockimport pytestfrom openviking.storage.queuefs.process_result import ProcessOutcome+from openviking.storage.queuefs.semantic_executor import SemanticTreeStats+from openviking.storage.queuefs.semantic_lock import SemanticLockScopefrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor+from openviking.storage.queuefs.semantic_work import SemanticMessageWorkdef _processor():@@ -68,6 +71,73 @@ async def test_memory_semantic_directory_does_not_release_borrowed_lock(monkeypaassert pathlock.release_calls == []+async def test_durable_plan_waits_for_embeddings_before_releasing_handoff_lock(monkeypatch):+ from openviking.storage.context_update_plan import (+ SemanticPlan,+ SemanticTreeEntry,+ SemanticTreeSnapshot,+ )++ events = []+ pathlock = _FakePathLock()+ lease = {"id": "durable-plan-lock"}+ plan = SemanticPlan(+ "viking://resources/demo",+ "resource",+ SemanticTreeSnapshot((SemanticTreeEntry("", "directory", "unchanged", "aggregate"),)),+ )+ msg = SemanticMsg(+ uri=plan.root_uri,+ context_type="resource",+ telemetry_id="durable-plan",+ lock_handoff={"owner_id": "producer"},+ plan=plan,+ )++ class Tracker:+ async def wait_for_embeddings(self, telemetry_id, **kwargs):+ assert telemetry_id == msg.telemetry_id+ events.append("embeddings-settled")++ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_work.get_request_wait_tracker", lambda: Tracker()+ )+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_lock.get_viking_fs",+ lambda: _FakeVikingFS(pathlock),+ )++ work = SemanticMessageWork(SimpleNamespace(), msg, caller_lock=None)+ work.scope = SemanticLockScope(lease, _owned=True)++ await work.finish_processing(True)++ assert events == ["embeddings-settled"]+ assert pathlock.release_calls == ["durable-plan-lock"]+++def test_semantic_tree_stats_aggregate_multiple_plans_for_one_request():+ telemetry_id = "reindex-request-stats"+ SemanticProcessor._cache_tree_stats(+ telemetry_id,+ "viking://resources/one",+ SemanticTreeStats(total_nodes=2, done_nodes=2, indexed_records=2, failures=["one failed"]),+ )+ SemanticProcessor._cache_tree_stats(+ telemetry_id,+ "viking://resources/two",+ SemanticTreeStats(total_nodes=3, done_nodes=3, indexed_records=3, failures=["two failed"]),+ )++ stats = SemanticProcessor.consume_tree_stats(telemetry_id=telemetry_id)++ assert stats.total_nodes == 5+ assert stats.done_nodes == 5+ assert stats.indexed_records == 5+ assert stats.failures == ["one failed", "two failed"]++@pytest.mark.asyncio@pytest.mark.parametrize("context_type", ["resource", "memory", "skill"])async def test_missing_root_is_acked_before_lock_scope_is_resolved(monkeypatch, context_type):diff --git a/tests/test_config_loader.py b/tests/test_config_loader.pyindex c6cefcce5..bf5042242 100644--- a/tests/test_config_loader.py+++ b/tests/test_config_loader.py@@ -130,6 +130,7 @@ def test_runtime_concurrency_uses_scope_specific_defaults():assert config.queue_workers.external_parse.max_concurrent == 4assert config.queue_workers.add_resource.max_concurrent == 4+ assert config.queue_workers.reindex.max_concurrent == 4assert config.queue_workers.add_resource.file_operation_concurrency == 16assert config.queue_workers.add_resource.file_vectorization_concurrency == 8assert config.queue_workers.session_commit.max_concurrent == 8@@ -154,6 +155,7 @@ def test_runtime_concurrency_accepts_separate_values():"file_operation_concurrency": 20,"file_vectorization_concurrency": 12,},+ "reindex": {"max_concurrent": 6},"session_commit": {"max_concurrent": 50},"external_task": {"max_concurrent": 11},},@@ -163,6 +165,7 @@ def test_runtime_concurrency_accepts_separate_values():assert config.queue_workers.external_parse.max_concurrent == 9assert config.queue_workers.add_resource.max_concurrent == 7+ assert config.queue_workers.reindex.max_concurrent == 6assert config.queue_workers.add_resource.file_operation_concurrency == 20assert config.queue_workers.add_resource.file_vectorization_concurrency == 12assert config.queue_workers.session_commit.max_concurrent == 50@@ -173,7 +176,7 @@ def test_runtime_concurrency_accepts_separate_values():@pytest.mark.parametrize("value", [0, -1])def test_queue_worker_concurrency_rejects_non_positive_value(value):with pytest.raises(ValueError) as exc_info:- QueueWorkersConfig(add_resource={"max_concurrent": value})+ QueueWorkersConfig(reindex={"max_concurrent": value})assert exc_info.value.errors()[0]["type"] == "greater_than"diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.pyindex e64b7263c..4af2395f0 100644--- a/tests/unit/test_vectorize_file_strategy.py+++ b/tests/unit/test_vectorize_file_strategy.py@@ -325,6 +325,33 @@ async def test_vectorize_image_downsamples_large_embedding_input(monkeypatch):assert fs.read_file_bytes_calls == 1+async def test_vectorize_image_uses_supplied_bytes_without_second_read(monkeypatch):+ queue = DummyQueue()+ original = _jpeg_bytes(20, 20)+ fs = DummyFS(b"must not be read")+ monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))+ monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs)+ monkeypatch.setattr(+ embedding_utils,+ "get_openviking_config",+ lambda: types.SimpleNamespace(+ embedding=types.SimpleNamespace(text_source="content_only", max_input_tokens=1000),+ image=ImageConfig(preview_max_dimension=64, max_file_size_mb=100.0),+ ),+ )++ await embedding_utils.vectorize_file(+ file_path="viking://resources/docs/image.jpg",+ summary_dict={"name": "image.jpg", "summary": "image"},+ parent_uri="viking://resources/docs",+ ctx=DummyReq(),+ file_content=original,+ )++ assert fs.read_file_bytes_calls == 0++@pytest.mark.asyncioasync def test_vectorize_file_registers_request_wait_with_embedding_msg_id(monkeypatch):queue = DummyQueueWithId()@@ -678,6 +705,26 @@ async def test_vectorize_directory_meta_applies_merge_per_level(monkeypatch):assert queue.items[1].action.value == "upsert"+async def test_vectorize_directory_meta_writes_per_level_source_md5(monkeypatch):+ queue = DummyQueue()+ monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))+ monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: DummyFS("ignored"))++ await embedding_utils.vectorize_directory_meta(+ uri="viking://user/default/resources/demo",+ abstract="demo abstract",+ overview="demo overview",+ ctx=DummyReq(),+ md5s={0: "abstract-md5", 1: "overview-md5"},+ )++ assert [item.context_data["md5"] for item in queue.items] == [+ "abstract-md5",+ "overview-md5",+ ]++@pytest.mark.asyncioasync def test_vectorize_directory_meta_l1_abstract_is_overview(monkeypatch):"""L1 records must carry the overview in the abstract scalar so Rerank