Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(tasks): support task cancellation (#3577)
Qin Haojiefd42b1ad92c9
diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rsindex 26165efac..b487d5a4a 100644--- a/crates/ov_cli/src/client.rs+++ b/crates/ov_cli/src/client.rs@@ -1176,6 +1176,11 @@ impl HttpClient {self.get(&path, &[]).await}+ pub async fn cancel_task(&self, task_id: &str) -> Result<serde_json::Value> {+ let path = format!("/api/v1/tasks/{}/cancel", task_id);+ self.post(&path, &serde_json::json!({})).await+ }+pub async fn list_tasks(&self,task_type: Option<&str>,diff --git a/crates/ov_cli/src/commands/task.rs b/crates/ov_cli/src/commands/task.rsindex 45f77132b..d809fa8eb 100644--- a/crates/ov_cli/src/commands/task.rs+++ b/crates/ov_cli/src/commands/task.rs@@ -13,6 +13,17 @@ pub async fn status(Ok(())}+pub async fn cancel(+ client: &HttpClient,+ task_id: &str,+ output_format: OutputFormat,+ compact: bool,+) -> Result<()> {+ let result = client.cancel_task(task_id).await?;+ output_success(&result, output_format, compact);+ Ok(())+}+pub async fn list(client: &HttpClient,task_type: Option<&str>,diff --git a/crates/ov_cli/src/error_ui.rs b/crates/ov_cli/src/error_ui.rsindex 3ae263fc0..94227a427 100644--- a/crates/ov_cli/src/error_ui.rs+++ b/crates/ov_cli/src/error_ui.rs@@ -511,7 +511,7 @@ fn task_help_command(program: &str, tokens: &[&str]) -> String {}_ => format!("{program} task watch --help"),},- Some("status" | "list") => format!("{program} task {} --help", tokens[2]),+ Some("status" | "cancel" | "list") => format!("{program} task {} --help", tokens[2]),_ => format!("{program} task --help"),}}diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rsindex 97d3a089b..b35115dd5 100644--- a/crates/ov_cli/src/help_ui.rs+++ b/crates/ov_cli/src/help_ui.rs@@ -916,6 +916,10 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[label: "ov task status <task-id>",description: "Inspect one task.",},+ HelpItem {+ label: "ov task cancel <task-id>",+ description: "Cancel one task.",+ },],next_steps: &[HelpItem {label: "ov wait",@@ -3119,6 +3123,7 @@ mod tests {.expect("task help should render"),);assert!(task.contains("status <task-id>"));+ assert!(task.contains("cancel <task-id>"));assert!(task.contains("list"));}diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rsindex b21371f5a..1cdbeb746 100644--- a/crates/ov_cli/src/main.rs+++ b/crates/ov_cli/src/main.rs@@ -1161,12 +1161,18 @@ enum TaskCommands {#[arg(value_name = "task-id")]task_id: String,},+ /// Cancel a task+ Cancel {+ /// Task ID returned by add-resource/add-skill+ #[arg(value_name = "task-id")]+ task_id: String,+ },/// List all tracked tasksList {/// Filter by task type (e.g. add_resource, add_skill, session_commit, reindex)#[arg(long, value_name = "type")]task_type: Option<String>,- /// Filter by status (pending, running, completed, failed)+ /// Filter by status (pending, running, cancelling, completed, failed, cancelled)#[arg(long, value_name = "status")]status: Option<String>,},@@ -2272,7 +2278,7 @@ fn command_tokens_for_config_gate(args: &[OsString]) -> Vec<String> {fn known_task_command_requires_config(tokens: &[String]) -> bool {match tokens.get(1).map(String::as_str) {- Some("status" | "list") => true,+ Some("status" | "cancel" | "list") => true,Some("watch") => match tokens.get(2).map(String::as_str) {None => true,Some(token) => is_watch_subcommand(token),@@ -3094,6 +3100,10 @@ async fn main() {let client = ctx.get_client();commands::task::status(&client, &task_id, ctx.output_format, ctx.compact).await}+ TaskCommands::Cancel { task_id } => {+ let client = ctx.get_client();+ commands::task::cancel(&client, &task_id, ctx.output_format, ctx.compact).await+ }TaskCommands::List { task_type, status } => {let client = ctx.get_client();commands::task::list(@@ -3840,6 +3850,7 @@ mod tests {for args in [&["ov", "find"][..],&["ov", "task", "status"],+ &["ov", "task", "cancel"],&["ov", "task", "watch", "show"],&["ov", "config", "validate"],&["ov", "config", "show"],@@ -3902,6 +3913,7 @@ mod tests {&["ov", "config", "show"],&["ov", "config", "validate"],&["ov", "task", "status"],+ &["ov", "task", "cancel"],&["ov", "task", "list"],&["ov", "task", "watch"],&["ov", "task", "watch", "ls"],diff --git a/crates/ov_cli/src/output.rs b/crates/ov_cli/src/output.rsindex 85c3e62fd..42d98bb34 100644--- a/crates/ov_cli/src/output.rs+++ b/crates/ov_cli/src/output.rs@@ -919,7 +919,7 @@ fn table_value_tone(trimmed: &str) -> TableValueTone {TableValueTone::Success}"running" | "in_progress" | "in-progress" | "pending" | "queued" | "processing"- | "checking" | "warning" => TableValueTone::Warning,+ | "cancelling" | "checking" | "warning" => TableValueTone::Warning,"unhealthy" | "failed" | "error" | "false" | "cancelled" | "canceled" | "timeout"| "timed_out" | "unreachable" => TableValueTone::Error,"unknown" | "null" | "(empty)" => TableValueTone::Muted,@@ -1045,6 +1045,7 @@ mod tests {assert_eq!(table_value_tone("running"), TableValueTone::Warning);assert_eq!(table_value_tone("pending"), TableValueTone::Warning);+ assert_eq!(table_value_tone("cancelling"), TableValueTone::Warning);assert_eq!(table_value_tone("queued"), TableValueTone::Warning);assert_eq!(table_value_tone("processing"), TableValueTone::Warning);diff --git a/crates/ragfs/src/plugins/queuefs/backend.rs b/crates/ragfs/src/plugins/queuefs/backend.rsindex 13e88bf01..12f40f6d3 100644--- a/crates/ragfs/src/plugins/queuefs/backend.rs+++ b/crates/ragfs/src/plugins/queuefs/backend.rs@@ -161,6 +161,9 @@ pub trait QueueBackend: Send + Sync {/// Get the number of messages in the queuefn size(&self, queue_name: &str) -> Result<usize>;+ /// List every unacknowledged message without changing queue state+ fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>>;+/// Clear all messages from the queuefn clear(&mut self, queue_name: &str) -> Result<()>;@@ -289,6 +292,14 @@ impl QueueBackend for MemoryBackend {Ok(queue.messages.len())}+ fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {+ let queue = self+ .queues+ .get(queue_name)+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;+ Ok(queue.messages.iter().cloned().collect())+ }+fn clear(&mut self, queue_name: &str) -> Result<()> {let queue = self.queues@@ -691,6 +702,34 @@ impl QueueBackend for SQLiteQueueBackend {Ok(count as usize)}+ fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;++ Self::require_queue_exists(&conn, queue_name)?;+ let mut stmt = conn+ .prepare(+ "SELECT data FROM queue_messages+ WHERE queue_name = ?1 AND status IN ('pending', 'processing')+ ORDER BY id",+ )+ .map_err(|e| Error::internal(format!("sqlite list unacked prepare error: {}", e)))?;+ let rows = stmt+ .query_map(params![queue_name], |row| read_sqlite_text(row, 0))+ .map_err(|e| Error::internal(format!("sqlite list unacked query error: {}", e)))?;++ let mut messages = Vec::new();+ for row in rows {+ let raw_data =+ row.map_err(|e| Error::internal(format!("sqlite list unacked row error: {}", e)))?;+ let stored: StoredMessage = serde_json::from_str(&raw_data)?;+ messages.push(stored.into_message());+ }+ Ok(messages)+ }+fn clear(&mut self, queue_name: &str) -> Result<()> {let conn = self.conn@@ -920,6 +959,9 @@ mod tests {let first = backend.dequeue("test").unwrap().unwrap();assert_eq!(first.data, b"message 1");assert_eq!(backend.size("test").unwrap(), 1);+ let unacked = backend.list_unacked("test").unwrap();+ assert_eq!(unacked.len(), 2);+ assert_eq!(unacked[0].id, msg1_id);assert!(backend.ack("test", &msg1_id).unwrap());let second = backend.dequeue("test").unwrap().unwrap();diff --git a/crates/ragfs/src/plugins/queuefs/mod.rs b/crates/ragfs/src/plugins/queuefs/mod.rsindex 12724272a..dc9cc88e0 100644--- a/crates/ragfs/src/plugins/queuefs/mod.rs+++ b/crates/ragfs/src/plugins/queuefs/mod.rs@@ -6,6 +6,7 @@//! - `/queue_name/dequeue` - Read from this file to remove and return the first message//! - `/queue_name/peek` - Read from this file to view the first message without removing it//! - `/queue_name/size` - Read from this file to get the current queue size+//! - `/queue_name/messages` - Read all unacknowledged messages without changing queue state//! - `/queue_name/clear` - Write to this file to clear all messages from the queue//! - `/queue_name/ack` - Write message ID to this file to acknowledge and delete it@@ -48,6 +49,10 @@ const CONTROL_FILES: &[ControlFileSpec] = &[name: "size",mode: 0o444,},+ ControlFileSpec {+ name: "messages",+ mode: 0o444,+ },ControlFileSpec {name: "clear",mode: 0o222,@@ -251,8 +256,19 @@ impl FileSystem for QueueFileSystem {let size = backend.size(&queue_name)?;Ok(size.to_string().into_bytes())}+ "messages" => {+ let messages = backend+ .list_unacked(&queue_name)?+ .into_iter()+ .map(|msg| QueueMessage {+ id: msg.id,+ data: String::from_utf8_lossy(&msg.data).to_string(),+ })+ .collect::<Vec<_>>();+ Ok(serde_json::to_vec(&messages)?)+ }_ => Err(Error::InvalidOperation(format!(- "Cannot read from '{}'. Use dequeue, peek, or size",+ "Cannot read from '{}'. Use dequeue, peek, size, or messages",operation))),}@@ -780,13 +796,14 @@ mod tests {// Queue directory should list control fileslet entries = fs.read_dir("/test").await.unwrap();- assert_eq!(entries.len(), 6);+ assert_eq!(entries.len(), 7);let names: Vec<String> = entries.iter().map(|e| e.name.clone()).collect();assert!(names.contains(&"enqueue".to_string()));assert!(names.contains(&"dequeue".to_string()));assert!(names.contains(&"peek".to_string()));assert!(names.contains(&"size".to_string()));+ assert!(names.contains(&"messages".to_string()));assert!(names.contains(&"clear".to_string()));assert!(names.contains(&"ack".to_string()));}diff --git a/docs/en/api/01-overview.md b/docs/en/api/01-overview.mdindex 72c5cb385..abef9d5ce 100644--- a/docs/en/api/01-overview.md+++ b/docs/en/api/01-overview.md@@ -518,6 +518,7 @@ This catalog follows the routes actually mounted by the server. Each group headi| Method | Path | Description ||--------|------|-------------|| GET | `/api/v1/tasks/{task_id}` | Get a background task |+| POST | `/api/v1/tasks/{task_id}/cancel` | Cancel a background task || GET | `/api/v1/tasks` | List background tasks || GET | `/api/v1/observer/queue` | Queue status || GET | `/api/v1/observer/vikingdb` | VikingDB status |diff --git a/docs/en/api/17-tasks.md b/docs/en/api/17-tasks.mdindex 39472fd6b..6d84835d4 100644--- a/docs/en/api/17-tasks.md+++ b/docs/en/api/17-tasks.md@@ -13,8 +13,10 @@ Query background task status for APIs that return `task_id`, such as session com**Task Statuses:**- `pending`: Task waiting to execute- `running`: Task in progress+- `cancelling`: Cancellation requested; waiting for the task's durable queue messages and in-process work to settle- `completed`: Task successfully completed- `failed`: Task failed+- `cancelled`: Task cancelled**Code Entries:**- `openviking/server/routers/tasks.py:get_task()` - HTTP route@@ -138,6 +140,102 @@ ov task status uuid-xxx---+### cancel_task()++#### 1. API Implementation Introduction++Request cooperative cancellation of a background task. The operation immediately prevents the task from creating new QueueFS work and cancels its active in-process work; writes that already completed are not rolled back. If durable messages or in-process work remain, the operation first returns `cancelling`. The task becomes `cancelled` only after all owned work settles.++Repeated cancellation of a task in `cancelling` or `cancelled` is idempotent.++**Supported Task Types:**+- `add_resource`+- `session_commit`+- `admin_reindex`+- `snapshot_restore_reindex`++**Code Entries:**+- `openviking/server/routers/tasks.py:cancel_task()` - HTTP route+- `openviking/service/task_tracker.py:TaskTracker.cancel()` - task lifecycle+- `crates/ov_cli/src/commands/task.rs:cancel()` - CLI command++#### 2. Interface and Parameter Description++| Parameter | Type | Required | Default | Description |+|-----------|------|----------|---------|-------------|+| task_id | str | Yes | - | Background task ID to cancel |++Only the current user who owns the task can cancel it. ROOT identities cannot cancel tasks.++#### 3. Usage Examples++**Python SDK**++```python+task = await client.cancel_task("uuid-xxx")+print(task["status"])+```++**TypeScript SDK**++```typescript+const task = await client.cancelTask("uuid-xxx");+console.log(task.status);+```++**Go SDK**++```go+task, err := client.CancelTask(ctx, "uuid-xxx")+if err != nil {+ return err+}+fmt.Println(task["status"])+```++**HTTP API**++```http+POST /api/v1/tasks/{task_id}/cancel+```++```bash+curl -X POST http://localhost:1933/api/v1/tasks/uuid-xxx/cancel \+ -H "X-API-Key: your-key"+```++**CLI**++```bash+ov task cancel uuid-xxx+```++**Response Example**++```json+{+ "status": "ok",+ "result": {+ "task_id": "uuid-xxx",+ "task_type": "add_resource",+ "status": "cancelling",+ "resource_id": "viking://resources/guide",+ "stage": "processing_queue",+ "result": null,+ "error": null+ }+}+```++If the task has no remaining work, the response status can be `cancelled` immediately. Otherwise, continue polling with `get_task()` until the status becomes `cancelled`.++**Error Handling:**+- `NOT_FOUND` (404): the task does not exist, has expired, or belongs to another user+- `PERMISSION_DENIED` (403): a ROOT identity attempts to cancel a task+- `FAILED_PRECONDITION` (412): the task type does not support cancellation, or the task is already `completed`/`failed`++---+### list_tasks()#### 1. API Implementation Introduction@@ -155,7 +253,7 @@ List background tasks visible to the current caller, supporting filtering by typ| Parameter | Type | Required | Default | Description ||-----------|------|----------|---------|-------------|| task_type | str | No | None | Filter by task type, for example `session_commit` |-| status | str | No | None | Filter by task status: `pending`, `running`, `completed`, `failed` |+| status | str | No | None | Filter by task status: `pending`, `running`, `cancelling`, `completed`, `failed`, `cancelled` || resource_id | str | No | None | Filter by task resource ID, for example a session ID || limit | int | No | 50 | Maximum number of task records to return |diff --git a/docs/zh/api/01-overview.md b/docs/zh/api/01-overview.mdindex 72574eee3..7bd68e349 100644--- a/docs/zh/api/01-overview.md+++ b/docs/zh/api/01-overview.md@@ -513,6 +513,7 @@ JSON 输出 - 错误:| 方法 | 路径 | 说明 ||------|------|------|| GET | `/api/v1/tasks/{task_id}` | 获取后台任务 |+| POST | `/api/v1/tasks/{task_id}/cancel` | 取消后台任务 || GET | `/api/v1/tasks` | 列出后台任务 || GET | `/api/v1/observer/queue` | 队列状态 || GET | `/api/v1/observer/vikingdb` | VikingDB 状态 |diff --git a/docs/zh/api/17-tasks.md b/docs/zh/api/17-tasks.mdindex 37dff6130..1726c478d 100644--- a/docs/zh/api/17-tasks.md+++ b/docs/zh/api/17-tasks.md@@ -13,8 +13,10 @@**任务状态**:- `pending`: 任务等待执行- `running`: 任务执行中+- `cancelling`: 已请求取消,正在等待该任务的持久化队列消息和进程内工作结束- `completed`: 任务成功完成- `failed`: 任务失败+- `cancelled`: 任务已取消**代码入口**:- `openviking/server/routers/tasks.py:get_task()` - HTTP 路由@@ -136,6 +138,102 @@ ov task status uuid-xxx---+### cancel_task()++#### 1. API 实现介绍++请求协作式取消后台任务。接口会立即阻止该任务产生新的 QueueFS work,并取消仍在运行的进程内工作;已经完成的写入不会回滚。当任务仍有待处理的持久化消息或进程内工作时,接口先返回 `cancelling`,全部收敛后任务才进入 `cancelled`。++重复取消处于 `cancelling` 或 `cancelled` 状态的任务是幂等的。++**支持的任务类型**:+- `add_resource`+- `session_commit`+- `admin_reindex`+- `snapshot_restore_reindex`++**代码入口**:+- `openviking/server/routers/tasks.py:cancel_task()` - HTTP 路由+- `openviking/service/task_tracker.py:TaskTracker.cancel()` - 任务生命周期+- `crates/ov_cli/src/commands/task.rs:cancel()` - CLI 命令++#### 2. 接口和参数说明++| 参数 | 类型 | 必填 | 默认值 | 说明 |+|------|------|------|--------|------|+| task_id | str | 是 | - | 要取消的后台任务 ID |++只有任务所属的当前用户可以取消任务。ROOT 身份不能执行取消操作。++#### 3. 使用示例++**Python SDK**++```python+task = await client.cancel_task("uuid-xxx")+print(task["status"])+```++**TypeScript SDK**++```typescript+const task = await client.cancelTask("uuid-xxx");+console.log(task.status);+```++**Go SDK**++```go+task, err := client.CancelTask(ctx, "uuid-xxx")+if err != nil {+ return err+}+fmt.Println(task["status"])+```++**HTTP API**++```http+POST /api/v1/tasks/{task_id}/cancel+```++```bash+curl -X POST http://localhost:1933/api/v1/tasks/uuid-xxx/cancel \+ -H "X-API-Key: your-key"+```++**CLI**++```bash+ov task cancel uuid-xxx+```++**响应示例**++```json+{+ "status": "ok",+ "result": {+ "task_id": "uuid-xxx",+ "task_type": "add_resource",+ "status": "cancelling",+ "resource_id": "viking://resources/guide",+ "stage": "processing_queue",+ "result": null,+ "error": null+ }+}+```++如果任务没有剩余 work,响应中的状态可以直接为 `cancelled`。否则继续通过 `get_task()` 查询,直到状态变为 `cancelled`。++**错误处理**:+- `NOT_FOUND`(404):任务不存在、已过期或不属于当前用户+- `PERMISSION_DENIED`(403):ROOT 身份请求取消任务+- `FAILED_PRECONDITION`(412):任务类型不支持取消,或任务已经 `completed`/`failed`++---+### list_tasks()#### 1. API 实现介绍@@ -153,7 +251,7 @@ ov task status uuid-xxx| 参数 | 类型 | 必填 | 默认值 | 说明 ||------|------|------|--------|------|| task_type | str | 否 | None | 按任务类型过滤,例如 `session_commit` |-| status | str | 否 | None | 按任务状态过滤:`pending`、`running`、`completed`、`failed` |+| status | str | 否 | None | 按任务状态过滤:`pending`、`running`、`cancelling`、`completed`、`failed`、`cancelled` || resource_id | str | 否 | None | 按资源 ID 过滤,例如会话 ID || limit | int | 否 | 50 | 最多返回的任务条数 |diff --git a/openviking/async_client.py b/openviking/async_client.pyindex 883a6ac31..1d98c40af 100644--- a/openviking/async_client.py+++ b/openviking/async_client.py@@ -276,6 +276,11 @@ class AsyncOpenViking:await self._ensure_initialized()return await self._client.get_task(task_id)+ async def cancel_task(self, task_id: str) -> Optional[Dict[str, Any]]:+ """Cancel a background task."""+ await self._ensure_initialized()+ return await self._client.cancel_task(task_id)+async def list_tasks(self,task_type: Optional[str] = None,diff --git a/openviking/client/local.py b/openviking/client/local.pyindex ee6463f0b..4d3afbcc5 100644--- a/openviking/client/local.py+++ b/openviking/client/local.py@@ -28,7 +28,7 @@ from openviking.utils.image_search import normalize_client_image_inputfrom openviking.utils.search_filters import SearchContextTypeInput, merge_search_filterfrom openviking.utils.tags import build_search_tags_filterfrom openviking_cli.client.base import BaseClient-from openviking_cli.exceptions import InvalidArgumentError, NotFoundError+from openviking_cli.exceptions import InvalidArgumentError, NotFoundError, PermissionDeniedErrorfrom openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils import run_async@@ -942,6 +942,17 @@ class LocalClient(BaseClient):"""Query background task status."""return await self._service.sessions.get_commit_task(task_id, self._ctx)+ async def cancel_task(self, task_id: str) -> Optional[Dict[str, Any]]:+ """Cancel a background task."""+ if self._ctx.role == Role.ROOT:+ raise PermissionDeniedError("ROOT may not cancel tasks")+ task = await get_task_tracker().cancel(+ task_id,+ account_id=self._ctx.account_id,+ user_id=self._ctx.user.user_id,+ )+ return task.to_dict() if task else None+async def list_tasks(self,task_type: Optional[str] = None,diff --git a/openviking/metrics/collectors/task_tracker.py b/openviking/metrics/collectors/task_tracker.pyindex 93a270ca9..b55049963 100644--- a/openviking/metrics/collectors/task_tracker.py+++ b/openviking/metrics/collectors/task_tracker.py@@ -17,7 +17,7 @@ class TaskTrackerCollector(StateMetricCollector):"""Export task tracker backlog and completion counts grouped by task type.- The datasource returns a snapshot of task counts, so all four families are exposed as gauges+ The datasource returns a snapshot of task counts, so all families are exposed as gaugeskeyed only by `task_type` rather than as cumulative counters."""@@ -28,12 +28,14 @@ class TaskTrackerCollector(StateMetricCollector):# rule: <METRICS_NAMESPACE>_<DOMAIN>_running# e.g.: openviking_task_runningRUNNING: ClassVar[str] = MetricCollector.metric_name(DOMAIN, "running")+ CANCELLING: ClassVar[str] = MetricCollector.metric_name(DOMAIN, "cancelling")# rule: <METRICS_NAMESPACE>_<DOMAIN>_completed# e.g.: openviking_task_completedCOMPLETED: ClassVar[str] = MetricCollector.metric_name(DOMAIN, "completed")# rule: <METRICS_NAMESPACE>_<DOMAIN>_failed# e.g.: openviking_task_failedFAILED: ClassVar[str] = MetricCollector.metric_name(DOMAIN, "failed")+ CANCELLED: ClassVar[str] = MetricCollector.metric_name(DOMAIN, "cancelled")data_source: TaskStateDataSourceconfig: CollectorConfig = CollectorConfig(timeout_seconds=0.5)@@ -43,11 +45,18 @@ class TaskTrackerCollector(StateMetricCollector):return self.data_source.read_task_counts()def collect_hook(self, registry, metric_input) -> None:- """Translate the task-count snapshot into pending, running, completed, and failed gauges."""+ """Translate the task-count snapshot into lifecycle-state gauges."""counts_by_type = metric_input# Snapshot semantics: types can disappear entirely, so clear previous series to avoid# exporting stale non-zero gauges forever.- for metric_name in (self.PENDING, self.RUNNING, self.COMPLETED, self.FAILED):+ for metric_name in (+ self.PENDING,+ self.RUNNING,+ self.CANCELLING,+ self.COMPLETED,+ self.FAILED,+ self.CANCELLED,+ ):registry.gauge_delete_matching(metric_name, match_labels={})for task_type, counts in counts_by_type.items():labels = {"task_type": str(task_type)}@@ -63,6 +72,12 @@ class TaskTrackerCollector(StateMetricCollector):labels=labels,label_names=("task_type",),)+ registry.set_gauge(+ self.CANCELLING,+ float(counts.get("cancelling", 0)),+ labels=labels,+ label_names=("task_type",),+ )registry.set_gauge(self.COMPLETED,float(counts.get("completed", 0)),@@ -75,3 +90,9 @@ class TaskTrackerCollector(StateMetricCollector):labels=labels,label_names=("task_type",),)+ registry.set_gauge(+ self.CANCELLED,+ float(counts.get("cancelled", 0)),+ labels=labels,+ label_names=("task_type",),+ )diff --git a/openviking/server/routers/tasks.py b/openviking/server/routers/tasks.pyindex adfc292d1..ccc52e141 100644--- a/openviking/server/routers/tasks.py+++ b/openviking/server/routers/tasks.py@@ -16,7 +16,11 @@ from openviking.server.identity import RequestContext, Rolefrom openviking.server.models import Responsefrom openviking.service.task_store import SYSTEM_TASK_ACCOUNT_ID, SYSTEM_TASK_USER_IDfrom openviking.service.task_tracker import get_task_tracker-from openviking_cli.exceptions import OpenVikingError+from openviking_cli.exceptions import (+ FailedPreconditionError,+ OpenVikingError,+ PermissionDeniedError,+)router = APIRouter(prefix="/api/v1", tags=["tasks"])@@ -51,11 +55,38 @@ async def get_task(return Response(status="ok", result=task.to_dict())[email protected]("/tasks/{task_id}/cancel")+async def cancel_task(+ task_id: str,+ _ctx: RequestContext = Depends(get_request_context),+):+ """Request cooperative cancellation of a background task."""+ if _ctx.role == Role.ROOT:+ raise PermissionDeniedError("ROOT may not cancel tasks")+ tracker = get_task_tracker()+ try:+ task = await tracker.cancel(+ task_id,+ account_id=_ctx.account_id,+ user_id=_ctx.user.user_id,+ )+ except ValueError as exc:+ raise FailedPreconditionError(str(exc)) from exc+ if task is None:+ raise OpenVikingError(+ "Task not found or expired",+ code="NOT_FOUND",+ details={"resource": task_id, "type": "task"},+ )+ return Response(status="ok", result=task.to_dict())++@router.get("/tasks")async def list_tasks(task_type: Optional[str] = Query(None, description="Filter by task type (e.g. session_commit)"),status: Optional[str] = Query(- None, description="Filter by status (pending/running/completed/failed)"+ None,+ description="Filter by status (pending/running/cancelling/completed/failed/cancelled)",),resource_id: Optional[str] = Query(None, description="Filter by resource ID (e.g. session_id)"),limit: int = Query(50, le=200, description="Max results"),diff --git a/openviking/service/core.py b/openviking/service/core.pyindex 3bac8ba76..5953e7543 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -23,7 +23,7 @@ from openviking.service.resource_memory_link_service import ResourceMemoryLinkSefrom openviking.service.resource_service import ResourceServicefrom openviking.service.search_service import SearchServicefrom openviking.service.session_service import SessionService-from openviking.service.task_tracker import set_task_tracker+from openviking.service.task_tracker import get_task_tracker, set_task_trackerfrom openviking.session import create_session_compressorfrom openviking.storage.vikingdb_manager import VikingDBManagerfrom openviking.storage.collection_schemas import init_context_collection@@ -365,8 +365,6 @@ class OpenVikingService:resource_service=self._resource_service,viking_fs=self._viking_fs,)- await self._watch_scheduler.start()- logger.info("WatchScheduler started")# Wire up sub-servicesself._fs_service.set_dependencies(@@ -429,6 +427,14 @@ class OpenVikingService:),allow_create=True,)+ await self._queue_manager.prepare_task_tracking(get_task_tracker())++ # Do not let watches produce queue work while task ownership is being+ # rebuilt from QueueFS. Consumers start only after the scheduler is ready.+ await self._watch_scheduler.start()+ logger.info("WatchScheduler started")++ if self._queue_manager:self._queue_manager.start()logger.info("QueueManager workers started")diff --git a/openviking/service/reindex_executor.py b/openviking/service/reindex_executor.pyindex 9789a178f..ca06b658c 100644--- a/openviking/service/reindex_executor.py+++ b/openviking/service/reindex_executor.py@@ -26,6 +26,7 @@ from openviking.core.namespace import (from openviking.server.dependencies import get_servicefrom openviking.server.identity import RequestContextfrom openviking.service.task_tracker import get_task_tracker+from openviking.service.task_work_index import bind_task_contextfrom openviking.session.memory.utils.memory_file_utils import MemoryFileUtilsfrom openviking.storage.expr import And, Eq, Or, PathScopefrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter@@ -531,21 +532,26 @@ class ReindexExecutor:ctx: RequestContext,) -> None:tracker = get_task_tracker()- await tracker.start(task_id, account_id=ctx.account_id, user_id=ctx.user.user_id)+ tracker.register_running_task(task_id)try:- result = await self._run(- uri=uri,- object_type=object_type,- mode=mode,- dry_run=dry_run,- ctx=ctx,- )+ await tracker.start(task_id, account_id=ctx.account_id, user_id=ctx.user.user_id)+ with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):+ result = await self._run(+ uri=uri,+ object_type=object_type,+ mode=mode,+ dry_run=dry_run,+ ctx=ctx,+ )await tracker.complete(task_id,result,account_id=ctx.account_id,user_id=ctx.user.user_id,)+ except asyncio.CancelledError:+ # TaskWorkIndex finalizes after this active task and its queue work settle.+ returnexcept Exception as exc:await tracker.fail(task_id,@@ -553,6 +559,8 @@ class ReindexExecutor:account_id=ctx.account_id,user_id=ctx.user.user_id,)+ finally:+ await tracker.unregister_running_task(task_id)async def _prune_orphan_vectors(self,diff --git a/openviking/service/resource_memory_link_service.py b/openviking/service/resource_memory_link_service.pyindex b460da0b7..804a08d85 100644--- a/openviking/service/resource_memory_link_service.py+++ b/openviking/service/resource_memory_link_service.py@@ -33,8 +33,8 @@ from openviking.session.memory.utils.resource_refs import (resource_ref_matches,unlink_resource_references_from_memory,)-from openviking.storage.vikingdb_manager import VikingDBManagerfrom openviking.storage.viking_fs import VikingFS, get_viking_fs+from openviking.storage.vikingdb_manager import VikingDBManagerfrom openviking_cli.exceptions import NotFoundErrorfrom openviking_cli.utils import get_logger@@ -336,29 +336,22 @@ class ResourceMemoryLinkService:ctx: RequestContext,timeout: Optional[float],) -> Dict[str, Any]:- from openviking.service.task_tracker import get_task_tracker-- async def _poll() -> Dict[str, Any]:- tracker = get_task_tracker()- while True:- task = await tracker.get(- task_id,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- if task is None:- raise RuntimeError(f"session commit task not found: {task_id}")- status = task.status.value if hasattr(task.status, "value") else str(task.status)- if status == "completed":- return task.to_dict()- if status == "failed":- raise RuntimeError(task.error or f"session commit task failed: {task_id}")- await asyncio.sleep(0.1)-- return await asyncio.wait_for(- _poll(),- timeout=timeout or _RESOURCE_REASON_COMMIT_TIMEOUT_SECONDS,+ from openviking.service.task_tracker import TaskStatus, get_task_tracker++ task = await get_task_tracker().wait(+ task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ timeout=(+ _RESOURCE_REASON_COMMIT_TIMEOUT_SECONDS if timeout is None else timeout+ ),+ poll_interval=0.1,)+ if task.status == TaskStatus.COMPLETED:+ return task.to_dict()+ if task.status == TaskStatus.FAILED:+ raise RuntimeError(task.error)+ raise RuntimeError(f"session commit task cancelled: {task_id}")async def before_resource_delete(self,diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.pyindex 535a42812..a5cff97fd 100644--- a/openviking/service/resource_service.py+++ b/openviking/service/resource_service.py@@ -40,16 +40,13 @@ from openviking.server.user_config import (effective_resource_add_target,effective_skill_add_target,)-from openviking.storage.vikingdb_manager import VikingDBManagerfrom openviking.storage.queuefs import QueueManager, get_queue_managerfrom openviking.storage.viking_fs import VikingFS-from openviking.telemetry import get_current_telemetry+from openviking.storage.vikingdb_manager import VikingDBManager+from openviking.telemetry import get_current_telemetry, register_telemetry, unregister_telemetryfrom openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.telemetry.resource_summary import (build_queue_status_payload,- record_resource_wait_metrics,- register_wait_telemetry,- unregister_wait_telemetry,)from openviking.utils import is_git_repo_url, parse_code_hosting_urlfrom openviking.utils.ingest_options import IngestOptions@@ -455,6 +452,7 @@ class ResourceService:account_id=msg.account_id,user_id=msg.user_id,task_id=msg.task_id,+ meta={"source_path": msg.source_path},)await tracker.update_stage(task.task_id,@@ -500,14 +498,14 @@ class ResourceService:from openviking.parse.understanding_api import PREPARED_RESPONSE_ID_ARGinternal_kwargs[PREPARED_RESPONSE_ID_ARG] = msg.understanding_response_id- return await self._execute_resource_ingestion(+ result = await self._execute_resource_ingestion(path=msg.path,ctx=ctx,to=target_uri,parent=parent_uri,reason=msg.reason,instruction=msg.instruction,- wait=True,+ defer_post_processing=False,timeout=msg.timeout,build_index=msg.build_index,summarize=msg.summarize,@@ -531,46 +529,26 @@ class ResourceService:create_parent=msg.create_parent,**internal_kwargs,)-- telemetry_id = get_current_telemetry().telemetry_id- ingest_tag_kwargs = self._add_resource_ingest_tag_kwargs(- tags=msg.tags,- tag_mode=msg.tag_mode,- )- request_wait_tracker = get_request_wait_tracker()- request_wait_tracker.register_request(telemetry_id)- try:stage_result = stage_callback("processing_queue")if inspect.isawaitable(stage_result):await stage_result- result = await self._resource_processor.finish_prepared_resource(- msg.prepared,- ctx=ctx,- resource_lock=resource_lock,- summarize=msg.summarize,- build_index=msg.build_index,- processing_mode=msg.processing_mode,- **ingest_tag_kwargs,- )- await request_wait_tracker.wait_for_request(- telemetry_id,- timeout=msg.timeout,- )- status = request_wait_tracker.build_queue_status(telemetry_id)- result["queue_status"] = status- await self._link_resource_reason_memory(- result=result,- ctx=ctx,- reason=msg.reason,- source_name=msg.source_name,- timeout=msg.timeout,- )return result- except TimeoutError as exc:- raise DeadlineExceededError("queue processing", msg.timeout) from exc- finally:- request_wait_tracker.cleanup(telemetry_id)- unregister_wait_telemetry(telemetry_id)++ stage_result = stage_callback("processing_queue")+ if inspect.isawaitable(stage_result):+ await stage_result+ return await self._resource_processor.finish_prepared_resource(+ msg.prepared,+ ctx=ctx,+ resource_lock=resource_lock,+ summarize=msg.summarize,+ build_index=msg.build_index,+ processing_mode=msg.processing_mode,+ **self._add_resource_ingest_tag_kwargs(+ tags=msg.tags,+ tag_mode=msg.tag_mode,+ ),+ )async def reacquire_add_resource_job_lock(self,@@ -667,6 +645,7 @@ class ResourceService:msg = AddResourceMsg(task_id=task_id,path=path,+ source_path=(source_name or "") if kwargs.get("temp_file_id") else path,root_uri=root_uri,telemetry_id=get_current_telemetry().telemetry_id or None,account_id=ctx.account_id,@@ -1013,14 +992,35 @@ class ResourceService:**kwargs,)- if not wait and is_git_repo_url(path):- return await self.enqueue_git_add_resource(+ if is_git_repo_url(path):+ result = await self.enqueue_git_add_resource(+ path=path,+ ctx=ctx,+ to=to,+ parent=parent,+ reason=reason,+ instruction=instruction,+ timeout=timeout,+ build_index=build_index,+ summarize=summarize,+ processing_mode=processing_mode,+ watch_interval=watch_interval,+ manage_watch=manage_watch,+ tags=tags,+ tag_mode=tag_mode,+ allow_local_path_resolution=allow_local_path_resolution,+ enforce_public_remote_targets=enforce_public_remote_targets,+ **kwargs,+ )+ else:+ result = await self._execute_resource_ingestion(path=path,ctx=ctx,to=to,parent=parent,reason=reason,instruction=instruction,+ defer_post_processing=True,timeout=timeout,build_index=build_index,summarize=summarize,@@ -1031,41 +1031,46 @@ class ResourceService:tag_mode=tag_mode,allow_local_path_resolution=allow_local_path_resolution,enforce_public_remote_targets=enforce_public_remote_targets,+ watch_auth_state=normalized_args.watch_auth_state,+ parser_args=normalized_args.processor_kwargs,**kwargs,)+ get_current_telemetry().set("resource.flags.wait", wait)+ if not wait:+ return result+ from openviking.service.task_tracker import TaskStatus, get_task_tracker- return await self._execute_resource_ingestion(- path=path,- ctx=ctx,- to=to,- parent=parent,- reason=reason,- instruction=instruction,- wait=wait,- timeout=timeout,- build_index=build_index,- summarize=summarize,- processing_mode=processing_mode,- watch_interval=watch_interval,- manage_watch=manage_watch,- tags=tags,- tag_mode=tag_mode,- allow_local_path_resolution=allow_local_path_resolution,- enforce_public_remote_targets=enforce_public_remote_targets,- watch_auth_state=normalized_args.watch_auth_state,- parser_args=normalized_args.processor_kwargs,- **kwargs,- )+ task_id = result["task_id"]+ try:+ task = await get_task_tracker().wait(+ task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ timeout=timeout,+ )+ except TimeoutError as exc:+ raise DeadlineExceededError("add resource task", timeout) from exc++ if task.status == TaskStatus.COMPLETED:+ completed = dict(task.result)+ completed.pop("task_id", None)+ return completed+ if task.status == TaskStatus.CANCELLED:+ return {"status": "cancelled"}+ return {+ "status": "error",+ "errors": [task.error],+ }async def _execute_resource_ingestion(self,path: str,ctx: RequestContext,+ defer_post_processing: bool,to: Optional[str] = None,parent: Optional[str] = None,reason: str = "",instruction: str = "",- wait: bool = False,timeout: Optional[float] = None,build_index: bool = True,summarize: bool = False,@@ -1086,20 +1091,17 @@ class ResourceService:self._ensure_initialized()request_start = time.perf_counter()telemetry = get_current_telemetry()- telemetry_id = register_wait_telemetry(wait)- request_wait_tracker = get_request_wait_tracker()+ telemetry_id = telemetry.telemetry_id+ register_telemetry(telemetry)job_enqueued = Falsedeferred_lock: Optional[Dict[str, Any]] = Noneingest_tag_kwargs = self._add_resource_ingest_tag_kwargs(tags=tags,tag_mode=tag_mode,)- if telemetry_id:- request_wait_tracker.register_request(telemetry_id)watch_manager = self._get_watch_manager()watch_enabled = bool(watch_manager and manage_watch and watch_interval > 0)- telemetry.set("resource.flags.wait", wait)telemetry.set("resource.flags.build_index", build_index)telemetry.set("resource.flags.summarize", summarize)telemetry.set("resource.flags.watch_enabled", watch_enabled)@@ -1120,7 +1122,7 @@ class ResourceService:kwargs["resource_lock"] = resource_lockasync_understanding_candidate = (- not wait+ defer_post_processingand not is_git_repo_url(path)and not allow_local_path_resolutionand self._resource_processor is not None@@ -1248,6 +1250,7 @@ class ResourceService:task_id=str(uuid4()),telemetry_id=telemetry_id or None,path=path,+ source_path=(source_name or "") if kwargs.get("temp_file_id") else path,root_uri=root_uri,account_id=ctx.account_id,user_id=ctx.user.user_id,@@ -1333,7 +1336,7 @@ class ResourceService:stage_callback=stage_callback,allow_local_path_resolution=allow_local_path_resolution,prepared_resource=prepared_resource,- defer_post_processing=not wait,+ defer_post_processing=defer_post_processing,**ingest_tag_kwargs,**kwargs,)@@ -1343,54 +1346,7 @@ class ResourceService:return resultprepared = result.pop("_post_process", None)deferred_lock = result.pop("_resource_lock", None)- if wait:- if stage_callback is not None:- stage_result = stage_callback("processing_queue")- if inspect.isawaitable(stage_result):- await stage_result- wait_start = time.perf_counter()- try:- with telemetry.measure("resource.wait"):- if telemetry_id:- await request_wait_tracker.wait_for_request(- telemetry_id,- timeout=timeout,- poll_interval=0.05,- )- status = request_wait_tracker.build_queue_status(telemetry_id)- else:- qm = get_queue_manager()- status = build_queue_status_payload(- await qm.wait_complete(timeout=timeout)- )- except TimeoutError as exc:- telemetry.set_error(- "resource_service.wait_complete",- "DEADLINE_EXCEEDED",- str(exc),- )- raise DeadlineExceededError("queue processing", timeout) from exc- queue_wait_duration_ms = round((time.perf_counter() - wait_start) * 1000, 3)- try:- from openviking.metrics.datasources.resource import (- ResourceIngestionEventDataSource,- )-- ResourceIngestionEventDataSource.record_wait(- operation="queue_processing",- duration_seconds=float(queue_wait_duration_ms) / 1000.0,- account_id=getattr(ctx, "account_id", None),- )- except Exception:- pass- result["queue_status"] = status- record_resource_wait_metrics(- telemetry_id=telemetry_id,- queue_status=status,- root_uri=result.get("root_uri"),- )- telemetry.set("queue.wait.duration_ms", queue_wait_duration_ms)- if not wait:+ if defer_post_processing:from openviking.storage.queuefs.add_resource_msg import AddResourceMsgroot_uri = result.get("root_uri", "")@@ -1401,6 +1357,11 @@ class ResourceService:task_id=str(uuid4()),root_uri=root_uri,prepared=prepared,+ source_path=str(+ (kwargs.get("source_name") or "")+ if kwargs.get("temp_file_id")+ else result.get("source_path") or ""+ ),telemetry_id=telemetry_id or None,account_id=ctx.account_id,user_id=ctx.user.user_id,@@ -1452,14 +1413,6 @@ class ResourceService:watch_auth_state=watch_auth_state,ctx=ctx,)- if wait:- await self._link_resource_reason_memory(- result=result,- ctx=ctx,- reason=reason,- source_name=kwargs.get("source_name"),- timeout=timeout,- )return resultexcept Exception as exc:telemetry.set_error(@@ -1475,9 +1428,8 @@ class ResourceService:"resource.request.duration_ms",round((time.perf_counter() - request_start) * 1000, 3),)- if wait or not telemetry_id or not job_enqueued:- get_request_wait_tracker().cleanup(telemetry_id)- unregister_wait_telemetry(telemetry_id)+ if not telemetry_id or (defer_post_processing and not job_enqueued):+ unregister_telemetry(telemetry_id)if deferred_lock is not None:await self._release_lock_ref(deferred_lock)@@ -1544,7 +1496,7 @@ class ResourceService:await task_tracker.fail(task_id, str(exc), account_id=account_id, user_id=user_id)finally:request_wait_tracker.cleanup(telemetry_id)- unregister_wait_telemetry(telemetry_id)+ unregister_telemetry(telemetry_id)# ── Connector routing ──@@ -1794,7 +1746,7 @@ class ResourceService:finally:if wait or not telemetry_id or not monitor_started:request_wait_tracker.cleanup(telemetry_id)- unregister_wait_telemetry(telemetry_id)+ unregister_telemetry(telemetry_id)async def build_index(self, resource_uris: List[str], ctx: RequestContext, **kwargsdiff --git a/openviking/service/task_store.py b/openviking/service/task_store.pyindex e2aea069e..741c81aad 100644--- a/openviking/service/task_store.py+++ b/openviking/service/task_store.py@@ -150,6 +150,7 @@ def _task_to_payload(task: Any) -> Dict[str, Any]:"resource_id": task.resource_id,"account_id": task.account_id,"user_id": task.user_id,+ "meta": deepcopy(task.meta),"stage": task.stage,"result": deepcopy(task.result),"error": task.error,diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.pyindex 07b0a94ea..c1bc7e827 100644--- a/openviking/service/task_tracker.py+++ b/openviking/service/task_tracker.py@@ -25,6 +25,7 @@ from typing import Any, Dict, List, Optionalfrom uuid import uuid4from openviking.service.task_store import TaskStore+from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndexfrom openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)@@ -35,11 +36,21 @@ class TaskStatus(str, Enum):PENDING = "pending"RUNNING = "running"+ CANCELLING = "cancelling"COMPLETED = "completed"FAILED = "failed"+ CANCELLED = "cancelled"-_TERMINAL_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED)+_TERMINAL_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED)+_ACTIVE_STATUSES = (TaskStatus.PENDING, TaskStatus.RUNNING, TaskStatus.CANCELLING)++_CANCELLABLE_TASK_TYPES = {+ "add_resource",+ "session_commit",+ "admin_reindex",+ "snapshot_restore_reindex",+}@dataclass@@ -54,6 +65,7 @@ class TaskRecord:resource_id: Optional[str] = None # e.g. session_idaccount_id: Optional[str] = Noneuser_id: Optional[str] = None+ meta: Dict[str, Any] = field(default_factory=dict)stage: Optional[str] = Noneresult: Optional[Dict[str, Any]] = Noneerror: Optional[str] = None@@ -65,6 +77,7 @@ class TaskRecord:d["created_at_iso"] = datetime.fromtimestamp(self.created_at, tz=timezone.utc).isoformat()d["updated_at_iso"] = datetime.fromtimestamp(self.updated_at, tz=timezone.utc).isoformat()d["result"] = _sanitize_task_result(d.get("result"))+ d["meta"] = _sanitize_task_result(d.get("meta"))d.pop("account_id", None)d.pop("user_id", None)return d@@ -153,6 +166,9 @@ class TaskTracker:self._lock = threading.Lock()self._async_lock = asyncio.Lock()self._cleanup_task: Optional[asyncio.Task] = None+ self._runtime_loop: Optional[asyncio.AbstractEventLoop] = None+ self._work_index = TaskWorkIndex()+ self._install_work_index_callbacks()logger.info("[TaskTracker] Initialized (store=%s, max_tasks=%d)",self._store.__class__.__name__,@@ -161,6 +177,54 @@ class TaskTracker:# ── Lifecycle ──+ def _install_work_index_callbacks(self) -> None:+ self._work_index.set_callbacks(+ finalize_before_ack=self._finalize_before_ack,+ is_cancellation_requested=self.is_cancellation_requested,+ )++ def attach_work_index(self, work_index: TaskWorkIndex) -> None:+ """Use the QueueManager-owned index as the task lifecycle authority."""+ self._work_index = work_index+ self._runtime_loop = asyncio.get_running_loop()+ self._install_work_index_callbacks()++ async def restore_work_tasks(self, owners: Dict[str, tuple[str, str]]) -> None:+ """Restore task records referenced by rebuilt QueueFS work."""+ for task_id, (account_id, user_id) in owners.items():+ await self.get(task_id, account_id=account_id, user_id=user_id)++ async def _finalize_before_ack(self, metadata: QueueTaskMetadata) -> None:+ """Persist the terminal state before QueueFS removes the last recovery message."""+ loop = self._runtime_loop+ if loop is None or loop.is_closed():+ raise RuntimeError("TaskTracker runtime loop is unavailable during QueueFS ACK")+ if asyncio.get_running_loop() is loop:+ await self._finalize_task(+ metadata.task_id,+ account_id=metadata.account_id or None,+ user_id=metadata.user_id or None,+ )+ return+ future = asyncio.run_coroutine_threadsafe(+ self._finalize_task(+ metadata.task_id,+ account_id=metadata.account_id or None,+ user_id=metadata.user_id or None,+ ),+ loop,+ )+ await asyncio.wrap_future(future)++ def is_cancellation_requested(self, task_id: str) -> bool:+ """Fast thread-safe status check used by queue workers."""+ with self._lock:+ task = self._tasks.get(task_id)+ return task is not None and task.status in (+ TaskStatus.CANCELLING,+ TaskStatus.CANCELLED,+ )+def start_cleanup_loop(self) -> None:"""Start the background TTL cleanup coroutine.@@ -169,6 +233,7 @@ class TaskTracker:"""if self._cleanup_task is not None and not self._cleanup_task.done():return+ self._runtime_loop = asyncio.get_running_loop()self._cleanup_task = asyncio.create_task(self._cleanup_loop())logger.debug("[TaskTracker] Cleanup loop started")@@ -195,10 +260,9 @@ class TaskTracker:with self._lock:expired_ids = []for tid, t in self._tasks.items():- if (- t.status == TaskStatus.COMPLETED- and (now - t.updated_at) > self.TTL_COMPLETED- ):+ if t.status in (TaskStatus.COMPLETED, TaskStatus.CANCELLED) and (+ now - t.updated_at+ ) > self.TTL_COMPLETED:expired_ids.append(tid)elif t.status == TaskStatus.FAILED and (now - t.updated_at) > self.TTL_FAILED:expired_ids.append(tid)@@ -244,6 +308,7 @@ class TaskTracker:account_id: str,user_id: str,task_id: Optional[str] = None,+ meta: Optional[Dict[str, Any]] = None,) -> TaskRecord:"""Register a new pending task. Returns a snapshot copy."""self._validate_owner(account_id, user_id)@@ -253,6 +318,7 @@ class TaskTracker:resource_id=resource_id,account_id=account_id,user_id=user_id,+ meta=dict(meta or {}),)async with self._async_lock:if task_id:@@ -303,7 +369,7 @@ class TaskTracker:t.task_type == task_typeand t.resource_id == resource_idand self._matches_owner(t, account_id, user_id)- and t.status in (TaskStatus.PENDING, TaskStatus.RUNNING)+ and t.status in _ACTIVE_STATUSESfor t in tasks)if has_active:@@ -336,7 +402,7 @@ class TaskTracker:"""Transition task to RUNNING."""async with self._async_lock:task = await self._load_for_update(task_id, account_id, user_id)- if task and task.status not in _TERMINAL_STATUSES:+ if task and task.status in (TaskStatus.PENDING, TaskStatus.RUNNING):task.status = TaskStatus.RUNNINGif stage is not None:task.stage = stage@@ -355,7 +421,7 @@ class TaskTracker:"""Update task stage without changing its lifecycle status."""async with self._async_lock:task = await self._load_for_update(task_id, account_id, user_id)- if task and task.status not in _TERMINAL_STATUSES:+ if task and task.status in (TaskStatus.PENDING, TaskStatus.RUNNING):task.stage = stagetask.updated_at = time.time()await self._store.update(task)@@ -365,52 +431,164 @@ class TaskTracker:async def complete(self,task_id: str,- result: Optional[Dict[str, Any]] = None,+ result: Dict[str, Any],account_id: Optional[str] = None,user_id: Optional[str] = None,*,resource_id: Optional[str] = None,) -> None:- """Transition task to COMPLETED with optional result."""- transitioned = False+ """Record successful completion and finalize after owned work settles."""+ await self._record_outcome(+ task_id,+ account_id,+ user_id,+ result=result,+ resource_id=resource_id,+ )++ async def fail(+ self,+ task_id: str,+ error: str,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,+ ) -> None:+ """Record failure and finalize after owned work settles."""+ await self._record_outcome(task_id, account_id, user_id, error=error)++ async def _record_outcome(+ self,+ task_id: str,+ account_id: Optional[str],+ user_id: Optional[str],+ *,+ result: Optional[Dict[str, Any]] = None,+ error: Optional[str] = None,+ resource_id: Optional[str] = None,+ ) -> None:async with self._async_lock:task = await self._load_for_update(task_id, account_id, user_id)- if task and task.status not in _TERMINAL_STATUSES:- task.status = TaskStatus.COMPLETED- task.stage = "completed"- task.result = result- if resource_id is not None:- task.resource_id = resource_id+ if task and task.status in (TaskStatus.PENDING, TaskStatus.RUNNING):+ if result is not None:+ task.result = result+ if resource_id is not None:+ task.resource_id = resource_id+ if error is not None and task.error is None:+ task.error = _sanitize_error(error)+ work_error = self._work_index.failure(task_id)+ if work_error and task.error is None:+ task.error = _sanitize_error(work_error)task.updated_at = time.time()await self._store.update(task)with self._lock:self._tasks[task.task_id] = task- transitioned = True- if transitioned:- logger.info("[TaskTracker] Task %s completed", task_id)+ await self._finalize_task(task_id, account_id=account_id, user_id=user_id)- async def fail(+ async def cancel(self,task_id: str,- error: str,account_id: Optional[str] = None,user_id: Optional[str] = None,- ) -> None:- """Transition task to FAILED with sanitized error."""- transitioned = False+ ) -> Optional[TaskRecord]:+ """Request cooperative cancellation and return the current task snapshot."""async with self._async_lock:task = await self._load_for_update(task_id, account_id, user_id)- if task and task.status not in _TERMINAL_STATUSES:- task.status = TaskStatus.FAILED- task.stage = "failed"- task.error = _sanitize_error(error)+ if task is None:+ return None+ if task.status == TaskStatus.CANCELLED:+ return self._copy(task)+ if task.status != TaskStatus.CANCELLING:+ if task.task_type not in _CANCELLABLE_TASK_TYPES:+ raise ValueError(f"Task type '{task.task_type}' does not support cancellation")+ if task.status in _TERMINAL_STATUSES:+ raise ValueError(f"Task is already {task.status.value}")++ task.status = TaskStatus.CANCELLINGtask.updated_at = time.time()await self._store.update(task)with self._lock:self._tasks[task.task_id] = task- transitioned = True- if transitioned:- logger.warning("[TaskTracker] Task %s failed: %s", task_id, _sanitize_error(error))+ self._work_index.cancel_active(task_id)+ await self._finalize_task(+ task_id,+ account_id=account_id,+ user_id=user_id,+ )+ return await self.get(task_id, account_id=account_id, user_id=user_id)++ async def _finalize_task(+ self,+ task_id: str,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,+ ) -> None:+ """Apply the task's terminal outcome after all owned work is settled."""+ if self._work_index.has_work(task_id):+ return+ async with self._async_lock:+ task = await self._load_for_update(task_id, account_id, user_id)+ if task and task.status in _ACTIVE_STATUSES:+ if self._work_index.has_work(task_id):+ return+ updated = deepcopy(task)+ if updated.status != TaskStatus.CANCELLING:+ work_error = self._work_index.failure(task_id)+ if work_error and updated.error is None:+ updated.error = _sanitize_error(work_error)+ if updated.status == TaskStatus.CANCELLING:+ updated.status = TaskStatus.CANCELLED+ elif updated.error is not None:+ updated.status = TaskStatus.FAILED+ elif updated.result is not None:+ updated.status = TaskStatus.COMPLETED+ else:+ return+ updated.stage = updated.status.value+ updated.updated_at = time.time()+ await self._store.update(updated)+ with self._lock:+ self._tasks[updated.task_id] = updated+ self._work_index.clear_failure(task_id)+ logger.info("[TaskTracker] Task %s %s", task_id, updated.status.value)++ async def wait(+ self,+ task_id: str,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,+ timeout: Optional[float] = None,+ poll_interval: float = 0.05,+ ) -> TaskRecord:+ """Wait for one task's terminal state without changing its lifecycle."""+ async def _poll() -> TaskRecord:+ while True:+ task = await self.get(task_id, account_id=account_id, user_id=user_id)+ if task.status in _TERMINAL_STATUSES:+ return task+ await asyncio.sleep(poll_interval)++ if timeout is None:+ return await _poll()+ return await asyncio.wait_for(_poll(), timeout)++ async def wait_for_descendants(self, task_id: str, current_work_id: str) -> None:+ """Wait on the same durable work index used by completion and cancellation."""+ while self._work_index.has_work(task_id, exclude_work_id=current_work_id):+ await asyncio.sleep(0.05)++ def register_running_task(self, task_id: str) -> None:+ """Register the current asyncio task so cancellation can interrupt it."""+ self._runtime_loop = self._runtime_loop or asyncio.get_running_loop()+ active_task = asyncio.current_task()+ if active_task is not None:+ self._work_index.register_active(task_id, active_task)++ async def unregister_running_task(self, task_id: str) -> None:+ """Release direct background work and persist its terminal outcome."""+ active_task = asyncio.current_task()+ if active_task is not None:+ self._work_index.unregister_active(task_id, active_task)+ await self._finalize_task(task_id)async def get(self,@@ -481,7 +659,7 @@ class TaskTracker:t.task_type == task_typeand t.resource_id == resource_idand self._matches_owner(t, account_id, user_id)- and t.status in (TaskStatus.PENDING, TaskStatus.RUNNING)+ and t.status in _ACTIVE_STATUSESfor t in tasks)@@ -528,6 +706,7 @@ class TaskTracker:def _copy(task: TaskRecord) -> TaskRecord:"""Return a defensive copy of a TaskRecord."""copied = deepcopy(task)+ copied.meta = _sanitize_task_result(copied.meta)copied.result = _sanitize_task_result(copied.result)return copieddiff --git a/openviking/service/task_work_index.py b/openviking/service/task_work_index.pynew file mode 100644index 000000000..e35139b63--- /dev/null+++ b/openviking/service/task_work_index.py@@ -0,0 +1,310 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Runtime index of durable queue work owned by a tracked task.++QueueFS remains the durable source of truth. This index is rebuilt from all+unacknowledged queue messages during startup and then maintained by enqueue/ACK+hooks. It deliberately does not persist a second counter beside QueueFS.+"""++from __future__ import annotations++import asyncio+import json+import threading+from contextlib import contextmanager+from contextvars import ContextVar+from dataclasses import dataclass+from typing import Any, Awaitable, Callable, Dict, Iterable, Iterator, Mapping, Optional+from uuid import uuid4++TASK_WORK_ID_FIELD = "_task_work_id"+++class TaskWorkRejected(Exception):+ """Task cancellation prevented descendant queue work from being created."""+++@dataclass(frozen=True)+class TaskExecutionContext:+ task_id: str+ account_id: str+ user_id: str+++@dataclass(frozen=True)+class QueueTaskMetadata:+ task_id: str+ work_id: str+ account_id: str = ""+ user_id: str = ""+++_current_task_context: ContextVar[Optional[TaskExecutionContext]] = ContextVar(+ "openviking_task_execution_context",+ default=None,+)+++@contextmanager+def bind_task_context(+ task_id: str,+ account_id: str,+ user_id: str,+) -> Iterator[None]:+ """Propagate task ownership to queue messages produced by this coroutine."""+ token = _current_task_context.set(+ TaskExecutionContext(+ task_id=str(task_id),+ account_id=str(account_id),+ user_id=str(user_id),+ )+ )+ try:+ yield+ finally:+ _current_task_context.reset(token)+++def get_task_context() -> Optional[TaskExecutionContext]:+ return _current_task_context.get()+++def _payload_dict(message: Any) -> Optional[Dict[str, Any]]:+ if not isinstance(message, dict):+ return None+ payload: Any = message.get("data", message)+ if isinstance(payload, str):+ try:+ payload = json.loads(payload)+ except (TypeError, ValueError):+ return None+ return payload if isinstance(payload, dict) else None+++def _owner_from_payload(payload: Mapping[str, Any]) -> tuple[str, str]:+ account_id = payload.get("account_id")+ user_id = payload.get("user_id")++ user = payload.get("user")+ if isinstance(user, dict):+ account_id = account_id or user.get("account_id")+ user_id = user_id or user.get("user_id")++ context_data = payload.get("context_data")+ if isinstance(context_data, dict):+ account_id = account_id or context_data.get("account_id")+ user_id = user_id or context_data.get("owner_user_id")+ return str(account_id or ""), str(user_id or "")+++def prepare_task_payload(+ data: Dict[str, Any],+) -> tuple[Dict[str, Any], Optional[QueueTaskMetadata]]:+ """Copy a payload and attach durable task metadata when it belongs to a task."""+ payload = dict(data)+ current = _current_task_context.get()+ task_id = payload.get("task_id") or (current.task_id if current is not None else "")+ if not task_id:+ return payload, None++ task_id = str(task_id)+ account_id, user_id = _owner_from_payload(payload)+ if current is not None and current.task_id == task_id:+ account_id = account_id or current.account_id+ user_id = user_id or current.user_id++ work_id = str(payload.get(TASK_WORK_ID_FIELD) or uuid4())+ payload["task_id"] = task_id+ payload[TASK_WORK_ID_FIELD] = work_id+ if account_id and user_id:+ payload.setdefault("account_id", account_id)+ payload.setdefault("user_id", user_id)+ return payload, QueueTaskMetadata(task_id, work_id, account_id, user_id)+++def extract_task_metadata(message: Any) -> Optional[QueueTaskMetadata]:+ """Read task metadata from either a QueueFS envelope or its inner payload."""+ payload = _payload_dict(message)+ if payload is None:+ return None+ task_id = payload.get("task_id")+ work_id = payload.get(TASK_WORK_ID_FIELD)+ if not work_id and isinstance(message, dict) and "data" in message and message.get("id"):+ # Messages produced before task work IDs were introduced can still be+ # rebuilt from their stable QueueFS envelope ID.+ work_id = f"queuefs:{message['id']}"+ if not task_id or not work_id:+ return None+ account_id, user_id = _owner_from_payload(payload)+ return QueueTaskMetadata(str(task_id), str(work_id), account_id, user_id)+++class TaskWorkIndex:+ """Thread-safe, rebuildable index of persistent and currently active task work."""++ def __init__(self) -> None:+ self._lock = threading.Lock()+ self._work: Dict[str, set[tuple[str, str]]] = {}+ self._active: Dict[str, set[asyncio.Task[Any]]] = {}+ self._failures: Dict[str, str] = {}+ self._finalize_before_ack: Optional[Callable[[QueueTaskMetadata], Awaitable[None]]] = None+ self._is_cancellation_requested: Optional[Callable[[str], bool]] = None++ def set_callbacks(+ self,+ *,+ finalize_before_ack: Callable[[QueueTaskMetadata], Awaitable[None]],+ is_cancellation_requested: Callable[[str], bool],+ ) -> None:+ self._finalize_before_ack = finalize_before_ack+ self._is_cancellation_requested = is_cancellation_requested++ def rebuild(+ self,+ snapshots: Mapping[str, Iterable[Any]],+ ) -> Dict[str, tuple[str, str]]:+ """Rebuild work from QueueFS and return owners needed to restore task records."""+ work: Dict[str, set[tuple[str, str]]] = {}+ owners: Dict[str, tuple[str, str]] = {}+ for queue_name, messages in snapshots.items():+ for message in messages:+ metadata = extract_task_metadata(message)+ if metadata is None:+ continue+ work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))+ if metadata.account_id and metadata.user_id:+ owners[metadata.task_id] = (metadata.account_id, metadata.user_id)+ with self._lock:+ self._work = work+ self._failures = {}+ return owners++ def register(self, queue_name: str, metadata: Optional[QueueTaskMetadata]) -> bool:+ """Atomically reject cancelled work or add it to the runtime index."""+ if metadata is None:+ return True+ with self._lock:+ if self.cancellation_requested(metadata.task_id):+ return False+ self._work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))+ return True++ def _remove_work(+ self,+ queue_name: str,+ metadata: QueueTaskMetadata,+ ) -> tuple[bool, bool]:+ """Remove one work item and report whether it existed and made its task idle."""+ removed = False+ with self._lock:+ entries = self._work.get(metadata.task_id)+ if entries is not None:+ entry = (queue_name, metadata.work_id)+ removed = entry in entries+ entries.discard(entry)+ if not entries:+ self._work.pop(metadata.task_id, None)+ became_idle = (+ removed+ and not self._work.get(metadata.task_id)+ and not self._active.get(metadata.task_id)+ )+ return removed, became_idle++ async def discard(self, queue_name: str, message: Any) -> None:+ """Discard work that was never durably enqueued and finalize if it was last."""+ metadata = (+ message if isinstance(message, QueueTaskMetadata) else extract_task_metadata(message)+ )+ if metadata is None:+ return+ _removed, became_idle = self._remove_work(queue_name, metadata)+ callback = self._finalize_before_ack+ if became_idle and callback is not None:+ await callback(metadata)++ async def prepare_ack(+ self,+ queue_name: str,+ message: Any,+ ) -> Optional[QueueTaskMetadata]:+ """Remove work provisionally and finalize its task before the last durable ACK."""+ metadata = (+ message if isinstance(message, QueueTaskMetadata) else extract_task_metadata(message)+ )+ if metadata is None:+ return None++ removed, became_idle = self._remove_work(queue_name, metadata)+ if not removed:+ return None+ if not became_idle:+ return metadata++ callback = self._finalize_before_ack+ if callback is None:+ return metadata+ try:+ await callback(metadata)+ except BaseException:+ self.rollback_ack(queue_name, metadata)+ raise+ return metadata++ def rollback_ack(self, queue_name: str, metadata: QueueTaskMetadata) -> None:+ """Restore provisionally removed work when finalization or ACK fails."""+ with self._lock:+ self._work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))++ def has_work(self, task_id: str, exclude_work_id: Optional[str] = None) -> bool:+ with self._lock:+ if exclude_work_id is not None:+ return any(+ work_id != exclude_work_id+ for _queue_name, work_id in self._work.get(task_id, ())+ )+ return bool(self._work.get(task_id) or self._active.get(task_id))++ def cancellation_requested(self, task_id: str) -> bool:+ callback = self._is_cancellation_requested+ return bool(callback is not None and callback(task_id))++ def record_failure(self, task_id: str, error: str) -> None:+ """Record the first failed work item owned by a task."""+ with self._lock:+ self._failures.setdefault(task_id, error)++ def failure(self, task_id: str) -> Optional[str]:+ with self._lock:+ return self._failures.get(task_id)++ def clear_failure(self, task_id: str) -> None:+ with self._lock:+ self._failures.pop(task_id, None)++ def register_active(+ self,+ task_id: str,+ active_task: asyncio.Task[Any],+ ) -> None:+ with self._lock:+ cancelled = self.cancellation_requested(task_id)+ if not cancelled:+ self._active.setdefault(task_id, set()).add(active_task)+ if cancelled:+ active_task.get_loop().call_soon(active_task.cancel)++ def unregister_active(self, task_id: str, active_task: asyncio.Task[Any]) -> None:+ with self._lock:+ entries = self._active.get(task_id)+ if entries is not None:+ entries.discard(active_task)+ if not entries:+ self._active.pop(task_id, None)++ def cancel_active(self, task_id: str) -> None:+ with self._lock:+ active = list(self._active.get(task_id, ()))+ for task in active:+ task.get_loop().call_soon_threadsafe(task.cancel)diff --git a/openviking/session/session.py b/openviking/session/session.pyindex b5c702048..383dfd6d2 100644--- a/openviking/session/session.py+++ b/openviking/session/session.py@@ -2014,6 +2014,14 @@ class Session:"budget_exceeded": retention_plan.budget_exceeded if retention_plan else False,}+ async def finalize_cancelled_commit(self, archive_uri: str) -> None:+ """Make a cancelled queued commit terminal without discarding its raw archive."""+ await self._write_failed_marker(+ archive_uri,+ stage="cancelled",+ error="session commit cancelled",+ )+async def resume_queued_commit(self, msg: "SessionCommitMsg") -> None:"""Run one durable Phase 2 job from its archived messages."""from openviking.service.task_tracker import get_task_tracker@@ -2650,6 +2658,11 @@ class Session:)logger.info(f"Session {self.session_id} memory extraction completed")except asyncio.CancelledError:+ await self._write_failed_marker(+ archive_uri,+ stage="cancelled",+ error="session commit cancelled",+ )raiseexcept Exception as e:await self._write_failed_marker(diff --git a/openviking/session/train/components/session_commit.py b/openviking/session/train/components/session_commit.pyindex e927bb1c5..50eab2323 100644--- a/openviking/session/train/components/session_commit.py+++ b/openviking/session/train/components/session_commit.py@@ -277,7 +277,7 @@ class SessionCommitPolicyTrainer:)while True:task = await self.client.get_task(task_id)- if task and task.get("status") in {"completed", "failed"}:+ if task and task.get("status") in {"completed", "failed", "cancelled"}:return taskif deadline is not None and asyncio.get_running_loop().time() >= deadline:return {"task_id": task_id, "status": "timeout", "error": "commit task timeout"}@@ -375,6 +375,8 @@ def _task_error(task: dict[str, Any] | None) -> str | None:return Noneif task.get("status") == "failed":return str(task.get("error") or "task failed")+ if task.get("status") == "cancelled":+ return str(task.get("error") or "task cancelled")if task.get("status") == "timeout":return str(task.get("error") or "task timeout")return Nonediff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex 946dd2278..8c7f3f6d3 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -849,6 +849,28 @@ class TextEmbeddingHandler(DequeueHandlerBase):elif report_success:self.report_success()+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ """Settle semantic joins when a queued embedding is cancelled."""+ embedding_msg: Optional[EmbeddingMsg] = None+ try:+ if data:+ payload = data.get("data", data)+ if isinstance(payload, str):+ payload = json.loads(payload)+ embedding_msg = EmbeddingMsg.from_dict(payload)+ except (KeyError, TypeError, ValueError) as exc:+ self.report_error(str(exc), data)+ return None++ if embedding_msg is not None:+ self._record_request_success(embedding_msg)+ if embedding_msg.semantic_msg_id:+ from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker++ await EmbeddingTaskTracker.get_instance().decrement(embedding_msg.semantic_msg_id)+ self.report_success()+ return None+@staticmethoddef _record_request_success(embedding_msg: EmbeddingMsg) -> None:tracker = get_request_wait_tracker()diff --git a/openviking/storage/queuefs/add_resource_msg.py b/openviking/storage/queuefs/add_resource_msg.pyindex dbc485e8a..78a4b6b2d 100644--- a/openviking/storage/queuefs/add_resource_msg.py+++ b/openviking/storage/queuefs/add_resource_msg.py@@ -16,6 +16,7 @@ class AddResourceMsg:user_id: strrole: strpath: str = ""+ source_path: str = ""telemetry_id: Optional[str] = Noneprepared: Optional[Dict[str, Any]] = Nonelock_handoff: Optional[Dict[str, Any]] = None@@ -80,6 +81,7 @@ class AddResourceMsg:return cls(task_id=str(task_id),path=str(path or ""),+ source_path=str(data.get("source_path") or path or ""),root_uri=str(root_uri),account_id=str(data.get("account_id", "default")),user_id=str(data.get("user_id", "default")),diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.pyindex 771c2102b..6345a38bd 100644--- a/openviking/storage/queuefs/add_resource_processor.py+++ b/openviking/storage/queuefs/add_resource_processor.py@@ -11,10 +11,11 @@ from typing import Any, Dict, Optionalfrom openviking.observability.context import bind_execution_contextfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.task_tracker import TaskStatus, get_task_tracker+from openviking.service.task_work_index import bind_task_context, extract_task_metadatafrom openviking.storage.queuefs.add_resource_msg import AddResourceMsgfrom openviking.storage.queuefs.named_queue import DequeueHandlerBase-from openviking.telemetry import bind_telemetry, resolve_telemetry-from openviking.telemetry.resource_summary import summarize_queue_errors+from openviking.telemetry import bind_telemetry, resolve_telemetry, unregister_telemetry+from openviking.telemetry.resource_summary import record_resource_queue_metricsfrom openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils.logger import get_logger@@ -51,6 +52,15 @@ class AddResourceProcessor(DequeueHandlerBase):except Exception:raise handoff_error+ async def _release_cancelled_handoff(self, msg: AddResourceMsg) -> None:+ if msg.lock_handoff is None:+ return+ try:+ lock = await self._viking_fs._async_agfs.pathlock_adopt(msg.lock_handoff)+ await self._viking_fs._async_agfs.pathlock_release(lock)+ except Exception as exc:+ logger.warning("[AddResource] Failed to release cancelled lock handoff: %s", exc)+async def _requeue_lock_handoff(self, msg: AddResourceMsg, exc: Exception) -> bool:if msg.lock_handoff_retry >= 2:return False@@ -70,6 +80,7 @@ class AddResourceProcessor(DequeueHandlerBase):return Trueasync def _process(self, msg: AddResourceMsg, data: Dict[str, Any]) -> None:+ telemetry_id = msg.telemetry_id or ""ctx = RequestContext(user=UserIdentifier(msg.account_id, msg.user_id),role=Role(msg.role),@@ -82,8 +93,17 @@ class AddResourceProcessor(DequeueHandlerBase):account_id=ctx.account_id,user_id=ctx.user.user_id,task_id=msg.task_id,+ meta={"source_path": msg.source_path},)- if task.status in (TaskStatus.COMPLETED, TaskStatus.FAILED):+ if task.status in (+ TaskStatus.CANCELLING,+ TaskStatus.COMPLETED,+ TaskStatus.FAILED,+ TaskStatus.CANCELLED,+ ):+ if task.status in (TaskStatus.CANCELLING, TaskStatus.CANCELLED):+ await self._release_cancelled_handoff(msg)+ unregister_telemetry(telemetry_id)self.report_success()return None@@ -100,9 +120,9 @@ class AddResourceProcessor(DequeueHandlerBase):user_id=ctx.user.user_id,)self.report_error(f"Invalid lock_handoff: {exc}", data)+ unregister_telemetry(telemetry_id)return None- telemetry_id = msg.telemetry_id or ""telemetry = resolve_telemetry(telemetry_id) if telemetry_id else Noneif telemetry is None:from openviking.telemetry.operation import OperationTelemetry@@ -119,8 +139,13 @@ class AddResourceProcessor(DequeueHandlerBase):user_id=ctx.user.user_id,)- with bind_execution_context(), bind_telemetry(telemetry):+ with (+ bind_execution_context(),+ bind_telemetry(telemetry),+ bind_task_context(msg.task_id, ctx.account_id, ctx.user.user_id),+ ):try:+ metadata = extract_task_metadata(data)await tracker.start(msg.task_id,account_id=ctx.account_id,@@ -143,16 +168,19 @@ class AddResourceProcessor(DequeueHandlerBase):)self.report_error("resource processing failed", data)return None- queue_errors = summarize_queue_errors(result.get("queue_status"))- if queue_errors:- await tracker.fail(- msg.task_id,- "queue processing failed: " + "; ".join(queue_errors),- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- self.report_error("queue processing failed", data)- return None+ await tracker.wait_for_descendants(msg.task_id, metadata.work_id)+ record_resource_queue_metrics(+ telemetry=telemetry,+ telemetry_id=telemetry_id,+ root_uri=result.get("root_uri"),+ )+ await self._resource_service._link_resource_reason_memory(+ result=result,+ ctx=ctx,+ reason=msg.reason,+ source_name=msg.source_name,+ timeout=msg.timeout,+ )await tracker.complete(msg.task_id,result,@@ -162,9 +190,6 @@ class AddResourceProcessor(DequeueHandlerBase):)self.report_success()return None- except asyncio.CancelledError:- # Leave both task and QueueFS message active; RecoverStale owns restart recovery.- raiseexcept Exception as exc:await tracker.fail(msg.task_id,@@ -175,10 +200,30 @@ class AddResourceProcessor(DequeueHandlerBase):self.report_error(str(exc), data)return Nonefinally:+ unregister_telemetry(telemetry_id)with suppress(Exception):if resource_lock is not None:await self._viking_fs._async_agfs.pathlock_release(resource_lock)+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ """Release an enqueue-time lock before ACKing cancelled work."""+ try:+ payload = data.get("data", data) if isinstance(data, dict) else data+ if isinstance(payload, str):+ payload = json.loads(payload)+ msg = AddResourceMsg.from_dict(payload)+ except (json.JSONDecodeError, TypeError, ValueError) as exc:+ self.report_error(str(exc), data)+ return None+ future = asyncio.run_coroutine_threadsafe(+ self._release_cancelled_handoff(msg),+ self._service_loop,+ )+ await asyncio.wrap_future(future)+ unregister_telemetry(msg.telemetry_id or "")+ self.report_success()+ return None+async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:if not data:return Nonediff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.pyindex 9b22a1743..4383c42df 100644--- a/openviking/storage/queuefs/named_queue.py+++ b/openviking/storage/queuefs/named_queue.py@@ -1,6 +1,7 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0import abc+import asyncioimport jsonimport threadingfrom dataclasses import dataclass, field@@ -9,6 +10,13 @@ from typing import Any, Callable, Dict, List, Optional, Unionfrom openviking.pyagfs import AGFSSyncClientProtocol, AsyncAGFSClientfrom openviking.pyagfs.exceptions import AGFSAlreadyExistsError, AGFSNotFoundError+from openviking.service.task_work_index import (+ TaskWorkIndex,+ TaskWorkRejected,+ bind_task_context,+ extract_task_metadata,+ prepare_task_payload,+)from openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)@@ -89,6 +97,11 @@ class DequeueHandlerBase(abc.ABC):if self._error_callback:self._error_callback(error_msg, data)+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ """Discard task work cancelled before its handler starts."""+ self.report_success()+ return None+@abc.abstractmethodasync def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:"""Called after message dequeue. Returns None to discard message."""@@ -109,6 +122,7 @@ class NamedQueue:name: str,enqueue_hook: Optional[EnqueueHookBase] = None,dequeue_handler: Optional[DequeueHandlerBase] = None,+ task_work_index: Optional[TaskWorkIndex] = None,):self.name = nameself.path = f"{mount_point}/{name}"@@ -116,6 +130,7 @@ class NamedQueue:self._async_agfs = AsyncAGFSClient(agfs)self._enqueue_hook = enqueue_hookself._dequeue_handler = dequeue_handler+ self._task_work_index = task_work_indexself._initialized = False# Status tracking@@ -152,6 +167,10 @@ class NamedQueue:def _on_process_error(self, error_msg: str, data: Optional[Dict[str, Any]] = None) -> None:"""Called on processing failure."""+ if self._task_work_index is not None and data is not None:+ metadata = extract_task_metadata(data)+ if metadata is not None:+ self._task_work_index.record_failure(metadata.task_id, error_msg)with self._lock:self._in_progress -= 1self._error_count += 1@@ -210,24 +229,57 @@ class NamedQueue:data = await self._enqueue_hook.on_enqueue(data)if isinstance(data, dict):- data = json.dumps(data)+ data, task_metadata = prepare_task_payload(data)+ else:+ task_metadata = None++ if self._task_work_index is not None and not self._task_work_index.register(+ self.name, task_metadata+ ):+ logger.info(+ "[NamedQueue] Skip enqueue for cancelling task %s on %s",+ task_metadata.task_id,+ self.name,+ )+ raise TaskWorkRejected(+ f"Task {task_metadata.task_id} is cancelling; rejected work for {self.name}"+ )- msg_id = await self._async_agfs.write(enqueue_file, data.encode("utf-8"))+ try:+ if isinstance(data, dict):+ data = json.dumps(data)++ msg_id = await self._async_agfs.write(enqueue_file, data.encode("utf-8"))+ except BaseException:+ if self._task_work_index is not None and task_metadata is not None:+ await self._task_work_index.discard(self.name, task_metadata)+ raisereturn msg_id if isinstance(msg_id, str) else str(msg_id)- async def ack(self, msg_id: str) -> None:+ async def ack(self, msg_id: str, message: Optional[Dict[str, Any]] = None) -> None:"""Acknowledge successful processing of a message (deletes it from persistent storage).Must be called after the dequeue handler finishes processing a message.+ Task-owned work is provisionally removed from the runtime index first so+ the last message can persist its task's terminal state before deletion.If not called (e.g. process crashes), the message will be automaticallyre-queued on the next startup via RecoverStale."""if not msg_id:returnack_file = f"{self.path}/ack"+ prepared = Nonetry:+ if self._task_work_index is not None and message is not None:+ prepared = await self._task_work_index.prepare_ack(self.name, message)await self._async_agfs.write(ack_file, msg_id.encode("utf-8"))+ except asyncio.CancelledError:+ if self._task_work_index is not None and prepared is not None:+ self._task_work_index.rollback_ack(self.name, prepared)+ raiseexcept Exception as e:+ if self._task_work_index is not None and prepared is not None:+ self._task_work_index.rollback_ack(self.name, prepared)logger.warning(f"[NamedQueue] Ack failed for {self.name} msg_id={msg_id}: {e}")async def _read_queue_message(self) -> Optional[Dict[str, Any]]:@@ -266,13 +318,14 @@ class NamedQueue:return None# Capture message ID before passing data to handler (handler may modify it)msg_id = data.get("id", "") if isinstance(data, dict) else ""+ raw_data = dataif self._dequeue_handler:self._on_dequeue_start()- data = await self._dequeue_handler.on_dequeue(data)+ data = await self.process_dequeued(data)# Ack unconditionally after handler returns (success or handled error).# If on_dequeue raises, the exception propagates and ack is skipped —# the message will be recovered on next startup.- await self.ack(msg_id)+ await self.ack(msg_id, raw_data)return dataexcept Exception as e:logger.debug(f"[NamedQueue] Dequeue failed for {self.name}: {e}")@@ -293,9 +346,29 @@ class NamedQueue:NOTE: caller must call _on_dequeue_start() before invoking this methodso that in_progress is incremented atomically with the dequeue."""- if self._dequeue_handler:+ if self._dequeue_handler is None:+ return data++ metadata = extract_task_metadata(data)+ if metadata is None or self._task_work_index is None:return await self._dequeue_handler.on_dequeue(data)- return data++ active_task = asyncio.current_task()+ with bind_task_context(metadata.task_id, metadata.account_id, metadata.user_id):+ if self._task_work_index.cancellation_requested(metadata.task_id):+ return await self._dequeue_handler.on_cancelled(data)+ if active_task is not None:+ self._task_work_index.register_active(metadata.task_id, active_task)+ try:+ return await self._dequeue_handler.on_dequeue(data)+ except asyncio.CancelledError:+ if self._task_work_index.cancellation_requested(metadata.task_id):+ self._on_process_success()+ return None+ raise+ finally:+ if active_task is not None:+ self._task_work_index.unregister_active(metadata.task_id, active_task)async def peek(self) -> Optional[Dict[str, Any]]:"""Peek at head message without removing."""@@ -336,14 +409,45 @@ class NamedQueue:except (AGFSNotFoundError, FileNotFoundError):return 0+ async def snapshot(self) -> List[Dict[str, Any]]:+ """Return all unacknowledged messages without changing queue state."""+ await self._ensure_initialized()+ try:+ content = await self._async_agfs.read(f"{self.path}/messages")+ except (AGFSNotFoundError, FileNotFoundError):+ return []+ if not content:+ return []+ if isinstance(content, bytes):+ content = content.decode("utf-8")+ elif hasattr(content, "content") and content.content is not None:+ content = content.content.decode("utf-8")+ parsed = json.loads(content)+ return parsed if isinstance(parsed, list) else []+async def clear(self) -> bool:"""Clear queue."""await self._ensure_initialized()clear_file = f"{self.path}/clear"+ messages = await self.snapshot() if self._task_work_index is not None else []+ prepared = []try:+ if self._task_work_index is not None:+ for message in messages:+ metadata = await self._task_work_index.prepare_ack(self.name, message)+ if metadata is not None:+ prepared.append(metadata)await self._async_agfs.write(clear_file, b"")return True+ except asyncio.CancelledError:+ if self._task_work_index is not None:+ for metadata in prepared:+ self._task_work_index.rollback_ack(self.name, metadata)+ raiseexcept Exception as e:+ if self._task_work_index is not None:+ for metadata in prepared:+ self._task_work_index.rollback_ack(self.name, metadata)logger.error(f"[NamedQueue] Clear failed for {self.name}: {e}")return Falsediff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 13f1bb0a6..c8dd4a09a 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -12,6 +12,7 @@ import timeimport tracebackfrom typing import Any, Dict, Optional, Set, Union+from openviking.service.task_work_index import TaskWorkIndexfrom openviking_cli.utils.logger import get_loggerfrom .embedding_queue import EmbeddingQueue@@ -95,6 +96,7 @@ class QueueManager:self._queue_threads: Dict[str, threading.Thread] = {}self._queue_stop_events: Dict[str, threading.Event] = {}self._poll_interval = 0.2+ self._task_work_index = TaskWorkIndex()atexit.register(self.stop)logger.info(@@ -114,6 +116,13 @@ class QueueManager:logger.info(f"[QueueManager] mount_point={self.mount_point} Started")+ async def prepare_task_tracking(self, tracker: Any) -> None:+ """Rebuild task work from QueueFS before any consumer starts."""+ snapshots = {name: await queue.snapshot() for name, queue in self._queues.items()}+ owners = self._task_work_index.rebuild(snapshots)+ tracker.attach_work_index(self._task_work_index)+ await tracker.restore_work_tasks(owners)+def setup_standard_queues(self, vector_store: Any, start: bool = True) -> None:"""Setup standard queues (Embedding and Semantic) with their handlers.@@ -223,7 +232,7 @@ class QueueManager:try:await queue.process_dequeued(data)# Ack after successful processing (delete from persistent storage).- await queue.ack(msg_id)+ await queue.ack(msg_id, data)except Exception as e:# Handler did not call report_error; decrement in_progress manually.# Do NOT ack — let RecoverStale re-queue on next startup.@@ -320,6 +329,7 @@ class QueueManager:name,enqueue_hook=enqueue_hook,dequeue_handler=dequeue_handler,+ task_work_index=self._task_work_index,)elif name == self.SEMANTIC:self._queues[name] = SemanticQueue(@@ -328,6 +338,7 @@ class QueueManager:name,enqueue_hook=enqueue_hook,dequeue_handler=dequeue_handler,+ task_work_index=self._task_work_index,)else:self._queues[name] = NamedQueue(@@ -336,6 +347,7 @@ class QueueManager:name,enqueue_hook=enqueue_hook,dequeue_handler=dequeue_handler,+ task_work_index=self._task_work_index,)if self._started:self._start_queue_worker(self._queues[name])diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pyindex 07ad10faa..d2f8617aa 100644--- a/openviking/storage/queuefs/semantic_dag.py+++ b/openviking/storage/queuefs/semantic_dag.py@@ -8,11 +8,12 @@ from dataclasses import dataclass, fieldfrom typing import Any, ClassVar, Dict, List, Optional, Setfrom weakref import WeakKeyDictionary-from openviking.utils.ingest_options import IngestOptionsfrom openviking.server.identity import RequestContext+from openviking.service.task_work_index import bind_task_context, get_task_contextfrom openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecarsfrom openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fsfrom openviking.telemetry.request_wait_tracker import get_request_wait_tracker+from openviking.utils.ingest_options import IngestOptionsfrom openviking_cli.utils import VikingURIfrom openviking_cli.utils.logger import get_logger@@ -124,6 +125,7 @@ class SemanticNodeScheduler:returncontinue+ item.executor._start_scheduled_work()try:if not item.executor.closed:await item.executor._run_work(item.work)@@ -132,6 +134,7 @@ class SemanticNodeScheduler:except Exception as exc:item.executor.fail(exc)finally:+ item.executor._finish_scheduled_work()self._queue.task_done()@@ -191,6 +194,7 @@ class SemanticDagExecutor:self._ingest_options = IngestOptions.from_value(ingest_options)self._coalesce_key = coalesce_keyself._coalesce_version = coalesce_version+ self._task_context = get_task_context()self._stale = Falseself._changed_paths = {path for key in ("added", "modified", "deleted") for path in self._changes.get(key, [])@@ -203,6 +207,9 @@ class SemanticDagExecutor:self._root_uri: Optional[str] = Noneself._root_done: Optional[asyncio.Event] = Noneself._scheduler: Optional[SemanticNodeScheduler] = None+ self._active_scheduled_work = 0+ self._active_work_idle = asyncio.Event()+ self._active_work_idle.set()self._closed = Falseself._failure: Optional[Exception] = Noneself._stats = DagStats()@@ -260,9 +267,11 @@ class SemanticDagExecutor:logger.error(f"Error in on_complete callback: {e}", exc_info=True)except BaseException:self._closed = True+ await self._active_work_idle.wait()raisefinally:self._closed = True+ await self._active_work_idle.wait()self._unregister_active()def _schedule_work(self, work: DagWork) -> None:@@ -272,6 +281,15 @@ class SemanticDagExecutor:self._scheduler = get_semantic_node_scheduler(self._node_concurrency)self._scheduler.submit(self, work)+ def _start_scheduled_work(self) -> None:+ self._active_scheduled_work += 1+ self._active_work_idle.clear()++ def _finish_scheduled_work(self) -> None:+ self._active_scheduled_work = max(0, self._active_scheduled_work - 1)+ if self._active_scheduled_work == 0:+ self._active_work_idle.set()+def _schedule_dir(self, dir_uri: str, parent_uri: Optional[str]) -> None:if self._closed:return@@ -337,6 +355,17 @@ class SemanticDagExecutor:return statsasync def _run_work(self, work: DagWork) -> None:+ if self._task_context is not None:+ with bind_task_context(+ self._task_context.task_id,+ self._task_context.account_id,+ self._task_context.user_id,+ ):+ await self._run_work_bound(work)+ return+ await self._run_work_bound(work)++ async def _run_work_bound(self, work: DagWork) -> None:if work.kind == "vectorize":await self._run_vectorize_work(work.vectorize_task)return@@ -676,6 +705,8 @@ class SemanticDagExecutor:self._stats.done_nodes += 1self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)+ if self._closed:+ returntry:if need_vectorize:use_summary = self._is_code_repo and bool(summary_dict.get("summary"))@@ -813,6 +844,9 @@ class SemanticDagExecutor:)overview, abstract = self._processor._normalize_overview_generation(overview)+ if self._closed:+ return+# Write directly, protected by the outer semantic lock.try:wrote = await self._write_directory_semantics(dir_uri, overview, abstract)diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex e453e8858..467c1407d 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -511,6 +511,31 @@ class SemanticProcessor(DequeueHandlerBase):self.report_error(str(e), data)return None+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ """Release a queued semantic lock before cancelled work is ACKed."""+ try:+ import json++ payload = data.get("data", data) if isinstance(data, dict) else data+ if isinstance(payload, str):+ payload = json.loads(payload)+ msg = SemanticMsg.from_dict(payload)+ except (TypeError, ValueError) as exc:+ self.report_error(str(exc), data)+ return None++ if msg.telemetry_id and msg.id:+ get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)+ if msg.lock_handoff is not None:+ try:+ viking_fs = get_viking_fs()+ lock = await viking_fs._async_agfs.pathlock_adopt(msg.lock_handoff)+ await viking_fs._async_agfs.pathlock_release(lock)+ except Exception as exc:+ logger.warning("Failed to release cancelled semantic lock: %s", exc)+ self.report_success()+ return None+def get_dag_stats(self) -> Optional["DagStats"]:return SemanticDagExecutor.get_active_stats()diff --git a/openviking/storage/queuefs/session_commit_processor.py b/openviking/storage/queuefs/session_commit_processor.pyindex 03b98dc42..02eaf4b7f 100644--- a/openviking/storage/queuefs/session_commit_processor.py+++ b/openviking/storage/queuefs/session_commit_processor.py@@ -13,6 +13,7 @@ from openviking.observability.context import ()from openviking.server.identity import RequestContext, Rolefrom openviking.service.task_tracker import get_task_tracker+from openviking.service.task_work_index import bind_task_contextfrom openviking.storage.queuefs.named_queue import DequeueHandlerBasefrom openviking.storage.queuefs.session_commit_msg import SessionCommitMsgfrom openviking.telemetry.span_models import create_root_span_attributes@@ -31,6 +32,19 @@ class SessionCommitProcessor(DequeueHandlerBase):self._session_service = session_serviceself._service_loop = service_loop+ @staticmethod+ def _parse_message(data: Dict[str, Any]) -> tuple[SessionCommitMsg, RequestContext]:+ payload = data.get("data", data)+ if isinstance(payload, str):+ payload = json.loads(payload)+ msg = SessionCommitMsg.from_dict(payload)+ ctx = RequestContext(+ user=UserIdentifier.from_dict(msg.user),+ role=Role.USER,+ actor_peer_id=msg.actor_peer_id,+ )+ return msg, ctx+async def _process(self, msg: SessionCommitMsg, ctx: RequestContext) -> None:# Bind a root observability context so Phase-2 extraction VLM/embedding# token events are attributed to the committing account/user rather than@@ -70,24 +84,48 @@ class SessionCommitProcessor(DequeueHandlerBase):)returnawait session.load()- await session.resume_queued_commit(msg)+ with bind_task_context(msg.task_id, ctx.account_id, ctx.user.user_id):+ await session.resume_queued_commit(msg)finally:reset_root_observability_context(root_context_token)+ async def _finalize_cancelled(self, msg: SessionCommitMsg, ctx: RequestContext) -> None:+ session = self._session_service.session(+ ctx,+ msg.session_id,+ session_uri=msg.session_uri,+ )+ if await session.exists():+ await session.finalize_cancelled_commit(msg.archive_uri)++ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ if not data:+ return None++ try:+ msg, ctx = self._parse_message(data)+ except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:+ self.report_error(str(exc), data)+ return None++ future = asyncio.run_coroutine_threadsafe(+ self._finalize_cancelled(msg, ctx),+ self._service_loop,+ )+ try:+ await asyncio.wrap_future(future)+ self.report_success()+ except asyncio.CancelledError:+ future.cancel()+ raise+ return None+async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:if not data:return Nonetry:- payload = data.get("data", data)- if isinstance(payload, str):- payload = json.loads(payload)- msg = SessionCommitMsg.from_dict(payload)- ctx = RequestContext(- user=UserIdentifier.from_dict(msg.user),- role=Role.USER,- actor_peer_id=msg.actor_peer_id,- )+ msg, ctx = self._parse_message(data)except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:self.report_error(str(exc), data)return Nonediff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex 030272a8e..3d9ae81a2 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -4391,28 +4391,36 @@ class VikingFS:the "failures do not block" semantics."""from openviking.service.task_tracker import get_task_tracker+ from openviking.service.task_work_index import bind_task_contexttracker = get_task_tracker()- await tracker.start(- task_id,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- stage="reindexing",- )+ tracker.register_running_task(task_id)try:+ await tracker.start(+ task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ stage="reindexing",+ )from openviking.service.reindex_executor import get_reindex_executorexecutor = get_reindex_executor()- await asyncio.gather(*[- self._run_vector_rebuild(executor, op, uri, level, ctx)- for (op, uri, level) in tasks- ])+ with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):+ await asyncio.gather(+ *[+ self._run_vector_rebuild(executor, op, uri, level, ctx)+ for (op, uri, level) in tasks+ ]+ )await tracker.complete(task_id,{"status": "completed", "task_count": len(tasks)},account_id=ctx.account_id,user_id=ctx.user.user_id,)+ except asyncio.CancelledError:+ # TaskWorkIndex finalizes after this active task and its queue work settle.+ returnexcept Exception as exc:await tracker.fail(task_id,@@ -4420,6 +4428,8 @@ class VikingFS:account_id=ctx.account_id,user_id=ctx.user.user_id,)+ finally:+ await tracker.unregister_running_task(task_id)async def _run_vector_rebuild(self,diff --git a/openviking/sync_client.py b/openviking/sync_client.pyindex 24b3f3acc..657b2ebb3 100644--- a/openviking/sync_client.py+++ b/openviking/sync_client.py@@ -190,6 +190,10 @@ class SyncOpenViking:"""Query background task status."""return run_async(self._async_client.get_task(task_id))+ def cancel_task(self, task_id: str) -> Optional[Dict[str, Any]]:+ """Cancel a background task."""+ return run_async(self._async_client.cancel_task(task_id))+def list_tasks(self,task_type: Optional[str] = None,diff --git a/openviking/telemetry/resource_summary.py b/openviking/telemetry/resource_summary.pyindex b88ac1715..90d19834e 100644--- a/openviking/telemetry/resource_summary.py+++ b/openviking/telemetry/resource_summary.py@@ -4,11 +4,9 @@from __future__ import annotations-from typing import Any, Dict, List+from typing import Any, Dict-from .context import get_current_telemetryfrom .operation import OperationTelemetry-from .registry import register_telemetry, unregister_telemetrydef _consume_semantic_request_stats(telemetry_id: str):@@ -38,21 +36,6 @@ def _consume_semantic_dag_stats(telemetry_id: str, root_uri: str | None):return None-def register_wait_telemetry(wait: bool) -> str:- """Register current telemetry collector for async queue consumers when needed."""- handle = get_current_telemetry()- if not handle.telemetry_id:- return ""- if handle.enabled:- register_telemetry(handle)- return handle.telemetry_id---def unregister_wait_telemetry(telemetry_id: str) -> None:- """Unregister request-scoped telemetry handle."""- unregister_telemetry(telemetry_id)--def build_queue_status_payload(status: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:"""Convert queue status objects to response payload format."""return {@@ -66,93 +49,28 @@ def build_queue_status_payload(status: Dict[str, Any]) -> Dict[str, Dict[str, An}-def summarize_queue_errors(queue_status: Dict[str, Any] | None) -> List[str]:- """Return human-readable summaries for queue groups with recorded errors."""- if not queue_status:- return []-- summaries: List[str] = []- for name, status in queue_status.items():- if isinstance(status, dict):- raw_error_count = status.get("error_count", 0)- raw_errors = status.get("errors") or []- else:- raw_error_count = getattr(status, "error_count", 0)- raw_errors = getattr(status, "errors", []) or []-- try:- error_count = int(raw_error_count or 0)- except (TypeError, ValueError):- error_count = 0- if error_count <= 0:- continue-- messages: List[str] = []- for error in raw_errors[:3]:- if isinstance(error, dict):- message = error.get("message")- else:- message = getattr(error, "message", None) or str(error)- if message:- messages.append(str(message))-- summary = f"{name} error_count={error_count}"- if messages:- summary = f"{summary}: {', '.join(messages)}"- summaries.append(summary)-- return summaries---def _resolve_queue_group(- *,- explicit_stats: Any,- fallback_status: Any,-) -> Dict[str, int]:- if explicit_stats is not None:- return {- "processed": explicit_stats.processed,- "requeue_count": getattr(explicit_stats, "requeue_count", 0),- "error_count": explicit_stats.error_count,- }- if fallback_status is None:+def _queue_metrics(stats: Any) -> Dict[str, int]:+ if stats is None:return {"processed": 0, "requeue_count": 0, "error_count": 0}- if isinstance(fallback_status, dict):- return {- "processed": int(fallback_status.get("processed", 0) or 0),- "requeue_count": int(fallback_status.get("requeue_count", 0) or 0),- "error_count": int(fallback_status.get("error_count", 0) or 0),- }return {- "processed": fallback_status.processed,- "requeue_count": getattr(fallback_status, "requeue_count", 0),- "error_count": fallback_status.error_count,+ "processed": stats.processed,+ "requeue_count": stats.requeue_count,+ "error_count": stats.error_count,}-def record_resource_wait_metrics(+def record_resource_queue_metrics(*,- telemetry: OperationTelemetry | None = None,+ telemetry: OperationTelemetry,telemetry_id: str,- queue_status: Dict[str, Any],root_uri: str | None,-) -> Dict[str, Dict[str, int]]:+) -> None:"""Apply queue and DAG metrics to a resource operation collector."""- telemetry = telemetry or get_current_telemetry()if not telemetry.enabled:- return {- "semantic": {"processed": 0, "requeue_count": 0, "error_count": 0},- "embedding": {"processed": 0, "requeue_count": 0, "error_count": 0},- }+ return- semantic = _resolve_queue_group(- explicit_stats=_consume_semantic_request_stats(telemetry_id),- fallback_status=queue_status.get("Semantic"),- )- embedding = _resolve_queue_group(- explicit_stats=_consume_embedding_request_stats(telemetry_id),- fallback_status=queue_status.get("Embedding"),- )+ semantic = _queue_metrics(_consume_semantic_request_stats(telemetry_id))+ embedding = _queue_metrics(_consume_embedding_request_stats(telemetry_id))telemetry.set("queue.semantic.processed", semantic["processed"])telemetry.set("queue.semantic.requeue_count", semantic["requeue_count"])@@ -168,16 +86,7 @@ def record_resource_wait_metrics(telemetry.set("semantic_nodes.pending", dag_stats.pending_nodes)telemetry.set("semantic_nodes.running", dag_stats.in_progress_nodes)- return {- "semantic": semantic,- "embedding": embedding,- }--__all__ = ["build_queue_status_payload",- "record_resource_wait_metrics",- "register_wait_telemetry",- "summarize_queue_errors",- "unregister_wait_telemetry",+ "record_resource_queue_metrics",]diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex 0358a205e..9618d195c 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -13,18 +13,19 @@ from typing import Any, Dict, Optionalfrom charset_normalizer import from_bytesfrom openviking.core.context import Context, ContextLevel, ResourceContentType, Vectorize-from openviking.utils.ingest_options import IngestOptionsfrom openviking.core.namespace import (context_type_for_uri,is_session_uri,owner_space_for_uri,)from openviking.server.identity import RequestContext+from openviking.service.task_work_index import TaskWorkRejectedfrom openviking.storage.queuefs import get_queue_managerfrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverterfrom openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fsfrom openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.image_search import image_bytes_to_data_uri+from openviking.utils.ingest_options import IngestOptionsfrom openviking.utils.time_utils import parse_iso_datetimefrom openviking_cli.utils import VikingURI, get_loggerfrom openviking_cli.utils.config import get_openviking_config@@ -384,6 +385,9 @@ async def vectorize_directory_meta(await embedding_queue.enqueue(msg_abstract)enqueued += 1logger.debug(f"Enqueued directory L0 (abstract) for vectorization: {uri}")+ except TaskWorkRejected:+ logger.debug("Skipped directory vectorization for cancelling task: %s", uri)+ returnexcept Exception as e:logger.error(f"Failed to enqueue directory L0 (abstract) for vectorization: {uri}: {e}",@@ -418,6 +422,9 @@ async def vectorize_directory_meta(await embedding_queue.enqueue(msg_overview)enqueued += 1logger.debug(f"Enqueued directory L1 (overview) for vectorization: {uri}")+ except TaskWorkRejected:+ logger.debug("Skipped directory vectorization for cancelling task: %s", uri)+ returnexcept Exception as e:logger.error(f"Failed to enqueue directory L1 (overview) for vectorization: {uri}: {e}",@@ -582,6 +589,14 @@ async def vectorize_file(enqueued = Truelogger.debug(f"Enqueued file for vectorization: {file_path}")+ except TaskWorkRejected:+ logger.debug("Skipped file vectorization for cancelling task: %s", file_path)+ if registered_wait_root is not None:+ get_request_wait_tracker().mark_embedding_failed(+ registered_wait_root[0],+ registered_wait_root[1],+ f"Task cancellation skipped file vector for {file_path}",+ )except Exception as e:logger.error(f"Failed to vectorize file {file_path}: {e}", exc_info=True)if registered_wait_root is not None:diff --git a/openviking_cli/client/base.py b/openviking_cli/client/base.pyindex e2ed1ae86..1f2568f51 100644--- a/openviking_cli/client/base.py+++ b/openviking_cli/client/base.py@@ -450,6 +450,11 @@ class BaseClient(ABC):"""Query background task status."""...+ @abstractmethod+ async def cancel_task(self, task_id: str) -> Optional[Dict[str, Any]]:+ """Cancel a background task."""+ ...+@abstractmethodasync def list_tasks(self,diff --git a/sdk/go/sessions.go b/sdk/go/sessions.goindex ba740980f..ec4b625d4 100644--- a/sdk/go/sessions.go+++ b/sdk/go/sessions.go@@ -136,6 +136,23 @@ func (c *Client) GetTask(ctx context.Context, taskID string) (map[string]any, erreturn result, nil}+// CancelTask requests cooperative cancellation of a background task.+func (c *Client) CancelTask(+ ctx context.Context,+ taskID string,+) (map[string]any, error) {+ var result map[string]any+ err := c.doJSON(+ ctx,+ http.MethodPost,+ "/api/v1/tasks/"+url.PathEscape(taskID)+"/cancel",+ nil,+ nil,+ &result,+ )+ return result, err+}+// ListTasks lists background tasks visible to the caller.func (c *Client) ListTasks(ctx context.Context, opts *ListTasksOptions) ([]any, error) {query := url.Values{}diff --git a/sdk/python/openviking_sdk/client.py b/sdk/python/openviking_sdk/client.pyindex 40907d8ab..2773889f3 100644--- a/sdk/python/openviking_sdk/client.py+++ b/sdk/python/openviking_sdk/client.py@@ -1368,6 +1368,10 @@ class AsyncHTTPClient:return Nonereturn self._handle_response(response)+ async def cancel_task(self, task_id: str) -> Dict[str, Any]:+ response = await self._request("POST", f"/api/v1/tasks/{task_id}/cancel")+ return self._handle_response(response)+async def list_tasks(self,task_type: Optional[str] = None,@@ -2327,6 +2331,9 @@ class SyncHTTPClient:def get_task(self, task_id: str) -> Optional[Dict[str, Any]]:return run_async(self._async_client.get_task(task_id))+ def cancel_task(self, task_id: str) -> Dict[str, Any]:+ return run_async(self._async_client.cancel_task(task_id))+def list_tasks(self,task_type: Optional[str] = None,diff --git a/sdk/typescript/src/client.ts b/sdk/typescript/src/client.tsindex 28ae1a6ac..10f43d326 100644--- a/sdk/typescript/src/client.ts+++ b/sdk/typescript/src/client.ts@@ -754,6 +754,13 @@ export class OpenVikingClient {throw error;}}++ cancelTask(taskId: string): Promise<JsonObject> {+ return this.request(+ "POST",+ `/api/v1/tasks/${pathPart(taskId)}/cancel`,+ ) as Promise<JsonObject>;+ }/** List background tasks. */listTasks(options: TaskListOptions = {}): Promise<unknown[]> {return this.request("GET", "/api/v1/tasks", {diff --git a/tests/api_test/resources/slow/test_temp_upload_deep.py b/tests/api_test/resources/slow/test_temp_upload_deep.pyindex cd40d9d63..0edb998eb 100644--- a/tests/api_test/resources/slow/test_temp_upload_deep.py+++ b/tests/api_test/resources/slow/test_temp_upload_deep.py@@ -65,17 +65,3 @@ class TestTempUploadDeep:assert add_resp.status_code == 200, (f"upload empty file should return 200, got {add_resp.status_code}")-- def test_upload_result_has_queue_status(self, api_client):- with tempfile.TemporaryDirectory() as temp_dir:- test_file = os.path.join(temp_dir, f"queue_{uuid.uuid4().hex[:8]}.md")- with open(test_file, "w") as f:- f.write("# Queue status test\n\nContent for queue status check.")- add_resp = api_client.add_resource(path=test_file, wait=True)- assert add_resp.status_code == 200- result = add_resp.json().get("result", {})- assert "queue_status" in result, (- f"result should contain queue_status, got keys: {list(result.keys())}"- )- qs = result["queue_status"]- assert isinstance(qs, dict), f"queue_status should be dict, got {type(qs).__name__}"diff --git a/tests/client/test_resource_management.py b/tests/client/test_resource_management.pyindex 09cdbd261..3efd2df43 100644--- a/tests/client/test_resource_management.py+++ b/tests/client/test_resource_management.py@@ -36,7 +36,6 @@ class TestAddResource:print(result)assert "root_uri" in result- assert "queue_status" in resultasync def test_local_client_add_resource_with_wait_preserves_queue_status(self):"""Local SDK add_resource(wait=True) should keep queue_status and internal telemetry."""diff --git a/tests/parse/test_feishu_parser_api.py b/tests/parse/test_feishu_parser_api.pyindex 26fcfba7f..4bebc818b 100644--- a/tests/parse/test_feishu_parser_api.py+++ b/tests/parse/test_feishu_parser_api.py@@ -12,6 +12,7 @@ from openviking.parse.understanding_api import PREPARED_RESPONSE_ID_ARG, Understfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.resource_service import ResourceServicefrom openviking.service.task_tracker import TaskStatus+from openviking.service.task_work_index import TASK_WORK_ID_FIELDfrom openviking.storage.queuefs.add_resource_msg import AddResourceMsgfrom openviking.storage.queuefs.add_resource_processor import AddResourceProcessorfrom openviking.storage.queuefs.queue_manager import QueueManager@@ -788,7 +789,8 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):service = SimpleNamespace(execute_add_resource_job=AsyncMock(return_value={"status": "success", "root_uri": final_uri}- )+ ),+ _link_resource_reason_memory=AsyncMock(),)task_tracker = SimpleNamespace(create=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.PENDING)),@@ -796,6 +798,7 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):update_stage=AsyncMock(),complete=AsyncMock(),fail=AsyncMock(),+ wait_for_descendants=AsyncMock(),)monkeypatch.setattr("openviking.storage.queuefs.add_resource_processor.get_task_tracker",@@ -823,7 +826,9 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):understanding_response_id="response-1",)- await processor._process(msg, msg.to_dict())+ data = msg.to_dict()+ data[TASK_WORK_ID_FIELD] = "work-1"+ await processor._process(msg, data)task_tracker.create.assert_awaited_once_with("add_resource",@@ -831,6 +836,7 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):account_id="account-1",user_id="user-1",task_id="task-1",+ meta={"source_path": ""},)task_tracker.complete.assert_awaited_once_with("task-1",diff --git a/tests/server/test_request_wait_tracking.py b/tests/server/test_request_wait_tracking.pyindex d654e1de6..1a2232770 100644--- a/tests/server/test_request_wait_tracking.py+++ b/tests/server/test_request_wait_tracking.py@@ -81,94 +81,6 @@ class _FakeVikingFS:self.content.pop(uri, None)-async def test_add_resource_wait_uses_request_tracker(service, monkeypatch):- tracker = _FakeRequestWaitTracker(- {- "Semantic": {"processed": 1, "error_count": 0, "errors": []},- "Embedding": {"processed": 2, "error_count": 0, "errors": []},- }- )- ctx = RequestContext(user=service.user, role=Role.ROOT)- telemetry = OperationTelemetry(operation="resources.add_resource", enabled=True)-- async def _fake_process_resource(**kwargs):- del kwargs- return {"status": "success", "root_uri": "viking://resources/demo"}-- monkeypatch.setattr(- service.resources._resource_processor, "process_resource", _fake_process_resource- )- monkeypatch.setattr(- "openviking.service.resource_service.get_queue_manager",- lambda: _ExplodingQueueManager(),- )- monkeypatch.setattr(- "openviking.service.resource_service.get_request_wait_tracker",- lambda: tracker,- raising=False,- )-- with bind_telemetry(telemetry):- result = await service.resources.add_resource(- path="/tmp/demo.md",- ctx=ctx,- reason="request wait test",- wait=True,- timeout=12.0,- )-- assert result["queue_status"] == tracker.queue_status- assert tracker.registered_requests == [telemetry.telemetry_id]- assert tracker.wait_calls == [(telemetry.telemetry_id, 12.0)]- assert tracker.build_calls == [telemetry.telemetry_id]- assert tracker.cleaned == [telemetry.telemetry_id]---async def test_add_resource_wait_uses_request_tracker_when_telemetry_disabled(service, monkeypatch):- tracker = _FakeRequestWaitTracker(- {- "Semantic": {"processed": 1, "error_count": 0, "errors": []},- "Embedding": {"processed": 2, "error_count": 0, "errors": []},- }- )- ctx = RequestContext(user=service.user, role=Role.ROOT)- telemetry = OperationTelemetry(operation="resources.add_resource", enabled=False)-- async def _fake_process_resource(**kwargs):- del kwargs- return {"status": "success", "root_uri": "viking://resources/demo"}-- monkeypatch.setattr(- service.resources._resource_processor, "process_resource", _fake_process_resource- )- monkeypatch.setattr(- "openviking.service.resource_service.get_queue_manager",- lambda: _ExplodingQueueManager(),- )- monkeypatch.setattr(- "openviking.service.resource_service.get_request_wait_tracker",- lambda: tracker,- raising=False,- )-- with bind_telemetry(telemetry):- result = await service.resources.add_resource(- path="/tmp/demo.md",- ctx=ctx,- reason="request wait test",- wait=True,- timeout=12.0,- )-- assert result["queue_status"] == tracker.queue_status- assert tracker.registered_requests == [telemetry.telemetry_id]- assert tracker.wait_calls == [(telemetry.telemetry_id, 12.0)]- assert tracker.build_calls == [telemetry.telemetry_id]- assert tracker.cleaned == [telemetry.telemetry_id]--@pytest.mark.asyncioasync def test_add_skill_wait_uses_request_tracker(service, monkeypatch):tracker = _FakeRequestWaitTracker(diff --git a/tests/service/test_resource_service_connector.py b/tests/service/test_resource_service_connector.pyindex d9cec7699..16ebc0956 100644--- a/tests/service/test_resource_service_connector.py+++ b/tests/service/test_resource_service_connector.py@@ -831,7 +831,7 @@ async def test_add_resource_job_executes_frozen_route(ctx, service):call = service._execute_resource_ingestion.await_args.kwargsassert call["path"] == msg.pathassert call["to"] == msg.root_uri- assert call["wait"] is True+ assert call["defer_post_processing"] is Falseassert call["resource_lock"] is resource_lockassert call["parser_backend"] == "understanding"service._should_use_connector.assert_not_called()diff --git a/tests/service/test_resource_service_watch.py b/tests/service/test_resource_service_watch.pyindex b12295753..fbbf0852a 100644--- a/tests/service/test_resource_service_watch.py+++ b/tests/service/test_resource_service_watch.py@@ -9,13 +9,14 @@ from unittest.mock import AsyncMock, MagicMockimport pytestimport pytest_asyncio-from openviking.utils.ingest_options import IngestOptionsfrom openviking.resource.watch_manager import WatchManagerfrom openviking.server.identity import RequestContext, Rolefrom openviking.service import resource_service as resource_service_modulefrom openviking.service.resource_service import ResourceService+from openviking.service.task_tracker import TaskStatusfrom openviking.storage.content_write import ContentWriteCoordinatorfrom openviking.storage.queuefs.add_resource_msg import AddResourceMsg+from openviking.utils.ingest_options import IngestOptionsfrom openviking_cli.exceptions import ConflictError, InvalidArgumentErrorfrom openviking_cli.session.user_id import UserIdentifier@@ -82,6 +83,9 @@ class NoopTaskTracker:async def complete(self, *_args, **_kwargs):pass+ async def wait(self, *_args, **_kwargs):+ return SimpleNamespace(status=TaskStatus.COMPLETED, result={})+def count(self):return self._count@@ -101,6 +105,7 @@ def isolate_service_dependencies(monkeypatch: pytest.MonkeyPatch) -> None:lambda: task_tracker,)monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: False)+ monkeypatch.setattr("openviking.connector.routing.is_git_repo_url", lambda _path: False)@pytest_asyncio.fixture@@ -331,31 +336,7 @@ class TestWatchTaskCreation:finish_calls.append({"args": args, "kwargs": kwargs})return {"root_uri": "viking://resources/queued"}- class FakeRequestWaitTracker:- def register_request(self, *_args, **_kwargs):- pass-- async def wait_for_request(self, *_args, **_kwargs):- pass-- def build_queue_status(self, *_args, **_kwargs):- return {}-- def cleanup(self, *_args, **_kwargs):- pass-monkeypatch.setattr(ContentWriteCoordinator, "set_tags", fake_set_tags)- monkeypatch.setattr(- resource_service_module,- "get_request_wait_tracker",- lambda: FakeRequestWaitTracker(),- )- monkeypatch.setattr(- resource_service_module,- "unregister_wait_telemetry",- lambda *_args, **_kwargs: None,- )- resource_service._link_resource_reason_memory = AsyncMock()resource_service._resource_processor.finish_prepared_resource = AsyncMock(side_effect=fake_finish_prepared_resource)diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.pyindex aec37012c..6ba49edcb 100644--- a/tests/storage/test_semantic_dag_stats.py+++ b/tests/storage/test_semantic_dag_stats.py@@ -6,7 +6,14 @@ import asyncioimport pytestfrom openviking.server.identity import RequestContext, Role-from openviking.storage.queuefs.semantic_dag import DagStats, SemanticDagExecutor+from openviking.service.task_work_index import TaskWorkIndex, TaskWorkRejected+from openviking.storage.queuefs.named_queue import NamedQueue+from openviking.storage.queuefs.semantic_dag import (+ DagStats,+ DagWork,+ SemanticDagExecutor,+ SemanticNodeScheduler,+)from openviking_cli.session.user_id import UserIdentifier@@ -19,7 +26,7 @@ class _FakeVikingFS:del node_limitreturn self._tree.get(uri, [])- async def write_file(self, path, content, ctx=None):+ async def write_file(self, path, content, ctx=None, lock_handle=None):self.writes.append((path, content))def _uri_to_path(self, uri, ctx=None):@@ -86,6 +93,25 @@ class _DummyTracker:return None+class _ScheduledExecutor:+ def __init__(self, run) -> None:+ self.closed = False+ self.failure = None+ self._run = run++ def _start_scheduled_work(self) -> None:+ return None++ def _finish_scheduled_work(self) -> None:+ return None++ async def _run_work(self, _work) -> None:+ await self._run()++ def fail(self, exc: Exception) -> None:+ self.failure = exc++@pytest.mark.asyncioasync def test_semantic_dag_stats_collects_nodes(monkeypatch):root_uri = "viking://resources/root"@@ -211,6 +237,54 @@ async def test_semantic_dag_shares_node_scheduler_across_roots(monkeypatch):assert executor_b.get_stats().done_nodes == 21+async def test_task_work_rejection_does_not_stop_shared_semantic_worker():+ work_index = TaskWorkIndex()++ async def finalize_before_ack(_metadata):+ return None++ work_index.set_callbacks(+ finalize_before_ack=finalize_before_ack,+ is_cancellation_requested=lambda _task_id: True,+ )+ embedding_queue = NamedQueue(+ None,+ "/queue",+ "Embedding",+ task_work_index=work_index,+ )+ embedding_queue._initialized = True+ unrelated_ran = asyncio.Event()++ async def rejected_work() -> None:+ await embedding_queue.enqueue(+ {+ "task_id": "task-a",+ "account_id": "account-a",+ "user_id": "user-a",+ }+ )++ async def unrelated_work() -> None:+ unrelated_ran.set()++ rejected = _ScheduledExecutor(rejected_work)+ unrelated = _ScheduledExecutor(unrelated_work)+ scheduler = SemanticNodeScheduler(max_workers=1)+ scheduler.submit(rejected, DagWork(kind="vectorize", dir_uri="a"))+ scheduler.submit(unrelated, DagWork(kind="vectorize", dir_uri="b"))++ await asyncio.wait_for(unrelated_ran.wait(), timeout=0.5)+ await asyncio.wait_for(scheduler._queue.join(), timeout=0.5)+ await asyncio.sleep(scheduler._idle_timeout * 2)++ assert isinstance(rejected.failure, TaskWorkRejected)+ assert unrelated.failure is None+ assert scheduler._queue.empty()+ assert all(worker.done() for worker in scheduler._workers)++@pytest.mark.asyncioasync def test_semantic_dag_skip_vectorization_does_not_schedule_tasks(monkeypatch):root_uri = "viking://resources/root"diff --git a/tests/storage/test_session_commit_processor_identity.py b/tests/storage/test_session_commit_processor_identity.pyindex 2be193dbc..ebccb77c3 100644--- a/tests/storage/test_session_commit_processor_identity.py+++ b/tests/storage/test_session_commit_processor_identity.py@@ -9,9 +9,13 @@ worker binds the committing account/user (so tokens are not attributed to"""import asyncio+import concurrent.futures+import json+from unittest.mock import Mockfrom openviking.observability.context import get_root_observability_contextfrom openviking.server.identity import RequestContext, Role+from openviking.session.session import Sessionfrom openviking.storage.queuefs.session_commit_msg import SessionCommitMsgfrom openviking.storage.queuefs.session_commit_processor import SessionCommitProcessorfrom openviking_cli.session.user_id import UserIdentifier@@ -41,6 +45,25 @@ class _FakeSessionService:return _FakeSession(self._captured)+class _MemoryVikingFS:+ def __init__(self) -> None:+ self.files: dict[str, str] = {}++ async def stat(self, uri, ctx=None):+ return {"path": uri}++ async def write_file(self, uri, content, ctx=None, lease_ref=None):+ self.files[uri] = content+++class _SingleSessionService:+ def __init__(self, session: Session) -> None:+ self._session = session++ def session(self, ctx, session_id, session_uri=None):+ return self._session++def _make_msg() -> SessionCommitMsg:return SessionCommitMsg(task_id="task-1",@@ -75,3 +98,49 @@ async def test_process_resets_root_context_after_completion():await processor._process(_make_msg(), ctx)assert get_root_observability_context() is None+++async def test_cancelled_queued_commit_writes_terminal_marker_before_success(monkeypatch):+ msg = _make_msg()+ viking_fs = _MemoryVikingFS()+ session = Session(+ viking_fs=viking_fs,+ session_id=msg.session_id,+ session_uri=msg.session_uri,+ )+ processor = SessionCommitProcessor(+ _SingleSessionService(session),+ asyncio.get_running_loop(),+ )+ marker_uri = f"{msg.archive_uri}/.failed.json"+ on_success = Mock(side_effect=lambda: viking_fs.files[marker_uri])+ processor.set_callbacks(on_success, Mock(), Mock())++ def run_on_current_loop(coro, _loop):+ task = asyncio.create_task(coro)+ future: concurrent.futures.Future[None] = concurrent.futures.Future()++ def complete(completed: asyncio.Task) -> None:+ if completed.cancelled():+ future.cancel()+ return+ error = completed.exception()+ if error is not None:+ future.set_exception(error)+ else:+ future.set_result(completed.result())++ task.add_done_callback(complete)+ return future++ monkeypatch.setattr(+ "openviking.storage.queuefs.session_commit_processor.asyncio.run_coroutine_threadsafe",+ run_on_current_loop,+ )++ await processor.on_cancelled({"data": json.dumps(msg.to_dict())})++ marker = json.loads(viking_fs.files[marker_uri])+ assert marker["stage"] == "cancelled"+ assert marker["error"] == "session commit cancelled"+ on_success.assert_called_once_with()diff --git a/tests/telemetry/test_resource_summary.py b/tests/telemetry/test_resource_summary.pyindex 2fdd1cf01..e12127983 100644--- a/tests/telemetry/test_resource_summary.py+++ b/tests/telemetry/test_resource_summary.py@@ -1,24 +1,20 @@-from types import SimpleNamespacefrom openviking.telemetry.operation import OperationTelemetry-def test_record_resource_wait_metrics_collects_queue_and_dag_stats(monkeypatch):- from openviking.telemetry.resource_summary import record_resource_wait_metrics+def test_record_resource_queue_metrics_collects_queue_and_dag_stats(monkeypatch):+ from openviking.telemetry.resource_summary import record_resource_queue_metricstelemetry = OperationTelemetry(operation="resources.add_resource", enabled=True)telemetry_id = telemetry.telemetry_id- queue_status = {- "Semantic": SimpleNamespace(processed=3, error_count=1, errors=[]),- "Embedding": SimpleNamespace(processed=5, error_count=0, errors=[]),- }-class _SemanticStats:processed = 7+ requeue_count = 0error_count = 2class _EmbeddingStats:processed = 11+ requeue_count = 0error_count = 1class _DagStats:@@ -40,10 +36,9 @@ def test_record_resource_wait_metrics_collects_queue_and_dag_stats(monkeypatch):lambda _tid, _uri: _DagStats(),)- record_resource_wait_metrics(+ record_resource_queue_metrics(telemetry=telemetry,telemetry_id=telemetry_id,- queue_status=queue_status,root_uri="viking://resources/demo",)diff --git a/tests/test_telemetry_runtime.py b/tests/test_telemetry_runtime.pyindex 8774b9daf..6947f1e2f 100644--- a/tests/test_telemetry_runtime.py+++ b/tests/test_telemetry_runtime.py@@ -20,8 +20,6 @@ from openviking.observability.context import (reset_operation_observability_context,reset_root_observability_context,)-from openviking.server.identity import RequestContext, Role-from openviking.service.resource_service import ResourceServicefrom openviking.storage.collection_schemas import TextEmbeddingHandlerfrom openviking.storage.queuefs.semantic_dag import DagStatsfrom openviking.storage.queuefs.semantic_msg import SemanticMsg@@ -36,7 +34,6 @@ from openviking.telemetry.backends.memory import MemoryOperationTelemetryfrom openviking.telemetry.context import bind_telemetry, bind_telemetry_stagefrom openviking.telemetry.snapshot import TelemetrySnapshotfrom openviking.telemetry.span_models import OperationSpanAttributes, RootSpanAttributes-from openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils import logger as logger_module@@ -881,89 +878,6 @@ async def test_embedding_handler_binds_registered_operation_telemetry(monkeypatcassert summary["tokens"]["embedding"] == {"total": 9}-async def test_resource_service_add_resource_reports_queue_summary(monkeypatch):- telemetry = MemoryOperationTelemetry(operation="resources.add_resource", enabled=True)- queue_status = {- "Semantic": {- "processed": 2,- "requeue_count": 0,- "error_count": 1,- "errors": [],- },- "Embedding": {- "processed": 5,- "requeue_count": 0,- "error_count": 0,- "errors": [],- },- }-- class _DummyProcessor:- async def process_resource(self, **kwargs):- return {- "status": "success",- "root_uri": "viking://resources/demo",- }-- class _DummyRequestWaitTracker:- def register_request(self, telemetry_id: str) -> None:- del telemetry_id-- async def wait_for_request(self, telemetry_id: str, timeout=None) -> None:- del telemetry_id, timeout-- def build_queue_status(self, telemetry_id: str):- del telemetry_id- return queue_status-- def cleanup(self, telemetry_id: str) -> None:- del telemetry_id-- monkeypatch.setattr(- "openviking.service.resource_service.get_request_wait_tracker",- lambda: _DummyRequestWaitTracker(),- raising=False,- )-- class _DagStats:- total_nodes = 3- done_nodes = 2- pending_nodes = 1- in_progress_nodes = 0-- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.SemanticProcessor.consume_dag_stats",- classmethod(lambda cls, telemetry_id="", uri=None: _DagStats()),- )-- service = ResourceService(- vikingdb=object(),- viking_fs=object(),- resource_processor=_DummyProcessor(),- skill_processor=object(),- )- ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)-- with bind_telemetry(telemetry):- result = await service.add_resource(path="/tmp/demo.md", ctx=ctx, wait=True)-- assert result["root_uri"] == "viking://resources/demo"- telemetry_result = telemetry.finish()- summary = telemetry_result.summary- assert summary["queue"] == {- "semantic": {"processed": 2, "error_count": 1},- "embedding": {"processed": 5},- }- assert summary["semantic_nodes"] == {- "total": 3,- "done": 2,- "pending": 1,- }- assert "memory" not in summary- assert "errors" not in summary--def test_telemetry_summary_includes_only_memory_group_when_memory_metrics_exist():telemetry = MemoryOperationTelemetry(operation="session.commit", enabled=True)telemetry.record_token_usage("llm", 5, 3)diff --git a/web-studio/src/i18n/locales/en.ts b/web-studio/src/i18n/locales/en.tsindex f317457e2..d30f5d68f 100644--- a/web-studio/src/i18n/locales/en.ts+++ b/web-studio/src/i18n/locales/en.ts@@ -177,6 +177,8 @@ const en = {'Results returned by the API will appear here when the task completes.',noResultFailedDescription:'This task did not return a result. See the failure reason above.',+ noResultCancelledDescription:+ 'This task was cancelled before it returned a result.',},filters: {label: 'Filter',@@ -203,6 +205,8 @@ const en = {status: 'Status',},status: {+ cancelled: 'Cancelled',+ cancelling: 'Cancelling',completed: 'Completed',failed: 'Failed',pending: 'Pending',@@ -774,6 +778,7 @@ const en = {size: 'Size',},status: {+ cancelled: 'Cancelled',processing: 'Processing',success: 'Processed',failed: 'Processing failed',diff --git a/web-studio/src/i18n/locales/zh-CN.ts b/web-studio/src/i18n/locales/zh-CN.tsindex 88b4f54c4..b1504ea16 100644--- a/web-studio/src/i18n/locales/zh-CN.ts+++ b/web-studio/src/i18n/locales/zh-CN.ts@@ -169,6 +169,7 @@ const zhCN = {noResult: '暂无执行结果',noResultDescription: '任务完成后,接口返回的结果会显示在这里。',noResultFailedDescription: '该任务未返回结果,请查看上方失败原因。',+ noResultCancelledDescription: '该任务已取消,未返回执行结果。',},filters: {label: '筛选',@@ -194,6 +195,8 @@ const zhCN = {status: '状态',},status: {+ cancelled: '已取消',+ cancelling: '取消中',completed: '已完成',failed: '失败',pending: '等待中',@@ -746,6 +749,7 @@ const zhCN = {size: '大小',},status: {+ cancelled: '已取消',processing: '处理中',success: '处理成功',failed: '处理失败',diff --git a/web-studio/src/routes/resources/-components/upload-task-dialog.tsx b/web-studio/src/routes/resources/-components/upload-task-dialog.tsxindex cf5099702..e5ffb1b62 100644--- a/web-studio/src/routes/resources/-components/upload-task-dialog.tsx+++ b/web-studio/src/routes/resources/-components/upload-task-dialog.tsx@@ -32,6 +32,14 @@ function TaskStatusBadge({ status }: { status: ResourceUploadTask['status'] }) {)}+ if (status === 'cancelled') {+ return (+ <Badge variant="secondary" className="text-muted-foreground">+ {t('processingTasks.status.cancelled')}+ </Badge>+ )+ }+return (<Badge variant="secondary" className="bg-amber-500/12 text-amber-300">{t('processingTasks.status.processing')}diff --git a/web-studio/src/routes/resources/-hooks/use-resource-upload.tsx b/web-studio/src/routes/resources/-hooks/use-resource-upload.tsxindex d9a0043cb..b89fc1561 100644--- a/web-studio/src/routes/resources/-hooks/use-resource-upload.tsx+++ b/web-studio/src/routes/resources/-hooks/use-resource-upload.tsx@@ -16,6 +16,7 @@ import type {import type { TaskListResult, TaskRecord } from '@ov-server/api/v1/tasks'export type ResourceUploadTaskStatus =+ | 'cancelled'| 'pending'| 'uploading'| 'processing'@@ -208,6 +209,9 @@ function toUploadStatus(if (status === 'failed') {return 'failed'}+ if (status === 'cancelled') {+ return 'cancelled'+ }return 'processing'}@@ -224,7 +228,8 @@ function mergeServerTask(existing && existing.source !== 'server'? existing.fileName: getServerTaskName(record)- const isFinished = status === 'success' || status === 'failed'+ const isFinished =+ status === 'success' || status === 'failed' || status === 'cancelled'return {id: existing?.id ?? `server-${record.task_id}`,@@ -244,7 +249,9 @@ function mergeServerTask(errorMessage:status === 'failed'? record.error || existing?.errorMessage || 'Processing failed'- : null,+ : status === 'cancelled'+ ? record.error || 'Processing cancelled'+ : null,rootUri,}}@@ -695,6 +702,19 @@ export function ResourceUploadProvider({}: prev,)+ return+ }++ if (remoteTask.status === 'cancelled') {+ setRemoteState((prev) =>+ prev.taskId === remoteTask.serverTaskId+ ? {+ ...prev,+ phase: 'idle',+ error: remoteTask.errorMessage || 'Processing cancelled',+ }+ : prev,+ )}}, [remoteState.phase, remoteState.taskId, tasks])diff --git a/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx b/web-studio/src/routes/tasks/-components/task-detail-sheet.tsxindex 22928d8c3..46d80f75c 100644--- a/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx+++ b/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx@@ -65,7 +65,11 @@ export function TaskDetailSheet({queryKey: ['task-detail', identityScopeKey, taskId],refetchInterval: (query) => {const status = normalizeTaskStatus(query.state.data?.status)- return status === 'pending' || status === 'running' ? 3_000 : false+ return status === 'pending' ||+ status === 'running' ||+ status === 'cancelling'+ ? 3_000+ : false},})const task = detailQuery.data@@ -177,7 +181,9 @@ export function TaskDetailSheet({{t(normalizeTaskStatus(task.status) === 'failed'? 'detail.noResultFailedDescription'- : 'detail.noResultDescription',+ : normalizeTaskStatus(task.status) === 'cancelled'+ ? 'detail.noResultCancelledDescription'+ : 'detail.noResultDescription',)}</p></div>diff --git a/web-studio/src/routes/tasks/-lib/task-record.ts b/web-studio/src/routes/tasks/-lib/task-record.tsindex 86d5d42ad..776134846 100644--- a/web-studio/src/routes/tasks/-lib/task-record.ts+++ b/web-studio/src/routes/tasks/-lib/task-record.ts@@ -1,6 +1,8 @@import type { TaskTimestamp } from './task-time'export type TaskStatus =+ | 'cancelled'+ | 'cancelling'| 'completed'| 'failed'| 'pending'@@ -37,6 +39,8 @@ export function normalizeTasks(value: unknown): TaskRecord[] {export function normalizeTaskStatus(status: string | undefined): TaskStatus {if (+ status === 'cancelled' ||+ status === 'cancelling' ||status === 'completed' ||status === 'failed' ||status === 'pending' ||diff --git a/web-studio/src/routes/tasks/route.tsx b/web-studio/src/routes/tasks/route.tsxindex 6f26b5d23..b08af861c 100644--- a/web-studio/src/routes/tasks/route.tsx+++ b/web-studio/src/routes/tasks/route.tsx@@ -81,8 +81,10 @@ const TASK_TYPE_OPTIONS: Exclude<TaskTypeFilter, 'all'>[] = [const TASK_STATUS_OPTIONS: Exclude<TaskStatusFilter, 'all'>[] = ['pending','running',+ 'cancelling','completed','failed',+ 'cancelled',]async function fetchTasks(@@ -145,7 +147,7 @@ function TasksRoute() {const Icon =status === 'completed'? CheckCircle2Icon- : status === 'failed'+ : status === 'failed' || status === 'cancelled'? CircleXIcon: status === 'running'? LoaderCircleIcon@@ -162,7 +164,13 @@ function TasksRoute() {}className="gap-1 font-normal">- <Icon className={status === 'running' ? 'animate-spin' : undefined} />+ <Icon+ className={+ status === 'running' || status === 'cancelling'+ ? 'animate-spin'+ : undefined+ }+ />{t(`status.${status}`)}</Badge>)