Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(tasks): support task cancellation (#3577)

Qin Haojiefd42b1ad92c9
diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rs
index 26165efac..b487d5a4a 100644
--- a/crates/ov_cli/src/client.rs
+++ b/crates/ov_cli/src/client.rs
@@ -1176,6 +1176,11 @@ impl HttpClient {
self.get(&path, &[]).await
}
+ pub async fn cancel_task(&self, task_id: &str) -> Result<serde_json::Value> {
+ let path = format!("/api/v1/tasks/{}/cancel", task_id);
+ self.post(&path, &serde_json::json!({})).await
+ }
+
pub async fn list_tasks(
&self,
task_type: Option<&str>,
diff --git a/crates/ov_cli/src/commands/task.rs b/crates/ov_cli/src/commands/task.rs
index 45f77132b..d809fa8eb 100644
--- a/crates/ov_cli/src/commands/task.rs
+++ b/crates/ov_cli/src/commands/task.rs
@@ -13,6 +13,17 @@ pub async fn status(
Ok(())
}
+pub async fn cancel(
+ client: &HttpClient,
+ task_id: &str,
+ output_format: OutputFormat,
+ compact: bool,
+) -> Result<()> {
+ let result = client.cancel_task(task_id).await?;
+ output_success(&result, output_format, compact);
+ Ok(())
+}
+
pub async fn list(
client: &HttpClient,
task_type: Option<&str>,
diff --git a/crates/ov_cli/src/error_ui.rs b/crates/ov_cli/src/error_ui.rs
index 3ae263fc0..94227a427 100644
--- a/crates/ov_cli/src/error_ui.rs
+++ b/crates/ov_cli/src/error_ui.rs
@@ -511,7 +511,7 @@ fn task_help_command(program: &str, tokens: &[&str]) -> String {
}
_ => format!("{program} task watch --help"),
},
- Some("status" | "list") => format!("{program} task {} --help", tokens[2]),
+ Some("status" | "cancel" | "list") => format!("{program} task {} --help", tokens[2]),
_ => format!("{program} task --help"),
}
}
diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rs
index 97d3a089b..b35115dd5 100644
--- a/crates/ov_cli/src/help_ui.rs
+++ b/crates/ov_cli/src/help_ui.rs
@@ -916,6 +916,10 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[
label: "ov task status <task-id>",
description: "Inspect one task.",
},
+ HelpItem {
+ label: "ov task cancel <task-id>",
+ description: "Cancel one task.",
+ },
],
next_steps: &[HelpItem {
label: "ov wait",
@@ -3119,6 +3123,7 @@ mod tests {
.expect("task help should render"),
);
assert!(task.contains("status <task-id>"));
+ assert!(task.contains("cancel <task-id>"));
assert!(task.contains("list"));
}
diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs
index b21371f5a..1cdbeb746 100644
--- a/crates/ov_cli/src/main.rs
+++ b/crates/ov_cli/src/main.rs
@@ -1161,12 +1161,18 @@ enum TaskCommands {
#[arg(value_name = "task-id")]
task_id: String,
},
+ /// Cancel a task
+ Cancel {
+ /// Task ID returned by add-resource/add-skill
+ #[arg(value_name = "task-id")]
+ task_id: String,
+ },
/// List all tracked tasks
List {
/// Filter by task type (e.g. add_resource, add_skill, session_commit, reindex)
#[arg(long, value_name = "type")]
task_type: Option<String>,
- /// Filter by status (pending, running, completed, failed)
+ /// Filter by status (pending, running, cancelling, completed, failed, cancelled)
#[arg(long, value_name = "status")]
status: Option<String>,
},
@@ -2272,7 +2278,7 @@ fn command_tokens_for_config_gate(args: &[OsString]) -> Vec<String> {
fn known_task_command_requires_config(tokens: &[String]) -> bool {
match tokens.get(1).map(String::as_str) {
- Some("status" | "list") => true,
+ Some("status" | "cancel" | "list") => true,
Some("watch") => match tokens.get(2).map(String::as_str) {
None => true,
Some(token) => is_watch_subcommand(token),
@@ -3094,6 +3100,10 @@ async fn main() {
let client = ctx.get_client();
commands::task::status(&client, &task_id, ctx.output_format, ctx.compact).await
}
+ TaskCommands::Cancel { task_id } => {
+ let client = ctx.get_client();
+ commands::task::cancel(&client, &task_id, ctx.output_format, ctx.compact).await
+ }
TaskCommands::List { task_type, status } => {
let client = ctx.get_client();
commands::task::list(
@@ -3840,6 +3850,7 @@ mod tests {
for args in [
&["ov", "find"][..],
&["ov", "task", "status"],
+ &["ov", "task", "cancel"],
&["ov", "task", "watch", "show"],
&["ov", "config", "validate"],
&["ov", "config", "show"],
@@ -3902,6 +3913,7 @@ mod tests {
&["ov", "config", "show"],
&["ov", "config", "validate"],
&["ov", "task", "status"],
+ &["ov", "task", "cancel"],
&["ov", "task", "list"],
&["ov", "task", "watch"],
&["ov", "task", "watch", "ls"],
diff --git a/crates/ov_cli/src/output.rs b/crates/ov_cli/src/output.rs
index 85c3e62fd..42d98bb34 100644
--- a/crates/ov_cli/src/output.rs
+++ b/crates/ov_cli/src/output.rs
@@ -919,7 +919,7 @@ fn table_value_tone(trimmed: &str) -> TableValueTone {
TableValueTone::Success
}
"running" | "in_progress" | "in-progress" | "pending" | "queued" | "processing"
- | "checking" | "warning" => TableValueTone::Warning,
+ | "cancelling" | "checking" | "warning" => TableValueTone::Warning,
"unhealthy" | "failed" | "error" | "false" | "cancelled" | "canceled" | "timeout"
| "timed_out" | "unreachable" => TableValueTone::Error,
"unknown" | "null" | "(empty)" => TableValueTone::Muted,
@@ -1045,6 +1045,7 @@ mod tests {
assert_eq!(table_value_tone("running"), TableValueTone::Warning);
assert_eq!(table_value_tone("pending"), TableValueTone::Warning);
+ assert_eq!(table_value_tone("cancelling"), TableValueTone::Warning);
assert_eq!(table_value_tone("queued"), TableValueTone::Warning);
assert_eq!(table_value_tone("processing"), TableValueTone::Warning);
diff --git a/crates/ragfs/src/plugins/queuefs/backend.rs b/crates/ragfs/src/plugins/queuefs/backend.rs
index 13e88bf01..12f40f6d3 100644
--- a/crates/ragfs/src/plugins/queuefs/backend.rs
+++ b/crates/ragfs/src/plugins/queuefs/backend.rs
@@ -161,6 +161,9 @@ pub trait QueueBackend: Send + Sync {
/// Get the number of messages in the queue
fn size(&self, queue_name: &str) -> Result<usize>;
+ /// List every unacknowledged message without changing queue state
+ fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>>;
+
/// Clear all messages from the queue
fn clear(&mut self, queue_name: &str) -> Result<()>;
@@ -289,6 +292,14 @@ impl QueueBackend for MemoryBackend {
Ok(queue.messages.len())
}
+ fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {
+ let queue = self
+ .queues
+ .get(queue_name)
+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
+ Ok(queue.messages.iter().cloned().collect())
+ }
+
fn clear(&mut self, queue_name: &str) -> Result<()> {
let queue = self
.queues
@@ -691,6 +702,34 @@ impl QueueBackend for SQLiteQueueBackend {
Ok(count as usize)
}
+ fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
+
+ Self::require_queue_exists(&conn, queue_name)?;
+ let mut stmt = conn
+ .prepare(
+ "SELECT data FROM queue_messages
+ WHERE queue_name = ?1 AND status IN ('pending', 'processing')
+ ORDER BY id",
+ )
+ .map_err(|e| Error::internal(format!("sqlite list unacked prepare error: {}", e)))?;
+ let rows = stmt
+ .query_map(params![queue_name], |row| read_sqlite_text(row, 0))
+ .map_err(|e| Error::internal(format!("sqlite list unacked query error: {}", e)))?;
+
+ let mut messages = Vec::new();
+ for row in rows {
+ let raw_data =
+ row.map_err(|e| Error::internal(format!("sqlite list unacked row error: {}", e)))?;
+ let stored: StoredMessage = serde_json::from_str(&raw_data)?;
+ messages.push(stored.into_message());
+ }
+ Ok(messages)
+ }
+
fn clear(&mut self, queue_name: &str) -> Result<()> {
let conn = self
.conn
@@ -920,6 +959,9 @@ mod tests {
let first = backend.dequeue("test").unwrap().unwrap();
assert_eq!(first.data, b"message 1");
assert_eq!(backend.size("test").unwrap(), 1);
+ let unacked = backend.list_unacked("test").unwrap();
+ assert_eq!(unacked.len(), 2);
+ assert_eq!(unacked[0].id, msg1_id);
assert!(backend.ack("test", &msg1_id).unwrap());
let second = backend.dequeue("test").unwrap().unwrap();
diff --git a/crates/ragfs/src/plugins/queuefs/mod.rs b/crates/ragfs/src/plugins/queuefs/mod.rs
index 12724272a..dc9cc88e0 100644
--- a/crates/ragfs/src/plugins/queuefs/mod.rs
+++ b/crates/ragfs/src/plugins/queuefs/mod.rs
@@ -6,6 +6,7 @@
//! - `/queue_name/dequeue` - Read from this file to remove and return the first message
//! - `/queue_name/peek` - Read from this file to view the first message without removing it
//! - `/queue_name/size` - Read from this file to get the current queue size
+//! - `/queue_name/messages` - Read all unacknowledged messages without changing queue state
//! - `/queue_name/clear` - Write to this file to clear all messages from the queue
//! - `/queue_name/ack` - Write message ID to this file to acknowledge and delete it
@@ -48,6 +49,10 @@ const CONTROL_FILES: &[ControlFileSpec] = &[
name: "size",
mode: 0o444,
},
+ ControlFileSpec {
+ name: "messages",
+ mode: 0o444,
+ },
ControlFileSpec {
name: "clear",
mode: 0o222,
@@ -251,8 +256,19 @@ impl FileSystem for QueueFileSystem {
let size = backend.size(&queue_name)?;
Ok(size.to_string().into_bytes())
}
+ "messages" => {
+ let messages = backend
+ .list_unacked(&queue_name)?
+ .into_iter()
+ .map(|msg| QueueMessage {
+ id: msg.id,
+ data: String::from_utf8_lossy(&msg.data).to_string(),
+ })
+ .collect::<Vec<_>>();
+ Ok(serde_json::to_vec(&messages)?)
+ }
_ => Err(Error::InvalidOperation(format!(
- "Cannot read from '{}'. Use dequeue, peek, or size",
+ "Cannot read from '{}'. Use dequeue, peek, size, or messages",
operation
))),
}
@@ -780,13 +796,14 @@ mod tests {
// Queue directory should list control files
let entries = fs.read_dir("/test").await.unwrap();
- assert_eq!(entries.len(), 6);
+ assert_eq!(entries.len(), 7);
let names: Vec<String> = entries.iter().map(|e| e.name.clone()).collect();
assert!(names.contains(&"enqueue".to_string()));
assert!(names.contains(&"dequeue".to_string()));
assert!(names.contains(&"peek".to_string()));
assert!(names.contains(&"size".to_string()));
+ assert!(names.contains(&"messages".to_string()));
assert!(names.contains(&"clear".to_string()));
assert!(names.contains(&"ack".to_string()));
}
diff --git a/docs/en/api/01-overview.md b/docs/en/api/01-overview.md
index 72c5cb385..abef9d5ce 100644
--- a/docs/en/api/01-overview.md
+++ b/docs/en/api/01-overview.md
@@ -518,6 +518,7 @@ This catalog follows the routes actually mounted by the server. Each group headi
| Method | Path | Description |
|--------|------|-------------|
| GET | `/api/v1/tasks/{task_id}` | Get a background task |
+| POST | `/api/v1/tasks/{task_id}/cancel` | Cancel a background task |
| GET | `/api/v1/tasks` | List background tasks |
| GET | `/api/v1/observer/queue` | Queue status |
| GET | `/api/v1/observer/vikingdb` | VikingDB status |
diff --git a/docs/en/api/17-tasks.md b/docs/en/api/17-tasks.md
index 39472fd6b..6d84835d4 100644
--- a/docs/en/api/17-tasks.md
+++ b/docs/en/api/17-tasks.md
@@ -13,8 +13,10 @@ Query background task status for APIs that return `task_id`, such as session com
**Task Statuses:**
- `pending`: Task waiting to execute
- `running`: Task in progress
+- `cancelling`: Cancellation requested; waiting for the task's durable queue messages and in-process work to settle
- `completed`: Task successfully completed
- `failed`: Task failed
+- `cancelled`: Task cancelled
**Code Entries:**
- `openviking/server/routers/tasks.py:get_task()` - HTTP route
@@ -138,6 +140,102 @@ ov task status uuid-xxx
---
+### cancel_task()
+
+#### 1. API Implementation Introduction
+
+Request cooperative cancellation of a background task. The operation immediately prevents the task from creating new QueueFS work and cancels its active in-process work; writes that already completed are not rolled back. If durable messages or in-process work remain, the operation first returns `cancelling`. The task becomes `cancelled` only after all owned work settles.
+
+Repeated cancellation of a task in `cancelling` or `cancelled` is idempotent.
+
+**Supported Task Types:**
+- `add_resource`
+- `session_commit`
+- `admin_reindex`
+- `snapshot_restore_reindex`
+
+**Code Entries:**
+- `openviking/server/routers/tasks.py:cancel_task()` - HTTP route
+- `openviking/service/task_tracker.py:TaskTracker.cancel()` - task lifecycle
+- `crates/ov_cli/src/commands/task.rs:cancel()` - CLI command
+
+#### 2. Interface and Parameter Description
+
+| Parameter | Type | Required | Default | Description |
+|-----------|------|----------|---------|-------------|
+| task_id | str | Yes | - | Background task ID to cancel |
+
+Only the current user who owns the task can cancel it. ROOT identities cannot cancel tasks.
+
+#### 3. Usage Examples
+
+**Python SDK**
+
+```python
+task = await client.cancel_task("uuid-xxx")
+print(task["status"])
+```
+
+**TypeScript SDK**
+
+```typescript
+const task = await client.cancelTask("uuid-xxx");
+console.log(task.status);
+```
+
+**Go SDK**
+
+```go
+task, err := client.CancelTask(ctx, "uuid-xxx")
+if err != nil {
+ return err
+}
+fmt.Println(task["status"])
+```
+
+**HTTP API**
+
+```http
+POST /api/v1/tasks/{task_id}/cancel
+```
+
+```bash
+curl -X POST http://localhost:1933/api/v1/tasks/uuid-xxx/cancel \
+ -H "X-API-Key: your-key"
+```
+
+**CLI**
+
+```bash
+ov task cancel uuid-xxx
+```
+
+**Response Example**
+
+```json
+{
+ "status": "ok",
+ "result": {
+ "task_id": "uuid-xxx",
+ "task_type": "add_resource",
+ "status": "cancelling",
+ "resource_id": "viking://resources/guide",
+ "stage": "processing_queue",
+ "result": null,
+ "error": null
+ }
+}
+```
+
+If the task has no remaining work, the response status can be `cancelled` immediately. Otherwise, continue polling with `get_task()` until the status becomes `cancelled`.
+
+**Error Handling:**
+- `NOT_FOUND` (404): the task does not exist, has expired, or belongs to another user
+- `PERMISSION_DENIED` (403): a ROOT identity attempts to cancel a task
+- `FAILED_PRECONDITION` (412): the task type does not support cancellation, or the task is already `completed`/`failed`
+
+---
+
### list_tasks()
#### 1. API Implementation Introduction
@@ -155,7 +253,7 @@ List background tasks visible to the current caller, supporting filtering by typ
| Parameter | Type | Required | Default | Description |
|-----------|------|----------|---------|-------------|
| task_type | str | No | None | Filter by task type, for example `session_commit` |
-| status | str | No | None | Filter by task status: `pending`, `running`, `completed`, `failed` |
+| status | str | No | None | Filter by task status: `pending`, `running`, `cancelling`, `completed`, `failed`, `cancelled` |
| resource_id | str | No | None | Filter by task resource ID, for example a session ID |
| limit | int | No | 50 | Maximum number of task records to return |
diff --git a/docs/zh/api/01-overview.md b/docs/zh/api/01-overview.md
index 72574eee3..7bd68e349 100644
--- a/docs/zh/api/01-overview.md
+++ b/docs/zh/api/01-overview.md
@@ -513,6 +513,7 @@ JSON 输出 - 错误:
| 方法 | 路径 | 说明 |
|------|------|------|
| GET | `/api/v1/tasks/{task_id}` | 获取后台任务 |
+| POST | `/api/v1/tasks/{task_id}/cancel` | 取消后台任务 |
| GET | `/api/v1/tasks` | 列出后台任务 |
| GET | `/api/v1/observer/queue` | 队列状态 |
| GET | `/api/v1/observer/vikingdb` | VikingDB 状态 |
diff --git a/docs/zh/api/17-tasks.md b/docs/zh/api/17-tasks.md
index 37dff6130..1726c478d 100644
--- a/docs/zh/api/17-tasks.md
+++ b/docs/zh/api/17-tasks.md
@@ -13,8 +13,10 @@
**任务状态**:
- `pending`: 任务等待执行
- `running`: 任务执行中
+- `cancelling`: 已请求取消,正在等待该任务的持久化队列消息和进程内工作结束
- `completed`: 任务成功完成
- `failed`: 任务失败
+- `cancelled`: 任务已取消
**代码入口**:
- `openviking/server/routers/tasks.py:get_task()` - HTTP 路由
@@ -136,6 +138,102 @@ ov task status uuid-xxx
---
+### cancel_task()
+
+#### 1. API 实现介绍
+
+请求协作式取消后台任务。接口会立即阻止该任务产生新的 QueueFS work,并取消仍在运行的进程内工作;已经完成的写入不会回滚。当任务仍有待处理的持久化消息或进程内工作时,接口先返回 `cancelling`,全部收敛后任务才进入 `cancelled`。
+
+重复取消处于 `cancelling` 或 `cancelled` 状态的任务是幂等的。
+
+**支持的任务类型**:
+- `add_resource`
+- `session_commit`
+- `admin_reindex`
+- `snapshot_restore_reindex`
+
+**代码入口**:
+- `openviking/server/routers/tasks.py:cancel_task()` - HTTP 路由
+- `openviking/service/task_tracker.py:TaskTracker.cancel()` - 任务生命周期
+- `crates/ov_cli/src/commands/task.rs:cancel()` - CLI 命令
+
+#### 2. 接口和参数说明
+
+| 参数 | 类型 | 必填 | 默认值 | 说明 |
+|------|------|------|--------|------|
+| task_id | str | 是 | - | 要取消的后台任务 ID |
+
+只有任务所属的当前用户可以取消任务。ROOT 身份不能执行取消操作。
+
+#### 3. 使用示例
+
+**Python SDK**
+
+```python
+task = await client.cancel_task("uuid-xxx")
+print(task["status"])
+```
+
+**TypeScript SDK**
+
+```typescript
+const task = await client.cancelTask("uuid-xxx");
+console.log(task.status);
+```
+
+**Go SDK**
+
+```go
+task, err := client.CancelTask(ctx, "uuid-xxx")
+if err != nil {
+ return err
+}
+fmt.Println(task["status"])
+```
+
+**HTTP API**
+
+```http
+POST /api/v1/tasks/{task_id}/cancel
+```
+
+```bash
+curl -X POST http://localhost:1933/api/v1/tasks/uuid-xxx/cancel \
+ -H "X-API-Key: your-key"
+```
+
+**CLI**
+
+```bash
+ov task cancel uuid-xxx
+```
+
+**响应示例**
+
+```json
+{
+ "status": "ok",
+ "result": {
+ "task_id": "uuid-xxx",
+ "task_type": "add_resource",
+ "status": "cancelling",
+ "resource_id": "viking://resources/guide",
+ "stage": "processing_queue",
+ "result": null,
+ "error": null
+ }
+}
+```
+
+如果任务没有剩余 work,响应中的状态可以直接为 `cancelled`。否则继续通过 `get_task()` 查询,直到状态变为 `cancelled`。
+
+**错误处理**:
+- `NOT_FOUND`(404):任务不存在、已过期或不属于当前用户
+- `PERMISSION_DENIED`(403):ROOT 身份请求取消任务
+- `FAILED_PRECONDITION`(412):任务类型不支持取消,或任务已经 `completed`/`failed`
+
+---
+
### list_tasks()
#### 1. API 实现介绍
@@ -153,7 +251,7 @@ ov task status uuid-xxx
| 参数 | 类型 | 必填 | 默认值 | 说明 |
|------|------|------|--------|------|
| task_type | str | 否 | None | 按任务类型过滤,例如 `session_commit` |
-| status | str | 否 | None | 按任务状态过滤:`pending`、`running`、`completed`、`failed` |
+| status | str | 否 | None | 按任务状态过滤:`pending`、`running`、`cancelling`、`completed`、`failed`、`cancelled` |
| resource_id | str | 否 | None | 按资源 ID 过滤,例如会话 ID |
| limit | int | 否 | 50 | 最多返回的任务条数 |
diff --git a/openviking/async_client.py b/openviking/async_client.py
index 883a6ac31..1d98c40af 100644
--- a/openviking/async_client.py
+++ b/openviking/async_client.py
@@ -276,6 +276,11 @@ class AsyncOpenViking:
await self._ensure_initialized()
return await self._client.get_task(task_id)
+ async def cancel_task(self, task_id: str) -> Optional[Dict[str, Any]]:
+ """Cancel a background task."""
+ await self._ensure_initialized()
+ return await self._client.cancel_task(task_id)
+
async def list_tasks(
self,
task_type: Optional[str] = None,
diff --git a/openviking/client/local.py b/openviking/client/local.py
index ee6463f0b..4d3afbcc5 100644
--- a/openviking/client/local.py
+++ b/openviking/client/local.py
@@ -28,7 +28,7 @@ from openviking.utils.image_search import normalize_client_image_input
from openviking.utils.search_filters import SearchContextTypeInput, merge_search_filter
from openviking.utils.tags import build_search_tags_filter
from openviking_cli.client.base import BaseClient
-from openviking_cli.exceptions import InvalidArgumentError, NotFoundError
+from openviking_cli.exceptions import InvalidArgumentError, NotFoundError, PermissionDeniedError
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils import run_async
@@ -942,6 +942,17 @@ class LocalClient(BaseClient):
"""Query background task status."""
return await self._service.sessions.get_commit_task(task_id, self._ctx)
+ async def cancel_task(self, task_id: str) -> Optional[Dict[str, Any]]:
+ """Cancel a background task."""
+ if self._ctx.role == Role.ROOT:
+ raise PermissionDeniedError("ROOT may not cancel tasks")
+ task = await get_task_tracker().cancel(
+ task_id,
+ account_id=self._ctx.account_id,
+ user_id=self._ctx.user.user_id,
+ )
+ return task.to_dict() if task else None
+
async def list_tasks(
self,
task_type: Optional[str] = None,
diff --git a/openviking/metrics/collectors/task_tracker.py b/openviking/metrics/collectors/task_tracker.py
index 93a270ca9..b55049963 100644
--- a/openviking/metrics/collectors/task_tracker.py
+++ b/openviking/metrics/collectors/task_tracker.py
@@ -17,7 +17,7 @@ class TaskTrackerCollector(StateMetricCollector):
"""
Export task tracker backlog and completion counts grouped by task type.
- The datasource returns a snapshot of task counts, so all four families are exposed as gauges
+ The datasource returns a snapshot of task counts, so all families are exposed as gauges
keyed only by `task_type` rather than as cumulative counters.
"""
@@ -28,12 +28,14 @@ class TaskTrackerCollector(StateMetricCollector):
# rule: <METRICS_NAMESPACE>_<DOMAIN>_running
# e.g.: openviking_task_running
RUNNING: ClassVar[str] = MetricCollector.metric_name(DOMAIN, "running")
+ CANCELLING: ClassVar[str] = MetricCollector.metric_name(DOMAIN, "cancelling")
# rule: <METRICS_NAMESPACE>_<DOMAIN>_completed
# e.g.: openviking_task_completed
COMPLETED: ClassVar[str] = MetricCollector.metric_name(DOMAIN, "completed")
# rule: <METRICS_NAMESPACE>_<DOMAIN>_failed
# e.g.: openviking_task_failed
FAILED: ClassVar[str] = MetricCollector.metric_name(DOMAIN, "failed")
+ CANCELLED: ClassVar[str] = MetricCollector.metric_name(DOMAIN, "cancelled")
data_source: TaskStateDataSource
config: CollectorConfig = CollectorConfig(timeout_seconds=0.5)
@@ -43,11 +45,18 @@ class TaskTrackerCollector(StateMetricCollector):
return self.data_source.read_task_counts()
def collect_hook(self, registry, metric_input) -> None:
- """Translate the task-count snapshot into pending, running, completed, and failed gauges."""
+ """Translate the task-count snapshot into lifecycle-state gauges."""
counts_by_type = metric_input
# Snapshot semantics: types can disappear entirely, so clear previous series to avoid
# exporting stale non-zero gauges forever.
- for metric_name in (self.PENDING, self.RUNNING, self.COMPLETED, self.FAILED):
+ for metric_name in (
+ self.PENDING,
+ self.RUNNING,
+ self.CANCELLING,
+ self.COMPLETED,
+ self.FAILED,
+ self.CANCELLED,
+ ):
registry.gauge_delete_matching(metric_name, match_labels={})
for task_type, counts in counts_by_type.items():
labels = {"task_type": str(task_type)}
@@ -63,6 +72,12 @@ class TaskTrackerCollector(StateMetricCollector):
labels=labels,
label_names=("task_type",),
)
+ registry.set_gauge(
+ self.CANCELLING,
+ float(counts.get("cancelling", 0)),
+ labels=labels,
+ label_names=("task_type",),
+ )
registry.set_gauge(
self.COMPLETED,
float(counts.get("completed", 0)),
@@ -75,3 +90,9 @@ class TaskTrackerCollector(StateMetricCollector):
labels=labels,
label_names=("task_type",),
)
+ registry.set_gauge(
+ self.CANCELLED,
+ float(counts.get("cancelled", 0)),
+ labels=labels,
+ label_names=("task_type",),
+ )
diff --git a/openviking/server/routers/tasks.py b/openviking/server/routers/tasks.py
index adfc292d1..ccc52e141 100644
--- a/openviking/server/routers/tasks.py
+++ b/openviking/server/routers/tasks.py
@@ -16,7 +16,11 @@ from openviking.server.identity import RequestContext, Role
from openviking.server.models import Response
from openviking.service.task_store import SYSTEM_TASK_ACCOUNT_ID, SYSTEM_TASK_USER_ID
from openviking.service.task_tracker import get_task_tracker
-from openviking_cli.exceptions import OpenVikingError
+from openviking_cli.exceptions import (
+ FailedPreconditionError,
+ OpenVikingError,
+ PermissionDeniedError,
+)
router = APIRouter(prefix="/api/v1", tags=["tasks"])
@@ -51,11 +55,38 @@ async def get_task(
return Response(status="ok", result=task.to_dict())
[email protected]("/tasks/{task_id}/cancel")
+async def cancel_task(
+ task_id: str,
+ _ctx: RequestContext = Depends(get_request_context),
+):
+ """Request cooperative cancellation of a background task."""
+ if _ctx.role == Role.ROOT:
+ raise PermissionDeniedError("ROOT may not cancel tasks")
+ tracker = get_task_tracker()
+ try:
+ task = await tracker.cancel(
+ task_id,
+ account_id=_ctx.account_id,
+ user_id=_ctx.user.user_id,
+ )
+ except ValueError as exc:
+ raise FailedPreconditionError(str(exc)) from exc
+ if task is None:
+ raise OpenVikingError(
+ "Task not found or expired",
+ code="NOT_FOUND",
+ details={"resource": task_id, "type": "task"},
+ )
+ return Response(status="ok", result=task.to_dict())
+
+
@router.get("/tasks")
async def list_tasks(
task_type: Optional[str] = Query(None, description="Filter by task type (e.g. session_commit)"),
status: Optional[str] = Query(
- None, description="Filter by status (pending/running/completed/failed)"
+ None,
+ description="Filter by status (pending/running/cancelling/completed/failed/cancelled)",
),
resource_id: Optional[str] = Query(None, description="Filter by resource ID (e.g. session_id)"),
limit: int = Query(50, le=200, description="Max results"),
diff --git a/openviking/service/core.py b/openviking/service/core.py
index 3bac8ba76..5953e7543 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -23,7 +23,7 @@ from openviking.service.resource_memory_link_service import ResourceMemoryLinkSe
from openviking.service.resource_service import ResourceService
from openviking.service.search_service import SearchService
from openviking.service.session_service import SessionService
-from openviking.service.task_tracker import set_task_tracker
+from openviking.service.task_tracker import get_task_tracker, set_task_tracker
from openviking.session import create_session_compressor
from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.storage.collection_schemas import init_context_collection
@@ -365,8 +365,6 @@ class OpenVikingService:
resource_service=self._resource_service,
viking_fs=self._viking_fs,
)
- await self._watch_scheduler.start()
- logger.info("WatchScheduler started")
# Wire up sub-services
self._fs_service.set_dependencies(
@@ -429,6 +427,14 @@ class OpenVikingService:
),
allow_create=True,
)
+ await self._queue_manager.prepare_task_tracking(get_task_tracker())
+
+ # Do not let watches produce queue work while task ownership is being
+ # rebuilt from QueueFS. Consumers start only after the scheduler is ready.
+ await self._watch_scheduler.start()
+ logger.info("WatchScheduler started")
+
+ if self._queue_manager:
self._queue_manager.start()
logger.info("QueueManager workers started")
diff --git a/openviking/service/reindex_executor.py b/openviking/service/reindex_executor.py
index 9789a178f..ca06b658c 100644
--- a/openviking/service/reindex_executor.py
+++ b/openviking/service/reindex_executor.py
@@ -26,6 +26,7 @@ from openviking.core.namespace import (
from openviking.server.dependencies import get_service
from openviking.server.identity import RequestContext
from openviking.service.task_tracker import get_task_tracker
+from openviking.service.task_work_index import bind_task_context
from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils
from openviking.storage.expr import And, Eq, Or, PathScope
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
@@ -531,21 +532,26 @@ class ReindexExecutor:
ctx: RequestContext,
) -> None:
tracker = get_task_tracker()
- await tracker.start(task_id, account_id=ctx.account_id, user_id=ctx.user.user_id)
+ tracker.register_running_task(task_id)
try:
- result = await self._run(
- uri=uri,
- object_type=object_type,
- mode=mode,
- dry_run=dry_run,
- ctx=ctx,
- )
+ await tracker.start(task_id, account_id=ctx.account_id, user_id=ctx.user.user_id)
+ with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):
+ result = await self._run(
+ uri=uri,
+ object_type=object_type,
+ mode=mode,
+ dry_run=dry_run,
+ ctx=ctx,
+ )
await tracker.complete(
task_id,
result,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
+ except asyncio.CancelledError:
+ # TaskWorkIndex finalizes after this active task and its queue work settle.
+ return
except Exception as exc:
await tracker.fail(
task_id,
@@ -553,6 +559,8 @@ class ReindexExecutor:
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
+ finally:
+ await tracker.unregister_running_task(task_id)
async def _prune_orphan_vectors(
self,
diff --git a/openviking/service/resource_memory_link_service.py b/openviking/service/resource_memory_link_service.py
index b460da0b7..804a08d85 100644
--- a/openviking/service/resource_memory_link_service.py
+++ b/openviking/service/resource_memory_link_service.py
@@ -33,8 +33,8 @@ from openviking.session.memory.utils.resource_refs import (
resource_ref_matches,
unlink_resource_references_from_memory,
)
-from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.storage.viking_fs import VikingFS, get_viking_fs
+from openviking.storage.vikingdb_manager import VikingDBManager
from openviking_cli.exceptions import NotFoundError
from openviking_cli.utils import get_logger
@@ -336,29 +336,22 @@ class ResourceMemoryLinkService:
ctx: RequestContext,
timeout: Optional[float],
) -> Dict[str, Any]:
- from openviking.service.task_tracker import get_task_tracker
-
- async def _poll() -> Dict[str, Any]:
- tracker = get_task_tracker()
- while True:
- task = await tracker.get(
- task_id,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- if task is None:
- raise RuntimeError(f"session commit task not found: {task_id}")
- status = task.status.value if hasattr(task.status, "value") else str(task.status)
- if status == "completed":
- return task.to_dict()
- if status == "failed":
- raise RuntimeError(task.error or f"session commit task failed: {task_id}")
- await asyncio.sleep(0.1)
-
- return await asyncio.wait_for(
- _poll(),
- timeout=timeout or _RESOURCE_REASON_COMMIT_TIMEOUT_SECONDS,
+ from openviking.service.task_tracker import TaskStatus, get_task_tracker
+
+ task = await get_task_tracker().wait(
+ task_id,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ timeout=(
+ _RESOURCE_REASON_COMMIT_TIMEOUT_SECONDS if timeout is None else timeout
+ ),
+ poll_interval=0.1,
)
+ if task.status == TaskStatus.COMPLETED:
+ return task.to_dict()
+ if task.status == TaskStatus.FAILED:
+ raise RuntimeError(task.error)
+ raise RuntimeError(f"session commit task cancelled: {task_id}")
async def before_resource_delete(
self,
diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.py
index 535a42812..a5cff97fd 100644
--- a/openviking/service/resource_service.py
+++ b/openviking/service/resource_service.py
@@ -40,16 +40,13 @@ from openviking.server.user_config import (
effective_resource_add_target,
effective_skill_add_target,
)
-from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.storage.queuefs import QueueManager, get_queue_manager
from openviking.storage.viking_fs import VikingFS
-from openviking.telemetry import get_current_telemetry
+from openviking.storage.vikingdb_manager import VikingDBManager
+from openviking.telemetry import get_current_telemetry, register_telemetry, unregister_telemetry
from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.telemetry.resource_summary import (
build_queue_status_payload,
- record_resource_wait_metrics,
- register_wait_telemetry,
- unregister_wait_telemetry,
)
from openviking.utils import is_git_repo_url, parse_code_hosting_url
from openviking.utils.ingest_options import IngestOptions
@@ -455,6 +452,7 @@ class ResourceService:
account_id=msg.account_id,
user_id=msg.user_id,
task_id=msg.task_id,
+ meta={"source_path": msg.source_path},
)
await tracker.update_stage(
task.task_id,
@@ -500,14 +498,14 @@ class ResourceService:
from openviking.parse.understanding_api import PREPARED_RESPONSE_ID_ARG
internal_kwargs[PREPARED_RESPONSE_ID_ARG] = msg.understanding_response_id
- return await self._execute_resource_ingestion(
+ result = await self._execute_resource_ingestion(
path=msg.path,
ctx=ctx,
to=target_uri,
parent=parent_uri,
reason=msg.reason,
instruction=msg.instruction,
- wait=True,
+ defer_post_processing=False,
timeout=msg.timeout,
build_index=msg.build_index,
summarize=msg.summarize,
@@ -531,46 +529,26 @@ class ResourceService:
create_parent=msg.create_parent,
**internal_kwargs,
)
-
- telemetry_id = get_current_telemetry().telemetry_id
- ingest_tag_kwargs = self._add_resource_ingest_tag_kwargs(
- tags=msg.tags,
- tag_mode=msg.tag_mode,
- )
- request_wait_tracker = get_request_wait_tracker()
- request_wait_tracker.register_request(telemetry_id)
- try:
stage_result = stage_callback("processing_queue")
if inspect.isawaitable(stage_result):
await stage_result
- result = await self._resource_processor.finish_prepared_resource(
- msg.prepared,
- ctx=ctx,
- resource_lock=resource_lock,
- summarize=msg.summarize,
- build_index=msg.build_index,
- processing_mode=msg.processing_mode,
- **ingest_tag_kwargs,
- )
- await request_wait_tracker.wait_for_request(
- telemetry_id,
- timeout=msg.timeout,
- )
- status = request_wait_tracker.build_queue_status(telemetry_id)
- result["queue_status"] = status
- await self._link_resource_reason_memory(
- result=result,
- ctx=ctx,
- reason=msg.reason,
- source_name=msg.source_name,
- timeout=msg.timeout,
- )
return result
- except TimeoutError as exc:
- raise DeadlineExceededError("queue processing", msg.timeout) from exc
- finally:
- request_wait_tracker.cleanup(telemetry_id)
- unregister_wait_telemetry(telemetry_id)
+
+ stage_result = stage_callback("processing_queue")
+ if inspect.isawaitable(stage_result):
+ await stage_result
+ return await self._resource_processor.finish_prepared_resource(
+ msg.prepared,
+ ctx=ctx,
+ resource_lock=resource_lock,
+ summarize=msg.summarize,
+ build_index=msg.build_index,
+ processing_mode=msg.processing_mode,
+ **self._add_resource_ingest_tag_kwargs(
+ tags=msg.tags,
+ tag_mode=msg.tag_mode,
+ ),
+ )
async def reacquire_add_resource_job_lock(
self,
@@ -667,6 +645,7 @@ class ResourceService:
msg = AddResourceMsg(
task_id=task_id,
path=path,
+ source_path=(source_name or "") if kwargs.get("temp_file_id") else path,
root_uri=root_uri,
telemetry_id=get_current_telemetry().telemetry_id or None,
account_id=ctx.account_id,
@@ -1013,14 +992,35 @@ class ResourceService:
**kwargs,
)
- if not wait and is_git_repo_url(path):
- return await self.enqueue_git_add_resource(
+ if is_git_repo_url(path):
+ result = await self.enqueue_git_add_resource(
+ path=path,
+ ctx=ctx,
+ to=to,
+ parent=parent,
+ reason=reason,
+ instruction=instruction,
+ timeout=timeout,
+ build_index=build_index,
+ summarize=summarize,
+ processing_mode=processing_mode,
+ watch_interval=watch_interval,
+ manage_watch=manage_watch,
+ tags=tags,
+ tag_mode=tag_mode,
+ allow_local_path_resolution=allow_local_path_resolution,
+ enforce_public_remote_targets=enforce_public_remote_targets,
+ **kwargs,
+ )
+ else:
+ result = await self._execute_resource_ingestion(
path=path,
ctx=ctx,
to=to,
parent=parent,
reason=reason,
instruction=instruction,
+ defer_post_processing=True,
timeout=timeout,
build_index=build_index,
summarize=summarize,
@@ -1031,41 +1031,46 @@ class ResourceService:
tag_mode=tag_mode,
allow_local_path_resolution=allow_local_path_resolution,
enforce_public_remote_targets=enforce_public_remote_targets,
+ watch_auth_state=normalized_args.watch_auth_state,
+ parser_args=normalized_args.processor_kwargs,
**kwargs,
)
+ get_current_telemetry().set("resource.flags.wait", wait)
+ if not wait:
+ return result
+ from openviking.service.task_tracker import TaskStatus, get_task_tracker
- return await self._execute_resource_ingestion(
- path=path,
- ctx=ctx,
- to=to,
- parent=parent,
- reason=reason,
- instruction=instruction,
- wait=wait,
- timeout=timeout,
- build_index=build_index,
- summarize=summarize,
- processing_mode=processing_mode,
- watch_interval=watch_interval,
- manage_watch=manage_watch,
- tags=tags,
- tag_mode=tag_mode,
- allow_local_path_resolution=allow_local_path_resolution,
- enforce_public_remote_targets=enforce_public_remote_targets,
- watch_auth_state=normalized_args.watch_auth_state,
- parser_args=normalized_args.processor_kwargs,
- **kwargs,
- )
+ task_id = result["task_id"]
+ try:
+ task = await get_task_tracker().wait(
+ task_id,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ timeout=timeout,
+ )
+ except TimeoutError as exc:
+ raise DeadlineExceededError("add resource task", timeout) from exc
+
+ if task.status == TaskStatus.COMPLETED:
+ completed = dict(task.result)
+ completed.pop("task_id", None)
+ return completed
+ if task.status == TaskStatus.CANCELLED:
+ return {"status": "cancelled"}
+ return {
+ "status": "error",
+ "errors": [task.error],
+ }
async def _execute_resource_ingestion(
self,
path: str,
ctx: RequestContext,
+ defer_post_processing: bool,
to: Optional[str] = None,
parent: Optional[str] = None,
reason: str = "",
instruction: str = "",
- wait: bool = False,
timeout: Optional[float] = None,
build_index: bool = True,
summarize: bool = False,
@@ -1086,20 +1091,17 @@ class ResourceService:
self._ensure_initialized()
request_start = time.perf_counter()
telemetry = get_current_telemetry()
- telemetry_id = register_wait_telemetry(wait)
- request_wait_tracker = get_request_wait_tracker()
+ telemetry_id = telemetry.telemetry_id
+ register_telemetry(telemetry)
job_enqueued = False
deferred_lock: Optional[Dict[str, Any]] = None
ingest_tag_kwargs = self._add_resource_ingest_tag_kwargs(
tags=tags,
tag_mode=tag_mode,
)
- if telemetry_id:
- request_wait_tracker.register_request(telemetry_id)
watch_manager = self._get_watch_manager()
watch_enabled = bool(watch_manager and manage_watch and watch_interval > 0)
- telemetry.set("resource.flags.wait", wait)
telemetry.set("resource.flags.build_index", build_index)
telemetry.set("resource.flags.summarize", summarize)
telemetry.set("resource.flags.watch_enabled", watch_enabled)
@@ -1120,7 +1122,7 @@ class ResourceService:
kwargs["resource_lock"] = resource_lock
async_understanding_candidate = (
- not wait
+ defer_post_processing
and not is_git_repo_url(path)
and not allow_local_path_resolution
and self._resource_processor is not None
@@ -1248,6 +1250,7 @@ class ResourceService:
task_id=str(uuid4()),
telemetry_id=telemetry_id or None,
path=path,
+ source_path=(source_name or "") if kwargs.get("temp_file_id") else path,
root_uri=root_uri,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
@@ -1333,7 +1336,7 @@ class ResourceService:
stage_callback=stage_callback,
allow_local_path_resolution=allow_local_path_resolution,
prepared_resource=prepared_resource,
- defer_post_processing=not wait,
+ defer_post_processing=defer_post_processing,
**ingest_tag_kwargs,
**kwargs,
)
@@ -1343,54 +1346,7 @@ class ResourceService:
return result
prepared = result.pop("_post_process", None)
deferred_lock = result.pop("_resource_lock", None)
- if wait:
- if stage_callback is not None:
- stage_result = stage_callback("processing_queue")
- if inspect.isawaitable(stage_result):
- await stage_result
- wait_start = time.perf_counter()
- try:
- with telemetry.measure("resource.wait"):
- if telemetry_id:
- await request_wait_tracker.wait_for_request(
- telemetry_id,
- timeout=timeout,
- poll_interval=0.05,
- )
- status = request_wait_tracker.build_queue_status(telemetry_id)
- else:
- qm = get_queue_manager()
- status = build_queue_status_payload(
- await qm.wait_complete(timeout=timeout)
- )
- except TimeoutError as exc:
- telemetry.set_error(
- "resource_service.wait_complete",
- "DEADLINE_EXCEEDED",
- str(exc),
- )
- raise DeadlineExceededError("queue processing", timeout) from exc
- queue_wait_duration_ms = round((time.perf_counter() - wait_start) * 1000, 3)
- try:
- from openviking.metrics.datasources.resource import (
- ResourceIngestionEventDataSource,
- )
-
- ResourceIngestionEventDataSource.record_wait(
- operation="queue_processing",
- duration_seconds=float(queue_wait_duration_ms) / 1000.0,
- account_id=getattr(ctx, "account_id", None),
- )
- except Exception:
- pass
- result["queue_status"] = status
- record_resource_wait_metrics(
- telemetry_id=telemetry_id,
- queue_status=status,
- root_uri=result.get("root_uri"),
- )
- telemetry.set("queue.wait.duration_ms", queue_wait_duration_ms)
- if not wait:
+ if defer_post_processing:
from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
root_uri = result.get("root_uri", "")
@@ -1401,6 +1357,11 @@ class ResourceService:
task_id=str(uuid4()),
root_uri=root_uri,
prepared=prepared,
+ source_path=str(
+ (kwargs.get("source_name") or "")
+ if kwargs.get("temp_file_id")
+ else result.get("source_path") or ""
+ ),
telemetry_id=telemetry_id or None,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
@@ -1452,14 +1413,6 @@ class ResourceService:
watch_auth_state=watch_auth_state,
ctx=ctx,
)
- if wait:
- await self._link_resource_reason_memory(
- result=result,
- ctx=ctx,
- reason=reason,
- source_name=kwargs.get("source_name"),
- timeout=timeout,
- )
return result
except Exception as exc:
telemetry.set_error(
@@ -1475,9 +1428,8 @@ class ResourceService:
"resource.request.duration_ms",
round((time.perf_counter() - request_start) * 1000, 3),
)
- if wait or not telemetry_id or not job_enqueued:
- get_request_wait_tracker().cleanup(telemetry_id)
- unregister_wait_telemetry(telemetry_id)
+ if not telemetry_id or (defer_post_processing and not job_enqueued):
+ unregister_telemetry(telemetry_id)
if deferred_lock is not None:
await self._release_lock_ref(deferred_lock)
@@ -1544,7 +1496,7 @@ class ResourceService:
await task_tracker.fail(task_id, str(exc), account_id=account_id, user_id=user_id)
finally:
request_wait_tracker.cleanup(telemetry_id)
- unregister_wait_telemetry(telemetry_id)
+ unregister_telemetry(telemetry_id)
# ── Connector routing ──
@@ -1794,7 +1746,7 @@ class ResourceService:
finally:
if wait or not telemetry_id or not monitor_started:
request_wait_tracker.cleanup(telemetry_id)
- unregister_wait_telemetry(telemetry_id)
+ unregister_telemetry(telemetry_id)
async def build_index(
self, resource_uris: List[str], ctx: RequestContext, **kwargs
diff --git a/openviking/service/task_store.py b/openviking/service/task_store.py
index e2aea069e..741c81aad 100644
--- a/openviking/service/task_store.py
+++ b/openviking/service/task_store.py
@@ -150,6 +150,7 @@ def _task_to_payload(task: Any) -> Dict[str, Any]:
"resource_id": task.resource_id,
"account_id": task.account_id,
"user_id": task.user_id,
+ "meta": deepcopy(task.meta),
"stage": task.stage,
"result": deepcopy(task.result),
"error": task.error,
diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.py
index 07b0a94ea..c1bc7e827 100644
--- a/openviking/service/task_tracker.py
+++ b/openviking/service/task_tracker.py
@@ -25,6 +25,7 @@ from typing import Any, Dict, List, Optional
from uuid import uuid4
from openviking.service.task_store import TaskStore
+from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
@@ -35,11 +36,21 @@ class TaskStatus(str, Enum):
PENDING = "pending"
RUNNING = "running"
+ CANCELLING = "cancelling"
COMPLETED = "completed"
FAILED = "failed"
+ CANCELLED = "cancelled"
-_TERMINAL_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED)
+_TERMINAL_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED)
+_ACTIVE_STATUSES = (TaskStatus.PENDING, TaskStatus.RUNNING, TaskStatus.CANCELLING)
+
+_CANCELLABLE_TASK_TYPES = {
+ "add_resource",
+ "session_commit",
+ "admin_reindex",
+ "snapshot_restore_reindex",
+}
@dataclass
@@ -54,6 +65,7 @@ class TaskRecord:
resource_id: Optional[str] = None # e.g. session_id
account_id: Optional[str] = None
user_id: Optional[str] = None
+ meta: Dict[str, Any] = field(default_factory=dict)
stage: Optional[str] = None
result: Optional[Dict[str, Any]] = None
error: Optional[str] = None
@@ -65,6 +77,7 @@ class TaskRecord:
d["created_at_iso"] = datetime.fromtimestamp(self.created_at, tz=timezone.utc).isoformat()
d["updated_at_iso"] = datetime.fromtimestamp(self.updated_at, tz=timezone.utc).isoformat()
d["result"] = _sanitize_task_result(d.get("result"))
+ d["meta"] = _sanitize_task_result(d.get("meta"))
d.pop("account_id", None)
d.pop("user_id", None)
return d
@@ -153,6 +166,9 @@ class TaskTracker:
self._lock = threading.Lock()
self._async_lock = asyncio.Lock()
self._cleanup_task: Optional[asyncio.Task] = None
+ self._runtime_loop: Optional[asyncio.AbstractEventLoop] = None
+ self._work_index = TaskWorkIndex()
+ self._install_work_index_callbacks()
logger.info(
"[TaskTracker] Initialized (store=%s, max_tasks=%d)",
self._store.__class__.__name__,
@@ -161,6 +177,54 @@ class TaskTracker:
# ── Lifecycle ──
+ def _install_work_index_callbacks(self) -> None:
+ self._work_index.set_callbacks(
+ finalize_before_ack=self._finalize_before_ack,
+ is_cancellation_requested=self.is_cancellation_requested,
+ )
+
+ def attach_work_index(self, work_index: TaskWorkIndex) -> None:
+ """Use the QueueManager-owned index as the task lifecycle authority."""
+ self._work_index = work_index
+ self._runtime_loop = asyncio.get_running_loop()
+ self._install_work_index_callbacks()
+
+ async def restore_work_tasks(self, owners: Dict[str, tuple[str, str]]) -> None:
+ """Restore task records referenced by rebuilt QueueFS work."""
+ for task_id, (account_id, user_id) in owners.items():
+ await self.get(task_id, account_id=account_id, user_id=user_id)
+
+ async def _finalize_before_ack(self, metadata: QueueTaskMetadata) -> None:
+ """Persist the terminal state before QueueFS removes the last recovery message."""
+ loop = self._runtime_loop
+ if loop is None or loop.is_closed():
+ raise RuntimeError("TaskTracker runtime loop is unavailable during QueueFS ACK")
+ if asyncio.get_running_loop() is loop:
+ await self._finalize_task(
+ metadata.task_id,
+ account_id=metadata.account_id or None,
+ user_id=metadata.user_id or None,
+ )
+ return
+ future = asyncio.run_coroutine_threadsafe(
+ self._finalize_task(
+ metadata.task_id,
+ account_id=metadata.account_id or None,
+ user_id=metadata.user_id or None,
+ ),
+ loop,
+ )
+ await asyncio.wrap_future(future)
+
+ def is_cancellation_requested(self, task_id: str) -> bool:
+ """Fast thread-safe status check used by queue workers."""
+ with self._lock:
+ task = self._tasks.get(task_id)
+ return task is not None and task.status in (
+ TaskStatus.CANCELLING,
+ TaskStatus.CANCELLED,
+ )
+
def start_cleanup_loop(self) -> None:
"""Start the background TTL cleanup coroutine.
@@ -169,6 +233,7 @@ class TaskTracker:
"""
if self._cleanup_task is not None and not self._cleanup_task.done():
return
+ self._runtime_loop = asyncio.get_running_loop()
self._cleanup_task = asyncio.create_task(self._cleanup_loop())
logger.debug("[TaskTracker] Cleanup loop started")
@@ -195,10 +260,9 @@ class TaskTracker:
with self._lock:
expired_ids = []
for tid, t in self._tasks.items():
- if (
- t.status == TaskStatus.COMPLETED
- and (now - t.updated_at) > self.TTL_COMPLETED
- ):
+ if t.status in (TaskStatus.COMPLETED, TaskStatus.CANCELLED) and (
+ now - t.updated_at
+ ) > self.TTL_COMPLETED:
expired_ids.append(tid)
elif t.status == TaskStatus.FAILED and (now - t.updated_at) > self.TTL_FAILED:
expired_ids.append(tid)
@@ -244,6 +308,7 @@ class TaskTracker:
account_id: str,
user_id: str,
task_id: Optional[str] = None,
+ meta: Optional[Dict[str, Any]] = None,
) -> TaskRecord:
"""Register a new pending task. Returns a snapshot copy."""
self._validate_owner(account_id, user_id)
@@ -253,6 +318,7 @@ class TaskTracker:
resource_id=resource_id,
account_id=account_id,
user_id=user_id,
+ meta=dict(meta or {}),
)
async with self._async_lock:
if task_id:
@@ -303,7 +369,7 @@ class TaskTracker:
t.task_type == task_type
and t.resource_id == resource_id
and self._matches_owner(t, account_id, user_id)
- and t.status in (TaskStatus.PENDING, TaskStatus.RUNNING)
+ and t.status in _ACTIVE_STATUSES
for t in tasks
)
if has_active:
@@ -336,7 +402,7 @@ class TaskTracker:
"""Transition task to RUNNING."""
async with self._async_lock:
task = await self._load_for_update(task_id, account_id, user_id)
- if task and task.status not in _TERMINAL_STATUSES:
+ if task and task.status in (TaskStatus.PENDING, TaskStatus.RUNNING):
task.status = TaskStatus.RUNNING
if stage is not None:
task.stage = stage
@@ -355,7 +421,7 @@ class TaskTracker:
"""Update task stage without changing its lifecycle status."""
async with self._async_lock:
task = await self._load_for_update(task_id, account_id, user_id)
- if task and task.status not in _TERMINAL_STATUSES:
+ if task and task.status in (TaskStatus.PENDING, TaskStatus.RUNNING):
task.stage = stage
task.updated_at = time.time()
await self._store.update(task)
@@ -365,52 +431,164 @@ class TaskTracker:
async def complete(
self,
task_id: str,
- result: Optional[Dict[str, Any]] = None,
+ result: Dict[str, Any],
account_id: Optional[str] = None,
user_id: Optional[str] = None,
*,
resource_id: Optional[str] = None,
) -> None:
- """Transition task to COMPLETED with optional result."""
- transitioned = False
+ """Record successful completion and finalize after owned work settles."""
+ await self._record_outcome(
+ task_id,
+ account_id,
+ user_id,
+ result=result,
+ resource_id=resource_id,
+ )
+
+ async def fail(
+ self,
+ task_id: str,
+ error: str,
+ account_id: Optional[str] = None,
+ user_id: Optional[str] = None,
+ ) -> None:
+ """Record failure and finalize after owned work settles."""
+ await self._record_outcome(task_id, account_id, user_id, error=error)
+
+ async def _record_outcome(
+ self,
+ task_id: str,
+ account_id: Optional[str],
+ user_id: Optional[str],
+ *,
+ result: Optional[Dict[str, Any]] = None,
+ error: Optional[str] = None,
+ resource_id: Optional[str] = None,
+ ) -> None:
async with self._async_lock:
task = await self._load_for_update(task_id, account_id, user_id)
- if task and task.status not in _TERMINAL_STATUSES:
- task.status = TaskStatus.COMPLETED
- task.stage = "completed"
- task.result = result
- if resource_id is not None:
- task.resource_id = resource_id
+ if task and task.status in (TaskStatus.PENDING, TaskStatus.RUNNING):
+ if result is not None:
+ task.result = result
+ if resource_id is not None:
+ task.resource_id = resource_id
+ if error is not None and task.error is None:
+ task.error = _sanitize_error(error)
+ work_error = self._work_index.failure(task_id)
+ if work_error and task.error is None:
+ task.error = _sanitize_error(work_error)
task.updated_at = time.time()
await self._store.update(task)
with self._lock:
self._tasks[task.task_id] = task
- transitioned = True
- if transitioned:
- logger.info("[TaskTracker] Task %s completed", task_id)
+ await self._finalize_task(task_id, account_id=account_id, user_id=user_id)
- async def fail(
+ async def cancel(
self,
task_id: str,
- error: str,
account_id: Optional[str] = None,
user_id: Optional[str] = None,
- ) -> None:
- """Transition task to FAILED with sanitized error."""
- transitioned = False
+ ) -> Optional[TaskRecord]:
+ """Request cooperative cancellation and return the current task snapshot."""
async with self._async_lock:
task = await self._load_for_update(task_id, account_id, user_id)
- if task and task.status not in _TERMINAL_STATUSES:
- task.status = TaskStatus.FAILED
- task.stage = "failed"
- task.error = _sanitize_error(error)
+ if task is None:
+ return None
+ if task.status == TaskStatus.CANCELLED:
+ return self._copy(task)
+ if task.status != TaskStatus.CANCELLING:
+ if task.task_type not in _CANCELLABLE_TASK_TYPES:
+ raise ValueError(f"Task type '{task.task_type}' does not support cancellation")
+ if task.status in _TERMINAL_STATUSES:
+ raise ValueError(f"Task is already {task.status.value}")
+
+ task.status = TaskStatus.CANCELLING
task.updated_at = time.time()
await self._store.update(task)
with self._lock:
self._tasks[task.task_id] = task
- transitioned = True
- if transitioned:
- logger.warning("[TaskTracker] Task %s failed: %s", task_id, _sanitize_error(error))
+ self._work_index.cancel_active(task_id)
+ await self._finalize_task(
+ task_id,
+ account_id=account_id,
+ user_id=user_id,
+ )
+ return await self.get(task_id, account_id=account_id, user_id=user_id)
+
+ async def _finalize_task(
+ self,
+ task_id: str,
+ account_id: Optional[str] = None,
+ user_id: Optional[str] = None,
+ ) -> None:
+ """Apply the task's terminal outcome after all owned work is settled."""
+ if self._work_index.has_work(task_id):
+ return
+ async with self._async_lock:
+ task = await self._load_for_update(task_id, account_id, user_id)
+ if task and task.status in _ACTIVE_STATUSES:
+ if self._work_index.has_work(task_id):
+ return
+ updated = deepcopy(task)
+ if updated.status != TaskStatus.CANCELLING:
+ work_error = self._work_index.failure(task_id)
+ if work_error and updated.error is None:
+ updated.error = _sanitize_error(work_error)
+ if updated.status == TaskStatus.CANCELLING:
+ updated.status = TaskStatus.CANCELLED
+ elif updated.error is not None:
+ updated.status = TaskStatus.FAILED
+ elif updated.result is not None:
+ updated.status = TaskStatus.COMPLETED
+ else:
+ return
+ updated.stage = updated.status.value
+ updated.updated_at = time.time()
+ await self._store.update(updated)
+ with self._lock:
+ self._tasks[updated.task_id] = updated
+ self._work_index.clear_failure(task_id)
+ logger.info("[TaskTracker] Task %s %s", task_id, updated.status.value)
+
+ async def wait(
+ self,
+ task_id: str,
+ account_id: Optional[str] = None,
+ user_id: Optional[str] = None,
+ timeout: Optional[float] = None,
+ poll_interval: float = 0.05,
+ ) -> TaskRecord:
+ """Wait for one task's terminal state without changing its lifecycle."""
+ async def _poll() -> TaskRecord:
+ while True:
+ task = await self.get(task_id, account_id=account_id, user_id=user_id)
+ if task.status in _TERMINAL_STATUSES:
+ return task
+ await asyncio.sleep(poll_interval)
+
+ if timeout is None:
+ return await _poll()
+ return await asyncio.wait_for(_poll(), timeout)
+
+ async def wait_for_descendants(self, task_id: str, current_work_id: str) -> None:
+ """Wait on the same durable work index used by completion and cancellation."""
+ while self._work_index.has_work(task_id, exclude_work_id=current_work_id):
+ await asyncio.sleep(0.05)
+
+ def register_running_task(self, task_id: str) -> None:
+ """Register the current asyncio task so cancellation can interrupt it."""
+ self._runtime_loop = self._runtime_loop or asyncio.get_running_loop()
+ active_task = asyncio.current_task()
+ if active_task is not None:
+ self._work_index.register_active(task_id, active_task)
+
+ async def unregister_running_task(self, task_id: str) -> None:
+ """Release direct background work and persist its terminal outcome."""
+ active_task = asyncio.current_task()
+ if active_task is not None:
+ self._work_index.unregister_active(task_id, active_task)
+ await self._finalize_task(task_id)
async def get(
self,
@@ -481,7 +659,7 @@ class TaskTracker:
t.task_type == task_type
and t.resource_id == resource_id
and self._matches_owner(t, account_id, user_id)
- and t.status in (TaskStatus.PENDING, TaskStatus.RUNNING)
+ and t.status in _ACTIVE_STATUSES
for t in tasks
)
@@ -528,6 +706,7 @@ class TaskTracker:
def _copy(task: TaskRecord) -> TaskRecord:
"""Return a defensive copy of a TaskRecord."""
copied = deepcopy(task)
+ copied.meta = _sanitize_task_result(copied.meta)
copied.result = _sanitize_task_result(copied.result)
return copied
diff --git a/openviking/service/task_work_index.py b/openviking/service/task_work_index.py
new file mode 100644
index 000000000..e35139b63
--- /dev/null
+++ b/openviking/service/task_work_index.py
@@ -0,0 +1,310 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Runtime index of durable queue work owned by a tracked task.
+
+QueueFS remains the durable source of truth. This index is rebuilt from all
+unacknowledged queue messages during startup and then maintained by enqueue/ACK
+hooks. It deliberately does not persist a second counter beside QueueFS.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import threading
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass
+from typing import Any, Awaitable, Callable, Dict, Iterable, Iterator, Mapping, Optional
+from uuid import uuid4
+
+TASK_WORK_ID_FIELD = "_task_work_id"
+
+
+class TaskWorkRejected(Exception):
+ """Task cancellation prevented descendant queue work from being created."""
+
+
+@dataclass(frozen=True)
+class TaskExecutionContext:
+ task_id: str
+ account_id: str
+ user_id: str
+
+
+@dataclass(frozen=True)
+class QueueTaskMetadata:
+ task_id: str
+ work_id: str
+ account_id: str = ""
+ user_id: str = ""
+
+
+_current_task_context: ContextVar[Optional[TaskExecutionContext]] = ContextVar(
+ "openviking_task_execution_context",
+ default=None,
+)
+
+
+@contextmanager
+def bind_task_context(
+ task_id: str,
+ account_id: str,
+ user_id: str,
+) -> Iterator[None]:
+ """Propagate task ownership to queue messages produced by this coroutine."""
+ token = _current_task_context.set(
+ TaskExecutionContext(
+ task_id=str(task_id),
+ account_id=str(account_id),
+ user_id=str(user_id),
+ )
+ )
+ try:
+ yield
+ finally:
+ _current_task_context.reset(token)
+
+
+def get_task_context() -> Optional[TaskExecutionContext]:
+ return _current_task_context.get()
+
+
+def _payload_dict(message: Any) -> Optional[Dict[str, Any]]:
+ if not isinstance(message, dict):
+ return None
+ payload: Any = message.get("data", message)
+ if isinstance(payload, str):
+ try:
+ payload = json.loads(payload)
+ except (TypeError, ValueError):
+ return None
+ return payload if isinstance(payload, dict) else None
+
+
+def _owner_from_payload(payload: Mapping[str, Any]) -> tuple[str, str]:
+ account_id = payload.get("account_id")
+ user_id = payload.get("user_id")
+
+ user = payload.get("user")
+ if isinstance(user, dict):
+ account_id = account_id or user.get("account_id")
+ user_id = user_id or user.get("user_id")
+
+ context_data = payload.get("context_data")
+ if isinstance(context_data, dict):
+ account_id = account_id or context_data.get("account_id")
+ user_id = user_id or context_data.get("owner_user_id")
+ return str(account_id or ""), str(user_id or "")
+
+
+def prepare_task_payload(
+ data: Dict[str, Any],
+) -> tuple[Dict[str, Any], Optional[QueueTaskMetadata]]:
+ """Copy a payload and attach durable task metadata when it belongs to a task."""
+ payload = dict(data)
+ current = _current_task_context.get()
+ task_id = payload.get("task_id") or (current.task_id if current is not None else "")
+ if not task_id:
+ return payload, None
+
+ task_id = str(task_id)
+ account_id, user_id = _owner_from_payload(payload)
+ if current is not None and current.task_id == task_id:
+ account_id = account_id or current.account_id
+ user_id = user_id or current.user_id
+
+ work_id = str(payload.get(TASK_WORK_ID_FIELD) or uuid4())
+ payload["task_id"] = task_id
+ payload[TASK_WORK_ID_FIELD] = work_id
+ if account_id and user_id:
+ payload.setdefault("account_id", account_id)
+ payload.setdefault("user_id", user_id)
+ return payload, QueueTaskMetadata(task_id, work_id, account_id, user_id)
+
+
+def extract_task_metadata(message: Any) -> Optional[QueueTaskMetadata]:
+ """Read task metadata from either a QueueFS envelope or its inner payload."""
+ payload = _payload_dict(message)
+ if payload is None:
+ return None
+ task_id = payload.get("task_id")
+ work_id = payload.get(TASK_WORK_ID_FIELD)
+ if not work_id and isinstance(message, dict) and "data" in message and message.get("id"):
+ # Messages produced before task work IDs were introduced can still be
+ # rebuilt from their stable QueueFS envelope ID.
+ work_id = f"queuefs:{message['id']}"
+ if not task_id or not work_id:
+ return None
+ account_id, user_id = _owner_from_payload(payload)
+ return QueueTaskMetadata(str(task_id), str(work_id), account_id, user_id)
+
+
+class TaskWorkIndex:
+ """Thread-safe, rebuildable index of persistent and currently active task work."""
+
+ def __init__(self) -> None:
+ self._lock = threading.Lock()
+ self._work: Dict[str, set[tuple[str, str]]] = {}
+ self._active: Dict[str, set[asyncio.Task[Any]]] = {}
+ self._failures: Dict[str, str] = {}
+ self._finalize_before_ack: Optional[Callable[[QueueTaskMetadata], Awaitable[None]]] = None
+ self._is_cancellation_requested: Optional[Callable[[str], bool]] = None
+
+ def set_callbacks(
+ self,
+ *,
+ finalize_before_ack: Callable[[QueueTaskMetadata], Awaitable[None]],
+ is_cancellation_requested: Callable[[str], bool],
+ ) -> None:
+ self._finalize_before_ack = finalize_before_ack
+ self._is_cancellation_requested = is_cancellation_requested
+
+ def rebuild(
+ self,
+ snapshots: Mapping[str, Iterable[Any]],
+ ) -> Dict[str, tuple[str, str]]:
+ """Rebuild work from QueueFS and return owners needed to restore task records."""
+ work: Dict[str, set[tuple[str, str]]] = {}
+ owners: Dict[str, tuple[str, str]] = {}
+ for queue_name, messages in snapshots.items():
+ for message in messages:
+ metadata = extract_task_metadata(message)
+ if metadata is None:
+ continue
+ work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))
+ if metadata.account_id and metadata.user_id:
+ owners[metadata.task_id] = (metadata.account_id, metadata.user_id)
+ with self._lock:
+ self._work = work
+ self._failures = {}
+ return owners
+
+ def register(self, queue_name: str, metadata: Optional[QueueTaskMetadata]) -> bool:
+ """Atomically reject cancelled work or add it to the runtime index."""
+ if metadata is None:
+ return True
+ with self._lock:
+ if self.cancellation_requested(metadata.task_id):
+ return False
+ self._work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))
+ return True
+
+ def _remove_work(
+ self,
+ queue_name: str,
+ metadata: QueueTaskMetadata,
+ ) -> tuple[bool, bool]:
+ """Remove one work item and report whether it existed and made its task idle."""
+ removed = False
+ with self._lock:
+ entries = self._work.get(metadata.task_id)
+ if entries is not None:
+ entry = (queue_name, metadata.work_id)
+ removed = entry in entries
+ entries.discard(entry)
+ if not entries:
+ self._work.pop(metadata.task_id, None)
+ became_idle = (
+ removed
+ and not self._work.get(metadata.task_id)
+ and not self._active.get(metadata.task_id)
+ )
+ return removed, became_idle
+
+ async def discard(self, queue_name: str, message: Any) -> None:
+ """Discard work that was never durably enqueued and finalize if it was last."""
+ metadata = (
+ message if isinstance(message, QueueTaskMetadata) else extract_task_metadata(message)
+ )
+ if metadata is None:
+ return
+ _removed, became_idle = self._remove_work(queue_name, metadata)
+ callback = self._finalize_before_ack
+ if became_idle and callback is not None:
+ await callback(metadata)
+
+ async def prepare_ack(
+ self,
+ queue_name: str,
+ message: Any,
+ ) -> Optional[QueueTaskMetadata]:
+ """Remove work provisionally and finalize its task before the last durable ACK."""
+ metadata = (
+ message if isinstance(message, QueueTaskMetadata) else extract_task_metadata(message)
+ )
+ if metadata is None:
+ return None
+
+ removed, became_idle = self._remove_work(queue_name, metadata)
+ if not removed:
+ return None
+ if not became_idle:
+ return metadata
+
+ callback = self._finalize_before_ack
+ if callback is None:
+ return metadata
+ try:
+ await callback(metadata)
+ except BaseException:
+ self.rollback_ack(queue_name, metadata)
+ raise
+ return metadata
+
+ def rollback_ack(self, queue_name: str, metadata: QueueTaskMetadata) -> None:
+ """Restore provisionally removed work when finalization or ACK fails."""
+ with self._lock:
+ self._work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))
+
+ def has_work(self, task_id: str, exclude_work_id: Optional[str] = None) -> bool:
+ with self._lock:
+ if exclude_work_id is not None:
+ return any(
+ work_id != exclude_work_id
+ for _queue_name, work_id in self._work.get(task_id, ())
+ )
+ return bool(self._work.get(task_id) or self._active.get(task_id))
+
+ def cancellation_requested(self, task_id: str) -> bool:
+ callback = self._is_cancellation_requested
+ return bool(callback is not None and callback(task_id))
+
+ def record_failure(self, task_id: str, error: str) -> None:
+ """Record the first failed work item owned by a task."""
+ with self._lock:
+ self._failures.setdefault(task_id, error)
+
+ def failure(self, task_id: str) -> Optional[str]:
+ with self._lock:
+ return self._failures.get(task_id)
+
+ def clear_failure(self, task_id: str) -> None:
+ with self._lock:
+ self._failures.pop(task_id, None)
+
+ def register_active(
+ self,
+ task_id: str,
+ active_task: asyncio.Task[Any],
+ ) -> None:
+ with self._lock:
+ cancelled = self.cancellation_requested(task_id)
+ if not cancelled:
+ self._active.setdefault(task_id, set()).add(active_task)
+ if cancelled:
+ active_task.get_loop().call_soon(active_task.cancel)
+
+ def unregister_active(self, task_id: str, active_task: asyncio.Task[Any]) -> None:
+ with self._lock:
+ entries = self._active.get(task_id)
+ if entries is not None:
+ entries.discard(active_task)
+ if not entries:
+ self._active.pop(task_id, None)
+
+ def cancel_active(self, task_id: str) -> None:
+ with self._lock:
+ active = list(self._active.get(task_id, ()))
+ for task in active:
+ task.get_loop().call_soon_threadsafe(task.cancel)
diff --git a/openviking/session/session.py b/openviking/session/session.py
index b5c702048..383dfd6d2 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -2014,6 +2014,14 @@ class Session:
"budget_exceeded": retention_plan.budget_exceeded if retention_plan else False,
}
+ async def finalize_cancelled_commit(self, archive_uri: str) -> None:
+ """Make a cancelled queued commit terminal without discarding its raw archive."""
+ await self._write_failed_marker(
+ archive_uri,
+ stage="cancelled",
+ error="session commit cancelled",
+ )
+
async def resume_queued_commit(self, msg: "SessionCommitMsg") -> None:
"""Run one durable Phase 2 job from its archived messages."""
from openviking.service.task_tracker import get_task_tracker
@@ -2650,6 +2658,11 @@ class Session:
)
logger.info(f"Session {self.session_id} memory extraction completed")
except asyncio.CancelledError:
+ await self._write_failed_marker(
+ archive_uri,
+ stage="cancelled",
+ error="session commit cancelled",
+ )
raise
except Exception as e:
await self._write_failed_marker(
diff --git a/openviking/session/train/components/session_commit.py b/openviking/session/train/components/session_commit.py
index e927bb1c5..50eab2323 100644
--- a/openviking/session/train/components/session_commit.py
+++ b/openviking/session/train/components/session_commit.py
@@ -277,7 +277,7 @@ class SessionCommitPolicyTrainer:
)
while True:
task = await self.client.get_task(task_id)
- if task and task.get("status") in {"completed", "failed"}:
+ if task and task.get("status") in {"completed", "failed", "cancelled"}:
return task
if deadline is not None and asyncio.get_running_loop().time() >= deadline:
return {"task_id": task_id, "status": "timeout", "error": "commit task timeout"}
@@ -375,6 +375,8 @@ def _task_error(task: dict[str, Any] | None) -> str | None:
return None
if task.get("status") == "failed":
return str(task.get("error") or "task failed")
+ if task.get("status") == "cancelled":
+ return str(task.get("error") or "task cancelled")
if task.get("status") == "timeout":
return str(task.get("error") or "task timeout")
return None
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index 946dd2278..8c7f3f6d3 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -849,6 +849,28 @@ class TextEmbeddingHandler(DequeueHandlerBase):
elif report_success:
self.report_success()
+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ """Settle semantic joins when a queued embedding is cancelled."""
+ embedding_msg: Optional[EmbeddingMsg] = None
+ try:
+ if data:
+ payload = data.get("data", data)
+ if isinstance(payload, str):
+ payload = json.loads(payload)
+ embedding_msg = EmbeddingMsg.from_dict(payload)
+ except (KeyError, TypeError, ValueError) as exc:
+ self.report_error(str(exc), data)
+ return None
+
+ if embedding_msg is not None:
+ self._record_request_success(embedding_msg)
+ if embedding_msg.semantic_msg_id:
+ from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker
+
+ await EmbeddingTaskTracker.get_instance().decrement(embedding_msg.semantic_msg_id)
+ self.report_success()
+ return None
+
@staticmethod
def _record_request_success(embedding_msg: EmbeddingMsg) -> None:
tracker = get_request_wait_tracker()
diff --git a/openviking/storage/queuefs/add_resource_msg.py b/openviking/storage/queuefs/add_resource_msg.py
index dbc485e8a..78a4b6b2d 100644
--- a/openviking/storage/queuefs/add_resource_msg.py
+++ b/openviking/storage/queuefs/add_resource_msg.py
@@ -16,6 +16,7 @@ class AddResourceMsg:
user_id: str
role: str
path: str = ""
+ source_path: str = ""
telemetry_id: Optional[str] = None
prepared: Optional[Dict[str, Any]] = None
lock_handoff: Optional[Dict[str, Any]] = None
@@ -80,6 +81,7 @@ class AddResourceMsg:
return cls(
task_id=str(task_id),
path=str(path or ""),
+ source_path=str(data.get("source_path") or path or ""),
root_uri=str(root_uri),
account_id=str(data.get("account_id", "default")),
user_id=str(data.get("user_id", "default")),
diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.py
index 771c2102b..6345a38bd 100644
--- a/openviking/storage/queuefs/add_resource_processor.py
+++ b/openviking/storage/queuefs/add_resource_processor.py
@@ -11,10 +11,11 @@ from typing import Any, Dict, Optional
from openviking.observability.context import bind_execution_context
from openviking.server.identity import RequestContext, Role
from openviking.service.task_tracker import TaskStatus, get_task_tracker
+from openviking.service.task_work_index import bind_task_context, extract_task_metadata
from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
-from openviking.telemetry import bind_telemetry, resolve_telemetry
-from openviking.telemetry.resource_summary import summarize_queue_errors
+from openviking.telemetry import bind_telemetry, resolve_telemetry, unregister_telemetry
+from openviking.telemetry.resource_summary import record_resource_queue_metrics
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils.logger import get_logger
@@ -51,6 +52,15 @@ class AddResourceProcessor(DequeueHandlerBase):
except Exception:
raise handoff_error
+ async def _release_cancelled_handoff(self, msg: AddResourceMsg) -> None:
+ if msg.lock_handoff is None:
+ return
+ try:
+ lock = await self._viking_fs._async_agfs.pathlock_adopt(msg.lock_handoff)
+ await self._viking_fs._async_agfs.pathlock_release(lock)
+ except Exception as exc:
+ logger.warning("[AddResource] Failed to release cancelled lock handoff: %s", exc)
+
async def _requeue_lock_handoff(self, msg: AddResourceMsg, exc: Exception) -> bool:
if msg.lock_handoff_retry >= 2:
return False
@@ -70,6 +80,7 @@ class AddResourceProcessor(DequeueHandlerBase):
return True
async def _process(self, msg: AddResourceMsg, data: Dict[str, Any]) -> None:
+ telemetry_id = msg.telemetry_id or ""
ctx = RequestContext(
user=UserIdentifier(msg.account_id, msg.user_id),
role=Role(msg.role),
@@ -82,8 +93,17 @@ class AddResourceProcessor(DequeueHandlerBase):
account_id=ctx.account_id,
user_id=ctx.user.user_id,
task_id=msg.task_id,
+ meta={"source_path": msg.source_path},
)
- if task.status in (TaskStatus.COMPLETED, TaskStatus.FAILED):
+ if task.status in (
+ TaskStatus.CANCELLING,
+ TaskStatus.COMPLETED,
+ TaskStatus.FAILED,
+ TaskStatus.CANCELLED,
+ ):
+ if task.status in (TaskStatus.CANCELLING, TaskStatus.CANCELLED):
+ await self._release_cancelled_handoff(msg)
+ unregister_telemetry(telemetry_id)
self.report_success()
return None
@@ -100,9 +120,9 @@ class AddResourceProcessor(DequeueHandlerBase):
user_id=ctx.user.user_id,
)
self.report_error(f"Invalid lock_handoff: {exc}", data)
+ unregister_telemetry(telemetry_id)
return None
- telemetry_id = msg.telemetry_id or ""
telemetry = resolve_telemetry(telemetry_id) if telemetry_id else None
if telemetry is None:
from openviking.telemetry.operation import OperationTelemetry
@@ -119,8 +139,13 @@ class AddResourceProcessor(DequeueHandlerBase):
user_id=ctx.user.user_id,
)
- with bind_execution_context(), bind_telemetry(telemetry):
+ with (
+ bind_execution_context(),
+ bind_telemetry(telemetry),
+ bind_task_context(msg.task_id, ctx.account_id, ctx.user.user_id),
+ ):
try:
+ metadata = extract_task_metadata(data)
await tracker.start(
msg.task_id,
account_id=ctx.account_id,
@@ -143,16 +168,19 @@ class AddResourceProcessor(DequeueHandlerBase):
)
self.report_error("resource processing failed", data)
return None
- queue_errors = summarize_queue_errors(result.get("queue_status"))
- if queue_errors:
- await tracker.fail(
- msg.task_id,
- "queue processing failed: " + "; ".join(queue_errors),
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- self.report_error("queue processing failed", data)
- return None
+ await tracker.wait_for_descendants(msg.task_id, metadata.work_id)
+ record_resource_queue_metrics(
+ telemetry=telemetry,
+ telemetry_id=telemetry_id,
+ root_uri=result.get("root_uri"),
+ )
+ await self._resource_service._link_resource_reason_memory(
+ result=result,
+ ctx=ctx,
+ reason=msg.reason,
+ source_name=msg.source_name,
+ timeout=msg.timeout,
+ )
await tracker.complete(
msg.task_id,
result,
@@ -162,9 +190,6 @@ class AddResourceProcessor(DequeueHandlerBase):
)
self.report_success()
return None
- except asyncio.CancelledError:
- # Leave both task and QueueFS message active; RecoverStale owns restart recovery.
- raise
except Exception as exc:
await tracker.fail(
msg.task_id,
@@ -175,10 +200,30 @@ class AddResourceProcessor(DequeueHandlerBase):
self.report_error(str(exc), data)
return None
finally:
+ unregister_telemetry(telemetry_id)
with suppress(Exception):
if resource_lock is not None:
await self._viking_fs._async_agfs.pathlock_release(resource_lock)
+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ """Release an enqueue-time lock before ACKing cancelled work."""
+ try:
+ payload = data.get("data", data) if isinstance(data, dict) else data
+ if isinstance(payload, str):
+ payload = json.loads(payload)
+ msg = AddResourceMsg.from_dict(payload)
+ except (json.JSONDecodeError, TypeError, ValueError) as exc:
+ self.report_error(str(exc), data)
+ return None
+ future = asyncio.run_coroutine_threadsafe(
+ self._release_cancelled_handoff(msg),
+ self._service_loop,
+ )
+ await asyncio.wrap_future(future)
+ unregister_telemetry(msg.telemetry_id or "")
+ self.report_success()
+ return None
+
async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
if not data:
return None
diff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.py
index 9b22a1743..4383c42df 100644
--- a/openviking/storage/queuefs/named_queue.py
+++ b/openviking/storage/queuefs/named_queue.py
@@ -1,6 +1,7 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
import abc
+import asyncio
import json
import threading
from dataclasses import dataclass, field
@@ -9,6 +10,13 @@ from typing import Any, Callable, Dict, List, Optional, Union
from openviking.pyagfs import AGFSSyncClientProtocol, AsyncAGFSClient
from openviking.pyagfs.exceptions import AGFSAlreadyExistsError, AGFSNotFoundError
+from openviking.service.task_work_index import (
+ TaskWorkIndex,
+ TaskWorkRejected,
+ bind_task_context,
+ extract_task_metadata,
+ prepare_task_payload,
+)
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
@@ -89,6 +97,11 @@ class DequeueHandlerBase(abc.ABC):
if self._error_callback:
self._error_callback(error_msg, data)
+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ """Discard task work cancelled before its handler starts."""
+ self.report_success()
+ return None
+
@abc.abstractmethod
async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
"""Called after message dequeue. Returns None to discard message."""
@@ -109,6 +122,7 @@ class NamedQueue:
name: str,
enqueue_hook: Optional[EnqueueHookBase] = None,
dequeue_handler: Optional[DequeueHandlerBase] = None,
+ task_work_index: Optional[TaskWorkIndex] = None,
):
self.name = name
self.path = f"{mount_point}/{name}"
@@ -116,6 +130,7 @@ class NamedQueue:
self._async_agfs = AsyncAGFSClient(agfs)
self._enqueue_hook = enqueue_hook
self._dequeue_handler = dequeue_handler
+ self._task_work_index = task_work_index
self._initialized = False
# Status tracking
@@ -152,6 +167,10 @@ class NamedQueue:
def _on_process_error(self, error_msg: str, data: Optional[Dict[str, Any]] = None) -> None:
"""Called on processing failure."""
+ if self._task_work_index is not None and data is not None:
+ metadata = extract_task_metadata(data)
+ if metadata is not None:
+ self._task_work_index.record_failure(metadata.task_id, error_msg)
with self._lock:
self._in_progress -= 1
self._error_count += 1
@@ -210,24 +229,57 @@ class NamedQueue:
data = await self._enqueue_hook.on_enqueue(data)
if isinstance(data, dict):
- data = json.dumps(data)
+ data, task_metadata = prepare_task_payload(data)
+ else:
+ task_metadata = None
+
+ if self._task_work_index is not None and not self._task_work_index.register(
+ self.name, task_metadata
+ ):
+ logger.info(
+ "[NamedQueue] Skip enqueue for cancelling task %s on %s",
+ task_metadata.task_id,
+ self.name,
+ )
+ raise TaskWorkRejected(
+ f"Task {task_metadata.task_id} is cancelling; rejected work for {self.name}"
+ )
- msg_id = await self._async_agfs.write(enqueue_file, data.encode("utf-8"))
+ try:
+ if isinstance(data, dict):
+ data = json.dumps(data)
+
+ msg_id = await self._async_agfs.write(enqueue_file, data.encode("utf-8"))
+ except BaseException:
+ if self._task_work_index is not None and task_metadata is not None:
+ await self._task_work_index.discard(self.name, task_metadata)
+ raise
return msg_id if isinstance(msg_id, str) else str(msg_id)
- async def ack(self, msg_id: str) -> None:
+ async def ack(self, msg_id: str, message: Optional[Dict[str, Any]] = None) -> None:
"""Acknowledge successful processing of a message (deletes it from persistent storage).
Must be called after the dequeue handler finishes processing a message.
+ Task-owned work is provisionally removed from the runtime index first so
+ the last message can persist its task's terminal state before deletion.
If not called (e.g. process crashes), the message will be automatically
re-queued on the next startup via RecoverStale.
"""
if not msg_id:
return
ack_file = f"{self.path}/ack"
+ prepared = None
try:
+ if self._task_work_index is not None and message is not None:
+ prepared = await self._task_work_index.prepare_ack(self.name, message)
await self._async_agfs.write(ack_file, msg_id.encode("utf-8"))
+ except asyncio.CancelledError:
+ if self._task_work_index is not None and prepared is not None:
+ self._task_work_index.rollback_ack(self.name, prepared)
+ raise
except Exception as e:
+ if self._task_work_index is not None and prepared is not None:
+ self._task_work_index.rollback_ack(self.name, prepared)
logger.warning(f"[NamedQueue] Ack failed for {self.name} msg_id={msg_id}: {e}")
async def _read_queue_message(self) -> Optional[Dict[str, Any]]:
@@ -266,13 +318,14 @@ class NamedQueue:
return None
# Capture message ID before passing data to handler (handler may modify it)
msg_id = data.get("id", "") if isinstance(data, dict) else ""
+ raw_data = data
if self._dequeue_handler:
self._on_dequeue_start()
- data = await self._dequeue_handler.on_dequeue(data)
+ data = await self.process_dequeued(data)
# Ack unconditionally after handler returns (success or handled error).
# If on_dequeue raises, the exception propagates and ack is skipped —
# the message will be recovered on next startup.
- await self.ack(msg_id)
+ await self.ack(msg_id, raw_data)
return data
except Exception as e:
logger.debug(f"[NamedQueue] Dequeue failed for {self.name}: {e}")
@@ -293,9 +346,29 @@ class NamedQueue:
NOTE: caller must call _on_dequeue_start() before invoking this method
so that in_progress is incremented atomically with the dequeue.
"""
- if self._dequeue_handler:
+ if self._dequeue_handler is None:
+ return data
+
+ metadata = extract_task_metadata(data)
+ if metadata is None or self._task_work_index is None:
return await self._dequeue_handler.on_dequeue(data)
- return data
+
+ active_task = asyncio.current_task()
+ with bind_task_context(metadata.task_id, metadata.account_id, metadata.user_id):
+ if self._task_work_index.cancellation_requested(metadata.task_id):
+ return await self._dequeue_handler.on_cancelled(data)
+ if active_task is not None:
+ self._task_work_index.register_active(metadata.task_id, active_task)
+ try:
+ return await self._dequeue_handler.on_dequeue(data)
+ except asyncio.CancelledError:
+ if self._task_work_index.cancellation_requested(metadata.task_id):
+ self._on_process_success()
+ return None
+ raise
+ finally:
+ if active_task is not None:
+ self._task_work_index.unregister_active(metadata.task_id, active_task)
async def peek(self) -> Optional[Dict[str, Any]]:
"""Peek at head message without removing."""
@@ -336,14 +409,45 @@ class NamedQueue:
except (AGFSNotFoundError, FileNotFoundError):
return 0
+ async def snapshot(self) -> List[Dict[str, Any]]:
+ """Return all unacknowledged messages without changing queue state."""
+ await self._ensure_initialized()
+ try:
+ content = await self._async_agfs.read(f"{self.path}/messages")
+ except (AGFSNotFoundError, FileNotFoundError):
+ return []
+ if not content:
+ return []
+ if isinstance(content, bytes):
+ content = content.decode("utf-8")
+ elif hasattr(content, "content") and content.content is not None:
+ content = content.content.decode("utf-8")
+ parsed = json.loads(content)
+ return parsed if isinstance(parsed, list) else []
+
async def clear(self) -> bool:
"""Clear queue."""
await self._ensure_initialized()
clear_file = f"{self.path}/clear"
+ messages = await self.snapshot() if self._task_work_index is not None else []
+ prepared = []
try:
+ if self._task_work_index is not None:
+ for message in messages:
+ metadata = await self._task_work_index.prepare_ack(self.name, message)
+ if metadata is not None:
+ prepared.append(metadata)
await self._async_agfs.write(clear_file, b"")
return True
+ except asyncio.CancelledError:
+ if self._task_work_index is not None:
+ for metadata in prepared:
+ self._task_work_index.rollback_ack(self.name, metadata)
+ raise
except Exception as e:
+ if self._task_work_index is not None:
+ for metadata in prepared:
+ self._task_work_index.rollback_ack(self.name, metadata)
logger.error(f"[NamedQueue] Clear failed for {self.name}: {e}")
return False
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index 13f1bb0a6..c8dd4a09a 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -12,6 +12,7 @@ import time
import traceback
from typing import Any, Dict, Optional, Set, Union
+from openviking.service.task_work_index import TaskWorkIndex
from openviking_cli.utils.logger import get_logger
from .embedding_queue import EmbeddingQueue
@@ -95,6 +96,7 @@ class QueueManager:
self._queue_threads: Dict[str, threading.Thread] = {}
self._queue_stop_events: Dict[str, threading.Event] = {}
self._poll_interval = 0.2
+ self._task_work_index = TaskWorkIndex()
atexit.register(self.stop)
logger.info(
@@ -114,6 +116,13 @@ class QueueManager:
logger.info(f"[QueueManager] mount_point={self.mount_point} Started")
+ async def prepare_task_tracking(self, tracker: Any) -> None:
+ """Rebuild task work from QueueFS before any consumer starts."""
+ snapshots = {name: await queue.snapshot() for name, queue in self._queues.items()}
+ owners = self._task_work_index.rebuild(snapshots)
+ tracker.attach_work_index(self._task_work_index)
+ await tracker.restore_work_tasks(owners)
+
def setup_standard_queues(self, vector_store: Any, start: bool = True) -> None:
"""
Setup standard queues (Embedding and Semantic) with their handlers.
@@ -223,7 +232,7 @@ class QueueManager:
try:
await queue.process_dequeued(data)
# Ack after successful processing (delete from persistent storage).
- await queue.ack(msg_id)
+ await queue.ack(msg_id, data)
except Exception as e:
# Handler did not call report_error; decrement in_progress manually.
# Do NOT ack — let RecoverStale re-queue on next startup.
@@ -320,6 +329,7 @@ class QueueManager:
name,
enqueue_hook=enqueue_hook,
dequeue_handler=dequeue_handler,
+ task_work_index=self._task_work_index,
)
elif name == self.SEMANTIC:
self._queues[name] = SemanticQueue(
@@ -328,6 +338,7 @@ class QueueManager:
name,
enqueue_hook=enqueue_hook,
dequeue_handler=dequeue_handler,
+ task_work_index=self._task_work_index,
)
else:
self._queues[name] = NamedQueue(
@@ -336,6 +347,7 @@ class QueueManager:
name,
enqueue_hook=enqueue_hook,
dequeue_handler=dequeue_handler,
+ task_work_index=self._task_work_index,
)
if self._started:
self._start_queue_worker(self._queues[name])
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.py
index 07ad10faa..d2f8617aa 100644
--- a/openviking/storage/queuefs/semantic_dag.py
+++ b/openviking/storage/queuefs/semantic_dag.py
@@ -8,11 +8,12 @@ from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Optional, Set
from weakref import WeakKeyDictionary
-from openviking.utils.ingest_options import IngestOptions
from openviking.server.identity import RequestContext
+from openviking.service.task_work_index import bind_task_context, get_task_context
from openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecars
from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
+from openviking.utils.ingest_options import IngestOptions
from openviking_cli.utils import VikingURI
from openviking_cli.utils.logger import get_logger
@@ -124,6 +125,7 @@ class SemanticNodeScheduler:
return
continue
+ item.executor._start_scheduled_work()
try:
if not item.executor.closed:
await item.executor._run_work(item.work)
@@ -132,6 +134,7 @@ class SemanticNodeScheduler:
except Exception as exc:
item.executor.fail(exc)
finally:
+ item.executor._finish_scheduled_work()
self._queue.task_done()
@@ -191,6 +194,7 @@ class SemanticDagExecutor:
self._ingest_options = IngestOptions.from_value(ingest_options)
self._coalesce_key = coalesce_key
self._coalesce_version = coalesce_version
+ self._task_context = get_task_context()
self._stale = False
self._changed_paths = {
path for key in ("added", "modified", "deleted") for path in self._changes.get(key, [])
@@ -203,6 +207,9 @@ class SemanticDagExecutor:
self._root_uri: Optional[str] = None
self._root_done: Optional[asyncio.Event] = None
self._scheduler: Optional[SemanticNodeScheduler] = None
+ self._active_scheduled_work = 0
+ self._active_work_idle = asyncio.Event()
+ self._active_work_idle.set()
self._closed = False
self._failure: Optional[Exception] = None
self._stats = DagStats()
@@ -260,9 +267,11 @@ class SemanticDagExecutor:
logger.error(f"Error in on_complete callback: {e}", exc_info=True)
except BaseException:
self._closed = True
+ await self._active_work_idle.wait()
raise
finally:
self._closed = True
+ await self._active_work_idle.wait()
self._unregister_active()
def _schedule_work(self, work: DagWork) -> None:
@@ -272,6 +281,15 @@ class SemanticDagExecutor:
self._scheduler = get_semantic_node_scheduler(self._node_concurrency)
self._scheduler.submit(self, work)
+ def _start_scheduled_work(self) -> None:
+ self._active_scheduled_work += 1
+ self._active_work_idle.clear()
+
+ def _finish_scheduled_work(self) -> None:
+ self._active_scheduled_work = max(0, self._active_scheduled_work - 1)
+ if self._active_scheduled_work == 0:
+ self._active_work_idle.set()
+
def _schedule_dir(self, dir_uri: str, parent_uri: Optional[str]) -> None:
if self._closed:
return
@@ -337,6 +355,17 @@ class SemanticDagExecutor:
return stats
async def _run_work(self, work: DagWork) -> None:
+ if self._task_context is not None:
+ with bind_task_context(
+ self._task_context.task_id,
+ self._task_context.account_id,
+ self._task_context.user_id,
+ ):
+ await self._run_work_bound(work)
+ return
+ await self._run_work_bound(work)
+
+ async def _run_work_bound(self, work: DagWork) -> None:
if work.kind == "vectorize":
await self._run_vectorize_work(work.vectorize_task)
return
@@ -676,6 +705,8 @@ class SemanticDagExecutor:
self._stats.done_nodes += 1
self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)
+ if self._closed:
+ return
try:
if need_vectorize:
use_summary = self._is_code_repo and bool(summary_dict.get("summary"))
@@ -813,6 +844,9 @@ class SemanticDagExecutor:
)
overview, abstract = self._processor._normalize_overview_generation(overview)
+ if self._closed:
+ return
+
# Write directly, protected by the outer semantic lock.
try:
wrote = await self._write_directory_semantics(dir_uri, overview, abstract)
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index e453e8858..467c1407d 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -511,6 +511,31 @@ class SemanticProcessor(DequeueHandlerBase):
self.report_error(str(e), data)
return None
+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ """Release a queued semantic lock before cancelled work is ACKed."""
+ try:
+ import json
+
+ payload = data.get("data", data) if isinstance(data, dict) else data
+ if isinstance(payload, str):
+ payload = json.loads(payload)
+ msg = SemanticMsg.from_dict(payload)
+ except (TypeError, ValueError) as exc:
+ self.report_error(str(exc), data)
+ return None
+
+ if msg.telemetry_id and msg.id:
+ get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
+ if msg.lock_handoff is not None:
+ try:
+ viking_fs = get_viking_fs()
+ lock = await viking_fs._async_agfs.pathlock_adopt(msg.lock_handoff)
+ await viking_fs._async_agfs.pathlock_release(lock)
+ except Exception as exc:
+ logger.warning("Failed to release cancelled semantic lock: %s", exc)
+ self.report_success()
+ return None
+
def get_dag_stats(self) -> Optional["DagStats"]:
return SemanticDagExecutor.get_active_stats()
diff --git a/openviking/storage/queuefs/session_commit_processor.py b/openviking/storage/queuefs/session_commit_processor.py
index 03b98dc42..02eaf4b7f 100644
--- a/openviking/storage/queuefs/session_commit_processor.py
+++ b/openviking/storage/queuefs/session_commit_processor.py
@@ -13,6 +13,7 @@ from openviking.observability.context import (
)
from openviking.server.identity import RequestContext, Role
from openviking.service.task_tracker import get_task_tracker
+from openviking.service.task_work_index import bind_task_context
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
from openviking.storage.queuefs.session_commit_msg import SessionCommitMsg
from openviking.telemetry.span_models import create_root_span_attributes
@@ -31,6 +32,19 @@ class SessionCommitProcessor(DequeueHandlerBase):
self._session_service = session_service
self._service_loop = service_loop
+ @staticmethod
+ def _parse_message(data: Dict[str, Any]) -> tuple[SessionCommitMsg, RequestContext]:
+ payload = data.get("data", data)
+ if isinstance(payload, str):
+ payload = json.loads(payload)
+ msg = SessionCommitMsg.from_dict(payload)
+ ctx = RequestContext(
+ user=UserIdentifier.from_dict(msg.user),
+ role=Role.USER,
+ actor_peer_id=msg.actor_peer_id,
+ )
+ return msg, ctx
+
async def _process(self, msg: SessionCommitMsg, ctx: RequestContext) -> None:
# Bind a root observability context so Phase-2 extraction VLM/embedding
# token events are attributed to the committing account/user rather than
@@ -70,24 +84,48 @@ class SessionCommitProcessor(DequeueHandlerBase):
)
return
await session.load()
- await session.resume_queued_commit(msg)
+ with bind_task_context(msg.task_id, ctx.account_id, ctx.user.user_id):
+ await session.resume_queued_commit(msg)
finally:
reset_root_observability_context(root_context_token)
+ async def _finalize_cancelled(self, msg: SessionCommitMsg, ctx: RequestContext) -> None:
+ session = self._session_service.session(
+ ctx,
+ msg.session_id,
+ session_uri=msg.session_uri,
+ )
+ if await session.exists():
+ await session.finalize_cancelled_commit(msg.archive_uri)
+
+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ if not data:
+ return None
+
+ try:
+ msg, ctx = self._parse_message(data)
+ except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:
+ self.report_error(str(exc), data)
+ return None
+
+ future = asyncio.run_coroutine_threadsafe(
+ self._finalize_cancelled(msg, ctx),
+ self._service_loop,
+ )
+ try:
+ await asyncio.wrap_future(future)
+ self.report_success()
+ except asyncio.CancelledError:
+ future.cancel()
+ raise
+ return None
+
async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
if not data:
return None
try:
- payload = data.get("data", data)
- if isinstance(payload, str):
- payload = json.loads(payload)
- msg = SessionCommitMsg.from_dict(payload)
- ctx = RequestContext(
- user=UserIdentifier.from_dict(msg.user),
- role=Role.USER,
- actor_peer_id=msg.actor_peer_id,
- )
+ msg, ctx = self._parse_message(data)
except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:
self.report_error(str(exc), data)
return None
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 030272a8e..3d9ae81a2 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -4391,28 +4391,36 @@ class VikingFS:
the "failures do not block" semantics.
"""
from openviking.service.task_tracker import get_task_tracker
+ from openviking.service.task_work_index import bind_task_context
tracker = get_task_tracker()
- await tracker.start(
- task_id,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- stage="reindexing",
- )
+ tracker.register_running_task(task_id)
try:
+ await tracker.start(
+ task_id,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ stage="reindexing",
+ )
from openviking.service.reindex_executor import get_reindex_executor
executor = get_reindex_executor()
- await asyncio.gather(*[
- self._run_vector_rebuild(executor, op, uri, level, ctx)
- for (op, uri, level) in tasks
- ])
+ with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):
+ await asyncio.gather(
+ *[
+ self._run_vector_rebuild(executor, op, uri, level, ctx)
+ for (op, uri, level) in tasks
+ ]
+ )
await tracker.complete(
task_id,
{"status": "completed", "task_count": len(tasks)},
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
+ except asyncio.CancelledError:
+ # TaskWorkIndex finalizes after this active task and its queue work settle.
+ return
except Exception as exc:
await tracker.fail(
task_id,
@@ -4420,6 +4428,8 @@ class VikingFS:
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
+ finally:
+ await tracker.unregister_running_task(task_id)
async def _run_vector_rebuild(
self,
diff --git a/openviking/sync_client.py b/openviking/sync_client.py
index 24b3f3acc..657b2ebb3 100644
--- a/openviking/sync_client.py
+++ b/openviking/sync_client.py
@@ -190,6 +190,10 @@ class SyncOpenViking:
"""Query background task status."""
return run_async(self._async_client.get_task(task_id))
+ def cancel_task(self, task_id: str) -> Optional[Dict[str, Any]]:
+ """Cancel a background task."""
+ return run_async(self._async_client.cancel_task(task_id))
+
def list_tasks(
self,
task_type: Optional[str] = None,
diff --git a/openviking/telemetry/resource_summary.py b/openviking/telemetry/resource_summary.py
index b88ac1715..90d19834e 100644
--- a/openviking/telemetry/resource_summary.py
+++ b/openviking/telemetry/resource_summary.py
@@ -4,11 +4,9 @@
from __future__ import annotations
-from typing import Any, Dict, List
+from typing import Any, Dict
-from .context import get_current_telemetry
from .operation import OperationTelemetry
-from .registry import register_telemetry, unregister_telemetry
def _consume_semantic_request_stats(telemetry_id: str):
@@ -38,21 +36,6 @@ def _consume_semantic_dag_stats(telemetry_id: str, root_uri: str | None):
return None
-def register_wait_telemetry(wait: bool) -> str:
- """Register current telemetry collector for async queue consumers when needed."""
- handle = get_current_telemetry()
- if not handle.telemetry_id:
- return ""
- if handle.enabled:
- register_telemetry(handle)
- return handle.telemetry_id
-
-
-def unregister_wait_telemetry(telemetry_id: str) -> None:
- """Unregister request-scoped telemetry handle."""
- unregister_telemetry(telemetry_id)
-
-
def build_queue_status_payload(status: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
"""Convert queue status objects to response payload format."""
return {
@@ -66,93 +49,28 @@ def build_queue_status_payload(status: Dict[str, Any]) -> Dict[str, Dict[str, An
}
-def summarize_queue_errors(queue_status: Dict[str, Any] | None) -> List[str]:
- """Return human-readable summaries for queue groups with recorded errors."""
- if not queue_status:
- return []
-
- summaries: List[str] = []
- for name, status in queue_status.items():
- if isinstance(status, dict):
- raw_error_count = status.get("error_count", 0)
- raw_errors = status.get("errors") or []
- else:
- raw_error_count = getattr(status, "error_count", 0)
- raw_errors = getattr(status, "errors", []) or []
-
- try:
- error_count = int(raw_error_count or 0)
- except (TypeError, ValueError):
- error_count = 0
- if error_count <= 0:
- continue
-
- messages: List[str] = []
- for error in raw_errors[:3]:
- if isinstance(error, dict):
- message = error.get("message")
- else:
- message = getattr(error, "message", None) or str(error)
- if message:
- messages.append(str(message))
-
- summary = f"{name} error_count={error_count}"
- if messages:
- summary = f"{summary}: {', '.join(messages)}"
- summaries.append(summary)
-
- return summaries
-
-
-def _resolve_queue_group(
- *,
- explicit_stats: Any,
- fallback_status: Any,
-) -> Dict[str, int]:
- if explicit_stats is not None:
- return {
- "processed": explicit_stats.processed,
- "requeue_count": getattr(explicit_stats, "requeue_count", 0),
- "error_count": explicit_stats.error_count,
- }
- if fallback_status is None:
+def _queue_metrics(stats: Any) -> Dict[str, int]:
+ if stats is None:
return {"processed": 0, "requeue_count": 0, "error_count": 0}
- if isinstance(fallback_status, dict):
- return {
- "processed": int(fallback_status.get("processed", 0) or 0),
- "requeue_count": int(fallback_status.get("requeue_count", 0) or 0),
- "error_count": int(fallback_status.get("error_count", 0) or 0),
- }
return {
- "processed": fallback_status.processed,
- "requeue_count": getattr(fallback_status, "requeue_count", 0),
- "error_count": fallback_status.error_count,
+ "processed": stats.processed,
+ "requeue_count": stats.requeue_count,
+ "error_count": stats.error_count,
}
-def record_resource_wait_metrics(
+def record_resource_queue_metrics(
*,
- telemetry: OperationTelemetry | None = None,
+ telemetry: OperationTelemetry,
telemetry_id: str,
- queue_status: Dict[str, Any],
root_uri: str | None,
-) -> Dict[str, Dict[str, int]]:
+) -> None:
"""Apply queue and DAG metrics to a resource operation collector."""
- telemetry = telemetry or get_current_telemetry()
if not telemetry.enabled:
- return {
- "semantic": {"processed": 0, "requeue_count": 0, "error_count": 0},
- "embedding": {"processed": 0, "requeue_count": 0, "error_count": 0},
- }
+ return
- semantic = _resolve_queue_group(
- explicit_stats=_consume_semantic_request_stats(telemetry_id),
- fallback_status=queue_status.get("Semantic"),
- )
- embedding = _resolve_queue_group(
- explicit_stats=_consume_embedding_request_stats(telemetry_id),
- fallback_status=queue_status.get("Embedding"),
- )
+ semantic = _queue_metrics(_consume_semantic_request_stats(telemetry_id))
+ embedding = _queue_metrics(_consume_embedding_request_stats(telemetry_id))
telemetry.set("queue.semantic.processed", semantic["processed"])
telemetry.set("queue.semantic.requeue_count", semantic["requeue_count"])
@@ -168,16 +86,7 @@ def record_resource_wait_metrics(
telemetry.set("semantic_nodes.pending", dag_stats.pending_nodes)
telemetry.set("semantic_nodes.running", dag_stats.in_progress_nodes)
- return {
- "semantic": semantic,
- "embedding": embedding,
- }
-
-
__all__ = [
"build_queue_status_payload",
- "record_resource_wait_metrics",
- "register_wait_telemetry",
- "summarize_queue_errors",
- "unregister_wait_telemetry",
+ "record_resource_queue_metrics",
]
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index 0358a205e..9618d195c 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -13,18 +13,19 @@ from typing import Any, Dict, Optional
from charset_normalizer import from_bytes
from openviking.core.context import Context, ContextLevel, ResourceContentType, Vectorize
-from openviking.utils.ingest_options import IngestOptions
from openviking.core.namespace import (
context_type_for_uri,
is_session_uri,
owner_space_for_uri,
)
from openviking.server.identity import RequestContext
+from openviking.service.task_work_index import TaskWorkRejected
from openviking.storage.queuefs import get_queue_manager
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.image_search import image_bytes_to_data_uri
+from openviking.utils.ingest_options import IngestOptions
from openviking.utils.time_utils import parse_iso_datetime
from openviking_cli.utils import VikingURI, get_logger
from openviking_cli.utils.config import get_openviking_config
@@ -384,6 +385,9 @@ async def vectorize_directory_meta(
await embedding_queue.enqueue(msg_abstract)
enqueued += 1
logger.debug(f"Enqueued directory L0 (abstract) for vectorization: {uri}")
+ except TaskWorkRejected:
+ logger.debug("Skipped directory vectorization for cancelling task: %s", uri)
+ return
except Exception as e:
logger.error(
f"Failed to enqueue directory L0 (abstract) for vectorization: {uri}: {e}",
@@ -418,6 +422,9 @@ async def vectorize_directory_meta(
await embedding_queue.enqueue(msg_overview)
enqueued += 1
logger.debug(f"Enqueued directory L1 (overview) for vectorization: {uri}")
+ except TaskWorkRejected:
+ logger.debug("Skipped directory vectorization for cancelling task: %s", uri)
+ return
except Exception as e:
logger.error(
f"Failed to enqueue directory L1 (overview) for vectorization: {uri}: {e}",
@@ -582,6 +589,14 @@ async def vectorize_file(
enqueued = True
logger.debug(f"Enqueued file for vectorization: {file_path}")
+ except TaskWorkRejected:
+ logger.debug("Skipped file vectorization for cancelling task: %s", file_path)
+ if registered_wait_root is not None:
+ get_request_wait_tracker().mark_embedding_failed(
+ registered_wait_root[0],
+ registered_wait_root[1],
+ f"Task cancellation skipped file vector for {file_path}",
+ )
except Exception as e:
logger.error(f"Failed to vectorize file {file_path}: {e}", exc_info=True)
if registered_wait_root is not None:
diff --git a/openviking_cli/client/base.py b/openviking_cli/client/base.py
index e2ed1ae86..1f2568f51 100644
--- a/openviking_cli/client/base.py
+++ b/openviking_cli/client/base.py
@@ -450,6 +450,11 @@ class BaseClient(ABC):
"""Query background task status."""
...
+ @abstractmethod
+ async def cancel_task(self, task_id: str) -> Optional[Dict[str, Any]]:
+ """Cancel a background task."""
+ ...
+
@abstractmethod
async def list_tasks(
self,
diff --git a/sdk/go/sessions.go b/sdk/go/sessions.go
index ba740980f..ec4b625d4 100644
--- a/sdk/go/sessions.go
+++ b/sdk/go/sessions.go
@@ -136,6 +136,23 @@ func (c *Client) GetTask(ctx context.Context, taskID string) (map[string]any, er
return result, nil
}
+// CancelTask requests cooperative cancellation of a background task.
+func (c *Client) CancelTask(
+ ctx context.Context,
+ taskID string,
+) (map[string]any, error) {
+ var result map[string]any
+ err := c.doJSON(
+ ctx,
+ http.MethodPost,
+ "/api/v1/tasks/"+url.PathEscape(taskID)+"/cancel",
+ nil,
+ nil,
+ &result,
+ )
+ return result, err
+}
+
// ListTasks lists background tasks visible to the caller.
func (c *Client) ListTasks(ctx context.Context, opts *ListTasksOptions) ([]any, error) {
query := url.Values{}
diff --git a/sdk/python/openviking_sdk/client.py b/sdk/python/openviking_sdk/client.py
index 40907d8ab..2773889f3 100644
--- a/sdk/python/openviking_sdk/client.py
+++ b/sdk/python/openviking_sdk/client.py
@@ -1368,6 +1368,10 @@ class AsyncHTTPClient:
return None
return self._handle_response(response)
+ async def cancel_task(self, task_id: str) -> Dict[str, Any]:
+ response = await self._request("POST", f"/api/v1/tasks/{task_id}/cancel")
+ return self._handle_response(response)
+
async def list_tasks(
self,
task_type: Optional[str] = None,
@@ -2327,6 +2331,9 @@ class SyncHTTPClient:
def get_task(self, task_id: str) -> Optional[Dict[str, Any]]:
return run_async(self._async_client.get_task(task_id))
+ def cancel_task(self, task_id: str) -> Dict[str, Any]:
+ return run_async(self._async_client.cancel_task(task_id))
+
def list_tasks(
self,
task_type: Optional[str] = None,
diff --git a/sdk/typescript/src/client.ts b/sdk/typescript/src/client.ts
index 28ae1a6ac..10f43d326 100644
--- a/sdk/typescript/src/client.ts
+++ b/sdk/typescript/src/client.ts
@@ -754,6 +754,13 @@ export class OpenVikingClient {
throw error;
}
}
+
+ cancelTask(taskId: string): Promise<JsonObject> {
+ return this.request(
+ "POST",
+ `/api/v1/tasks/${pathPart(taskId)}/cancel`,
+ ) as Promise<JsonObject>;
+ }
/** List background tasks. */
listTasks(options: TaskListOptions = {}): Promise<unknown[]> {
return this.request("GET", "/api/v1/tasks", {
diff --git a/tests/api_test/resources/slow/test_temp_upload_deep.py b/tests/api_test/resources/slow/test_temp_upload_deep.py
index cd40d9d63..0edb998eb 100644
--- a/tests/api_test/resources/slow/test_temp_upload_deep.py
+++ b/tests/api_test/resources/slow/test_temp_upload_deep.py
@@ -65,17 +65,3 @@ class TestTempUploadDeep:
assert add_resp.status_code == 200, (
f"upload empty file should return 200, got {add_resp.status_code}"
)
-
- def test_upload_result_has_queue_status(self, api_client):
- with tempfile.TemporaryDirectory() as temp_dir:
- test_file = os.path.join(temp_dir, f"queue_{uuid.uuid4().hex[:8]}.md")
- with open(test_file, "w") as f:
- f.write("# Queue status test\n\nContent for queue status check.")
- add_resp = api_client.add_resource(path=test_file, wait=True)
- assert add_resp.status_code == 200
- result = add_resp.json().get("result", {})
- assert "queue_status" in result, (
- f"result should contain queue_status, got keys: {list(result.keys())}"
- )
- qs = result["queue_status"]
- assert isinstance(qs, dict), f"queue_status should be dict, got {type(qs).__name__}"
diff --git a/tests/client/test_resource_management.py b/tests/client/test_resource_management.py
index 09cdbd261..3efd2df43 100644
--- a/tests/client/test_resource_management.py
+++ b/tests/client/test_resource_management.py
@@ -36,7 +36,6 @@ class TestAddResource:
print(result)
assert "root_uri" in result
- assert "queue_status" in result
async def test_local_client_add_resource_with_wait_preserves_queue_status(self):
"""Local SDK add_resource(wait=True) should keep queue_status and internal telemetry."""
diff --git a/tests/parse/test_feishu_parser_api.py b/tests/parse/test_feishu_parser_api.py
index 26fcfba7f..4bebc818b 100644
--- a/tests/parse/test_feishu_parser_api.py
+++ b/tests/parse/test_feishu_parser_api.py
@@ -12,6 +12,7 @@ from openviking.parse.understanding_api import PREPARED_RESPONSE_ID_ARG, Underst
from openviking.server.identity import RequestContext, Role
from openviking.service.resource_service import ResourceService
from openviking.service.task_tracker import TaskStatus
+from openviking.service.task_work_index import TASK_WORK_ID_FIELD
from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
from openviking.storage.queuefs.add_resource_processor import AddResourceProcessor
from openviking.storage.queuefs.queue_manager import QueueManager
@@ -788,7 +789,8 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):
service = SimpleNamespace(
execute_add_resource_job=AsyncMock(
return_value={"status": "success", "root_uri": final_uri}
- )
+ ),
+ _link_resource_reason_memory=AsyncMock(),
)
task_tracker = SimpleNamespace(
create=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.PENDING)),
@@ -796,6 +798,7 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):
update_stage=AsyncMock(),
complete=AsyncMock(),
fail=AsyncMock(),
+ wait_for_descendants=AsyncMock(),
)
monkeypatch.setattr(
"openviking.storage.queuefs.add_resource_processor.get_task_tracker",
@@ -823,7 +826,9 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):
understanding_response_id="response-1",
)
- await processor._process(msg, msg.to_dict())
+ data = msg.to_dict()
+ data[TASK_WORK_ID_FIELD] = "work-1"
+ await processor._process(msg, data)
task_tracker.create.assert_awaited_once_with(
"add_resource",
@@ -831,6 +836,7 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):
account_id="account-1",
user_id="user-1",
task_id="task-1",
+ meta={"source_path": ""},
)
task_tracker.complete.assert_awaited_once_with(
"task-1",
diff --git a/tests/server/test_request_wait_tracking.py b/tests/server/test_request_wait_tracking.py
index d654e1de6..1a2232770 100644
--- a/tests/server/test_request_wait_tracking.py
+++ b/tests/server/test_request_wait_tracking.py
@@ -81,94 +81,6 @@ class _FakeVikingFS:
self.content.pop(uri, None)
-async def test_add_resource_wait_uses_request_tracker(service, monkeypatch):
- tracker = _FakeRequestWaitTracker(
- {
- "Semantic": {"processed": 1, "error_count": 0, "errors": []},
- "Embedding": {"processed": 2, "error_count": 0, "errors": []},
- }
- )
- ctx = RequestContext(user=service.user, role=Role.ROOT)
- telemetry = OperationTelemetry(operation="resources.add_resource", enabled=True)
-
- async def _fake_process_resource(**kwargs):
- del kwargs
- return {"status": "success", "root_uri": "viking://resources/demo"}
-
- monkeypatch.setattr(
- service.resources._resource_processor, "process_resource", _fake_process_resource
- )
- monkeypatch.setattr(
- "openviking.service.resource_service.get_queue_manager",
- lambda: _ExplodingQueueManager(),
- )
- monkeypatch.setattr(
- "openviking.service.resource_service.get_request_wait_tracker",
- lambda: tracker,
- raising=False,
- )
-
- with bind_telemetry(telemetry):
- result = await service.resources.add_resource(
- path="/tmp/demo.md",
- ctx=ctx,
- reason="request wait test",
- wait=True,
- timeout=12.0,
- )
-
- assert result["queue_status"] == tracker.queue_status
- assert tracker.registered_requests == [telemetry.telemetry_id]
- assert tracker.wait_calls == [(telemetry.telemetry_id, 12.0)]
- assert tracker.build_calls == [telemetry.telemetry_id]
- assert tracker.cleaned == [telemetry.telemetry_id]
-
-
-async def test_add_resource_wait_uses_request_tracker_when_telemetry_disabled(service, monkeypatch):
- tracker = _FakeRequestWaitTracker(
- {
- "Semantic": {"processed": 1, "error_count": 0, "errors": []},
- "Embedding": {"processed": 2, "error_count": 0, "errors": []},
- }
- )
- ctx = RequestContext(user=service.user, role=Role.ROOT)
- telemetry = OperationTelemetry(operation="resources.add_resource", enabled=False)
-
- async def _fake_process_resource(**kwargs):
- del kwargs
- return {"status": "success", "root_uri": "viking://resources/demo"}
-
- monkeypatch.setattr(
- service.resources._resource_processor, "process_resource", _fake_process_resource
- )
- monkeypatch.setattr(
- "openviking.service.resource_service.get_queue_manager",
- lambda: _ExplodingQueueManager(),
- )
- monkeypatch.setattr(
- "openviking.service.resource_service.get_request_wait_tracker",
- lambda: tracker,
- raising=False,
- )
-
- with bind_telemetry(telemetry):
- result = await service.resources.add_resource(
- path="/tmp/demo.md",
- ctx=ctx,
- reason="request wait test",
- wait=True,
- timeout=12.0,
- )
-
- assert result["queue_status"] == tracker.queue_status
- assert tracker.registered_requests == [telemetry.telemetry_id]
- assert tracker.wait_calls == [(telemetry.telemetry_id, 12.0)]
- assert tracker.build_calls == [telemetry.telemetry_id]
- assert tracker.cleaned == [telemetry.telemetry_id]
-
-
@pytest.mark.asyncio
async def test_add_skill_wait_uses_request_tracker(service, monkeypatch):
tracker = _FakeRequestWaitTracker(
diff --git a/tests/service/test_resource_service_connector.py b/tests/service/test_resource_service_connector.py
index d9cec7699..16ebc0956 100644
--- a/tests/service/test_resource_service_connector.py
+++ b/tests/service/test_resource_service_connector.py
@@ -831,7 +831,7 @@ async def test_add_resource_job_executes_frozen_route(ctx, service):
call = service._execute_resource_ingestion.await_args.kwargs
assert call["path"] == msg.path
assert call["to"] == msg.root_uri
- assert call["wait"] is True
+ assert call["defer_post_processing"] is False
assert call["resource_lock"] is resource_lock
assert call["parser_backend"] == "understanding"
service._should_use_connector.assert_not_called()
diff --git a/tests/service/test_resource_service_watch.py b/tests/service/test_resource_service_watch.py
index b12295753..fbbf0852a 100644
--- a/tests/service/test_resource_service_watch.py
+++ b/tests/service/test_resource_service_watch.py
@@ -9,13 +9,14 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
import pytest_asyncio
-from openviking.utils.ingest_options import IngestOptions
from openviking.resource.watch_manager import WatchManager
from openviking.server.identity import RequestContext, Role
from openviking.service import resource_service as resource_service_module
from openviking.service.resource_service import ResourceService
+from openviking.service.task_tracker import TaskStatus
from openviking.storage.content_write import ContentWriteCoordinator
from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
+from openviking.utils.ingest_options import IngestOptions
from openviking_cli.exceptions import ConflictError, InvalidArgumentError
from openviking_cli.session.user_id import UserIdentifier
@@ -82,6 +83,9 @@ class NoopTaskTracker:
async def complete(self, *_args, **_kwargs):
pass
+ async def wait(self, *_args, **_kwargs):
+ return SimpleNamespace(status=TaskStatus.COMPLETED, result={})
+
def count(self):
return self._count
@@ -101,6 +105,7 @@ def isolate_service_dependencies(monkeypatch: pytest.MonkeyPatch) -> None:
lambda: task_tracker,
)
monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: False)
+ monkeypatch.setattr("openviking.connector.routing.is_git_repo_url", lambda _path: False)
@pytest_asyncio.fixture
@@ -331,31 +336,7 @@ class TestWatchTaskCreation:
finish_calls.append({"args": args, "kwargs": kwargs})
return {"root_uri": "viking://resources/queued"}
- class FakeRequestWaitTracker:
- def register_request(self, *_args, **_kwargs):
- pass
-
- async def wait_for_request(self, *_args, **_kwargs):
- pass
-
- def build_queue_status(self, *_args, **_kwargs):
- return {}
-
- def cleanup(self, *_args, **_kwargs):
- pass
-
monkeypatch.setattr(ContentWriteCoordinator, "set_tags", fake_set_tags)
- monkeypatch.setattr(
- resource_service_module,
- "get_request_wait_tracker",
- lambda: FakeRequestWaitTracker(),
- )
- monkeypatch.setattr(
- resource_service_module,
- "unregister_wait_telemetry",
- lambda *_args, **_kwargs: None,
- )
- resource_service._link_resource_reason_memory = AsyncMock()
resource_service._resource_processor.finish_prepared_resource = AsyncMock(
side_effect=fake_finish_prepared_resource
)
diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.py
index aec37012c..6ba49edcb 100644
--- a/tests/storage/test_semantic_dag_stats.py
+++ b/tests/storage/test_semantic_dag_stats.py
@@ -6,7 +6,14 @@ import asyncio
import pytest
from openviking.server.identity import RequestContext, Role
-from openviking.storage.queuefs.semantic_dag import DagStats, SemanticDagExecutor
+from openviking.service.task_work_index import TaskWorkIndex, TaskWorkRejected
+from openviking.storage.queuefs.named_queue import NamedQueue
+from openviking.storage.queuefs.semantic_dag import (
+ DagStats,
+ DagWork,
+ SemanticDagExecutor,
+ SemanticNodeScheduler,
+)
from openviking_cli.session.user_id import UserIdentifier
@@ -19,7 +26,7 @@ class _FakeVikingFS:
del node_limit
return self._tree.get(uri, [])
- async def write_file(self, path, content, ctx=None):
+ async def write_file(self, path, content, ctx=None, lock_handle=None):
self.writes.append((path, content))
def _uri_to_path(self, uri, ctx=None):
@@ -86,6 +93,25 @@ class _DummyTracker:
return None
+class _ScheduledExecutor:
+ def __init__(self, run) -> None:
+ self.closed = False
+ self.failure = None
+ self._run = run
+
+ def _start_scheduled_work(self) -> None:
+ return None
+
+ def _finish_scheduled_work(self) -> None:
+ return None
+
+ async def _run_work(self, _work) -> None:
+ await self._run()
+
+ def fail(self, exc: Exception) -> None:
+ self.failure = exc
+
+
@pytest.mark.asyncio
async def test_semantic_dag_stats_collects_nodes(monkeypatch):
root_uri = "viking://resources/root"
@@ -211,6 +237,54 @@ async def test_semantic_dag_shares_node_scheduler_across_roots(monkeypatch):
assert executor_b.get_stats().done_nodes == 21
+async def test_task_work_rejection_does_not_stop_shared_semantic_worker():
+ work_index = TaskWorkIndex()
+
+ async def finalize_before_ack(_metadata):
+ return None
+
+ work_index.set_callbacks(
+ finalize_before_ack=finalize_before_ack,
+ is_cancellation_requested=lambda _task_id: True,
+ )
+ embedding_queue = NamedQueue(
+ None,
+ "/queue",
+ "Embedding",
+ task_work_index=work_index,
+ )
+ embedding_queue._initialized = True
+ unrelated_ran = asyncio.Event()
+
+ async def rejected_work() -> None:
+ await embedding_queue.enqueue(
+ {
+ "task_id": "task-a",
+ "account_id": "account-a",
+ "user_id": "user-a",
+ }
+ )
+
+ async def unrelated_work() -> None:
+ unrelated_ran.set()
+
+ rejected = _ScheduledExecutor(rejected_work)
+ unrelated = _ScheduledExecutor(unrelated_work)
+ scheduler = SemanticNodeScheduler(max_workers=1)
+ scheduler.submit(rejected, DagWork(kind="vectorize", dir_uri="a"))
+ scheduler.submit(unrelated, DagWork(kind="vectorize", dir_uri="b"))
+
+ await asyncio.wait_for(unrelated_ran.wait(), timeout=0.5)
+ await asyncio.wait_for(scheduler._queue.join(), timeout=0.5)
+ await asyncio.sleep(scheduler._idle_timeout * 2)
+
+ assert isinstance(rejected.failure, TaskWorkRejected)
+ assert unrelated.failure is None
+ assert scheduler._queue.empty()
+ assert all(worker.done() for worker in scheduler._workers)
+
+
@pytest.mark.asyncio
async def test_semantic_dag_skip_vectorization_does_not_schedule_tasks(monkeypatch):
root_uri = "viking://resources/root"
diff --git a/tests/storage/test_session_commit_processor_identity.py b/tests/storage/test_session_commit_processor_identity.py
index 2be193dbc..ebccb77c3 100644
--- a/tests/storage/test_session_commit_processor_identity.py
+++ b/tests/storage/test_session_commit_processor_identity.py
@@ -9,9 +9,13 @@ worker binds the committing account/user (so tokens are not attributed to
"""
import asyncio
+import concurrent.futures
+import json
+from unittest.mock import Mock
from openviking.observability.context import get_root_observability_context
from openviking.server.identity import RequestContext, Role
+from openviking.session.session import Session
from openviking.storage.queuefs.session_commit_msg import SessionCommitMsg
from openviking.storage.queuefs.session_commit_processor import SessionCommitProcessor
from openviking_cli.session.user_id import UserIdentifier
@@ -41,6 +45,25 @@ class _FakeSessionService:
return _FakeSession(self._captured)
+class _MemoryVikingFS:
+ def __init__(self) -> None:
+ self.files: dict[str, str] = {}
+
+ async def stat(self, uri, ctx=None):
+ return {"path": uri}
+
+ async def write_file(self, uri, content, ctx=None, lease_ref=None):
+ self.files[uri] = content
+
+
+class _SingleSessionService:
+ def __init__(self, session: Session) -> None:
+ self._session = session
+
+ def session(self, ctx, session_id, session_uri=None):
+ return self._session
+
+
def _make_msg() -> SessionCommitMsg:
return SessionCommitMsg(
task_id="task-1",
@@ -75,3 +98,49 @@ async def test_process_resets_root_context_after_completion():
await processor._process(_make_msg(), ctx)
assert get_root_observability_context() is None
+
+
+async def test_cancelled_queued_commit_writes_terminal_marker_before_success(monkeypatch):
+ msg = _make_msg()
+ viking_fs = _MemoryVikingFS()
+ session = Session(
+ viking_fs=viking_fs,
+ session_id=msg.session_id,
+ session_uri=msg.session_uri,
+ )
+ processor = SessionCommitProcessor(
+ _SingleSessionService(session),
+ asyncio.get_running_loop(),
+ )
+ marker_uri = f"{msg.archive_uri}/.failed.json"
+ on_success = Mock(side_effect=lambda: viking_fs.files[marker_uri])
+ processor.set_callbacks(on_success, Mock(), Mock())
+
+ def run_on_current_loop(coro, _loop):
+ task = asyncio.create_task(coro)
+ future: concurrent.futures.Future[None] = concurrent.futures.Future()
+
+ def complete(completed: asyncio.Task) -> None:
+ if completed.cancelled():
+ future.cancel()
+ return
+ error = completed.exception()
+ if error is not None:
+ future.set_exception(error)
+ else:
+ future.set_result(completed.result())
+
+ task.add_done_callback(complete)
+ return future
+
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.session_commit_processor.asyncio.run_coroutine_threadsafe",
+ run_on_current_loop,
+ )
+
+ await processor.on_cancelled({"data": json.dumps(msg.to_dict())})
+
+ marker = json.loads(viking_fs.files[marker_uri])
+ assert marker["stage"] == "cancelled"
+ assert marker["error"] == "session commit cancelled"
+ on_success.assert_called_once_with()
diff --git a/tests/telemetry/test_resource_summary.py b/tests/telemetry/test_resource_summary.py
index 2fdd1cf01..e12127983 100644
--- a/tests/telemetry/test_resource_summary.py
+++ b/tests/telemetry/test_resource_summary.py
@@ -1,24 +1,20 @@
-from types import SimpleNamespace
from openviking.telemetry.operation import OperationTelemetry
-def test_record_resource_wait_metrics_collects_queue_and_dag_stats(monkeypatch):
- from openviking.telemetry.resource_summary import record_resource_wait_metrics
+def test_record_resource_queue_metrics_collects_queue_and_dag_stats(monkeypatch):
+ from openviking.telemetry.resource_summary import record_resource_queue_metrics
telemetry = OperationTelemetry(operation="resources.add_resource", enabled=True)
telemetry_id = telemetry.telemetry_id
- queue_status = {
- "Semantic": SimpleNamespace(processed=3, error_count=1, errors=[]),
- "Embedding": SimpleNamespace(processed=5, error_count=0, errors=[]),
- }
-
class _SemanticStats:
processed = 7
+ requeue_count = 0
error_count = 2
class _EmbeddingStats:
processed = 11
+ requeue_count = 0
error_count = 1
class _DagStats:
@@ -40,10 +36,9 @@ def test_record_resource_wait_metrics_collects_queue_and_dag_stats(monkeypatch):
lambda _tid, _uri: _DagStats(),
)
- record_resource_wait_metrics(
+ record_resource_queue_metrics(
telemetry=telemetry,
telemetry_id=telemetry_id,
- queue_status=queue_status,
root_uri="viking://resources/demo",
)
diff --git a/tests/test_telemetry_runtime.py b/tests/test_telemetry_runtime.py
index 8774b9daf..6947f1e2f 100644
--- a/tests/test_telemetry_runtime.py
+++ b/tests/test_telemetry_runtime.py
@@ -20,8 +20,6 @@ from openviking.observability.context import (
reset_operation_observability_context,
reset_root_observability_context,
)
-from openviking.server.identity import RequestContext, Role
-from openviking.service.resource_service import ResourceService
from openviking.storage.collection_schemas import TextEmbeddingHandler
from openviking.storage.queuefs.semantic_dag import DagStats
from openviking.storage.queuefs.semantic_msg import SemanticMsg
@@ -36,7 +34,6 @@ from openviking.telemetry.backends.memory import MemoryOperationTelemetry
from openviking.telemetry.context import bind_telemetry, bind_telemetry_stage
from openviking.telemetry.snapshot import TelemetrySnapshot
from openviking.telemetry.span_models import OperationSpanAttributes, RootSpanAttributes
-from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils import logger as logger_module
@@ -881,89 +878,6 @@ async def test_embedding_handler_binds_registered_operation_telemetry(monkeypatc
assert summary["tokens"]["embedding"] == {"total": 9}
-async def test_resource_service_add_resource_reports_queue_summary(monkeypatch):
- telemetry = MemoryOperationTelemetry(operation="resources.add_resource", enabled=True)
- queue_status = {
- "Semantic": {
- "processed": 2,
- "requeue_count": 0,
- "error_count": 1,
- "errors": [],
- },
- "Embedding": {
- "processed": 5,
- "requeue_count": 0,
- "error_count": 0,
- "errors": [],
- },
- }
-
- class _DummyProcessor:
- async def process_resource(self, **kwargs):
- return {
- "status": "success",
- "root_uri": "viking://resources/demo",
- }
-
- class _DummyRequestWaitTracker:
- def register_request(self, telemetry_id: str) -> None:
- del telemetry_id
-
- async def wait_for_request(self, telemetry_id: str, timeout=None) -> None:
- del telemetry_id, timeout
-
- def build_queue_status(self, telemetry_id: str):
- del telemetry_id
- return queue_status
-
- def cleanup(self, telemetry_id: str) -> None:
- del telemetry_id
-
- monkeypatch.setattr(
- "openviking.service.resource_service.get_request_wait_tracker",
- lambda: _DummyRequestWaitTracker(),
- raising=False,
- )
-
- class _DagStats:
- total_nodes = 3
- done_nodes = 2
- pending_nodes = 1
- in_progress_nodes = 0
-
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_processor.SemanticProcessor.consume_dag_stats",
- classmethod(lambda cls, telemetry_id="", uri=None: _DagStats()),
- )
-
- service = ResourceService(
- vikingdb=object(),
- viking_fs=object(),
- resource_processor=_DummyProcessor(),
- skill_processor=object(),
- )
- ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
-
- with bind_telemetry(telemetry):
- result = await service.add_resource(path="/tmp/demo.md", ctx=ctx, wait=True)
-
- assert result["root_uri"] == "viking://resources/demo"
- telemetry_result = telemetry.finish()
- summary = telemetry_result.summary
- assert summary["queue"] == {
- "semantic": {"processed": 2, "error_count": 1},
- "embedding": {"processed": 5},
- }
- assert summary["semantic_nodes"] == {
- "total": 3,
- "done": 2,
- "pending": 1,
- }
- assert "memory" not in summary
- assert "errors" not in summary
-
-
def test_telemetry_summary_includes_only_memory_group_when_memory_metrics_exist():
telemetry = MemoryOperationTelemetry(operation="session.commit", enabled=True)
telemetry.record_token_usage("llm", 5, 3)
diff --git a/web-studio/src/i18n/locales/en.ts b/web-studio/src/i18n/locales/en.ts
index f317457e2..d30f5d68f 100644
--- a/web-studio/src/i18n/locales/en.ts
+++ b/web-studio/src/i18n/locales/en.ts
@@ -177,6 +177,8 @@ const en = {
'Results returned by the API will appear here when the task completes.',
noResultFailedDescription:
'This task did not return a result. See the failure reason above.',
+ noResultCancelledDescription:
+ 'This task was cancelled before it returned a result.',
},
filters: {
label: 'Filter',
@@ -203,6 +205,8 @@ const en = {
status: 'Status',
},
status: {
+ cancelled: 'Cancelled',
+ cancelling: 'Cancelling',
completed: 'Completed',
failed: 'Failed',
pending: 'Pending',
@@ -774,6 +778,7 @@ const en = {
size: 'Size',
},
status: {
+ cancelled: 'Cancelled',
processing: 'Processing',
success: 'Processed',
failed: 'Processing failed',
diff --git a/web-studio/src/i18n/locales/zh-CN.ts b/web-studio/src/i18n/locales/zh-CN.ts
index 88b4f54c4..b1504ea16 100644
--- a/web-studio/src/i18n/locales/zh-CN.ts
+++ b/web-studio/src/i18n/locales/zh-CN.ts
@@ -169,6 +169,7 @@ const zhCN = {
noResult: '暂无执行结果',
noResultDescription: '任务完成后,接口返回的结果会显示在这里。',
noResultFailedDescription: '该任务未返回结果,请查看上方失败原因。',
+ noResultCancelledDescription: '该任务已取消,未返回执行结果。',
},
filters: {
label: '筛选',
@@ -194,6 +195,8 @@ const zhCN = {
status: '状态',
},
status: {
+ cancelled: '已取消',
+ cancelling: '取消中',
completed: '已完成',
failed: '失败',
pending: '等待中',
@@ -746,6 +749,7 @@ const zhCN = {
size: '大小',
},
status: {
+ cancelled: '已取消',
processing: '处理中',
success: '处理成功',
failed: '处理失败',
diff --git a/web-studio/src/routes/resources/-components/upload-task-dialog.tsx b/web-studio/src/routes/resources/-components/upload-task-dialog.tsx
index cf5099702..e5ffb1b62 100644
--- a/web-studio/src/routes/resources/-components/upload-task-dialog.tsx
+++ b/web-studio/src/routes/resources/-components/upload-task-dialog.tsx
@@ -32,6 +32,14 @@ function TaskStatusBadge({ status }: { status: ResourceUploadTask['status'] }) {
)
}
+ if (status === 'cancelled') {
+ return (
+ <Badge variant="secondary" className="text-muted-foreground">
+ {t('processingTasks.status.cancelled')}
+ </Badge>
+ )
+ }
+
return (
<Badge variant="secondary" className="bg-amber-500/12 text-amber-300">
{t('processingTasks.status.processing')}
diff --git a/web-studio/src/routes/resources/-hooks/use-resource-upload.tsx b/web-studio/src/routes/resources/-hooks/use-resource-upload.tsx
index d9a0043cb..b89fc1561 100644
--- a/web-studio/src/routes/resources/-hooks/use-resource-upload.tsx
+++ b/web-studio/src/routes/resources/-hooks/use-resource-upload.tsx
@@ -16,6 +16,7 @@ import type {
import type { TaskListResult, TaskRecord } from '@ov-server/api/v1/tasks'
export type ResourceUploadTaskStatus =
+ | 'cancelled'
| 'pending'
| 'uploading'
| 'processing'
@@ -208,6 +209,9 @@ function toUploadStatus(
if (status === 'failed') {
return 'failed'
}
+ if (status === 'cancelled') {
+ return 'cancelled'
+ }
return 'processing'
}
@@ -224,7 +228,8 @@ function mergeServerTask(
existing && existing.source !== 'server'
? existing.fileName
: getServerTaskName(record)
- const isFinished = status === 'success' || status === 'failed'
+ const isFinished =
+ status === 'success' || status === 'failed' || status === 'cancelled'
return {
id: existing?.id ?? `server-${record.task_id}`,
@@ -244,7 +249,9 @@ function mergeServerTask(
errorMessage:
status === 'failed'
? record.error || existing?.errorMessage || 'Processing failed'
- : null,
+ : status === 'cancelled'
+ ? record.error || 'Processing cancelled'
+ : null,
rootUri,
}
}
@@ -695,6 +702,19 @@ export function ResourceUploadProvider({
}
: prev,
)
+ return
+ }
+
+ if (remoteTask.status === 'cancelled') {
+ setRemoteState((prev) =>
+ prev.taskId === remoteTask.serverTaskId
+ ? {
+ ...prev,
+ phase: 'idle',
+ error: remoteTask.errorMessage || 'Processing cancelled',
+ }
+ : prev,
+ )
}
}, [remoteState.phase, remoteState.taskId, tasks])
diff --git a/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx b/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx
index 22928d8c3..46d80f75c 100644
--- a/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx
+++ b/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx
@@ -65,7 +65,11 @@ export function TaskDetailSheet({
queryKey: ['task-detail', identityScopeKey, taskId],
refetchInterval: (query) => {
const status = normalizeTaskStatus(query.state.data?.status)
- return status === 'pending' || status === 'running' ? 3_000 : false
+ return status === 'pending' ||
+ status === 'running' ||
+ status === 'cancelling'
+ ? 3_000
+ : false
},
})
const task = detailQuery.data
@@ -177,7 +181,9 @@ export function TaskDetailSheet({
{t(
normalizeTaskStatus(task.status) === 'failed'
? 'detail.noResultFailedDescription'
- : 'detail.noResultDescription',
+ : normalizeTaskStatus(task.status) === 'cancelled'
+ ? 'detail.noResultCancelledDescription'
+ : 'detail.noResultDescription',
)}
</p>
</div>
diff --git a/web-studio/src/routes/tasks/-lib/task-record.ts b/web-studio/src/routes/tasks/-lib/task-record.ts
index 86d5d42ad..776134846 100644
--- a/web-studio/src/routes/tasks/-lib/task-record.ts
+++ b/web-studio/src/routes/tasks/-lib/task-record.ts
@@ -1,6 +1,8 @@
import type { TaskTimestamp } from './task-time'
export type TaskStatus =
+ | 'cancelled'
+ | 'cancelling'
| 'completed'
| 'failed'
| 'pending'
@@ -37,6 +39,8 @@ export function normalizeTasks(value: unknown): TaskRecord[] {
export function normalizeTaskStatus(status: string | undefined): TaskStatus {
if (
+ status === 'cancelled' ||
+ status === 'cancelling' ||
status === 'completed' ||
status === 'failed' ||
status === 'pending' ||
diff --git a/web-studio/src/routes/tasks/route.tsx b/web-studio/src/routes/tasks/route.tsx
index 6f26b5d23..b08af861c 100644
--- a/web-studio/src/routes/tasks/route.tsx
+++ b/web-studio/src/routes/tasks/route.tsx
@@ -81,8 +81,10 @@ const TASK_TYPE_OPTIONS: Exclude<TaskTypeFilter, 'all'>[] = [
const TASK_STATUS_OPTIONS: Exclude<TaskStatusFilter, 'all'>[] = [
'pending',
'running',
+ 'cancelling',
'completed',
'failed',
+ 'cancelled',
]
async function fetchTasks(
@@ -145,7 +147,7 @@ function TasksRoute() {
const Icon =
status === 'completed'
? CheckCircle2Icon
- : status === 'failed'
+ : status === 'failed' || status === 'cancelled'
? CircleXIcon
: status === 'running'
? LoaderCircleIcon
@@ -162,7 +164,13 @@ function TasksRoute() {
}
className="gap-1 font-normal"
>
- <Icon className={status === 'running' ? 'animate-spin' : undefined} />
+ <Icon
+ className={
+ status === 'running' || status === 'cancelling'
+ ? 'animate-spin'
+ : undefined
+ }
+ />
{t(`status.${status}`)}
</Badge>
)