Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(queuefs): make completion status backend-owned (#5000)
runyunzhou7df74141c01a
diff --git a/crates/ragfs/src/plugins/queuefs/backend.rs b/crates/ragfs/src/plugins/queuefs/backend.rsindex 50b79acb5..67dac8898 100644--- a/crates/ragfs/src/plugins/queuefs/backend.rs+++ b/crates/ragfs/src/plugins/queuefs/backend.rs@@ -35,6 +35,15 @@ impl Message {}}+/// Current queue occupancy.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]+pub struct QueueState {+ /// Messages available for dequeue.+ pub pending: usize,+ /// Messages dequeued but not acknowledged.+ pub processing: usize,+}+#[derive(Debug, Clone, Serialize, Deserialize)]pub(super) struct StoredMessage {id: String,@@ -161,6 +170,9 @@ pub trait QueueBackend: Send + Sync {/// Get the number of messages in the queuefn size(&self, queue_name: &str) -> Result<usize>;+ /// Get pending and processing message counts from one backend snapshot+ fn status(&self, queue_name: &str) -> Result<QueueState>;+/// List every unacknowledged message without changing queue statefn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>>;@@ -193,6 +205,7 @@ impl Default for SQLiteQueueOptions {/// A single queue with its messagesstruct Queue {messages: VecDeque<Message>,+ processing: VecDeque<Message>,last_enqueue_time: SystemTime,}@@ -200,6 +213,7 @@ impl Queue {fn new() -> Self {Self {messages: VecDeque::new(),+ processing: VecDeque::new(),last_enqueue_time: SystemTime::UNIX_EPOCH,}}@@ -271,7 +285,11 @@ impl QueueBackend for MemoryBackend {.get_mut(queue_name).ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;- Ok(queue.messages.pop_front())+ let message = queue.messages.pop_front();+ if let Some(message) = &message {+ queue.processing.push_back(message.clone());+ }+ Ok(message)}fn peek(&self, queue_name: &str) -> Result<Option<Message>> {@@ -292,12 +310,28 @@ impl QueueBackend for MemoryBackend {Ok(queue.messages.len())}+ fn status(&self, queue_name: &str) -> Result<QueueState> {+ let queue = self+ .queues+ .get(queue_name)+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;+ Ok(QueueState {+ pending: queue.messages.len(),+ processing: queue.processing.len(),+ })+ }+fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {let queue = self.queues.get(queue_name).ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;- Ok(queue.messages.iter().cloned().collect())+ Ok(queue+ .messages+ .iter()+ .chain(queue.processing.iter())+ .cloned()+ .collect())}fn clear(&mut self, queue_name: &str) -> Result<()> {@@ -307,6 +341,7 @@ impl QueueBackend for MemoryBackend {.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;queue.messages.clear();+ queue.processing.clear();Ok(())}@@ -325,10 +360,9 @@ impl QueueBackend for MemoryBackend {.get_mut(queue_name).ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;- // Find and remove message by ID- let original_len = queue.messages.len();- queue.messages.retain(|msg| msg.id != msg_id);- Ok(queue.messages.len() != original_len)+ let original_len = queue.processing.len();+ queue.processing.retain(|msg| msg.id != msg_id);+ Ok(queue.processing.len() != original_len)}}@@ -702,6 +736,30 @@ impl QueueBackend for SQLiteQueueBackend {Ok(count as usize)}+ fn status(&self, queue_name: &str) -> Result<QueueState> {+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;++ Self::require_queue_exists(&conn, queue_name)?;++ let (pending, processing): (i64, i64) = conn+ .query_row(+ "SELECT+ COALESCE(SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END), 0),+ COALESCE(SUM(CASE WHEN status = 'processing' THEN 1 ELSE 0 END), 0)+ FROM queue_messages WHERE queue_name = ?1",+ params![queue_name],+ |row| Ok((row.get(0)?, row.get(1)?)),+ )+ .map_err(|e| Error::internal(format!("sqlite status query error: {}", e)))?;+ Ok(QueueState {+ pending: pending as usize,+ processing: processing as usize,+ })+ }+fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {let conn = self.conn@@ -866,11 +924,27 @@ mod tests {let dequeued1 = backend.dequeue("test").unwrap().unwrap();assert_eq!(dequeued1.data, b"message 1");+ assert_eq!(+ backend.status("test").unwrap(),+ QueueState {+ pending: 1,+ processing: 1,+ }+ );+ assert!(backend.ack("test", &dequeued1.id).unwrap());let dequeued2 = backend.dequeue("test").unwrap().unwrap();assert_eq!(dequeued2.data, b"message 2");+ assert!(backend.ack("test", &dequeued2.id).unwrap());assert_eq!(backend.size("test").unwrap(), 0);+ assert_eq!(+ backend.status("test").unwrap(),+ QueueState {+ pending: 0,+ processing: 0,+ }+ );assert!(backend.dequeue("test").unwrap().is_none());}@@ -941,6 +1015,7 @@ mod tests {assert!(backend.dequeue("nonexistent").is_err());assert!(backend.peek("nonexistent").is_err());assert!(backend.size("nonexistent").is_err());+ assert!(backend.status("nonexistent").is_err());assert!(backend.clear("nonexistent").is_err());}@@ -955,14 +1030,35 @@ mod tests {backend.enqueue("test", msg1).unwrap();backend.enqueue("test", msg2).unwrap();+ assert_eq!(+ backend.status("test").unwrap(),+ QueueState {+ pending: 2,+ processing: 0,+ }+ );let first = backend.dequeue("test").unwrap().unwrap();assert_eq!(first.data, b"message 1");assert_eq!(backend.size("test").unwrap(), 1);+ assert_eq!(+ backend.status("test").unwrap(),+ QueueState {+ pending: 1,+ processing: 1,+ }+ );let unacked = backend.list_unacked("test").unwrap();assert_eq!(unacked.len(), 2);assert_eq!(unacked[0].id, msg1_id);assert!(backend.ack("test", &msg1_id).unwrap());+ assert_eq!(+ backend.status("test").unwrap(),+ QueueState {+ pending: 1,+ processing: 0,+ }+ );let second = backend.dequeue("test").unwrap().unwrap();assert_eq!(second.data, b"message 2");diff --git a/crates/ragfs/src/plugins/queuefs/cache_backend.rs b/crates/ragfs/src/plugins/queuefs/cache_backend.rsindex 5c89aafff..f96759afd 100644--- a/crates/ragfs/src/plugins/queuefs/cache_backend.rs+++ b/crates/ragfs/src/plugins/queuefs/cache_backend.rs@@ -1,10 +1,10 @@-use super::backend::{Message, StoredMessage};+use super::backend::{Message, QueueState, StoredMessage};use super::cache_protocol::{heartbeat_key, instance_key_prefix, last_enqueue_time_from_pending_payloads, queue_key_prefix,queue_names_key, unix_secs, QueueKeys, ACK_SCRIPT, CLEAR_SCRIPT, CREATE_QUEUE_SCRIPT,DEQUEUE_SCRIPT, ENQUEUE_SCRIPT, HEARTBEAT_INTERVAL_SECS, HEARTBEAT_TTL_SECS,LIST_UNACKED_SCRIPT, PEEK_SCRIPT, RECOVER_STALE_SCRIPT, REMOVE_QUEUE_SCRIPT,- STARTUP_RECOVERY_SWEEPS,+ STARTUP_RECOVERY_SWEEPS, STATUS_SCRIPT,};use crate::cache_runtime::{CacheError, CacheOperation, CacheRuntime, Expiration, ScriptDefinition, ScriptRequest,@@ -25,6 +25,7 @@ const REMOVE_QUEUE_ID: &str = "queuefs.remove_queue.v1";const ENQUEUE_ID: &str = "queuefs.enqueue.v1";const DEQUEUE_ID: &str = "queuefs.dequeue.v1";const PEEK_ID: &str = "queuefs.peek.v1";+const STATUS_ID: &str = "queuefs.status.v1";const LIST_UNACKED_ID: &str = "queuefs.list_unacked.v1";const ACK_ID: &str = "queuefs.ack.v1";const CLEAR_ID: &str = "queuefs.clear.v1";@@ -51,6 +52,10 @@ const SCRIPT_DEFINITIONS: &[ScriptDefinition] = &[id: PEEK_ID,redis_lua: PEEK_SCRIPT,},+ ScriptDefinition {+ id: STATUS_ID,+ redis_lua: STATUS_SCRIPT,+ },ScriptDefinition {id: LIST_UNACKED_ID,redis_lua: LIST_UNACKED_SCRIPT,@@ -334,6 +339,42 @@ impl CacheQueueStorage {.map_err(|_| Error::internal("redis size returned an invalid value"))}+ pub(super) async fn status(&self, queue_name: &str) -> Result<QueueState> {+ let keys = QueueKeys::new(&self.key_prefix, queue_name);+ let value = self+ .execute(+ "status",+ STATUS_ID,+ vec![+ queue_names_key(&self.key_prefix),+ keys.pending,+ keys.processing,+ ],+ vec![bytes(queue_name)],+ )+ .await?;+ if matches!(value, ScriptValue::Null) {+ return Err(Error::NotFound(format!(+ "queue '{}' not found",+ queue_name+ )));+ }+ let ScriptValue::Array(values) = value else {+ return Err(invalid_result("queue status", value));+ };+ if values.len() != 2 {+ return Err(Error::internal(+ "redis queue status returned an invalid value",+ ));+ }+ Ok(QueueState {+ pending: usize::try_from(integer(values[0].clone())?)+ .map_err(|_| Error::internal("redis pending count is invalid"))?,+ processing: usize::try_from(integer(values[1].clone())?)+ .map_err(|_| Error::internal("redis processing count is invalid"))?,+ })+ }+pub(super) async fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {self.require_queue(queue_name).await?;let keys = QueueKeys::new(&self.key_prefix, queue_name);diff --git a/crates/ragfs/src/plugins/queuefs/cache_protocol.rs b/crates/ragfs/src/plugins/queuefs/cache_protocol.rsindex 28963f943..e2fd2d427 100644--- a/crates/ragfs/src/plugins/queuefs/cache_protocol.rs+++ b/crates/ragfs/src/plugins/queuefs/cache_protocol.rs@@ -78,6 +78,13 @@ endreturn payload"#;+pub(super) const STATUS_SCRIPT: &str = r#"+if redis.call('SISMEMBER', KEYS[1], ARGV[1]) == 0 then+ return nil+end+return {redis.call('LLEN', KEYS[2]), redis.call('ZCARD', KEYS[3])}+"#;+pub(super) const LIST_UNACKED_SCRIPT: &str = r#"local result = {}local pending = redis.call('LRANGE', KEYS[1], 0, -1)diff --git a/crates/ragfs/src/plugins/queuefs/mod.rs b/crates/ragfs/src/plugins/queuefs/mod.rsindex b0659bd75..bce789b6a 100644--- a/crates/ragfs/src/plugins/queuefs/mod.rs+++ b/crates/ragfs/src/plugins/queuefs/mod.rs@@ -6,6 +6,7 @@//! - `/queue_name/dequeue` - Read from this file to remove and return the first message//! - `/queue_name/peek` - Read from this file to view the first message without removing it//! - `/queue_name/size` - Read from this file to get the current queue size+//! - `/queue_name/status` - Read pending and processing message counts//! - `/queue_name/messages` - Read all unacknowledged messages without changing queue state//! - `/queue_name/clear` - Write to this file to clear all messages from the queue//! - `/queue_name/ack` - Write message ID to this file to acknowledge and delete it@@ -55,6 +56,10 @@ const CONTROL_FILES: &[ControlFileSpec] = &[name: "size",mode: 0o444,},+ ControlFileSpec {+ name: "status",+ mode: 0o444,+ },ControlFileSpec {name: "messages",mode: 0o444,@@ -179,6 +184,14 @@ impl QueueStorage {}}+ async fn status(&self, queue_name: &str) -> Result<backend::QueueState> {+ match self {+ Self::Local(backend) => backend.lock().await.status(queue_name),+ #[cfg(feature = "cache")]+ Self::Cache(storage) => storage.status(queue_name).await,+ }+ }+async fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {match self {Self::Local(backend) => backend.lock().await.list_unacked(queue_name),@@ -383,6 +396,10 @@ impl FileSystem for QueueFileSystem {let size = self.storage.size(&queue_name).await?;Ok(size.to_string().into_bytes())}+ "status" => {+ let status = self.storage.status(&queue_name).await?;+ Ok(serde_json::to_vec(&status)?)+ }"messages" => {let messages = self.storage@@ -397,7 +414,7 @@ impl FileSystem for QueueFileSystem {Ok(serde_json::to_vec(&messages)?)}_ => Err(Error::InvalidOperation(format!(- "Cannot read from '{}'. Use dequeue, peek, size, or messages",+ "Cannot read from '{}'. Use dequeue, peek, size, status, or messages",operation))),}@@ -812,6 +829,7 @@ impl ServicePlugin for QueueFSPlugin {- dequeue: Read to remove and return the first message\n\- peek: Read to view the first message without removing it\n\- size: Read to get the current queue size\n\+ - status: Read pending and processing message counts\n\- clear: Write to clear all messages from the queue\n\- ack: Write message id to acknowledge and delete it\n\\n\@@ -1008,13 +1026,14 @@ mod tests {.read_dir("/test", None, None, None, None).await.unwrap();- assert_eq!(entries.len(), 7);+ assert_eq!(entries.len(), 8);let names: Vec<String> = entries.iter().map(|e| e.name.clone()).collect();assert!(names.contains(&"enqueue".to_string()));assert!(names.contains(&"dequeue".to_string()));assert!(names.contains(&"peek".to_string()));assert!(names.contains(&"size".to_string()));+ assert!(names.contains(&"status".to_string()));assert!(names.contains(&"messages".to_string()));assert!(names.contains(&"clear".to_string()));assert!(names.contains(&"ack".to_string()));@@ -1227,11 +1246,20 @@ mod tests {let dequeued: TestQueueMessage =serde_json::from_slice(&fs.read("/Semantic/dequeue", 0, 0).await.unwrap()).unwrap();assert_eq!(dequeued.data, "payload");+ assert_eq!(+ fs.read("/Semantic/status", 0, 0).await.unwrap(),+ br#"{"pending":0,"processing":1}"#+ );fs.write("/Semantic/ack", dequeued.id.as_bytes(), 0, WriteFlag::None).await.unwrap();+ assert_eq!(+ fs.read("/Semantic/status", 0, 0).await.unwrap(),+ br#"{"pending":0,"processing":0}"#+ );assert_eq!(fs.read("/Semantic/size", 0, 0).await.unwrap(), b"0");fs.remove_all("/Semantic").await.unwrap();+ assert!(fs.read("/Semantic/status", 0, 0).await.is_err());drop(fs);runtime.close().await.unwrap();}diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.tsindex 416e9a86b..c9c9905b0 100644--- a/docs/.vitepress/config.ts+++ b/docs/.vitepress/config.ts@@ -345,7 +345,8 @@ const conceptsSidebar = {['11-multi-tenant.md', 'Multi-Tenancy'],['12-metrics.md', 'Metrics'],['13-privacy.md', 'Privacy'],- ['14-multi-write-storage.md', 'Multi-Write Storage']+ ['14-multi-write-storage.md', 'Multi-Write Storage'],+ ['16-queue-lifecycle.md', 'Queue State and Completion']]},{@@ -382,7 +383,8 @@ const conceptsSidebar = {['11-multi-tenant.md', '多租户'],['12-metrics.md', '监控指标'],['13-privacy.md', '隐私配置'],- ['14-multi-write-storage.md', '多写存储']+ ['14-multi-write-storage.md', '多写存储'],+ ['16-queue-lifecycle.md', '队列状态与完成语义']]},{diff --git a/docs/en/concepts/16-queue-lifecycle.md b/docs/en/concepts/16-queue-lifecycle.mdnew file mode 100644index 000000000..41d5594eb--- /dev/null+++ b/docs/en/concepts/16-queue-lifecycle.md@@ -0,0 +1,104 @@+# Queue State and Completion Semantics++## Problem++Queue completion is currently inferred from two independent sources:++- QueueFS `/size`, which reports messages still pending for dequeue.+- Python `NamedQueue._in_progress`, which reports work observed by the current+ process.++These values cannot be read or updated atomically. After QueueFS moves a message+from `pending` to `processing`, another thread, event loop, or process can+observe `size == 0` while the `_in_progress` value it reads is zero. It may+therefore report the queue as complete before the message is acknowledged.++Moving the local counter update before or after task creation only narrows some+timing windows. It cannot make the backend state transition and local counter+update one atomic operation, so it does not strictly solve the problem across+threads, event loops, or processes.++## Previous Backend Behavior++SQLite and cache-backed queues implement an acknowledgement lifecycle:++```text+enqueue -> pending -> dequeue -> processing -> ack -> removed+```++Before this change, the memory backend did not implement the same lifecycle.+Its `dequeue` operation removed the message from its only queue immediately.+Although the backend exposed an `ack` method, the dequeued message was no+longer stored, so a later ACK normally had nothing to remove. The memory+backend therefore had no effective `processing` state or ACK lifecycle.++## State Model++Queue length is not one scalar. QueueFS must maintain these current-state+gauges:++| Field | Meaning |+| --- | --- |+| `pending` | Messages available for dequeue |+| `processing` | Messages dequeued but not yet acknowledged |+| `unacked` | `pending + processing` |++A queue is complete only when:++```text+pending == 0 && processing == 0+```++Equivalently, `unacked == 0`.++An unacknowledged message remains incomplete when its handler fails, its ACK+fails, or its worker exits. It becomes pending again through recovery or leaves+the queue through a successful ACK.++## Ownership++QueueFS is the owner of queue lifecycle state. Each backend must update+`pending` and `processing` together with enqueue, dequeue, ACK, clear, and+recovery operations.++Python must not combine backend `pending` with a process-local counter to decide+completion. A local worker counter may still be exposed as runtime telemetry,+but it is not queue length and is not authoritative.++The following cumulative counters are also not queue length:++- `processed`+- `requeue_count`+- `error_count`++They describe processing outcomes rather than current queue occupancy. They can+remain in the processing or metrics layer for this fix. Moving them into+QueueFS would require an explicit backend outcome protocol because the backend+cannot infer a handler result from dequeue or ACK alone.++## Backend Contract++QueueFS should expose one atomic status operation:++```json+{+ "pending": 3,+ "processing": 2+}+```++`unacked` is derived as `pending + processing`. The existing `/size` operation+can continue to mean `pending` for compatibility and worker scheduling.+QueueFS control names are reserved path components: a queue name cannot end in+`enqueue`, `dequeue`, `peek`, `size`, `status`, `messages`, `clear`, or `ack`.++Backend requirements:++- **SQLite:** read both counts from one database snapshot.+- **Cache:** return `LLEN(pending)` and `ZCARD(processing)` from one Lua script.+- **Memory:** add a processing collection; dequeue moves a message into it and+ ACK removes the message from it.++`NamedQueue.get_status()` consumes this backend snapshot. `wait_complete()` and+`is_all_complete()` use only the backend-owned `pending` and `processing`+values.diff --git a/docs/zh/concepts/16-queue-lifecycle.md b/docs/zh/concepts/16-queue-lifecycle.mdnew file mode 100644index 000000000..b87ea77ab--- /dev/null+++ b/docs/zh/concepts/16-queue-lifecycle.md@@ -0,0 +1,95 @@+# 队列状态与完成语义++## 问题++当前队列完成状态由两个相互独立的数据源推断:++- QueueFS `/size`:返回仍可被 dequeue 的 pending 消息数。+- Python `NamedQueue._in_progress`:返回当前进程观察到的执行中任务数。++这两个值无法原子读取或更新。QueueFS 将消息从 `pending` 移到+`processing` 后,另一个线程、event loop 或进程可能同时观察到 `size == 0`,+且其读取到的 `_in_progress == 0`,从而在消息 ACK 前错误地判断队列已经完成。++将本地计数放到 task 创建之前或之后,只能缩小部分时序窗口。它无法让 backend+状态迁移与本地计数更新成为同一个原子操作,因此无法严格解决跨线程、跨 event loop+或跨进程的并发判断问题。++## 改动前的后端行为++SQLite 和缓存后端实现了 ACK 生命周期:++```text+enqueue -> pending -> dequeue -> processing -> ack -> removed+```++改动前,MemoryBackend 没有实现相同的生命周期。它的 `dequeue` 会直接从唯一的+队列中删除消息。虽然接口上存在 `ack` 方法,但被 dequeue 的消息已经不再保存,+后续 ACK 通常找不到可删除的消息。因此,MemoryBackend 实际上没有有效的+`processing` 状态和 ACK 生命周期。++## 状态模型++队列长度不是单一数字。QueueFS 必须维护以下当前状态指标:++| 字段 | 含义 |+| --- | --- |+| `pending` | 尚未 dequeue、可被 worker 获取的消息数 |+| `processing` | 已 dequeue、尚未 ACK 的消息数 |+| `unacked` | `pending + processing` |++队列完成条件必须是:++```text+pending == 0 && processing == 0+```++等价于 `unacked == 0`。++当 handler 失败、ACK 失败或 worker 退出时,只要消息仍未 ACK,就不能视为完成。+消息只能通过恢复流程重新进入 pending,或通过成功 ACK 离开队列。++## 状态归属++QueueFS 是队列生命周期状态的 Owner。各 backend 必须在 enqueue、dequeue、ACK、+clear 和 recovery 操作中维护 `pending` 与 `processing`。++Python 不应再组合 backend 的 `pending` 和进程内计数来判断完成。本地 worker+计数仍可作为运行时观测指标,但它不属于队列长度,也不是完成状态的权威来源。++以下累计计数同样不属于队列长度:++- `processed`+- `requeue_count`+- `error_count`++它们描述的是处理结果,而不是当前队列占用。本次修复中可以继续由处理层或指标层+维护。如果需要将它们下沉到 QueueFS,必须先定义显式的处理结果协议,因为 backend+无法仅根据 dequeue 或 ACK 推断 handler 的处理结果。++## Backend 契约++QueueFS 应提供一个原子状态操作:++```json+{+ "pending": 3,+ "processing": 2+}+```++`unacked` 由 `pending + processing` 派生。为保持兼容和 worker 调度语义,现有+`/size` 可以继续表示 `pending`。+QueueFS 控制文件名属于保留路径段,队列名不能以 `enqueue`、`dequeue`、`peek`、+`size`、`status`、`messages`、`clear` 或 `ack` 结尾。++各后端要求:++- **SQLite:** 在同一个数据库快照中读取两个计数。+- **Cache:** 通过一个 Lua 脚本返回 `LLEN(pending)` 和+ `ZCARD(processing)`。+- **Memory:** 增加 processing 集合;dequeue 将消息移入该集合,ACK 从该集合+ 删除消息。++`NamedQueue.get_status()` 只消费 backend 返回的状态快照。`wait_complete()` 和+`is_all_complete()` 只使用 backend 维护的 `pending` 与 `processing` 判断完成。diff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.pyindex b7da44d2d..7090ddaa9 100644--- a/openviking/storage/queuefs/named_queue.py+++ b/openviking/storage/queuefs/named_queue.py@@ -135,7 +135,6 @@ class NamedQueue:# Status trackingself._lock = threading.Lock()- self._in_progress = 0self._processed = 0self._requeue_count = 0self._error_count = 0@@ -154,15 +153,9 @@ class NamedQueue:on_error=self._on_process_error,)- def _on_dequeue_start(self) -> None:- """Called on dequeue."""- with self._lock:- self._in_progress += 1-def _on_process_success(self) -> None:"""Called on processing success."""with self._lock:- self._in_progress -= 1self._processed += 1def _on_process_requeue(self) -> None:@@ -177,7 +170,6 @@ class NamedQueue:if metadata is not None:self._task_work_index.record_failure(metadata.task_id, error_msg)with self._lock:- self._in_progress -= 1self._error_count += 1self._errors.append(QueueError(@@ -191,11 +183,11 @@ class NamedQueue:async def get_status(self) -> QueueStatus:"""Get queue status."""- pending = await self.size()+ backend_status = await self._read_backend_status()with self._lock:return QueueStatus(- pending=pending,- in_progress=self._in_progress,+ pending=backend_status["pending"],+ in_progress=backend_status["processing"],processed=self._processed,requeue_count=self._requeue_count,error_count=self._error_count,@@ -205,7 +197,6 @@ class NamedQueue:def reset_status(self) -> None:"""Reset status counters."""with self._lock:- self._in_progress = 0self._processed = 0self._requeue_count = 0self._error_count = 0@@ -325,7 +316,6 @@ class NamedQueue:msg_id = data.get("id", "") if isinstance(data, dict) else ""raw_data = dataif self._dequeue_handler:- self._on_dequeue_start()data = await self.process_dequeued(data)# Ack unconditionally after handler returns (success or handled error).# If on_dequeue raises, the exception propagates and ack is skipped —@@ -346,11 +336,7 @@ class NamedQueue:return Noneasync def process_dequeued(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]:- """Invoke the dequeue handler on already-fetched raw data.-- NOTE: caller must call _on_dequeue_start() before invoking this method- so that in_progress is incremented atomically with the dequeue.- """+ """Invoke the dequeue handler on already-fetched raw data."""if self._dequeue_handler is None:return data@@ -414,6 +400,19 @@ class NamedQueue:except (AGFSNotFoundError, FileNotFoundError):return 0+ async def _read_backend_status(self) -> Dict[str, int]:+ await self._ensure_initialized()+ content = await self._async_agfs.read(f"{self.path}/status")+ if isinstance(content, bytes):+ content = content.decode("utf-8")+ elif hasattr(content, "content") and content.content is not None:+ content = content.content.decode("utf-8")+ status = json.loads(content)+ return {+ "pending": int(status["pending"]),+ "processing": int(status["processing"]),+ }+async def snapshot(self) -> List[Dict[str, Any]]:"""Return all unacknowledged messages without changing queue state."""await self._ensure_initialized()diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 670e1a4b8..91cbb6a18 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -276,7 +276,6 @@ class QueueManager:# Ack after successful processing (delete from persistent storage).await queue.ack(msg_id, data)except Exception as e:- # Handler did not call report_error; decrement in_progress manually.# Do NOT ack — let RecoverStale re-queue on next startup.queue._on_process_error(str(e), data)logger.error(f"[QueueManager] Concurrent worker error for {queue.name}: {e}")@@ -287,18 +286,11 @@ class QueueManager:# While capacity remains, keep draining the queuewhile len(active_tasks) < max_concurrent:- try:- queue_size = await queue.size()- except Exception:- break- if not queue.has_dequeue_handler() or queue_size == 0:+ if not queue.has_dequeue_handler():breakdata = await queue.dequeue_raw()if data is None:break- # Increment before task creation to close the race window where- # size=0 and in_progress=0 between dequeue_raw() and task execution.- queue._on_dequeue_start()task = asyncio.create_task(process_one(data))active_tasks.add(task)logger.debug(diff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.pyindex ec12526a8..1511ce447 100644--- a/tests/storage/test_queue_manager.py+++ b/tests/storage/test_queue_manager.py@@ -5,7 +5,9 @@import osimport subprocessimport sys+from unittest.mock import AsyncMock+from openviking.storage.queuefs.named_queue import NamedQueuefrom openviking.storage.queuefs.queue_manager import QueueManager@@ -34,3 +36,24 @@ def test_queue_concurrency_uses_separate_configured_values() -> None:assert manager._max_concurrent_for_queue(manager.EXTERNAL_PARSE) == 9assert manager._max_concurrent_for_queue(manager.ADD_RESOURCE) == 7assert manager._max_concurrent_for_queue(manager.SESSION_COMMIT) == 5+++async def test_status_waits_for_processing_messages_from_other_workers(monkeypatch) -> None:+ client = AsyncMock()++ async def read(path: str):+ if path.endswith("/status"):+ return b'{"pending":0,"processing":1}'+ raise AssertionError(f"unexpected read: {path}")++ client.read.side_effect = read+ monkeypatch.setattr(+ "openviking.storage.queuefs.named_queue.AsyncAGFSClient",+ lambda _: client,+ )++ status = await NamedQueue(object(), "/queue", "Test").get_status()++ assert status.pending == 0+ assert status.in_progress == 1+ assert not status.is_complete