Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(queuefs): make completion status backend-owned (#5000)

runyunzhou7df74141c01a
diff --git a/crates/ragfs/src/plugins/queuefs/backend.rs b/crates/ragfs/src/plugins/queuefs/backend.rs
index 50b79acb5..67dac8898 100644
--- a/crates/ragfs/src/plugins/queuefs/backend.rs
+++ b/crates/ragfs/src/plugins/queuefs/backend.rs
@@ -35,6 +35,15 @@ impl Message {
}
}
+/// Current queue occupancy.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+pub struct QueueState {
+ /// Messages available for dequeue.
+ pub pending: usize,
+ /// Messages dequeued but not acknowledged.
+ pub processing: usize,
+}
+
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(super) struct StoredMessage {
id: String,
@@ -161,6 +170,9 @@ pub trait QueueBackend: Send + Sync {
/// Get the number of messages in the queue
fn size(&self, queue_name: &str) -> Result<usize>;
+ /// Get pending and processing message counts from one backend snapshot
+ fn status(&self, queue_name: &str) -> Result<QueueState>;
+
/// List every unacknowledged message without changing queue state
fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>>;
@@ -193,6 +205,7 @@ impl Default for SQLiteQueueOptions {
/// A single queue with its messages
struct Queue {
messages: VecDeque<Message>,
+ processing: VecDeque<Message>,
last_enqueue_time: SystemTime,
}
@@ -200,6 +213,7 @@ impl Queue {
fn new() -> Self {
Self {
messages: VecDeque::new(),
+ processing: VecDeque::new(),
last_enqueue_time: SystemTime::UNIX_EPOCH,
}
}
@@ -271,7 +285,11 @@ impl QueueBackend for MemoryBackend {
.get_mut(queue_name)
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
- Ok(queue.messages.pop_front())
+ let message = queue.messages.pop_front();
+ if let Some(message) = &message {
+ queue.processing.push_back(message.clone());
+ }
+ Ok(message)
}
fn peek(&self, queue_name: &str) -> Result<Option<Message>> {
@@ -292,12 +310,28 @@ impl QueueBackend for MemoryBackend {
Ok(queue.messages.len())
}
+ fn status(&self, queue_name: &str) -> Result<QueueState> {
+ let queue = self
+ .queues
+ .get(queue_name)
+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
+ Ok(QueueState {
+ pending: queue.messages.len(),
+ processing: queue.processing.len(),
+ })
+ }
+
fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {
let queue = self
.queues
.get(queue_name)
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
- Ok(queue.messages.iter().cloned().collect())
+ Ok(queue
+ .messages
+ .iter()
+ .chain(queue.processing.iter())
+ .cloned()
+ .collect())
}
fn clear(&mut self, queue_name: &str) -> Result<()> {
@@ -307,6 +341,7 @@ impl QueueBackend for MemoryBackend {
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
queue.messages.clear();
+ queue.processing.clear();
Ok(())
}
@@ -325,10 +360,9 @@ impl QueueBackend for MemoryBackend {
.get_mut(queue_name)
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
- // Find and remove message by ID
- let original_len = queue.messages.len();
- queue.messages.retain(|msg| msg.id != msg_id);
- Ok(queue.messages.len() != original_len)
+ let original_len = queue.processing.len();
+ queue.processing.retain(|msg| msg.id != msg_id);
+ Ok(queue.processing.len() != original_len)
}
}
@@ -702,6 +736,30 @@ impl QueueBackend for SQLiteQueueBackend {
Ok(count as usize)
}
+ fn status(&self, queue_name: &str) -> Result<QueueState> {
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
+
+ Self::require_queue_exists(&conn, queue_name)?;
+
+ let (pending, processing): (i64, i64) = conn
+ .query_row(
+ "SELECT
+ COALESCE(SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END), 0),
+ COALESCE(SUM(CASE WHEN status = 'processing' THEN 1 ELSE 0 END), 0)
+ FROM queue_messages WHERE queue_name = ?1",
+ params![queue_name],
+ |row| Ok((row.get(0)?, row.get(1)?)),
+ )
+ .map_err(|e| Error::internal(format!("sqlite status query error: {}", e)))?;
+ Ok(QueueState {
+ pending: pending as usize,
+ processing: processing as usize,
+ })
+ }
+
fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {
let conn = self
.conn
@@ -866,11 +924,27 @@ mod tests {
let dequeued1 = backend.dequeue("test").unwrap().unwrap();
assert_eq!(dequeued1.data, b"message 1");
+ assert_eq!(
+ backend.status("test").unwrap(),
+ QueueState {
+ pending: 1,
+ processing: 1,
+ }
+ );
+ assert!(backend.ack("test", &dequeued1.id).unwrap());
let dequeued2 = backend.dequeue("test").unwrap().unwrap();
assert_eq!(dequeued2.data, b"message 2");
+ assert!(backend.ack("test", &dequeued2.id).unwrap());
assert_eq!(backend.size("test").unwrap(), 0);
+ assert_eq!(
+ backend.status("test").unwrap(),
+ QueueState {
+ pending: 0,
+ processing: 0,
+ }
+ );
assert!(backend.dequeue("test").unwrap().is_none());
}
@@ -941,6 +1015,7 @@ mod tests {
assert!(backend.dequeue("nonexistent").is_err());
assert!(backend.peek("nonexistent").is_err());
assert!(backend.size("nonexistent").is_err());
+ assert!(backend.status("nonexistent").is_err());
assert!(backend.clear("nonexistent").is_err());
}
@@ -955,14 +1030,35 @@ mod tests {
backend.enqueue("test", msg1).unwrap();
backend.enqueue("test", msg2).unwrap();
+ assert_eq!(
+ backend.status("test").unwrap(),
+ QueueState {
+ pending: 2,
+ processing: 0,
+ }
+ );
let first = backend.dequeue("test").unwrap().unwrap();
assert_eq!(first.data, b"message 1");
assert_eq!(backend.size("test").unwrap(), 1);
+ assert_eq!(
+ backend.status("test").unwrap(),
+ QueueState {
+ pending: 1,
+ processing: 1,
+ }
+ );
let unacked = backend.list_unacked("test").unwrap();
assert_eq!(unacked.len(), 2);
assert_eq!(unacked[0].id, msg1_id);
assert!(backend.ack("test", &msg1_id).unwrap());
+ assert_eq!(
+ backend.status("test").unwrap(),
+ QueueState {
+ pending: 1,
+ processing: 0,
+ }
+ );
let second = backend.dequeue("test").unwrap().unwrap();
assert_eq!(second.data, b"message 2");
diff --git a/crates/ragfs/src/plugins/queuefs/cache_backend.rs b/crates/ragfs/src/plugins/queuefs/cache_backend.rs
index 5c89aafff..f96759afd 100644
--- a/crates/ragfs/src/plugins/queuefs/cache_backend.rs
+++ b/crates/ragfs/src/plugins/queuefs/cache_backend.rs
@@ -1,10 +1,10 @@
-use super::backend::{Message, StoredMessage};
+use super::backend::{Message, QueueState, StoredMessage};
use super::cache_protocol::{
heartbeat_key, instance_key_prefix, last_enqueue_time_from_pending_payloads, queue_key_prefix,
queue_names_key, unix_secs, QueueKeys, ACK_SCRIPT, CLEAR_SCRIPT, CREATE_QUEUE_SCRIPT,
DEQUEUE_SCRIPT, ENQUEUE_SCRIPT, HEARTBEAT_INTERVAL_SECS, HEARTBEAT_TTL_SECS,
LIST_UNACKED_SCRIPT, PEEK_SCRIPT, RECOVER_STALE_SCRIPT, REMOVE_QUEUE_SCRIPT,
- STARTUP_RECOVERY_SWEEPS,
+ STARTUP_RECOVERY_SWEEPS, STATUS_SCRIPT,
};
use crate::cache_runtime::{
CacheError, CacheOperation, CacheRuntime, Expiration, ScriptDefinition, ScriptRequest,
@@ -25,6 +25,7 @@ const REMOVE_QUEUE_ID: &str = "queuefs.remove_queue.v1";
const ENQUEUE_ID: &str = "queuefs.enqueue.v1";
const DEQUEUE_ID: &str = "queuefs.dequeue.v1";
const PEEK_ID: &str = "queuefs.peek.v1";
+const STATUS_ID: &str = "queuefs.status.v1";
const LIST_UNACKED_ID: &str = "queuefs.list_unacked.v1";
const ACK_ID: &str = "queuefs.ack.v1";
const CLEAR_ID: &str = "queuefs.clear.v1";
@@ -51,6 +52,10 @@ const SCRIPT_DEFINITIONS: &[ScriptDefinition] = &[
id: PEEK_ID,
redis_lua: PEEK_SCRIPT,
},
+ ScriptDefinition {
+ id: STATUS_ID,
+ redis_lua: STATUS_SCRIPT,
+ },
ScriptDefinition {
id: LIST_UNACKED_ID,
redis_lua: LIST_UNACKED_SCRIPT,
@@ -334,6 +339,42 @@ impl CacheQueueStorage {
.map_err(|_| Error::internal("redis size returned an invalid value"))
}
+ pub(super) async fn status(&self, queue_name: &str) -> Result<QueueState> {
+ let keys = QueueKeys::new(&self.key_prefix, queue_name);
+ let value = self
+ .execute(
+ "status",
+ STATUS_ID,
+ vec![
+ queue_names_key(&self.key_prefix),
+ keys.pending,
+ keys.processing,
+ ],
+ vec![bytes(queue_name)],
+ )
+ .await?;
+ if matches!(value, ScriptValue::Null) {
+ return Err(Error::NotFound(format!(
+ "queue '{}' not found",
+ queue_name
+ )));
+ }
+ let ScriptValue::Array(values) = value else {
+ return Err(invalid_result("queue status", value));
+ };
+ if values.len() != 2 {
+ return Err(Error::internal(
+ "redis queue status returned an invalid value",
+ ));
+ }
+ Ok(QueueState {
+ pending: usize::try_from(integer(values[0].clone())?)
+ .map_err(|_| Error::internal("redis pending count is invalid"))?,
+ processing: usize::try_from(integer(values[1].clone())?)
+ .map_err(|_| Error::internal("redis processing count is invalid"))?,
+ })
+ }
+
pub(super) async fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {
self.require_queue(queue_name).await?;
let keys = QueueKeys::new(&self.key_prefix, queue_name);
diff --git a/crates/ragfs/src/plugins/queuefs/cache_protocol.rs b/crates/ragfs/src/plugins/queuefs/cache_protocol.rs
index 28963f943..e2fd2d427 100644
--- a/crates/ragfs/src/plugins/queuefs/cache_protocol.rs
+++ b/crates/ragfs/src/plugins/queuefs/cache_protocol.rs
@@ -78,6 +78,13 @@ end
return payload
"#;
+pub(super) const STATUS_SCRIPT: &str = r#"
+if redis.call('SISMEMBER', KEYS[1], ARGV[1]) == 0 then
+ return nil
+end
+return {redis.call('LLEN', KEYS[2]), redis.call('ZCARD', KEYS[3])}
+"#;
+
pub(super) const LIST_UNACKED_SCRIPT: &str = r#"
local result = {}
local pending = redis.call('LRANGE', KEYS[1], 0, -1)
diff --git a/crates/ragfs/src/plugins/queuefs/mod.rs b/crates/ragfs/src/plugins/queuefs/mod.rs
index b0659bd75..bce789b6a 100644
--- a/crates/ragfs/src/plugins/queuefs/mod.rs
+++ b/crates/ragfs/src/plugins/queuefs/mod.rs
@@ -6,6 +6,7 @@
//! - `/queue_name/dequeue` - Read from this file to remove and return the first message
//! - `/queue_name/peek` - Read from this file to view the first message without removing it
//! - `/queue_name/size` - Read from this file to get the current queue size
+//! - `/queue_name/status` - Read pending and processing message counts
//! - `/queue_name/messages` - Read all unacknowledged messages without changing queue state
//! - `/queue_name/clear` - Write to this file to clear all messages from the queue
//! - `/queue_name/ack` - Write message ID to this file to acknowledge and delete it
@@ -55,6 +56,10 @@ const CONTROL_FILES: &[ControlFileSpec] = &[
name: "size",
mode: 0o444,
},
+ ControlFileSpec {
+ name: "status",
+ mode: 0o444,
+ },
ControlFileSpec {
name: "messages",
mode: 0o444,
@@ -179,6 +184,14 @@ impl QueueStorage {
}
}
+ async fn status(&self, queue_name: &str) -> Result<backend::QueueState> {
+ match self {
+ Self::Local(backend) => backend.lock().await.status(queue_name),
+ #[cfg(feature = "cache")]
+ Self::Cache(storage) => storage.status(queue_name).await,
+ }
+ }
+
async fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {
match self {
Self::Local(backend) => backend.lock().await.list_unacked(queue_name),
@@ -383,6 +396,10 @@ impl FileSystem for QueueFileSystem {
let size = self.storage.size(&queue_name).await?;
Ok(size.to_string().into_bytes())
}
+ "status" => {
+ let status = self.storage.status(&queue_name).await?;
+ Ok(serde_json::to_vec(&status)?)
+ }
"messages" => {
let messages = self
.storage
@@ -397,7 +414,7 @@ impl FileSystem for QueueFileSystem {
Ok(serde_json::to_vec(&messages)?)
}
_ => Err(Error::InvalidOperation(format!(
- "Cannot read from '{}'. Use dequeue, peek, size, or messages",
+ "Cannot read from '{}'. Use dequeue, peek, size, status, or messages",
operation
))),
}
@@ -812,6 +829,7 @@ impl ServicePlugin for QueueFSPlugin {
- dequeue: Read to remove and return the first message\n\
- peek: Read to view the first message without removing it\n\
- size: Read to get the current queue size\n\
+ - status: Read pending and processing message counts\n\
- clear: Write to clear all messages from the queue\n\
- ack: Write message id to acknowledge and delete it\n\
\n\
@@ -1008,13 +1026,14 @@ mod tests {
.read_dir("/test", None, None, None, None)
.await
.unwrap();
- assert_eq!(entries.len(), 7);
+ assert_eq!(entries.len(), 8);
let names: Vec<String> = entries.iter().map(|e| e.name.clone()).collect();
assert!(names.contains(&"enqueue".to_string()));
assert!(names.contains(&"dequeue".to_string()));
assert!(names.contains(&"peek".to_string()));
assert!(names.contains(&"size".to_string()));
+ assert!(names.contains(&"status".to_string()));
assert!(names.contains(&"messages".to_string()));
assert!(names.contains(&"clear".to_string()));
assert!(names.contains(&"ack".to_string()));
@@ -1227,11 +1246,20 @@ mod tests {
let dequeued: TestQueueMessage =
serde_json::from_slice(&fs.read("/Semantic/dequeue", 0, 0).await.unwrap()).unwrap();
assert_eq!(dequeued.data, "payload");
+ assert_eq!(
+ fs.read("/Semantic/status", 0, 0).await.unwrap(),
+ br#"{"pending":0,"processing":1}"#
+ );
fs.write("/Semantic/ack", dequeued.id.as_bytes(), 0, WriteFlag::None)
.await
.unwrap();
+ assert_eq!(
+ fs.read("/Semantic/status", 0, 0).await.unwrap(),
+ br#"{"pending":0,"processing":0}"#
+ );
assert_eq!(fs.read("/Semantic/size", 0, 0).await.unwrap(), b"0");
fs.remove_all("/Semantic").await.unwrap();
+ assert!(fs.read("/Semantic/status", 0, 0).await.is_err());
drop(fs);
runtime.close().await.unwrap();
}
diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts
index 416e9a86b..c9c9905b0 100644
--- a/docs/.vitepress/config.ts
+++ b/docs/.vitepress/config.ts
@@ -345,7 +345,8 @@ const conceptsSidebar = {
['11-multi-tenant.md', 'Multi-Tenancy'],
['12-metrics.md', 'Metrics'],
['13-privacy.md', 'Privacy'],
- ['14-multi-write-storage.md', 'Multi-Write Storage']
+ ['14-multi-write-storage.md', 'Multi-Write Storage'],
+ ['16-queue-lifecycle.md', 'Queue State and Completion']
]
},
{
@@ -382,7 +383,8 @@ const conceptsSidebar = {
['11-multi-tenant.md', '多租户'],
['12-metrics.md', '监控指标'],
['13-privacy.md', '隐私配置'],
- ['14-multi-write-storage.md', '多写存储']
+ ['14-multi-write-storage.md', '多写存储'],
+ ['16-queue-lifecycle.md', '队列状态与完成语义']
]
},
{
diff --git a/docs/en/concepts/16-queue-lifecycle.md b/docs/en/concepts/16-queue-lifecycle.md
new file mode 100644
index 000000000..41d5594eb
--- /dev/null
+++ b/docs/en/concepts/16-queue-lifecycle.md
@@ -0,0 +1,104 @@
+# Queue State and Completion Semantics
+
+## Problem
+
+Queue completion is currently inferred from two independent sources:
+
+- QueueFS `/size`, which reports messages still pending for dequeue.
+- Python `NamedQueue._in_progress`, which reports work observed by the current
+ process.
+
+These values cannot be read or updated atomically. After QueueFS moves a message
+from `pending` to `processing`, another thread, event loop, or process can
+observe `size == 0` while the `_in_progress` value it reads is zero. It may
+therefore report the queue as complete before the message is acknowledged.
+
+Moving the local counter update before or after task creation only narrows some
+timing windows. It cannot make the backend state transition and local counter
+update one atomic operation, so it does not strictly solve the problem across
+threads, event loops, or processes.
+
+## Previous Backend Behavior
+
+SQLite and cache-backed queues implement an acknowledgement lifecycle:
+
+```text
+enqueue -> pending -> dequeue -> processing -> ack -> removed
+```
+
+Before this change, the memory backend did not implement the same lifecycle.
+Its `dequeue` operation removed the message from its only queue immediately.
+Although the backend exposed an `ack` method, the dequeued message was no
+longer stored, so a later ACK normally had nothing to remove. The memory
+backend therefore had no effective `processing` state or ACK lifecycle.
+
+## State Model
+
+Queue length is not one scalar. QueueFS must maintain these current-state
+gauges:
+
+| Field | Meaning |
+| --- | --- |
+| `pending` | Messages available for dequeue |
+| `processing` | Messages dequeued but not yet acknowledged |
+| `unacked` | `pending + processing` |
+
+A queue is complete only when:
+
+```text
+pending == 0 && processing == 0
+```
+
+Equivalently, `unacked == 0`.
+
+An unacknowledged message remains incomplete when its handler fails, its ACK
+fails, or its worker exits. It becomes pending again through recovery or leaves
+the queue through a successful ACK.
+
+## Ownership
+
+QueueFS is the owner of queue lifecycle state. Each backend must update
+`pending` and `processing` together with enqueue, dequeue, ACK, clear, and
+recovery operations.
+
+Python must not combine backend `pending` with a process-local counter to decide
+completion. A local worker counter may still be exposed as runtime telemetry,
+but it is not queue length and is not authoritative.
+
+The following cumulative counters are also not queue length:
+
+- `processed`
+- `requeue_count`
+- `error_count`
+
+They describe processing outcomes rather than current queue occupancy. They can
+remain in the processing or metrics layer for this fix. Moving them into
+QueueFS would require an explicit backend outcome protocol because the backend
+cannot infer a handler result from dequeue or ACK alone.
+
+## Backend Contract
+
+QueueFS should expose one atomic status operation:
+
+```json
+{
+ "pending": 3,
+ "processing": 2
+}
+```
+
+`unacked` is derived as `pending + processing`. The existing `/size` operation
+can continue to mean `pending` for compatibility and worker scheduling.
+QueueFS control names are reserved path components: a queue name cannot end in
+`enqueue`, `dequeue`, `peek`, `size`, `status`, `messages`, `clear`, or `ack`.
+
+Backend requirements:
+
+- **SQLite:** read both counts from one database snapshot.
+- **Cache:** return `LLEN(pending)` and `ZCARD(processing)` from one Lua script.
+- **Memory:** add a processing collection; dequeue moves a message into it and
+ ACK removes the message from it.
+
+`NamedQueue.get_status()` consumes this backend snapshot. `wait_complete()` and
+`is_all_complete()` use only the backend-owned `pending` and `processing`
+values.
diff --git a/docs/zh/concepts/16-queue-lifecycle.md b/docs/zh/concepts/16-queue-lifecycle.md
new file mode 100644
index 000000000..b87ea77ab
--- /dev/null
+++ b/docs/zh/concepts/16-queue-lifecycle.md
@@ -0,0 +1,95 @@
+# 队列状态与完成语义
+
+## 问题
+
+当前队列完成状态由两个相互独立的数据源推断:
+
+- QueueFS `/size`:返回仍可被 dequeue 的 pending 消息数。
+- Python `NamedQueue._in_progress`:返回当前进程观察到的执行中任务数。
+
+这两个值无法原子读取或更新。QueueFS 将消息从 `pending` 移到
+`processing` 后,另一个线程、event loop 或进程可能同时观察到 `size == 0`,
+且其读取到的 `_in_progress == 0`,从而在消息 ACK 前错误地判断队列已经完成。
+
+将本地计数放到 task 创建之前或之后,只能缩小部分时序窗口。它无法让 backend
+状态迁移与本地计数更新成为同一个原子操作,因此无法严格解决跨线程、跨 event loop
+或跨进程的并发判断问题。
+
+## 改动前的后端行为
+
+SQLite 和缓存后端实现了 ACK 生命周期:
+
+```text
+enqueue -> pending -> dequeue -> processing -> ack -> removed
+```
+
+改动前,MemoryBackend 没有实现相同的生命周期。它的 `dequeue` 会直接从唯一的
+队列中删除消息。虽然接口上存在 `ack` 方法,但被 dequeue 的消息已经不再保存,
+后续 ACK 通常找不到可删除的消息。因此,MemoryBackend 实际上没有有效的
+`processing` 状态和 ACK 生命周期。
+
+## 状态模型
+
+队列长度不是单一数字。QueueFS 必须维护以下当前状态指标:
+
+| 字段 | 含义 |
+| --- | --- |
+| `pending` | 尚未 dequeue、可被 worker 获取的消息数 |
+| `processing` | 已 dequeue、尚未 ACK 的消息数 |
+| `unacked` | `pending + processing` |
+
+队列完成条件必须是:
+
+```text
+pending == 0 && processing == 0
+```
+
+等价于 `unacked == 0`。
+
+当 handler 失败、ACK 失败或 worker 退出时,只要消息仍未 ACK,就不能视为完成。
+消息只能通过恢复流程重新进入 pending,或通过成功 ACK 离开队列。
+
+## 状态归属
+
+QueueFS 是队列生命周期状态的 Owner。各 backend 必须在 enqueue、dequeue、ACK、
+clear 和 recovery 操作中维护 `pending` 与 `processing`。
+
+Python 不应再组合 backend 的 `pending` 和进程内计数来判断完成。本地 worker
+计数仍可作为运行时观测指标,但它不属于队列长度,也不是完成状态的权威来源。
+
+以下累计计数同样不属于队列长度:
+
+- `processed`
+- `requeue_count`
+- `error_count`
+
+它们描述的是处理结果,而不是当前队列占用。本次修复中可以继续由处理层或指标层
+维护。如果需要将它们下沉到 QueueFS,必须先定义显式的处理结果协议,因为 backend
+无法仅根据 dequeue 或 ACK 推断 handler 的处理结果。
+
+## Backend 契约
+
+QueueFS 应提供一个原子状态操作:
+
+```json
+{
+ "pending": 3,
+ "processing": 2
+}
+```
+
+`unacked` 由 `pending + processing` 派生。为保持兼容和 worker 调度语义,现有
+`/size` 可以继续表示 `pending`。
+QueueFS 控制文件名属于保留路径段,队列名不能以 `enqueue`、`dequeue`、`peek`、
+`size`、`status`、`messages`、`clear` 或 `ack` 结尾。
+
+各后端要求:
+
+- **SQLite:** 在同一个数据库快照中读取两个计数。
+- **Cache:** 通过一个 Lua 脚本返回 `LLEN(pending)` 和
+ `ZCARD(processing)`。
+- **Memory:** 增加 processing 集合;dequeue 将消息移入该集合,ACK 从该集合
+ 删除消息。
+
+`NamedQueue.get_status()` 只消费 backend 返回的状态快照。`wait_complete()` 和
+`is_all_complete()` 只使用 backend 维护的 `pending` 与 `processing` 判断完成。
diff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.py
index b7da44d2d..7090ddaa9 100644
--- a/openviking/storage/queuefs/named_queue.py
+++ b/openviking/storage/queuefs/named_queue.py
@@ -135,7 +135,6 @@ class NamedQueue:
# Status tracking
self._lock = threading.Lock()
- self._in_progress = 0
self._processed = 0
self._requeue_count = 0
self._error_count = 0
@@ -154,15 +153,9 @@ class NamedQueue:
on_error=self._on_process_error,
)
- def _on_dequeue_start(self) -> None:
- """Called on dequeue."""
- with self._lock:
- self._in_progress += 1
-
def _on_process_success(self) -> None:
"""Called on processing success."""
with self._lock:
- self._in_progress -= 1
self._processed += 1
def _on_process_requeue(self) -> None:
@@ -177,7 +170,6 @@ class NamedQueue:
if metadata is not None:
self._task_work_index.record_failure(metadata.task_id, error_msg)
with self._lock:
- self._in_progress -= 1
self._error_count += 1
self._errors.append(
QueueError(
@@ -191,11 +183,11 @@ class NamedQueue:
async def get_status(self) -> QueueStatus:
"""Get queue status."""
- pending = await self.size()
+ backend_status = await self._read_backend_status()
with self._lock:
return QueueStatus(
- pending=pending,
- in_progress=self._in_progress,
+ pending=backend_status["pending"],
+ in_progress=backend_status["processing"],
processed=self._processed,
requeue_count=self._requeue_count,
error_count=self._error_count,
@@ -205,7 +197,6 @@ class NamedQueue:
def reset_status(self) -> None:
"""Reset status counters."""
with self._lock:
- self._in_progress = 0
self._processed = 0
self._requeue_count = 0
self._error_count = 0
@@ -325,7 +316,6 @@ class NamedQueue:
msg_id = data.get("id", "") if isinstance(data, dict) else ""
raw_data = data
if self._dequeue_handler:
- self._on_dequeue_start()
data = await self.process_dequeued(data)
# Ack unconditionally after handler returns (success or handled error).
# If on_dequeue raises, the exception propagates and ack is skipped —
@@ -346,11 +336,7 @@ class NamedQueue:
return None
async def process_dequeued(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
- """Invoke the dequeue handler on already-fetched raw data.
-
- NOTE: caller must call _on_dequeue_start() before invoking this method
- so that in_progress is incremented atomically with the dequeue.
- """
+ """Invoke the dequeue handler on already-fetched raw data."""
if self._dequeue_handler is None:
return data
@@ -414,6 +400,19 @@ class NamedQueue:
except (AGFSNotFoundError, FileNotFoundError):
return 0
+ async def _read_backend_status(self) -> Dict[str, int]:
+ await self._ensure_initialized()
+ content = await self._async_agfs.read(f"{self.path}/status")
+ if isinstance(content, bytes):
+ content = content.decode("utf-8")
+ elif hasattr(content, "content") and content.content is not None:
+ content = content.content.decode("utf-8")
+ status = json.loads(content)
+ return {
+ "pending": int(status["pending"]),
+ "processing": int(status["processing"]),
+ }
+
async def snapshot(self) -> List[Dict[str, Any]]:
"""Return all unacknowledged messages without changing queue state."""
await self._ensure_initialized()
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index 670e1a4b8..91cbb6a18 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -276,7 +276,6 @@ class QueueManager:
# Ack after successful processing (delete from persistent storage).
await queue.ack(msg_id, data)
except Exception as e:
- # Handler did not call report_error; decrement in_progress manually.
# Do NOT ack — let RecoverStale re-queue on next startup.
queue._on_process_error(str(e), data)
logger.error(f"[QueueManager] Concurrent worker error for {queue.name}: {e}")
@@ -287,18 +286,11 @@ class QueueManager:
# While capacity remains, keep draining the queue
while len(active_tasks) < max_concurrent:
- try:
- queue_size = await queue.size()
- except Exception:
- break
- if not queue.has_dequeue_handler() or queue_size == 0:
+ if not queue.has_dequeue_handler():
break
data = await queue.dequeue_raw()
if data is None:
break
- # Increment before task creation to close the race window where
- # size=0 and in_progress=0 between dequeue_raw() and task execution.
- queue._on_dequeue_start()
task = asyncio.create_task(process_one(data))
active_tasks.add(task)
logger.debug(
diff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.py
index ec12526a8..1511ce447 100644
--- a/tests/storage/test_queue_manager.py
+++ b/tests/storage/test_queue_manager.py
@@ -5,7 +5,9 @@
import os
import subprocess
import sys
+from unittest.mock import AsyncMock
+from openviking.storage.queuefs.named_queue import NamedQueue
from openviking.storage.queuefs.queue_manager import QueueManager
@@ -34,3 +36,24 @@ def test_queue_concurrency_uses_separate_configured_values() -> None:
assert manager._max_concurrent_for_queue(manager.EXTERNAL_PARSE) == 9
assert manager._max_concurrent_for_queue(manager.ADD_RESOURCE) == 7
assert manager._max_concurrent_for_queue(manager.SESSION_COMMIT) == 5
+
+
+async def test_status_waits_for_processing_messages_from_other_workers(monkeypatch) -> None:
+ client = AsyncMock()
+
+ async def read(path: str):
+ if path.endswith("/status"):
+ return b'{"pending":0,"processing":1}'
+ raise AssertionError(f"unexpected read: {path}")
+
+ client.read.side_effect = read
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.named_queue.AsyncAGFSClient",
+ lambda _: client,
+ )
+
+ status = await NamedQueue(object(), "/queue", "Test").get_status()
+
+ assert status.pending == 0
+ assert status.in_progress == 1
+ assert not status.is_complete