Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(queuefs): add bounded redis startup stale-recovery sweeps (#3748)

baojun-zhang758fc7f0faff
diff --git a/crates/ragfs/src/plugins/queuefs/redis_backend.rs b/crates/ragfs/src/plugins/queuefs/redis_backend.rs
index 69c0f10a2..685224101 100644
--- a/crates/ragfs/src/plugins/queuefs/redis_backend.rs
+++ b/crates/ragfs/src/plugins/queuefs/redis_backend.rs
@@ -12,11 +12,12 @@ use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::mpsc::{self, RecvTimeoutError, Sender};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use uuid::Uuid;
const HEARTBEAT_TTL_SECS: u64 = 30;
const HEARTBEAT_INTERVAL_SECS: u64 = 10;
+const STARTUP_RECOVERY_SWEEPS: usize = 3;
const REDIS_POOL_MAX_SIZE: u32 = 2;
const REDIS_POOL_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(5);
const SENTINEL_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(30);
@@ -520,6 +521,8 @@ pub(super) struct RedisQueueBackend {
instance_id: String,
heartbeat_stop: Option<Sender<()>>,
heartbeat_thread: Option<JoinHandle<()>>,
+ startup_recovery_stop: Option<Sender<()>>,
+ startup_recovery_thread: Option<JoinHandle<()>>,
}
impl RedisQueueBackend {
@@ -536,9 +539,11 @@ impl RedisQueueBackend {
instance_id,
heartbeat_stop: None,
heartbeat_thread: None,
+ startup_recovery_stop: None,
+ startup_recovery_thread: None,
};
backend.start_heartbeat(heartbeat_key);
- backend.recover_stale()?;
+ backend.start_startup_recovery();
Ok(backend)
}
@@ -552,6 +557,8 @@ impl RedisQueueBackend {
}
/// Start the heartbeat thread using a separate pool checkout for every renewal.
+ /// `key` is this instance's heartbeat key in Redis.
+ /// Returns no value; the spawned thread is stored on `self`.
fn start_heartbeat(&mut self, key: String) {
let (sender, receiver) = mpsc::channel();
let pool = self.pool.clone();
@@ -569,19 +576,69 @@ impl RedisQueueBackend {
}));
}
+ /// Start a bounded startup recovery thread.
+ /// The thread runs at offsets 0, 30, and 60 seconds from startup unless stopped early.
+ /// Returns no value; the spawned thread is stored on `self`.
+ fn start_startup_recovery(&mut self) {
+ let (sender, receiver) = mpsc::channel();
+ let pool = self.pool.clone();
+ let key_prefix = self.key_prefix.clone();
+ self.startup_recovery_stop = Some(sender);
+ self.startup_recovery_thread = Some(std::thread::spawn(move || {
+ // ponytail: keep startup recovery bounded to the restart TTL window;
+ // upgrade to steady-state scanning only if deployments need long-lived handoff.
+ let started_at = Instant::now();
+ for sweep_index in 0..STARTUP_RECOVERY_SWEEPS {
+ let deadline = started_at + Self::startup_recovery_delay_before_sweep(sweep_index);
+ loop {
+ let now = Instant::now();
+ if now >= deadline {
+ break;
+ }
+ match receiver.recv_timeout(deadline.saturating_duration_since(now)) {
+ Ok(()) | Err(RecvTimeoutError::Disconnected) => return,
+ Err(RecvTimeoutError::Timeout) => {}
+ }
+ }
+ Self::run_startup_recovery_sweep(&pool, &key_prefix);
+ }
+ }));
+ }
+
+ /// Return the delay before one bounded startup recovery sweep.
+ /// `sweep_index` is zero-based and maps to the documented 0/30/60-second schedule.
+ /// Returns the sweep offset from startup.
+ fn startup_recovery_delay_before_sweep(sweep_index: usize) -> Duration {
+ Duration::from_secs(HEARTBEAT_TTL_SECS * sweep_index as u64)
+ }
+
+ /// Run one startup recovery sweep.
+ /// `pool` provides Redis connections and `key_prefix` selects the queue namespace.
+ /// Returns no value; recovery results are emitted through logs.
+ fn run_startup_recovery_sweep(pool: &RedisPool, key_prefix: &str) {
+ match Self::recover_stale(pool, key_prefix) {
+ Ok(recovered) => {
+ if recovered > 0 {
+ tracing::info!("queuefs redis recovered {recovered} stale message(s)");
+ }
+ }
+ Err(error) => tracing::warn!("queuefs redis startup recover_stale failed: {error}"),
+ }
+ }
+
/// Recover processing messages whose owning instance heartbeat has expired.
- fn recover_stale(&self) -> Result<usize> {
- let queue_names_key = queue_names_key(&self.key_prefix);
- let instance_key_prefix = instance_key_prefix(&self.key_prefix);
- let queues = self.with_connection("recover_stale list queues", |connection| {
+ fn recover_stale(pool: &RedisPool, key_prefix: &str) -> Result<usize> {
+ let queue_names_key = queue_names_key(key_prefix);
+ let instance_key_prefix = instance_key_prefix(key_prefix);
+ let queues = pool.execute("recover_stale list queues", |connection| {
redis::cmd("SMEMBERS")
.arg(&queue_names_key)
.query::<Vec<String>>(connection)
})?;
let mut recovered = 0;
for queue in queues {
- let keys = QueueKeys::new(&self.key_prefix, &queue);
- recovered += self.with_connection("recover_stale", |connection| {
+ let keys = QueueKeys::new(key_prefix, &queue);
+ recovered += pool.execute("recover_stale", |connection| {
redis::Script::new(RECOVER_STALE_SCRIPT)
.key(&keys.processing)
.key(&keys.pending)
@@ -629,9 +686,15 @@ impl Drop for RedisQueueBackend {
if let Some(sender) = self.heartbeat_stop.take() {
let _ = sender.send(());
}
+ if let Some(sender) = self.startup_recovery_stop.take() {
+ let _ = sender.send(());
+ }
if let Some(thread) = self.heartbeat_thread.take() {
let _ = thread.join();
}
+ if let Some(thread) = self.startup_recovery_thread.take() {
+ let _ = thread.join();
+ }
let _ = self.pool.try_execute(|connection| {
redis::cmd("DEL")
.arg(heartbeat_key(&self.key_prefix, &self.instance_id))
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md
index ac83acf12..28b7cc282 100644
--- a/docs/en/guides/01-configuration.md
+++ b/docs/en/guides/01-configuration.md
@@ -1129,6 +1129,7 @@ Notes:
- QueueFS defaults to `sqlite` even if the main AGFS storage backend is `local`, `s3`, or `memory`.
- `mode=shared` keeps the historical global queue namespace at `/queue`; `mode=worker` isolates each worker under `/queue/worker-<index|pid>`.
- `db_path` is only used when QueueFS backend is `sqlite` or `sqlite3`.
+- Redis backend runs three bounded `recover_stale` sweeps in a dedicated startup recovery thread at startup, 30 seconds, and 60 seconds to cover the heartbeat-expiry window after a container restart; it does not run long-lived periodic recovery.
- If both `storage.agfs.queuefs.db_path` and legacy `storage.agfs.queue_db_path` are set, `storage.agfs.queuefs.db_path` wins.
- If QueueFS backend is `memory`, any `db_path` or legacy `queue_db_path` is ignored.
diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md
index e1fe333f3..b32f83c27 100644
--- a/docs/zh/guides/01-configuration.md
+++ b/docs/zh/guides/01-configuration.md
@@ -1125,7 +1125,7 @@ QueueFS Redis 参数:
- `username` 和 `password` 用于 Redis 数据节点;`sentinel_username` 和 `sentinel_password` 仅用于 Sentinel 节点。
- Redis backend 使用 `{key_prefix}:ov:*` key;连接同一 Redis database 的不同业务必须配置不同的 `key_prefix`。
- Redis backend 的实例心跳 TTL 为 30 秒,每 10 秒续约一次。
-- Redis backend 仅在启动时按实例心跳状态执行一次 `recover_stale`,运行期间不周期恢复。
+- Redis backend 会在独立的 startup recovery 线程中按实例心跳状态执行三次有界 `recover_stale` 扫描,时间点分别为启动后立即、30 秒和 60 秒,用于覆盖容器异常退出后旧实例心跳尚未过期的恢复窗口;运行期间不做长期周期恢复。
- `tls_insecure_skip_verify=true` 时必须同时设置 `tls_enabled=true`。
- 如果同时设置了 `storage.agfs.queuefs.db_path` 和旧字段 `storage.agfs.queue_db_path`,以前者为准。
- 如果 QueueFS backend 为 `memory`,则 `db_path` 和旧字段 `queue_db_path` 都会被忽略。