Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(queuefs): add bounded redis startup stale-recovery sweeps (#3748)
baojun-zhang758fc7f0faff
diff --git a/crates/ragfs/src/plugins/queuefs/redis_backend.rs b/crates/ragfs/src/plugins/queuefs/redis_backend.rsindex 69c0f10a2..685224101 100644--- a/crates/ragfs/src/plugins/queuefs/redis_backend.rs+++ b/crates/ragfs/src/plugins/queuefs/redis_backend.rs@@ -12,11 +12,12 @@ use std::sync::atomic::{AtomicU64, Ordering};use std::sync::mpsc::{self, RecvTimeoutError, Sender};use std::sync::{Arc, Mutex};use std::thread::JoinHandle;-use std::time::{Duration, SystemTime, UNIX_EPOCH};+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};use uuid::Uuid;const HEARTBEAT_TTL_SECS: u64 = 30;const HEARTBEAT_INTERVAL_SECS: u64 = 10;+const STARTUP_RECOVERY_SWEEPS: usize = 3;const REDIS_POOL_MAX_SIZE: u32 = 2;const REDIS_POOL_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(5);const SENTINEL_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(30);@@ -520,6 +521,8 @@ pub(super) struct RedisQueueBackend {instance_id: String,heartbeat_stop: Option<Sender<()>>,heartbeat_thread: Option<JoinHandle<()>>,+ startup_recovery_stop: Option<Sender<()>>,+ startup_recovery_thread: Option<JoinHandle<()>>,}impl RedisQueueBackend {@@ -536,9 +539,11 @@ impl RedisQueueBackend {instance_id,heartbeat_stop: None,heartbeat_thread: None,+ startup_recovery_stop: None,+ startup_recovery_thread: None,};backend.start_heartbeat(heartbeat_key);- backend.recover_stale()?;+ backend.start_startup_recovery();Ok(backend)}@@ -552,6 +557,8 @@ impl RedisQueueBackend {}/// Start the heartbeat thread using a separate pool checkout for every renewal.+ /// `key` is this instance's heartbeat key in Redis.+ /// Returns no value; the spawned thread is stored on `self`.fn start_heartbeat(&mut self, key: String) {let (sender, receiver) = mpsc::channel();let pool = self.pool.clone();@@ -569,19 +576,69 @@ impl RedisQueueBackend {}));}+ /// Start a bounded startup recovery thread.+ /// The thread runs at offsets 0, 30, and 60 seconds from startup unless stopped early.+ /// Returns no value; the spawned thread is stored on `self`.+ fn start_startup_recovery(&mut self) {+ let (sender, receiver) = mpsc::channel();+ let pool = self.pool.clone();+ let key_prefix = self.key_prefix.clone();+ self.startup_recovery_stop = Some(sender);+ self.startup_recovery_thread = Some(std::thread::spawn(move || {+ // ponytail: keep startup recovery bounded to the restart TTL window;+ // upgrade to steady-state scanning only if deployments need long-lived handoff.+ let started_at = Instant::now();+ for sweep_index in 0..STARTUP_RECOVERY_SWEEPS {+ let deadline = started_at + Self::startup_recovery_delay_before_sweep(sweep_index);+ loop {+ let now = Instant::now();+ if now >= deadline {+ break;+ }+ match receiver.recv_timeout(deadline.saturating_duration_since(now)) {+ Ok(()) | Err(RecvTimeoutError::Disconnected) => return,+ Err(RecvTimeoutError::Timeout) => {}+ }+ }+ Self::run_startup_recovery_sweep(&pool, &key_prefix);+ }+ }));+ }++ /// Return the delay before one bounded startup recovery sweep.+ /// `sweep_index` is zero-based and maps to the documented 0/30/60-second schedule.+ /// Returns the sweep offset from startup.+ fn startup_recovery_delay_before_sweep(sweep_index: usize) -> Duration {+ Duration::from_secs(HEARTBEAT_TTL_SECS * sweep_index as u64)+ }++ /// Run one startup recovery sweep.+ /// `pool` provides Redis connections and `key_prefix` selects the queue namespace.+ /// Returns no value; recovery results are emitted through logs.+ fn run_startup_recovery_sweep(pool: &RedisPool, key_prefix: &str) {+ match Self::recover_stale(pool, key_prefix) {+ Ok(recovered) => {+ if recovered > 0 {+ tracing::info!("queuefs redis recovered {recovered} stale message(s)");+ }+ }+ Err(error) => tracing::warn!("queuefs redis startup recover_stale failed: {error}"),+ }+ }+/// Recover processing messages whose owning instance heartbeat has expired.- fn recover_stale(&self) -> Result<usize> {- let queue_names_key = queue_names_key(&self.key_prefix);- let instance_key_prefix = instance_key_prefix(&self.key_prefix);- let queues = self.with_connection("recover_stale list queues", |connection| {+ fn recover_stale(pool: &RedisPool, key_prefix: &str) -> Result<usize> {+ let queue_names_key = queue_names_key(key_prefix);+ let instance_key_prefix = instance_key_prefix(key_prefix);+ let queues = pool.execute("recover_stale list queues", |connection| {redis::cmd("SMEMBERS").arg(&queue_names_key).query::<Vec<String>>(connection)})?;let mut recovered = 0;for queue in queues {- let keys = QueueKeys::new(&self.key_prefix, &queue);- recovered += self.with_connection("recover_stale", |connection| {+ let keys = QueueKeys::new(key_prefix, &queue);+ recovered += pool.execute("recover_stale", |connection| {redis::Script::new(RECOVER_STALE_SCRIPT).key(&keys.processing).key(&keys.pending)@@ -629,9 +686,15 @@ impl Drop for RedisQueueBackend {if let Some(sender) = self.heartbeat_stop.take() {let _ = sender.send(());}+ if let Some(sender) = self.startup_recovery_stop.take() {+ let _ = sender.send(());+ }if let Some(thread) = self.heartbeat_thread.take() {let _ = thread.join();}+ if let Some(thread) = self.startup_recovery_thread.take() {+ let _ = thread.join();+ }let _ = self.pool.try_execute(|connection| {redis::cmd("DEL").arg(heartbeat_key(&self.key_prefix, &self.instance_id))diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex ac83acf12..28b7cc282 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -1129,6 +1129,7 @@ Notes:- QueueFS defaults to `sqlite` even if the main AGFS storage backend is `local`, `s3`, or `memory`.- `mode=shared` keeps the historical global queue namespace at `/queue`; `mode=worker` isolates each worker under `/queue/worker-<index|pid>`.- `db_path` is only used when QueueFS backend is `sqlite` or `sqlite3`.+- Redis backend runs three bounded `recover_stale` sweeps in a dedicated startup recovery thread at startup, 30 seconds, and 60 seconds to cover the heartbeat-expiry window after a container restart; it does not run long-lived periodic recovery.- If both `storage.agfs.queuefs.db_path` and legacy `storage.agfs.queue_db_path` are set, `storage.agfs.queuefs.db_path` wins.- If QueueFS backend is `memory`, any `db_path` or legacy `queue_db_path` is ignored.diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex e1fe333f3..b32f83c27 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -1125,7 +1125,7 @@ QueueFS Redis 参数:- `username` 和 `password` 用于 Redis 数据节点;`sentinel_username` 和 `sentinel_password` 仅用于 Sentinel 节点。- Redis backend 使用 `{key_prefix}:ov:*` key;连接同一 Redis database 的不同业务必须配置不同的 `key_prefix`。- Redis backend 的实例心跳 TTL 为 30 秒,每 10 秒续约一次。-- Redis backend 仅在启动时按实例心跳状态执行一次 `recover_stale`,运行期间不周期恢复。+- Redis backend 会在独立的 startup recovery 线程中按实例心跳状态执行三次有界 `recover_stale` 扫描,时间点分别为启动后立即、30 秒和 60 秒,用于覆盖容器异常退出后旧实例心跳尚未过期的恢复窗口;运行期间不做长期周期恢复。- `tls_insecure_skip_verify=true` 时必须同时设置 `tls_enabled=true`。- 如果同时设置了 `storage.agfs.queuefs.db_path` 和旧字段 `storage.agfs.queue_db_path`,以前者为准。- 如果 QueueFS backend 为 `memory`,则 `db_path` 和旧字段 `queue_db_path` 都会被忽略。