Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Fix/cross process recovery (#6281)

runyunzhouc476a3184b9d
diff --git a/crates/ragfs/src/lock/cache_provder.rs b/crates/ragfs/src/lock/cache_provder.rs
index f81df9607..9f5f27b97 100644
--- a/crates/ragfs/src/lock/cache_provder.rs
+++ b/crates/ragfs/src/lock/cache_provder.rs
@@ -1578,6 +1578,16 @@ return redis.call("HGET", KEYS[1], ARGV[2])
assert_eq!(adopted.lease.lock_paths, vec!["/local/account-a/a"]);
second.release(&adopted).await.unwrap();
+ tokio::time::timeout(Duration::from_secs(2), async {
+ loop {
+ if first.observe().await.active_locks == 0 {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .expect("producer must forget a released cross-manager handoff");
let legacy = PathLockHandoffRef {
lease_ref: None,
owner_id: "owner-a".to_string(),
diff --git a/crates/ragfs/src/lock/manager.rs b/crates/ragfs/src/lock/manager.rs
index e5a64e442..e9fe10b96 100644
--- a/crates/ragfs/src/lock/manager.rs
+++ b/crates/ragfs/src/lock/manager.rs
@@ -521,28 +521,62 @@ impl PathLockManager {
continue;
};
let lock_paths = entry.lease.lock_paths.clone();
- let mut all_ok = true;
+ let pending_handoff = entry.pending_handoff;
+ let mut any_ok = false;
+ let mut released_handoff_paths = Vec::new();
for lp in &lock_paths {
match refresh_provider
.refresh_token(lp, &owner.owner_id, now_ns)
.await
{
Ok(true) => {
+ any_ok = true;
debug!(lease_ref = %lease_ref, owner_id = %owner.owner_id, lock_path = %lp, "pathlock lease automatically refreshed");
}
Ok(false) => {
- all_ok = false;
- warn!(lease_ref = %lease_ref, owner_id = %owner.owner_id, lock_path = %lp, "pathlock lease automatic refresh failed");
+ if pending_handoff {
+ // A different process may have adopted and released this
+ // handoff. Its completed release makes the producer's
+ // local reference obsolete. Do not remove the provider
+ // token: another process may own it or have released it.
+ released_handoff_paths.push(lp.clone());
+ } else {
+ warn!(lease_ref = %lease_ref, owner_id = %owner.owner_id, lock_path = %lp, "pathlock lease automatic refresh failed");
+ }
}
Err(error) => {
- all_ok = false;
warn!(lease_ref = %lease_ref, owner_id = %owner.owner_id, lock_path = %lp, error = %error, "pathlock lease automatic refresh failed");
}
}
}
- if all_ok {
+ if any_ok {
owner_registry.touch(&lease_ref);
}
+ for lock_path in released_handoff_paths {
+ let is_last_path =
+ owner_registry.entries.get(&lease_ref).is_some_and(|entry| {
+ entry.lease.lock_paths.iter().all(|path| path == &lock_path)
+ });
+ if is_last_path {
+ if let Err(error) = refresh_registry.unpublish(&lease_ref, &owner) {
+ error!(
+ lease_ref = %lease_ref,
+ owner_id = %owner.owner_id,
+ error = %error,
+ "failed to forget released pathlock handoff"
+ );
+ continue;
+ }
+ }
+ owner_registry.remove_path(&lease_ref, &lock_path);
+ if is_last_path {
+ debug!(
+ lease_ref = %lease_ref,
+ owner_id = %owner.owner_id,
+ "forgot locally released pathlock handoff"
+ );
+ }
+ }
}
let stale_cutoff =
Instant::now() - Duration::from_secs_f64(refresh_config.lock_expire_secs * 2.0);
@@ -2712,6 +2746,99 @@ mod tests {
assert!(mgr.release(&adopted).await.is_ok());
}
+ #[tokio::test]
+ async fn pending_handoff_is_forgotten_after_cross_manager_release() {
+ let fs = Arc::new(MemFileSystem::new());
+ fs.mkdir("/data", 0o755).await.unwrap();
+ fs.mkdir("/data/sub", 0o755).await.unwrap();
+ let provider = Arc::new(crate::lock::provider::MemoryPathLockProvider::new());
+ let config = PathLockConfig {
+ lock_expire_secs: 0.3,
+ ..PathLockConfig::default()
+ };
+ let producer = PathLockManager::new(fs.clone(), provider.clone(), config.clone());
+ let consumer = PathLockManager::new(fs, provider.clone(), config);
+ let lease = producer
+ .acquire_tree("/data/sub", Duration::ZERO, None)
+ .await
+ .unwrap();
+ let lock_path = lease.lease.lock_paths[0].clone();
+ let handoff = producer.to_handoff(&lease);
+ producer.handoff(&lease).await.unwrap();
+
+ let adopted = consumer.adopt(&handoff).await.unwrap();
+ consumer.release(&adopted).await.unwrap();
+
+ // The producer sees the consumer's release on its next renewal and
+ // drops only its local pending-handoff registration. The old stale-GC
+ // path would retain this entry for two lock-expiry periods.
+ tokio::time::timeout(Duration::from_millis(400), async {
+ loop {
+ if producer.observe().await.active_locks == 0 {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .expect("producer must forget a released cross-manager handoff");
+ assert_eq!(provider.read_token(&lock_path).await.unwrap(), None);
+ }
+
+ #[tokio::test]
+ async fn pending_handoff_keeps_unreleased_paths_refreshing() {
+ let fs = Arc::new(MemFileSystem::new());
+ fs.mkdir("/data", 0o755).await.unwrap();
+ let provider = Arc::new(crate::lock::provider::MemoryPathLockProvider::new());
+ let config = PathLockConfig {
+ lock_expire_secs: 0.3,
+ ..PathLockConfig::default()
+ };
+ let producer = PathLockManager::new(fs.clone(), provider.clone(), config.clone());
+ let consumer = PathLockManager::new(fs, provider.clone(), config);
+ let lease = producer
+ .acquire_exact_batch(
+ &["/data/a".to_string(), "/data/b".to_string()],
+ Duration::ZERO,
+ None,
+ )
+ .await
+ .unwrap();
+ let first_lock_path = lease.lease.lock_paths[0].clone();
+ let second_lock_path = lease.lease.lock_paths[1].clone();
+ let handoff = producer.to_handoff(&lease);
+ producer.handoff(&lease).await.unwrap();
+
+ let adopted = consumer.adopt(&handoff).await.unwrap();
+ consumer
+ .release_selected(&adopted, &[first_lock_path.clone()])
+ .await
+ .unwrap();
+
+ // Wait beyond the producer stale-GC cutoff. The remaining path must
+ // retain the producer's local renewal state while the consumer holds it.
+ tokio::time::sleep(Duration::from_millis(700)).await;
+ assert_eq!(producer.observe().await.active_locks, 1);
+ assert_eq!(provider.read_token(&first_lock_path).await.unwrap(), None);
+ assert!(provider
+ .read_token(&second_lock_path)
+ .await
+ .unwrap()
+ .is_some());
+
+ consumer.release(&adopted).await.unwrap();
+ tokio::time::timeout(Duration::from_millis(400), async {
+ loop {
+ if producer.observe().await.active_locks == 0 {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .expect("producer must forget every released handoff path");
+ }
+
#[tokio::test]
async fn adopt_before_handoff_is_retryable() {
let mgr = make_manager().await;
diff --git a/crates/ragfs/src/plugins/queuefs/cache_backend.rs b/crates/ragfs/src/plugins/queuefs/cache_backend.rs
index 26d000526..45be5703c 100644
--- a/crates/ragfs/src/plugins/queuefs/cache_backend.rs
+++ b/crates/ragfs/src/plugins/queuefs/cache_backend.rs
@@ -1,10 +1,11 @@
use super::backend::{Message, QueueState, StoredMessage};
use super::cache_protocol::{
heartbeat_key, instance_key_prefix, last_enqueue_time_from_pending_payloads, queue_key_prefix,
- queue_names_key, unix_secs, QueueKeys, ACK_SCRIPT, CLEAR_SCRIPT, CREATE_QUEUE_SCRIPT,
- DEQUEUE_SCRIPT, ENQUEUE_SCRIPT, HEARTBEAT_INTERVAL_SECS, HEARTBEAT_TTL_SECS,
- LIST_UNACKED_SCRIPT, PEEK_SCRIPT, RECOVER_STALE_SCRIPT, REMOVE_QUEUE_SCRIPT,
- STARTUP_RECOVERY_SWEEPS, STATUS_SCRIPT,
+ queue_names_key, recovery_leader_key, unix_secs, QueueKeys, ACK_SCRIPT,
+ ACQUIRE_RECOVERY_LEADER_SCRIPT, CLEAR_SCRIPT, CREATE_QUEUE_SCRIPT, DEQUEUE_SCRIPT,
+ ENQUEUE_SCRIPT, HEARTBEAT_INTERVAL_SECS, HEARTBEAT_TTL_SECS, LIST_UNACKED_SCRIPT, PEEK_SCRIPT,
+ RECOVERY_INTERVAL_SECS, RECOVERY_LEADER_TTL_SECS, RECOVER_STALE_SCRIPT,
+ RELEASE_RECOVERY_LEADER_SCRIPT, REMOVE_QUEUE_SCRIPT, STATUS_SCRIPT,
};
use crate::cache_runtime::{
CacheError, CacheOperation, CacheRuntime, Expiration, ScriptDefinition, ScriptRequest,
@@ -17,7 +18,6 @@ use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio::sync::{watch, Mutex};
use tokio::task::JoinHandle;
-use tokio::time::Instant;
use uuid::Uuid;
const CREATE_QUEUE_ID: &str = "queuefs.create_queue.v1";
@@ -30,6 +30,8 @@ const LIST_UNACKED_ID: &str = "queuefs.list_unacked.v1";
const ACK_ID: &str = "queuefs.ack.v1";
const CLEAR_ID: &str = "queuefs.clear.v1";
const RECOVER_STALE_ID: &str = "queuefs.recover_stale.v1";
+const ACQUIRE_RECOVERY_LEADER_ID: &str = "queuefs.acquire_recovery_leader.v1";
+const RELEASE_RECOVERY_LEADER_ID: &str = "queuefs.release_recovery_leader.v1";
const SCRIPT_DEFINITIONS: &[ScriptDefinition] = &[
ScriptDefinition {
@@ -72,8 +74,35 @@ const SCRIPT_DEFINITIONS: &[ScriptDefinition] = &[
id: RECOVER_STALE_ID,
redis_lua: RECOVER_STALE_SCRIPT,
},
+ ScriptDefinition {
+ id: ACQUIRE_RECOVERY_LEADER_ID,
+ redis_lua: ACQUIRE_RECOVERY_LEADER_SCRIPT,
+ },
+ ScriptDefinition {
+ id: RELEASE_RECOVERY_LEADER_ID,
+ redis_lua: RELEASE_RECOVERY_LEADER_SCRIPT,
+ },
];
+#[derive(Clone, Copy)]
+struct BackgroundTiming {
+ heartbeat_interval: Duration,
+ heartbeat_ttl: Duration,
+ recovery_interval: Duration,
+ recovery_leader_ttl: Duration,
+}
+
+impl Default for BackgroundTiming {
+ fn default() -> Self {
+ Self {
+ heartbeat_interval: Duration::from_secs(HEARTBEAT_INTERVAL_SECS),
+ heartbeat_ttl: Duration::from_secs(HEARTBEAT_TTL_SECS),
+ recovery_interval: Duration::from_secs(RECOVERY_INTERVAL_SECS),
+ recovery_leader_ttl: Duration::from_secs(RECOVERY_LEADER_TTL_SECS),
+ }
+ }
+}
+
pub(super) struct CacheQueueStorage {
runtime: Arc<CacheRuntime>,
key_prefix: String,
@@ -87,6 +116,14 @@ pub(super) struct CacheQueueStorage {
impl CacheQueueStorage {
pub(super) async fn open(runtime: Arc<CacheRuntime>, key_prefix: String) -> Result<Self> {
+ Self::open_with_timing(runtime, key_prefix, BackgroundTiming::default()).await
+ }
+
+ async fn open_with_timing(
+ runtime: Arc<CacheRuntime>,
+ key_prefix: String,
+ timing: BackgroundTiming,
+ ) -> Result<Self> {
runtime
.require_operations(&[
CacheOperation::Set,
@@ -106,19 +143,22 @@ impl CacheQueueStorage {
}
let instance_id = Uuid::new_v4().to_string();
let heartbeat = heartbeat_key(&key_prefix, &instance_id);
- refresh_heartbeat(&runtime, &heartbeat).await?;
+ refresh_heartbeat(&runtime, &heartbeat, timing.heartbeat_ttl).await?;
let (heartbeat_stop, heartbeat_receiver) = watch::channel(false);
let heartbeat_task = tokio::spawn(run_heartbeat(
Arc::clone(&runtime),
heartbeat,
heartbeat_receiver,
+ timing,
));
let (recovery_stop, recovery_receiver) = watch::channel(false);
- let recovery_task = tokio::spawn(run_startup_recovery(
+ let recovery_task = tokio::spawn(run_continuous_recovery(
Arc::clone(&runtime),
key_prefix.clone(),
+ instance_id.clone(),
recovery_receiver,
+ timing,
));
Ok(Self {
@@ -354,10 +394,7 @@ impl CacheQueueStorage {
)
.await?;
if matches!(value, ScriptValue::Null) {
- return Err(Error::NotFound(format!(
- "queue '{}' not found",
- queue_name
- )));
+ return Err(Error::NotFound(format!("queue '{}' not found", queue_name)));
}
let ScriptValue::Array(values) = value else {
return Err(invalid_result("queue status", value));
@@ -499,13 +536,18 @@ impl Drop for CacheQueueStorage {
}
}
-async fn run_heartbeat(runtime: Arc<CacheRuntime>, key: String, mut stop: watch::Receiver<bool>) {
- let mut interval = tokio::time::interval(Duration::from_secs(HEARTBEAT_INTERVAL_SECS));
+async fn run_heartbeat(
+ runtime: Arc<CacheRuntime>,
+ key: String,
+ mut stop: watch::Receiver<bool>,
+ timing: BackgroundTiming,
+) {
+ let mut interval = tokio::time::interval(timing.heartbeat_interval);
interval.tick().await;
loop {
tokio::select! {
_ = interval.tick() => {
- if let Err(error) = refresh_heartbeat(&runtime, &key).await {
+ if let Err(error) = refresh_heartbeat(&runtime, &key, timing.heartbeat_ttl).await {
tracing::warn!("queuefs cache heartbeat failed: {error}");
}
}
@@ -518,13 +560,13 @@ async fn run_heartbeat(runtime: Arc<CacheRuntime>, key: String, mut stop: watch:
}
}
-async fn refresh_heartbeat(runtime: &CacheRuntime, key: &str) -> Result<()> {
+async fn refresh_heartbeat(runtime: &CacheRuntime, key: &str, ttl: Duration) -> Result<()> {
runtime
.set(
key,
Bytes::from_static(b"1"),
SetOptions {
- expiration: Some(Expiration::After(Duration::from_secs(HEARTBEAT_TTL_SECS))),
+ expiration: Some(Expiration::After(ttl)),
..SetOptions::default()
},
)
@@ -533,35 +575,56 @@ async fn refresh_heartbeat(runtime: &CacheRuntime, key: &str) -> Result<()> {
.map_err(|error| cache_error("heartbeat", error))
}
-async fn run_startup_recovery(
+async fn run_continuous_recovery(
runtime: Arc<CacheRuntime>,
key_prefix: String,
+ leader_token: String,
mut stop: watch::Receiver<bool>,
+ timing: BackgroundTiming,
) {
- let started_at = Instant::now();
- for sweep_index in 0..STARTUP_RECOVERY_SWEEPS {
- let deadline = started_at + startup_recovery_delay_before_sweep(sweep_index);
- if deadline > Instant::now() {
- tokio::select! {
- _ = tokio::time::sleep_until(deadline) => {}
- changed = stop.changed() => {
- if changed.is_err() || *stop.borrow() {
- return;
+ let mut interval = tokio::time::interval(timing.recovery_interval);
+ loop {
+ tokio::select! {
+ _ = interval.tick() => {
+ match acquire_or_renew_recovery_leader(
+ &runtime,
+ &key_prefix,
+ &leader_token,
+ timing.recovery_leader_ttl,
+ ).await {
+ Ok(true) => {
+ if let Err(error) = recover_stale(
+ &runtime,
+ &key_prefix,
+ &leader_token,
+ timing.recovery_leader_ttl,
+ ).await {
+ tracing::warn!("queuefs cache recover_stale failed: {error}");
+ }
}
+ Ok(false) => {}
+ Err(error) => tracing::warn!("queuefs cache recovery leader lease failed: {error}"),
+ }
+ }
+ changed = stop.changed() => {
+ if changed.is_err() || *stop.borrow() {
+ break;
}
}
- }
- if let Err(error) = recover_stale(&runtime, &key_prefix).await {
- tracing::warn!("queuefs cache startup recover_stale failed: {error}");
}
}
-}
-fn startup_recovery_delay_before_sweep(sweep_index: usize) -> Duration {
- Duration::from_secs(HEARTBEAT_TTL_SECS * sweep_index as u64)
+ if let Err(error) = release_recovery_leader(&runtime, &key_prefix, &leader_token).await {
+ tracing::warn!("queuefs cache recovery leader release failed: {error}");
+ }
}
-async fn recover_stale(runtime: &CacheRuntime, key_prefix: &str) -> Result<usize> {
+async fn recover_stale(
+ runtime: &CacheRuntime,
+ key_prefix: &str,
+ leader_token: &str,
+ leader_ttl: Duration,
+) -> Result<usize> {
let queues = runtime
.smembers(&queue_names_key(key_prefix))
.await
@@ -569,6 +632,9 @@ async fn recover_stale(runtime: &CacheRuntime, key_prefix: &str) -> Result<usize
.and_then(bytes_array_to_strings)?;
let mut recovered = 0;
for queue in queues {
+ if !acquire_or_renew_recovery_leader(runtime, key_prefix, leader_token, leader_ttl).await? {
+ break;
+ }
let keys = QueueKeys::new(key_prefix, &queue);
recovered += usize::try_from(integer(
execute_runtime(
@@ -585,6 +651,41 @@ async fn recover_stale(runtime: &CacheRuntime, key_prefix: &str) -> Result<usize
Ok(recovered)
}
+async fn acquire_or_renew_recovery_leader(
+ runtime: &CacheRuntime,
+ key_prefix: &str,
+ leader_token: &str,
+ ttl: Duration,
+) -> Result<bool> {
+ boolean(
+ execute_runtime(
+ runtime,
+ "acquire recovery leader",
+ ACQUIRE_RECOVERY_LEADER_ID,
+ vec![recovery_leader_key(key_prefix)],
+ vec![bytes(leader_token), bytes(ttl.as_millis().to_string())],
+ )
+ .await?,
+ )
+}
+
+async fn release_recovery_leader(
+ runtime: &CacheRuntime,
+ key_prefix: &str,
+ leader_token: &str,
+) -> Result<bool> {
+ boolean(
+ execute_runtime(
+ runtime,
+ "release recovery leader",
+ RELEASE_RECOVERY_LEADER_ID,
+ vec![recovery_leader_key(key_prefix)],
+ vec![bytes(leader_token)],
+ )
+ .await?,
+ )
+}
+
async fn execute_runtime(
runtime: &CacheRuntime,
operation: &str,
@@ -672,11 +773,8 @@ mod tests {
fn lifecycle_test_runtime() -> Arc<CacheRuntime> {
let provider = Arc::new(
- crate::cache_runtime::MemoryMockProvider::new().with_script_handler(|request| {
- panic!(
- "script execution is outside this lifecycle test's scope: {}",
- request.script_id
- )
+ crate::cache_runtime::MemoryMockProvider::new().with_script_handler(|_| {
+ crate::cache_runtime::ScriptResult::encode(&ScriptValue::Integer(1))
}),
);
CacheRuntime::memory_with_provider(provider)
@@ -697,18 +795,23 @@ mod tests {
}
#[test]
- fn startup_recovery_uses_original_zero_thirty_sixty_second_offsets() {
+ fn default_background_timing_matches_recovery_contract() {
+ let timing = BackgroundTiming::default();
+ assert_eq!(
+ timing.heartbeat_interval,
+ Duration::from_secs(HEARTBEAT_INTERVAL_SECS)
+ );
assert_eq!(
- startup_recovery_delay_before_sweep(0),
- Duration::from_secs(0)
+ timing.heartbeat_ttl,
+ Duration::from_secs(HEARTBEAT_TTL_SECS)
);
assert_eq!(
- startup_recovery_delay_before_sweep(1),
- Duration::from_secs(30)
+ timing.recovery_interval,
+ Duration::from_secs(RECOVERY_INTERVAL_SECS)
);
assert_eq!(
- startup_recovery_delay_before_sweep(2),
- Duration::from_secs(60)
+ timing.recovery_leader_ttl,
+ Duration::from_secs(RECOVERY_LEADER_TTL_SECS)
);
}
@@ -769,9 +872,13 @@ mod tests {
assert_eq!(first.dequeue("jobs").await.unwrap().unwrap().id, message_id);
first.shutdown().await.unwrap();
- let second = CacheQueueStorage::open(Arc::clone(&runtime), prefix.clone())
- .await
- .unwrap();
+ let second = CacheQueueStorage::open_with_timing(
+ Arc::clone(&runtime),
+ prefix.clone(),
+ fast_background_timing(),
+ )
+ .await
+ .unwrap();
let recovered = tokio::time::timeout(Duration::from_secs(2), async {
loop {
if let Some(message) = second.dequeue("jobs").await.unwrap() {
@@ -784,8 +891,87 @@ mod tests {
.unwrap();
assert_eq!(recovered.id, message_id);
+ second.remove_queue("jobs").await.unwrap();
second.shutdown().await.unwrap();
runtime.del(&[queue_names_key(&prefix)]).await.unwrap();
runtime.close().await.unwrap();
}
+
+ #[tokio::test]
+ async fn running_node_recovers_processing_after_peer_crash() {
+ let Ok(endpoint) = std::env::var("REDIS_URL") else {
+ return;
+ };
+ let runtime = CacheRuntime::redis(crate::cache_runtime::RedisProviderConfig {
+ endpoints: vec![endpoint],
+ connect_timeout_ms: 5_000,
+ command_timeout_ms: 1_000,
+ default_ttl_seconds: 60,
+ ..crate::cache_runtime::RedisProviderConfig::default()
+ })
+ .await
+ .unwrap();
+ let prefix = format!("queuefs-continuous-recovery-test:{}", Uuid::new_v4());
+ let survivor = CacheQueueStorage::open_with_timing(
+ Arc::clone(&runtime),
+ prefix.clone(),
+ fast_background_timing(),
+ )
+ .await
+ .unwrap();
+ let failed = CacheQueueStorage::open_with_timing(
+ Arc::clone(&runtime),
+ prefix.clone(),
+ fast_background_timing(),
+ )
+ .await
+ .unwrap();
+ failed.create_queue("jobs").await.unwrap();
+ let message = Message::new(b"payload".to_vec());
+ let message_id = message.id.clone();
+ failed.enqueue("jobs", message).await.unwrap();
+ assert_eq!(
+ failed.dequeue("jobs").await.unwrap().unwrap().id,
+ message_id
+ );
+
+ let heartbeat = heartbeat_key(&prefix, &failed.instance_id);
+ for task in [
+ failed.heartbeat_task.lock().await.take(),
+ failed.recovery_task.lock().await.take(),
+ ]
+ .into_iter()
+ .flatten()
+ {
+ task.abort();
+ }
+ runtime.del(&[heartbeat]).await.unwrap();
+ drop(failed);
+
+ let recovered = tokio::time::timeout(Duration::from_secs(2), async {
+ loop {
+ if let Some(message) = survivor.dequeue("jobs").await.unwrap() {
+ break message;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .unwrap();
+
+ assert_eq!(recovered.id, message_id);
+ survivor.remove_queue("jobs").await.unwrap();
+ survivor.shutdown().await.unwrap();
+ runtime.del(&[queue_names_key(&prefix)]).await.unwrap();
+ runtime.close().await.unwrap();
+ }
+
+ fn fast_background_timing() -> BackgroundTiming {
+ BackgroundTiming {
+ heartbeat_interval: Duration::from_millis(20),
+ heartbeat_ttl: Duration::from_millis(100),
+ recovery_interval: Duration::from_millis(20),
+ recovery_leader_ttl: Duration::from_millis(100),
+ }
+ }
}
diff --git a/crates/ragfs/src/plugins/queuefs/cache_protocol.rs b/crates/ragfs/src/plugins/queuefs/cache_protocol.rs
index e2fd2d427..31896c8c4 100644
--- a/crates/ragfs/src/plugins/queuefs/cache_protocol.rs
+++ b/crates/ragfs/src/plugins/queuefs/cache_protocol.rs
@@ -4,7 +4,8 @@ use std::time::{SystemTime, UNIX_EPOCH};
pub(super) const HEARTBEAT_TTL_SECS: u64 = 30;
pub(super) const HEARTBEAT_INTERVAL_SECS: u64 = 10;
-pub(super) const STARTUP_RECOVERY_SWEEPS: usize = 3;
+pub(super) const RECOVERY_INTERVAL_SECS: u64 = 10;
+pub(super) const RECOVERY_LEADER_TTL_SECS: u64 = 30;
pub(super) const CREATE_QUEUE_SCRIPT: &str = r#"
if redis.call('SADD', KEYS[1], ARGV[1]) == 0 then
@@ -156,6 +157,24 @@ end
return recovered
"#;
+pub(super) const ACQUIRE_RECOVERY_LEADER_SCRIPT: &str = r#"
+if redis.call('SET', KEYS[1], ARGV[1], 'NX', 'PX', ARGV[2]) then
+ return 1
+end
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+ redis.call('PEXPIRE', KEYS[1], ARGV[2])
+ return 1
+end
+return 0
+"#;
+
+pub(super) const RELEASE_RECOVERY_LEADER_SCRIPT: &str = r#"
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+ return redis.call('DEL', KEYS[1])
+end
+return 0
+"#;
+
pub(super) struct QueueKeys {
pub(super) meta: String,
pub(super) pending: String,
@@ -195,6 +214,10 @@ pub(super) fn heartbeat_key(key_prefix: &str, instance_id: &str) -> String {
format!("{}{instance_id}:alive", instance_key_prefix(key_prefix))
}
+pub(super) fn recovery_leader_key(key_prefix: &str) -> String {
+ format!("{}recovery:leader", queue_key_prefix(key_prefix))
+}
+
pub(super) fn unix_secs(time: SystemTime) -> u64 {
time.duration_since(UNIX_EPOCH)
.unwrap_or_default()
diff --git a/docs/en/api/01-overview.md b/docs/en/api/01-overview.md
index a3202e460..df46fbdd0 100644
--- a/docs/en/api/01-overview.md
+++ b/docs/en/api/01-overview.md
@@ -491,6 +491,7 @@ This catalog follows the routes actually mounted by the server. Each group headi
|--------|------|-------------|
| GET | `/api/v1/admin/configuration` | Get explicit Cluster runtime configuration |
| PATCH | `/api/v1/admin/configuration` | Update Cluster runtime configuration |
+| POST | `/api/v1/admin/configuration/test-model` | Test a draft model or embedding connection without saving it |
| POST | `/api/v1/admin/restart` | Request a graceful restart of a supported server process (ROOT only) |
| GET | `/api/v1/admin/accounts/{account_id}/configuration` | Get explicit Account runtime configuration |
| PATCH | `/api/v1/admin/accounts/{account_id}/configuration` | Update Account runtime configuration |
@@ -582,6 +583,8 @@ When the gateway runs inside the server, use the server's port; when it runs sta
| POST | `/bot/v1/chat` | Non-streaming VikingBot chat |
| POST | `/bot/v1/chat/stream` | Streaming VikingBot chat |
| POST | `/bot/v1/feedback` | Submit feedback for a VikingBot answer |
+| POST | `/bot/v1/sessions/{session_id}/interrupt` | Interrupt an active or queued VikingBot session |
+| POST | `/bot/v1/sessions/{session_id}/steer` | Send additional input to an active streaming session |
| POST | `/bot/v1/compile` | Retired; returns migration guidance for the new endpoint |
| GET | `/bot/v1/compile/{task_id}` | Retired; returns Task API migration guidance |
| POST | `/bot/v1/compile/{task_id}/cancel` | Retired; returns Task cancellation API migration guidance |
diff --git a/docs/en/api/08-admin.md b/docs/en/api/08-admin.md
index 182439a57..1a26c4944 100644
--- a/docs/en/api/08-admin.md
+++ b/docs/en/api/08-admin.md
@@ -539,6 +539,56 @@ Draft previews (`source=file&dry_run=true`) accept optional VLM/Embedding form `
other sections remain editable through the complete file.
File reads and previews return `form_readonly: true` when the document contains dollar signs, Windows `%VAR%` references, or escaped dollar/percent signs. Model form data is omitted and form patches are rejected. Use file mode to edit the original text, including quoted and unquoted environment references. Validation follows startup expansion of the original text; saves do not serialize the document. CLI startup arguments are reapplied during validation, as during restart.
+#### Draft Model Connection Test
+
+ROOT can test a draft VLM or Embedding connection without saving or publishing
+the submitted settings. This private Studio endpoint is intentionally hidden
+from the generated OpenAPI schema and always returns `Cache-Control: no-store`.
+
+```http
+POST /api/v1/admin/configuration/test-model
+Content-Type: application/json
+X-API-Key: <root-api-key>
+```
+
+| Field | Type | Required | Default | Description |
+|-------|------|----------|---------|-------------|
+| `kind` | string | Yes | - | `vlm` or `embedding` |
+| `config` | object | Yes | - | Draft provider credential or endpoint fields to test |
+| `fallback` | object | No | `{}` | Existing effective configuration used for omitted non-secret fields |
+| `thinking` | boolean | No | `null` | VLM thinking-mode override for the probe |
+| `mode` | string | No | `dense` | Embedding mode: `dense`, `sparse`, or `hybrid` |
+
+```bash
+curl -X POST http://localhost:1933/api/v1/admin/configuration/test-model \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: <root-api-key>" \
+ -d '{
+ "kind": "vlm",
+ "config": {
+ "provider": "openai",
+ "api_key": "draft-key"
+ },
+ "fallback": {
+ "model": "gpt-4.1"
+ }
+ }'
+```
+
+**Response Example**
+
+```json
+{
+ "status": "ok",
+ "result": {
+ "ok": true,
+ "ms": 120
+ }
+}
+```
+
+The endpoint redacts provider errors and does not persist the submitted draft.
+
#### Server Restart
Only ROOT can restart a single-worker `openviking-server` CLI process.
diff --git a/docs/en/api/24-vikingbot.md b/docs/en/api/24-vikingbot.md
index a07393d25..9aeb9da7f 100644
--- a/docs/en/api/24-vikingbot.md
+++ b/docs/en/api/24-vikingbot.md
@@ -179,6 +179,67 @@ curl -X POST http://localhost:1933/bot/v1/feedback \
A missing target response returns `404`. Rating feedback without `feedback_score` returns a request validation error.
+### interrupt_session()
+
+Interrupt an active or queued Bot session. OpenViking forwards the authenticated
+Account and User identity to the local VikingBot Gateway.
+
+```http
+POST /bot/v1/sessions/{session_id}/interrupt
+```
+
+**HTTP API**
+
+```bash
+curl -X POST http://localhost:1933/bot/v1/sessions/session-id/interrupt \
+ -H "X-API-Key: your-key"
+```
+
+**Response Example**
+
+The response body is forwarded from VikingBot. A typical response is:
+
+```json
+{
+ "interrupted": true
+}
+```
+
+### steer_session()
+
+Append user input to an active streaming Bot session. OpenViking strips any
+client-supplied `openviking_connection` and attaches the authenticated
+OpenViking connection before forwarding the request.
+
+```http
+POST /bot/v1/sessions/{session_id}/steer
+Content-Type: application/json
+```
+
+| Field | Type | Required | Description |
+|-------|------|----------|-------------|
+| `message` | string | Yes | Additional user input for the active session |
+| `openviking_connection` | object | No | Ignored when supplied by the client; OpenViking attaches the authenticated connection |
+
+**HTTP API**
+
+```bash
+curl -X POST http://localhost:1933/bot/v1/sessions/session-id/steer \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{"message":"Continue with the next step"}'
+```
+
+**Response Example**
+
+The response body is forwarded from VikingBot. A typical response is:
+
+```json
+{
+ "accepted": true
+}
+```
+
## Client Scope
The standard OpenViking Python, TypeScript, and Go SDKs do not currently wrap the Bot proxy. Chat is available through the `ov` CLI and HTTP. The VikingBot Gateway also exposes Session and Channel APIs; see the [VikingBot documentation](https://github.com/volcengine/OpenViking/blob/main/bot/README.md#http-api).
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md
index 1cade6e0f..2c7319612 100644
--- a/docs/en/guides/01-configuration.md
+++ b/docs/en/guides/01-configuration.md
@@ -1492,7 +1492,7 @@ Notes:
- All Redis reads go to the primary. Replica reads are not configurable.
- `tls_insecure_skip_verify=true` requires `rediss://` endpoints.
- QueueFS cache keys use `{service_name}:ov:*`; use different service names for deployments sharing one Redis cluster.
-- Redis backend runs three bounded `recover_stale` sweeps in a dedicated startup recovery thread at startup, 30 seconds, and 60 seconds to cover the heartbeat-expiry window after a container restart; it does not run long-lived periodic recovery.
+- Cache instances renew a 30-second heartbeat every 10 seconds. One live instance per QueueFS cache namespace holds a token-checked 30-second leader lease and scans for stale `processing` messages every 10 seconds. If that instance stops, another instance takes over after the lease expires; a normal shutdown removes its heartbeat so recovery can begin immediately.
- If both `storage.agfs.queuefs.db_path` and legacy `storage.agfs.queue_db_path` are set, `storage.agfs.queuefs.db_path` wins.
- If QueueFS backend is `memory`, any `db_path` or legacy `queue_db_path` is ignored.
diff --git a/docs/zh/api/01-overview.md b/docs/zh/api/01-overview.md
index afbaf3e95..6aee7b1a7 100644
--- a/docs/zh/api/01-overview.md
+++ b/docs/zh/api/01-overview.md
@@ -486,6 +486,7 @@ JSON 输出 - 错误:
|------|------|------|
| GET | `/api/v1/admin/configuration` | 获取 Cluster 层显式运行时配置 |
| PATCH | `/api/v1/admin/configuration` | 更新 Cluster 层运行时配置 |
+| POST | `/api/v1/admin/configuration/test-model` | 测试草稿模型或 Embedding 连接,不保存配置 |
| POST | `/api/v1/admin/restart` | 请求支持该能力的服务进程优雅重启(仅 ROOT) |
| GET | `/api/v1/admin/accounts/{account_id}/configuration` | 获取 Account 层显式运行时配置 |
| PATCH | `/api/v1/admin/accounts/{account_id}/configuration` | 更新 Account 层运行时配置 |
@@ -577,6 +578,8 @@ JSON 输出 - 错误:
| POST | `/bot/v1/chat` | VikingBot 非流式对话 |
| POST | `/bot/v1/chat/stream` | VikingBot 流式对话 |
| POST | `/bot/v1/feedback` | 提交 VikingBot 回答反馈 |
+| POST | `/bot/v1/sessions/{session_id}/interrupt` | 中断活跃或排队中的 VikingBot 会话 |
+| POST | `/bot/v1/sessions/{session_id}/steer` | 向活跃流式会话发送追加输入 |
| POST | `/bot/v1/compile` | 已停用;返回新接口迁移提示 |
| GET | `/bot/v1/compile/{task_id}` | 已停用;返回 Task 接口迁移提示 |
| POST | `/bot/v1/compile/{task_id}/cancel` | 已停用;返回 Task 取消接口迁移提示 |
diff --git a/docs/zh/api/08-admin.md b/docs/zh/api/08-admin.md
index b0e78ab3a..2849d5bee 100644
--- a/docs/zh/api/08-admin.md
+++ b/docs/zh/api/08-admin.md
@@ -451,6 +451,56 @@ PATCH 会先做结构校验,再构造合并后的配置:未知路径和运
文件含美元符号、Windows `%VAR%` 引用或转义的美元/百分号时,读取和预览返回 `form_readonly: true`,不返回模型表单数据,并拒绝表单修改。请使用文件模式编辑原文,包括带引号和未带引号的环境变量引用。验证沿用启动时对原文展开环境变量的流程,保存不会重新序列化文档。文件校验会应用 CLI 启动参数覆盖,与重启时的处理一致。
+#### 草稿模型连接测试
+
+ROOT 可以测试草稿 VLM 或 Embedding 连接,而不保存或发布提交的配置。
+这是 Studio 使用的私有接口,刻意不出现在生成的 OpenAPI schema 中,
+并始终返回 `Cache-Control: no-store`。
+
+```http
+POST /api/v1/admin/configuration/test-model
+Content-Type: application/json
+X-API-Key: <root-api-key>
+```
+
+| 字段 | 类型 | 必填 | 默认值 | 说明 |
+|------|------|------|--------|------|
+| `kind` | string | 是 | - | `vlm` 或 `embedding` |
+| `config` | object | 是 | - | 待测试的草稿 provider 凭证或端点字段 |
+| `fallback` | object | 否 | `{}` | 现有生效配置,用于补齐未提交的非敏感字段 |
+| `thinking` | boolean | 否 | `null` | 本次 VLM 探测的 thinking 模式覆盖 |
+| `mode` | string | 否 | `dense` | Embedding 模式:`dense`、`sparse` 或 `hybrid` |
+
+```bash
+curl -X POST http://localhost:1933/api/v1/admin/configuration/test-model \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: <root-api-key>" \
+ -d '{
+ "kind": "vlm",
+ "config": {
+ "provider": "openai",
+ "api_key": "draft-key"
+ },
+ "fallback": {
+ "model": "gpt-4.1"
+ }
+ }'
+```
+
+**响应示例**
+
+```json
+{
+ "status": "ok",
+ "result": {
+ "ok": true,
+ "ms": 120
+ }
+}
+```
+
+该接口会脱敏 provider 错误,并且不会持久化提交的草稿配置。
+
#### 服务端重启
仅 ROOT 可以重启通过单 worker `openviking-server` CLI 启动的进程。
diff --git a/docs/zh/api/24-vikingbot.md b/docs/zh/api/24-vikingbot.md
index a04074123..b03377e44 100644
--- a/docs/zh/api/24-vikingbot.md
+++ b/docs/zh/api/24-vikingbot.md
@@ -176,6 +176,66 @@ curl -X POST http://localhost:1933/bot/v1/feedback \
目标回复不存在时返回 `404`;`rating` 缺少 `feedback_score` 时返回请求校验错误。
+### interrupt_session()
+
+中断活跃或排队中的 Bot 会话。OpenViking 会把已认证的 Account 和 User
+身份转发给本地 VikingBot Gateway。
+
+```http
+POST /bot/v1/sessions/{session_id}/interrupt
+```
+
+**HTTP API**
+
+```bash
+curl -X POST http://localhost:1933/bot/v1/sessions/session-id/interrupt \
+ -H "X-API-Key: your-key"
+```
+
+**响应示例**
+
+响应体由 VikingBot 透传;典型响应如下:
+
+```json
+{
+ "interrupted": true
+}
+```
+
+### steer_session()
+
+向活跃的流式 Bot 会话追加用户输入。OpenViking 会丢弃客户端提交的
+`openviking_connection`,并在转发前附加已认证的 OpenViking 连接信息。
+
+```http
+POST /bot/v1/sessions/{session_id}/steer
+Content-Type: application/json
+```
+
+| 字段 | 类型 | 必填 | 说明 |
+|------|------|------|------|
+| `message` | string | 是 | 追加到活跃会话的用户输入 |
+| `openviking_connection` | object | 否 | 客户端传入时会被忽略;OpenViking 会附加已认证连接 |
+
+**HTTP API**
+
+```bash
+curl -X POST http://localhost:1933/bot/v1/sessions/session-id/steer \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{"message":"继续下一步"}'
+```
+
+**响应示例**
+
+响应体由 VikingBot 透传;典型响应如下:
+
+```json
+{
+ "accepted": true
+}
+```
+
## 客户端范围
标准 OpenViking Python、TypeScript 和 Go SDK 当前不封装 Bot 代理接口;Chat 可通过 `ov` CLI 与 HTTP 使用。VikingBot Gateway 自身还提供 Session 和 Channel API,详见 [VikingBot 文档](https://github.com/volcengine/OpenViking/blob/main/bot/README_CN.md#http-api)。
diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md
index 5400795bb..5822ac572 100644
--- a/docs/zh/guides/01-configuration.md
+++ b/docs/zh/guides/01-configuration.md
@@ -1477,8 +1477,7 @@ Redis 命名空间。
- Redis Sentinel 模式的 endpoints 是 Sentinel 节点,并且必须配置非空 `master_name`;master 发现和故障切换后的重连由 Fred RedisProvider 处理。
- `username` 和 `password` 用于 Redis 数据节点;`sentinel_username` 和 `sentinel_password` 仅用于 Sentinel 节点。
- Cache backend 使用 `{service_name}:ov:*` key;连接同一 Redis 集群的不同部署必须配置不同的 `service_name`。
-- Cache backend 的实例心跳 TTL 为 30 秒,每 10 秒续约一次。
-- Cache backend 会在独立的 startup recovery 任务中按实例心跳状态执行三次有界 `recover_stale` 扫描,时间点分别为启动后立即、30 秒和 60 秒,用于覆盖容器异常退出后旧实例心跳尚未过期的恢复窗口;正常关闭会先删除 heartbeat,使新实例可以立即恢复 processing 消息。
+- Cache backend 的实例心跳 TTL 为 30 秒,每 10 秒续约一次。同一 QueueFS cache namespace 中只有一个存活实例持有经 token 校验的 30 秒 leader lease,并每 10 秒扫描过期的 `processing` 消息;该实例停止后,其他实例会在 lease 到期后接管。正常关闭会删除 heartbeat,因此恢复可以立即开始。
- 所有 Redis 读命令都发送到主节点,不提供副本读配置。
- `tls_insecure_skip_verify=true` 时 endpoint 必须使用 `rediss://`。
- 如果同时设置了 `storage.agfs.queuefs.db_path` 和旧字段 `storage.agfs.queue_db_path`,以前者为准。
diff --git a/examples/memory-plugin-shared/install-tui.test.mjs b/examples/memory-plugin-shared/install-tui.test.mjs
index 96fd4b78a..8662f7579 100644
--- a/examples/memory-plugin-shared/install-tui.test.mjs
+++ b/examples/memory-plugin-shared/install-tui.test.mjs
@@ -93,6 +93,7 @@ PATH=/usr/bin:/bin
HOME=${JSON.stringify(home)}
refresh_available_harnesses
HAVE_CURSOR=0
+HAVE_TRAE_CN=0
INTERACTIVE=0
REQUESTED_HARNESSES=""
select_harnesses
@@ -114,6 +115,7 @@ PATH=/usr/bin:/bin
HOME=${JSON.stringify(home)}
refresh_available_harnesses
HAVE_CURSOR=0
+HAVE_TRAE_CN=0
INTERACTIVE=0
REQUESTED_HARNESSES=""
select_harnesses
@@ -143,6 +145,7 @@ TUI_CODEX_BINS="$CODEX_BINS"
add_detected_traecode_cli_alias
refresh_available_harnesses
HAVE_CURSOR=0
+HAVE_TRAE_CN=0
tui_reset_bin_selection
INTERACTIVE=0
REQUESTED_HARNESSES=""
diff --git a/tests/api_test/filesystem/test_fs_cp.py b/tests/api_test/filesystem/test_fs_cp.py
index ef3e19487..86d15ea52 100644
--- a/tests/api_test/filesystem/test_fs_cp.py
+++ b/tests/api_test/filesystem/test_fs_cp.py
@@ -4,6 +4,23 @@ import uuid
import pytest
+def write_or_skip_unavailable(api_client, uri, content):
+ response = api_client.fs_write(uri, content, mode="create", wait=True)
+ if response.status_code == 200:
+ return response
+ if os.getenv("HAS_SECRETS", "true").lower() != "true":
+ try:
+ error = response.json().get("error")
+ except Exception:
+ error = response.text
+ pytest.skip(
+ f"fs_write failed on this environment: {error}. "
+ "This may be due to AGFS service not being available."
+ )
+ assert response.status_code == 200, response.text
+ return response
+
+
class TestFsCp:
def test_cp_file_preserves_source_and_content(self, api_client):
suffix = uuid.uuid4().hex[:8]
@@ -11,8 +28,7 @@ class TestFsCp:
target = f"viking://resources/cp-target-{suffix}.md"
content = f"copy payload {suffix}"
try:
- write = api_client.fs_write(source, content, mode="create", wait=True)
- assert write.status_code == 200, write.text
+ write = write_or_skip_unavailable(api_client, source, content)
if os.getenv("HAS_SECRETS", "true").lower() == "true":
write_result = write.json().get("result", {})
@@ -85,12 +101,7 @@ class TestFsCp:
assert api_client.fs_mkdir(source).status_code == 200
assert api_client.fs_mkdir(f"{source}/nested").status_code == 200
assert api_client.fs_mkdir(f"{source}/empty").status_code == 200
- assert (
- api_client.fs_write(
- child, "recursive copy child", mode="create", wait=True
- ).status_code
- == 200
- )
+ write_or_skip_unavailable(api_client, child, "recursive copy child")
without_recursive = api_client.fs_cp(source, target)
assert without_recursive.status_code == 400, without_recursive.text
@@ -119,14 +130,8 @@ class TestFsCp:
source_content = f"new payload {suffix}"
target_content = f"old target content must be completely replaced {suffix}"
try:
- assert (
- api_client.fs_write(source, source_content, mode="create", wait=True).status_code
- == 200
- )
- assert (
- api_client.fs_write(target, target_content, mode="create", wait=True).status_code
- == 200
- )
+ write_or_skip_unavailable(api_client, source, source_content)
+ write_or_skip_unavailable(api_client, target, target_content)
copied = api_client.fs_cp(source, target)
assert copied.status_code == 200, copied.text
@@ -154,8 +159,7 @@ class TestFsCp:
try:
assert api_client.fs_mkdir(directory_uri).status_code == 200
for uri, content in ((file_uri, file_content), (child_uri, child_content)):
- write = api_client.fs_write(uri, content, mode="create", wait=True)
- assert write.status_code == 200, write.text
+ write_or_skip_unavailable(api_client, uri, content)
source, target = (
(directory_uri, file_uri) if source_is_dir else (file_uri, directory_uri)