Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
//! Queue Backend Abstraction
//!
//! This module provides a pluggable backend system for QueueFS, allowing different
//! storage implementations (memory, SQLite, etc.) while maintaining a consistent interface.
use crate::core::errors::{Error, Result};
use chrono::{DateTime, SecondsFormat, Utc};
use rusqlite::{params, types::ValueRef, Connection, Row};
use serde::{Deserialize, Deserializer, Serialize};
use std::collections::{HashMap, VecDeque};
use std::sync::Mutex;
use std::time::SystemTime;
use std::time::{Duration, UNIX_EPOCH};
use uuid::Uuid;
/// A message in the queue
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Message {
/// Unique identifier for the message
pub id: String,
/// Message data
pub data: Vec<u8>,
/// Timestamp when the message was enqueued
pub timestamp: SystemTime,
/// Whether `timestamp` came from a real persisted enqueue time.
pub timestamp_valid: bool,
}
impl Message {
/// Create a new message with the given data
pub fn new(data: Vec<u8>) -> Self {
Self {
id: Uuid::new_v4().to_string(),
data,
timestamp: SystemTime::now(),
timestamp_valid: true,
}
}
}
/// Current queue occupancy.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct QueueState {
/// Messages available for dequeue.
pub pending: usize,
/// Messages dequeued but not acknowledged.
pub processing: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(super) struct StoredMessage {
id: String,
#[serde(deserialize_with = "deserialize_stored_message_data")]
data: Vec<u8>,
#[serde(default)]
timestamp: Option<serde_json::Value>,
}
impl StoredMessage {
pub(super) fn from_message(msg: &Message) -> Self {
Self {
id: msg.id.clone(),
data: msg.data.clone(),
// Older readers already accept RFC3339 strings; nanoseconds preserve latency accuracy.
timestamp: Some(serde_json::Value::String(
DateTime::<Utc>::from(msg.timestamp)
.to_rfc3339_opts(SecondsFormat::Nanos, true),
)),
}
}
pub(super) fn into_message(self) -> Message {
let (timestamp, timestamp_valid) = parse_stored_timestamp(self.timestamp);
Message {
id: self.id,
data: self.data,
timestamp,
timestamp_valid,
}
}
}
fn deserialize_stored_message_data<'de, D>(
deserializer: D,
) -> std::result::Result<Vec<u8>, D::Error>
where
D: Deserializer<'de>,
{
let value = serde_json::Value::deserialize(deserializer)?;
match value {
// Legacy queue.db rows stored message bytes as a JSON string.
serde_json::Value::String(data) => Ok(data.into_bytes()),
// New rows keep arbitrary message bytes lossless as a JSON byte array.
serde_json::Value::Array(bytes) => bytes
.into_iter()
.map(|byte| {
byte.as_u64()
.and_then(|value| u8::try_from(value).ok())
.ok_or_else(|| {
serde::de::Error::custom(
"stored queue message data byte must be an integer in 0..=255",
)
})
})
.collect(),
other => Err(serde::de::Error::custom(format!(
"stored queue message data must be a string or byte array, got {other}"
))),
}
}
fn parse_stored_timestamp(raw: Option<serde_json::Value>) -> (SystemTime, bool) {
match raw {
Some(serde_json::Value::String(ts)) => DateTime::parse_from_rfc3339(&ts)
.ok()
.and_then(|dt| timestamp_from_parts(dt.timestamp(), dt.timestamp_subsec_nanos()))
.map(|timestamp| (timestamp, true))
.unwrap_or_else(|| (SystemTime::now(), false)),
Some(serde_json::Value::Number(num)) => num
.as_i64()
.and_then(|secs| timestamp_from_parts(secs, 0))
.map(|timestamp| (timestamp, true))
.unwrap_or_else(|| (SystemTime::now(), false)),
_ => (SystemTime::now(), false),
}
}
fn timestamp_from_parts(secs: i64, nanos: u32) -> Option<SystemTime> {
(secs > 0)
.then(|| Duration::new(secs as u64, nanos))
.and_then(|duration| UNIX_EPOCH.checked_add(duration))
}
fn unix_secs(time: SystemTime) -> i64 {
time.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs() as i64
}
fn read_sqlite_text(row: &Row<'_>, idx: usize) -> rusqlite::Result<String> {
match row.get_ref(idx)? {
ValueRef::Text(bytes) => Ok(String::from_utf8_lossy(bytes).into_owned()),
ValueRef::Blob(bytes) => Ok(String::from_utf8_lossy(bytes).into_owned()),
ValueRef::Null => Ok(String::new()),
other => Err(rusqlite::Error::InvalidColumnType(
idx,
row.as_ref()
.column_name(idx)
.map(|s| s.to_string())
.unwrap_or_else(|_| format!("column_{idx}")),
other.data_type(),
)),
}
}
/// Queue backend trait for pluggable storage implementations
pub trait QueueBackend: Send + Sync {
/// Create a new queue with the given name
fn create_queue(&mut self, name: &str) -> Result<()>;
/// Remove a queue and all its messages
fn remove_queue(&mut self, name: &str) -> Result<()>;
/// Check if a queue exists
fn queue_exists(&self, name: &str) -> bool;
/// List all queues with the given prefix
/// If prefix is empty, returns all queues
fn list_queues(&self, prefix: &str) -> Vec<String>;
/// Add a message to the queue
fn enqueue(&mut self, queue_name: &str, msg: Message) -> Result<()>;
/// Remove and return the first message from the queue
fn dequeue(&mut self, queue_name: &str) -> Result<Option<Message>>;
/// View the first message without removing it
fn peek(&self, queue_name: &str) -> Result<Option<Message>>;
/// Get the number of messages in the queue
fn size(&self, queue_name: &str) -> Result<usize>;
/// Get pending and processing message counts from one backend snapshot
fn status(&self, queue_name: &str) -> Result<QueueState>;
/// List every unacknowledged message without changing queue state
fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>>;
/// Clear all messages from the queue
fn clear(&mut self, queue_name: &str) -> Result<()>;
/// Get the last enqueue time for the queue
fn get_last_enqueue_time(&self, queue_name: &str) -> Result<SystemTime>;
/// Acknowledge (delete) a message by ID
fn ack(&mut self, queue_name: &str, msg_id: &str) -> Result<bool>;
}
/// Options for SQLite queue backend.
#[derive(Debug, Clone, Copy)]
pub struct SQLiteQueueOptions {
pub recover_stale_sec: i64,
pub busy_timeout_ms: u64,
}
impl Default for SQLiteQueueOptions {
fn default() -> Self {
Self {
recover_stale_sec: 0,
busy_timeout_ms: 5_000,
}
}
}
/// A single queue with its messages
struct Queue {
messages: VecDeque<Message>,
processing: VecDeque<Message>,
last_enqueue_time: SystemTime,
}
impl Queue {
fn new() -> Self {
Self {
messages: VecDeque::new(),
processing: VecDeque::new(),
last_enqueue_time: SystemTime::UNIX_EPOCH,
}
}
}
/// In-memory queue backend using HashMap
pub struct MemoryBackend {
queues: HashMap<String, Queue>,
}
impl MemoryBackend {
/// Create a new memory backend
pub fn new() -> Self {
Self {
queues: HashMap::new(),
}
}
}
impl QueueBackend for MemoryBackend {
fn create_queue(&mut self, name: &str) -> Result<()> {
if self.queues.contains_key(name) {
return Err(Error::AlreadyExists(format!(
"queue '{}' already exists",
name
)));
}
self.queues.insert(name.to_string(), Queue::new());
Ok(())
}
fn remove_queue(&mut self, name: &str) -> Result<()> {
if self.queues.remove(name).is_none() {
return Err(Error::NotFound(format!("queue '{}' not found", name)));
}
Ok(())
}
fn queue_exists(&self, name: &str) -> bool {
self.queues.contains_key(name)
}
fn list_queues(&self, prefix: &str) -> Vec<String> {
if prefix.is_empty() {
self.queues.keys().cloned().collect()
} else {
self.queues
.keys()
.filter(|name| name.starts_with(prefix))
.cloned()
.collect()
}
}
fn enqueue(&mut self, queue_name: &str, msg: Message) -> Result<()> {
let queue = self
.queues
.get_mut(queue_name)
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
queue.last_enqueue_time = SystemTime::now();
queue.messages.push_back(msg);
Ok(())
}
fn dequeue(&mut self, queue_name: &str) -> Result<Option<Message>> {
let queue = self
.queues
.get_mut(queue_name)
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
let message = queue.messages.pop_front();
if let Some(message) = &message {
queue.processing.push_back(message.clone());
}
Ok(message)
}
fn peek(&self, queue_name: &str) -> Result<Option<Message>> {
let queue = self
.queues
.get(queue_name)
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
Ok(queue.messages.front().cloned())
}
fn size(&self, queue_name: &str) -> Result<usize> {
let queue = self
.queues
.get(queue_name)
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
Ok(queue.messages.len())
}
fn status(&self, queue_name: &str) -> Result<QueueState> {
let queue = self
.queues
.get(queue_name)
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
Ok(QueueState {
pending: queue.messages.len(),
processing: queue.processing.len(),
})
}
fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {
let queue = self
.queues
.get(queue_name)
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
Ok(queue
.messages
.iter()
.chain(queue.processing.iter())
.cloned()
.collect())
}
fn clear(&mut self, queue_name: &str) -> Result<()> {
let queue = self
.queues
.get_mut(queue_name)
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
queue.messages.clear();
queue.processing.clear();
Ok(())
}
fn get_last_enqueue_time(&self, queue_name: &str) -> Result<SystemTime> {
let queue = self
.queues
.get(queue_name)
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
Ok(queue.last_enqueue_time)
}
fn ack(&mut self, queue_name: &str, msg_id: &str) -> Result<bool> {
let queue = self
.queues
.get_mut(queue_name)
.ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
let original_len = queue.processing.len();
queue.processing.retain(|msg| msg.id != msg_id);
Ok(queue.processing.len() != original_len)
}
}
/// SQLite queue backend with at-least-once delivery semantics.
pub struct SQLiteQueueBackend {
conn: Mutex<Connection>,
}
impl SQLiteQueueBackend {
pub fn open(db_path: &str, options: SQLiteQueueOptions) -> Result<Self> {
let conn = Connection::open(db_path)
.map_err(|e| Error::internal(format!("sqlite connection error: {}", e)))?;
if options.busy_timeout_ms > 0 {
conn.busy_timeout(Duration::from_millis(options.busy_timeout_ms))
.map_err(|e| Error::internal(format!("sqlite busy_timeout error: {}", e)))?;
}
if Self::is_new_database(&conn)? {
conn.execute_batch(
r#"
PRAGMA auto_vacuum=FULL;
VACUUM;
"#,
)
.map_err(|e| Error::internal(format!("sqlite auto_vacuum pragma error: {}", e)))?;
}
conn.execute_batch(
r#"
PRAGMA journal_mode=WAL;
PRAGMA synchronous=NORMAL;
"#,
)
.map_err(|e| Error::internal(format!("sqlite pragma error: {}", e)))?;
conn.execute_batch(
r#"
CREATE TABLE IF NOT EXISTS queue_metadata (
queue_name TEXT PRIMARY KEY,
created_at INTEGER DEFAULT (strftime('%s', 'now')),
last_updated INTEGER DEFAULT (strftime('%s', 'now'))
);
CREATE TABLE IF NOT EXISTS queue_messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
queue_name TEXT NOT NULL,
message_id TEXT NOT NULL,
data TEXT NOT NULL,
timestamp INTEGER NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
processing_started_at INTEGER,
created_at INTEGER DEFAULT (strftime('%s', 'now'))
);
"#,
)
.map_err(|e| Error::internal(format!("sqlite schema init error: {}", e)))?;
let backend = Self {
conn: Mutex::new(conn),
};
backend.run_migrations()?;
backend.recover_stale(options.recover_stale_sec)?;
Ok(backend)
}
fn is_new_database(conn: &Connection) -> Result<bool> {
let table_count: i64 = conn
.query_row(
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table'",
[],
|row| row.get(0),
)
.map_err(|e| Error::internal(format!("sqlite schema probe error: {}", e)))?;
Ok(table_count == 0)
}
fn run_migrations(&self) -> Result<()> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
for stmt in [
"ALTER TABLE queue_messages ADD COLUMN status TEXT NOT NULL DEFAULT 'pending'",
"ALTER TABLE queue_messages ADD COLUMN processing_started_at INTEGER",
"CREATE INDEX IF NOT EXISTS idx_queue_status ON queue_messages(queue_name, status, id)",
"CREATE INDEX IF NOT EXISTS idx_queue_message_id ON queue_messages(queue_name, message_id)",
] {
if let Err(err) = conn.execute(stmt, []) {
let text = err.to_string();
if !text.contains("duplicate column name") && !text.contains("already exists") {
return Err(Error::internal(format!(
"sqlite migration failed for '{}': {}",
stmt, text
)));
}
}
}
// Backfill queue metadata for legacy databases that only persisted queue_messages.
conn.execute(
"INSERT OR IGNORE INTO queue_metadata (queue_name)
SELECT DISTINCT queue_name
FROM queue_messages
WHERE queue_name IS NOT NULL AND queue_name != ''",
[],
)
.map_err(|e| {
Error::internal(format!(
"sqlite migration backfill queue metadata error: {}",
e
))
})?;
Ok(())
}
fn recover_stale(&self, stale_sec: i64) -> Result<usize> {
let cutoff = Utc::now().timestamp() - stale_sec.max(0);
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
let changed = if stale_sec <= 0 {
conn.execute(
"UPDATE queue_messages SET status = 'pending', processing_started_at = NULL WHERE status = 'processing'",
[],
)
} else {
conn.execute(
"UPDATE queue_messages SET status = 'pending', processing_started_at = NULL WHERE status = 'processing' AND processing_started_at <= ?1",
params![cutoff],
)
}
.map_err(|e| Error::internal(format!("sqlite recover stale error: {}", e)))?;
Ok(changed)
}
fn queue_known(conn: &Connection, queue_name: &str) -> Result<bool> {
let metadata_count: i64 = conn
.query_row(
"SELECT COUNT(*) FROM queue_metadata WHERE queue_name = ?1",
params![queue_name],
|row| row.get(0),
)
.map_err(|e| Error::internal(format!("sqlite queue metadata lookup error: {}", e)))?;
if metadata_count > 0 {
return Ok(true);
}
let message_count: i64 = conn
.query_row(
"SELECT COUNT(*) FROM queue_messages WHERE queue_name = ?1",
params![queue_name],
|row| row.get(0),
)
.map_err(|e| Error::internal(format!("sqlite queue messages lookup error: {}", e)))?;
Ok(message_count > 0)
}
fn require_queue_exists(conn: &Connection, queue_name: &str) -> Result<()> {
if Self::queue_known(conn, queue_name)? {
Ok(())
} else {
Err(Error::NotFound(format!("queue '{}' not found", queue_name)))
}
}
}
impl QueueBackend for SQLiteQueueBackend {
fn create_queue(&mut self, name: &str) -> Result<()> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
let changed = conn
.execute(
"INSERT OR IGNORE INTO queue_metadata (queue_name) VALUES (?1)",
params![name],
)
.map_err(|e| Error::internal(format!("sqlite create queue error: {}", e)))?;
if changed == 0 {
return Err(Error::AlreadyExists(format!(
"queue '{}' already exists",
name
)));
}
Ok(())
}
fn remove_queue(&mut self, name: &str) -> Result<()> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
let pattern = format!("{}/%", name);
let deleted_messages = conn
.execute(
"DELETE FROM queue_messages WHERE queue_name = ?1 OR queue_name LIKE ?2",
params![name, pattern],
)
.map_err(|e| Error::internal(format!("sqlite remove messages error: {}", e)))?;
let deleted_meta = conn
.execute(
"DELETE FROM queue_metadata WHERE queue_name = ?1 OR queue_name LIKE ?2",
params![name, pattern],
)
.map_err(|e| Error::internal(format!("sqlite remove metadata error: {}", e)))?;
if deleted_messages == 0 && deleted_meta == 0 {
return Err(Error::NotFound(format!("queue '{}' not found", name)));
}
Ok(())
}
fn queue_exists(&self, name: &str) -> bool {
let conn = match self.conn.lock() {
Ok(conn) => conn,
Err(_) => return false,
};
Self::queue_known(&conn, name).unwrap_or(false)
}
fn list_queues(&self, prefix: &str) -> Vec<String> {
let conn = match self.conn.lock() {
Ok(conn) => conn,
Err(_) => return Vec::new(),
};
if prefix.is_empty() {
let mut stmt =
match conn.prepare("SELECT queue_name FROM queue_metadata ORDER BY queue_name") {
Ok(stmt) => stmt,
Err(_) => return Vec::new(),
};
let rows = match stmt.query_map([], |row| row.get::<_, String>(0)) {
Ok(rows) => rows,
Err(_) => return Vec::new(),
};
return rows.filter_map(|row| row.ok()).collect();
}
let mut stmt = match conn.prepare(
"SELECT queue_name FROM queue_metadata
WHERE queue_name = ?1 OR queue_name LIKE ?2
ORDER BY queue_name",
) {
Ok(stmt) => stmt,
Err(_) => return Vec::new(),
};
let pattern = format!("{}/%", prefix);
let rows = match stmt.query_map(params![prefix, pattern], |row| row.get::<_, String>(0)) {
Ok(rows) => rows,
Err(_) => return Vec::new(),
};
rows.filter_map(|row| row.ok()).collect()
}
fn enqueue(&mut self, queue_name: &str, msg: Message) -> Result<()> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
Self::require_queue_exists(&conn, queue_name)?;
let stored = serde_json::to_string(&StoredMessage::from_message(&msg))?;
conn.execute(
"INSERT INTO queue_messages (queue_name, message_id, data, timestamp, status)
VALUES (?1, ?2, ?3, ?4, 'pending')",
params![queue_name, msg.id, stored, unix_secs(msg.timestamp)],
)
.map_err(|e| Error::internal(format!("sqlite enqueue error: {}", e)))?;
Ok(())
}
fn dequeue(&mut self, queue_name: &str) -> Result<Option<Message>> {
let mut conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
Self::require_queue_exists(&conn, queue_name)?;
let tx = conn
.transaction()
.map_err(|e| Error::internal(format!("sqlite transaction begin error: {}", e)))?;
let row = tx.query_row(
"SELECT id, data FROM queue_messages
WHERE queue_name = ?1 AND status = 'pending'
ORDER BY id LIMIT 1",
params![queue_name],
|row| Ok((row.get::<_, i64>(0)?, read_sqlite_text(row, 1)?)),
);
let (id, raw_data) = match row {
Ok(row) => row,
Err(rusqlite::Error::QueryReturnedNoRows) => return Ok(None),
Err(e) => {
return Err(Error::internal(format!(
"sqlite dequeue query error: {}",
e
)))
}
};
tx.execute(
"UPDATE queue_messages
SET status = 'processing', processing_started_at = ?1
WHERE id = ?2",
params![Utc::now().timestamp(), id],
)
.map_err(|e| Error::internal(format!("sqlite mark processing error: {}", e)))?;
tx.commit()
.map_err(|e| Error::internal(format!("sqlite transaction commit error: {}", e)))?;
let stored: StoredMessage =
serde_json::from_str(&raw_data).map_err(|e| Error::Serialization(e.to_string()))?;
Ok(Some(stored.into_message()))
}
fn peek(&self, queue_name: &str) -> Result<Option<Message>> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
Self::require_queue_exists(&conn, queue_name)?;
let raw_data = match conn.query_row(
"SELECT data FROM queue_messages
WHERE queue_name = ?1 AND status = 'pending'
ORDER BY id LIMIT 1",
params![queue_name],
|row| read_sqlite_text(row, 0),
) {
Ok(data) => data,
Err(rusqlite::Error::QueryReturnedNoRows) => return Ok(None),
Err(e) => return Err(Error::internal(format!("sqlite peek query error: {}", e))),
};
let stored: StoredMessage = serde_json::from_str(&raw_data)?;
Ok(Some(stored.into_message()))
}
fn size(&self, queue_name: &str) -> Result<usize> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
Self::require_queue_exists(&conn, queue_name)?;
let count: i64 = conn
.query_row(
"SELECT COUNT(*) FROM queue_messages
WHERE queue_name = ?1 AND status = 'pending'",
params![queue_name],
|row| row.get(0),
)
.map_err(|e| Error::internal(format!("sqlite size query error: {}", e)))?;
Ok(count as usize)
}
fn status(&self, queue_name: &str) -> Result<QueueState> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
Self::require_queue_exists(&conn, queue_name)?;
let (pending, processing): (i64, i64) = conn
.query_row(
"SELECT
COALESCE(SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN status = 'processing' THEN 1 ELSE 0 END), 0)
FROM queue_messages WHERE queue_name = ?1",
params![queue_name],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.map_err(|e| Error::internal(format!("sqlite status query error: {}", e)))?;
Ok(QueueState {
pending: pending as usize,
processing: processing as usize,
})
}
fn list_unacked(&self, queue_name: &str) -> Result<Vec<Message>> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
Self::require_queue_exists(&conn, queue_name)?;
let mut stmt = conn
.prepare(
"SELECT data FROM queue_messages
WHERE queue_name = ?1 AND status IN ('pending', 'processing')
ORDER BY id",
)
.map_err(|e| Error::internal(format!("sqlite list unacked prepare error: {}", e)))?;
let rows = stmt
.query_map(params![queue_name], |row| read_sqlite_text(row, 0))
.map_err(|e| Error::internal(format!("sqlite list unacked query error: {}", e)))?;
let mut messages = Vec::new();
for row in rows {
let raw_data =
row.map_err(|e| Error::internal(format!("sqlite list unacked row error: {}", e)))?;
let stored: StoredMessage = serde_json::from_str(&raw_data)?;
messages.push(stored.into_message());
}
Ok(messages)
}
fn clear(&mut self, queue_name: &str) -> Result<()> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
Self::require_queue_exists(&conn, queue_name)?;
conn.execute(
"DELETE FROM queue_messages WHERE queue_name = ?1",
params![queue_name],
)
.map_err(|e| Error::internal(format!("sqlite clear error: {}", e)))?;
Ok(())
}
fn get_last_enqueue_time(&self, queue_name: &str) -> Result<SystemTime> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
Self::require_queue_exists(&conn, queue_name)?;
let timestamp: Option<i64> = conn
.query_row(
"SELECT MAX(timestamp) FROM queue_messages
WHERE queue_name = ?1 AND status = 'pending'",
params![queue_name],
|row| row.get(0),
)
.map_err(|e| Error::internal(format!("sqlite last enqueue query error: {}", e)))?;
Ok(timestamp
.map(|secs| {
let secs_u64 = if secs <= 0 { 0 } else { secs as u64 };
UNIX_EPOCH + Duration::from_secs(secs_u64)
})
.unwrap_or(SystemTime::UNIX_EPOCH))
}
fn ack(&mut self, queue_name: &str, msg_id: &str) -> Result<bool> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(format!("sqlite mutex poisoned: {}", e)))?;
Self::require_queue_exists(&conn, queue_name)?;
let changed = conn
.execute(
"DELETE FROM queue_messages
WHERE queue_name = ?1 AND message_id = ?2 AND status = 'processing'",
params![queue_name, msg_id],
)
.map_err(|e| Error::internal(format!("sqlite ack error: {}", e)))?;
Ok(changed > 0)
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
use tempfile::TempDir;
/// Create an in-memory backend with a test queue.
fn memory_backend_with_queue(queue: &str) -> MemoryBackend {
let mut backend = MemoryBackend::new();
backend.create_queue(queue).unwrap();
backend
}
/// Create a SQLite backend backed by a temporary database file.
fn sqlite_backend() -> (TempDir, String, SQLiteQueueBackend) {
let dir = tempdir().unwrap();
let db_path = dir.path().join("queue.db");
let db_path_str = db_path.to_str().unwrap().to_string();
let backend =
SQLiteQueueBackend::open(&db_path_str, SQLiteQueueOptions::default()).unwrap();
(dir, db_path_str, backend)
}
#[test]
fn test_create_queue() {
let mut backend = MemoryBackend::new();
backend.create_queue("test").unwrap();
assert!(backend.queue_exists("test"));
// Creating duplicate should fail
let result = backend.create_queue("test");
assert!(result.is_err());
}
#[test]
fn test_remove_queue() {
let mut backend = memory_backend_with_queue("test");
backend.remove_queue("test").unwrap();
assert!(!backend.queue_exists("test"));
// Removing non-existent queue should fail
let result = backend.remove_queue("test");
assert!(result.is_err());
}
#[test]
fn test_list_queues() {
let mut backend = MemoryBackend::new();
backend.create_queue("queue1").unwrap();
backend.create_queue("queue2").unwrap();
backend.create_queue("logs/errors").unwrap();
let all = backend.list_queues("");
assert_eq!(all.len(), 3);
let logs = backend.list_queues("logs");
assert_eq!(logs.len(), 1);
assert_eq!(logs[0], "logs/errors");
}
#[test]
fn test_enqueue_dequeue() {
let mut backend = memory_backend_with_queue("test");
let msg1 = Message::new(b"message 1".to_vec());
let msg2 = Message::new(b"message 2".to_vec());
backend.enqueue("test", msg1.clone()).unwrap();
backend.enqueue("test", msg2.clone()).unwrap();
assert_eq!(backend.size("test").unwrap(), 2);
let dequeued1 = backend.dequeue("test").unwrap().unwrap();
assert_eq!(dequeued1.data, b"message 1");
assert_eq!(
backend.status("test").unwrap(),
QueueState {
pending: 1,
processing: 1,
}
);
assert!(backend.ack("test", &dequeued1.id).unwrap());
let dequeued2 = backend.dequeue("test").unwrap().unwrap();
assert_eq!(dequeued2.data, b"message 2");
assert!(backend.ack("test", &dequeued2.id).unwrap());
assert_eq!(backend.size("test").unwrap(), 0);
assert_eq!(
backend.status("test").unwrap(),
QueueState {
pending: 0,
processing: 0,
}
);
assert!(backend.dequeue("test").unwrap().is_none());
}
#[test]
fn test_peek() {
let mut backend = memory_backend_with_queue("test");
let msg = Message::new(b"test message".to_vec());
backend.enqueue("test", msg.clone()).unwrap();
let peeked1 = backend.peek("test").unwrap().unwrap();
assert_eq!(peeked1.data, b"test message");
let peeked2 = backend.peek("test").unwrap().unwrap();
assert_eq!(peeked2.data, b"test message");
// Size should still be 1
assert_eq!(backend.size("test").unwrap(), 1);
}
#[test]
fn test_clear() {
let mut backend = memory_backend_with_queue("test");
backend
.enqueue("test", Message::new(b"msg1".to_vec()))
.unwrap();
backend
.enqueue("test", Message::new(b"msg2".to_vec()))
.unwrap();
assert_eq!(backend.size("test").unwrap(), 2);
backend.clear("test").unwrap();
assert_eq!(backend.size("test").unwrap(), 0);
}
#[test]
fn test_multi_queue_isolation() {
let mut backend = MemoryBackend::new();
backend.create_queue("queue1").unwrap();
backend.create_queue("queue2").unwrap();
backend
.enqueue("queue1", Message::new(b"msg1".to_vec()))
.unwrap();
backend
.enqueue("queue2", Message::new(b"msg2".to_vec()))
.unwrap();
assert_eq!(backend.size("queue1").unwrap(), 1);
assert_eq!(backend.size("queue2").unwrap(), 1);
let msg1 = backend.dequeue("queue1").unwrap().unwrap();
assert_eq!(msg1.data, b"msg1");
// queue2 should be unaffected
assert_eq!(backend.size("queue2").unwrap(), 1);
}
#[test]
fn test_operations_on_nonexistent_queue() {
let mut backend = MemoryBackend::new();
assert!(backend
.enqueue("nonexistent", Message::new(b"data".to_vec()))
.is_err());
assert!(backend.dequeue("nonexistent").is_err());
assert!(backend.peek("nonexistent").is_err());
assert!(backend.size("nonexistent").is_err());
assert!(backend.status("nonexistent").is_err());
assert!(backend.clear("nonexistent").is_err());
}
#[test]
fn test_sqlite_backend_basic_flow() {
let (_dir, _db_path, mut backend) = sqlite_backend();
backend.create_queue("test").unwrap();
let msg1 = Message::new(b"message 1".to_vec());
let msg2 = Message::new(b"message 2".to_vec());
let msg1_id = msg1.id.clone();
backend.enqueue("test", msg1).unwrap();
backend.enqueue("test", msg2).unwrap();
assert_eq!(
backend.status("test").unwrap(),
QueueState {
pending: 2,
processing: 0,
}
);
let first = backend.dequeue("test").unwrap().unwrap();
assert_eq!(first.data, b"message 1");
assert_eq!(backend.size("test").unwrap(), 1);
assert_eq!(
backend.status("test").unwrap(),
QueueState {
pending: 1,
processing: 1,
}
);
let unacked = backend.list_unacked("test").unwrap();
assert_eq!(unacked.len(), 2);
assert_eq!(unacked[0].id, msg1_id);
assert!(backend.ack("test", &msg1_id).unwrap());
assert_eq!(
backend.status("test").unwrap(),
QueueState {
pending: 1,
processing: 0,
}
);
let second = backend.dequeue("test").unwrap().unwrap();
assert_eq!(second.data, b"message 2");
}
#[test]
fn test_sqlite_backend_preserves_non_utf8_payload_bytes() {
let (_dir, _db_path, mut backend) = sqlite_backend();
backend.create_queue("test").unwrap();
let payload = vec![0xff, 0x00, 0x80, b'a'];
backend
.enqueue("test", Message::new(payload.clone()))
.unwrap();
let peeked = backend.peek("test").unwrap().unwrap();
assert_eq!(peeked.data, payload);
let dequeued = backend.dequeue("test").unwrap().unwrap();
assert_eq!(dequeued.data, payload);
}
#[test]
fn test_stored_message_preserves_timestamp_validity() {
let missing = StoredMessage {
id: "missing".to_string(),
data: b"payload".to_vec(),
timestamp: None,
}
.into_message();
assert!(!missing.timestamp_valid);
let invalid = StoredMessage {
id: "invalid".to_string(),
data: b"payload".to_vec(),
timestamp: Some(serde_json::Value::String("not-a-time".to_string())),
}
.into_message();
assert!(!invalid.timestamp_valid);
let valid = StoredMessage {
id: "valid".to_string(),
data: b"payload".to_vec(),
timestamp: Some(serde_json::Value::Number(1234.into())),
}
.into_message();
assert!(valid.timestamp_valid);
assert_eq!(unix_secs(valid.timestamp), 1234);
let precise_timestamp = UNIX_EPOCH + Duration::new(1_234, 567_890_123);
let precise = StoredMessage::from_message(&Message {
id: "precise".to_string(),
data: b"payload".to_vec(),
timestamp: precise_timestamp,
timestamp_valid: true,
})
.into_message();
assert!(precise.timestamp_valid);
assert_eq!(precise.timestamp, precise_timestamp);
}
#[test]
fn test_sqlite_backend_preserves_subsecond_enqueue_timestamp() {
let (_dir, _db_path, mut backend) = sqlite_backend();
backend.create_queue("test").unwrap();
let timestamp = UNIX_EPOCH + Duration::new(1_700_000_000, 987_654_321);
backend
.enqueue(
"test",
Message {
id: "precise".to_string(),
data: b"payload".to_vec(),
timestamp,
timestamp_valid: true,
},
)
.unwrap();
let dequeued = backend.dequeue("test").unwrap().unwrap();
assert_eq!(dequeued.timestamp, timestamp);
}
#[test]
fn test_sqlite_backend_recover_stale() {
let (_dir, db_path_str, _backend) = sqlite_backend();
let msg_id = {
let mut backend =
SQLiteQueueBackend::open(&db_path_str, SQLiteQueueOptions::default()).unwrap();
backend.create_queue("test").unwrap();
let msg = Message::new(b"recover me".to_vec());
let msg_id = msg.id.clone();
backend.enqueue("test", msg).unwrap();
let dequeued = backend.dequeue("test").unwrap().unwrap();
assert_eq!(dequeued.id, msg_id);
msg_id
};
let mut reopened =
SQLiteQueueBackend::open(&db_path_str, SQLiteQueueOptions::default()).unwrap();
let recovered = reopened.dequeue("test").unwrap().unwrap();
assert_eq!(recovered.id, msg_id);
assert_eq!(recovered.data, b"recover me");
}
#[test]
fn test_sqlite_backend_dequeue_legacy_go_row() {
let (_dir, db_path_str, mut backend) = sqlite_backend();
backend.create_queue("Semantic").unwrap();
drop(backend);
let conn = Connection::open(&db_path_str).unwrap();
conn.execute(
"INSERT INTO queue_messages (queue_name, message_id, data, timestamp, status)
VALUES (?1, ?2, ?3, ?4, 'pending')",
params![
"Semantic",
"legacy-msg-id",
r#"{"id":"legacy-msg-id","data":"{\"id\":\"semantic-inner\",\"uri\":\"viking://resources/demo\",\"context_type\":\"resource\",\"status\":\"pending\",\"timestamp\":1776411350,\"recursive\":true,\"account_id\":\"default\",\"user_id\":\"default\",\"peer_id\":\"default\",\"role\":\"root\",\"skip_vectorization\":false,\"telemetry_id\":\"tm_demo\",\"target_uri\":null,\"lifecycle_lock_handle_id\":\"lock-demo\",\"is_code_repo\":false,\"changes\":null}","timestamp":"2026-04-17T15:37:39.287855+08:00"}"#,
1776411459_i64,
],
)
.unwrap();
drop(conn);
let mut reopened =
SQLiteQueueBackend::open(&db_path_str, SQLiteQueueOptions::default()).unwrap();
let msg = reopened.dequeue("Semantic").unwrap().unwrap();
let payload = String::from_utf8(msg.data).unwrap();
assert!(payload.contains("\"uri\":\"viking://resources/demo\""));
}
#[test]
fn test_sqlite_backend_backfills_queue_metadata_from_legacy_rows() {
let dir = tempdir().unwrap();
let db_path = dir.path().join("legacy.db");
let db_path_str = db_path.to_str().unwrap().to_string();
let conn = Connection::open(&db_path_str).unwrap();
conn.execute_batch(
r#"
CREATE TABLE queue_metadata (
queue_name TEXT PRIMARY KEY,
created_at INTEGER DEFAULT (strftime('%s', 'now')),
last_updated INTEGER DEFAULT (strftime('%s', 'now'))
);
CREATE TABLE queue_messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
queue_name TEXT NOT NULL,
message_id TEXT NOT NULL,
data TEXT NOT NULL,
timestamp INTEGER NOT NULL
);
INSERT INTO queue_messages (queue_name, message_id, data, timestamp)
VALUES ('legacy/semantic', 'legacy-id', '{"id":"legacy-id","data":"payload"}', 1776411459);
"#,
)
.unwrap();
drop(conn);
let backend =
SQLiteQueueBackend::open(&db_path_str, SQLiteQueueOptions::default()).unwrap();
assert!(backend.queue_exists("legacy/semantic"));
assert_eq!(backend.list_queues("legacy"), vec!["legacy/semantic"]);
drop(backend);
let conn = Connection::open(&db_path_str).unwrap();
let migrated_indexes = conn
.prepare(
"SELECT name FROM sqlite_master
WHERE type = 'index'
AND name IN ('idx_queue_message_id', 'idx_queue_status')
ORDER BY name",
)
.unwrap()
.query_map([], |row| row.get::<_, String>(0))
.unwrap()
.collect::<std::result::Result<Vec<_>, _>>()
.unwrap();
assert_eq!(
migrated_indexes,
vec![
"idx_queue_message_id".to_string(),
"idx_queue_status".to_string()
]
);
}
#[test]
fn test_sqlite_backend_enables_full_auto_vacuum_for_new_databases() {
let (_dir, db_path, _backend) = sqlite_backend();
let conn = Connection::open(db_path).unwrap();
let auto_vacuum: i64 = conn.query_row("PRAGMA auto_vacuum", [], |row| row.get(0)).unwrap();
assert_eq!(auto_vacuum, 1);
}
#[test]
fn test_sqlite_backend_does_not_rewrite_existing_auto_vacuum_mode() {
let dir = tempdir().unwrap();
let db_path = dir.path().join("legacy-auto-vacuum.db");
let db_path_str = db_path.to_str().unwrap().to_string();
let conn = Connection::open(&db_path_str).unwrap();
conn.execute_batch(
r#"
PRAGMA auto_vacuum=NONE;
CREATE TABLE queue_metadata (
queue_name TEXT PRIMARY KEY,
created_at INTEGER DEFAULT (strftime('%s', 'now')),
last_updated INTEGER DEFAULT (strftime('%s', 'now'))
);
CREATE TABLE queue_messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
queue_name TEXT NOT NULL,
message_id TEXT NOT NULL,
data TEXT NOT NULL,
timestamp INTEGER NOT NULL
);
"#,
)
.unwrap();
drop(conn);
let _backend = SQLiteQueueBackend::open(&db_path_str, SQLiteQueueOptions::default()).unwrap();
let conn = Connection::open(&db_path_str).unwrap();
let auto_vacuum: i64 = conn.query_row("PRAGMA auto_vacuum", [], |row| row.get(0)).unwrap();
assert_eq!(auto_vacuum, 0);
}
#[test]
fn test_sqlite_backend_operations_on_nonexistent_queue() {
let (_dir, _db_path, mut backend) = sqlite_backend();
assert!(backend
.enqueue("nonexistent", Message::new(b"data".to_vec()))
.is_err());
assert!(backend.dequeue("nonexistent").is_err());
assert!(backend.peek("nonexistent").is_err());
assert!(backend.size("nonexistent").is_err());
assert!(backend.clear("nonexistent").is_err());
assert!(backend.get_last_enqueue_time("nonexistent").is_err());
assert!(backend.ack("nonexistent", "msg-id").is_err());
}
}