Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
//! The companion's own durable queue: one private SQLite database per queue
//! directory. It never opens ai-memory's database or wiki.
//!
//! The binding fixes the destination, producer identity and scope. Pending
//! events keep their body values until acknowledgement. Receipts retain keys
//! and content hashes for duplicate recognition during the retry window.
//! Session records reject a native session changing agents within one queue.
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use anyhow::{Context, Result, bail};
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
use crate::fsguard;
use crate::identity::{MAX_BODY_BYTES, RETRY_WINDOW_MS, TERMINAL_EVENTS, ValidEvent};
/// Undelivered events allowed in one queue.
pub const MAX_PENDING_ITEMS: i64 = 50_000;
/// Undelivered body bytes allowed in one queue.
pub const MAX_PENDING_BYTES: i64 = 64 * 1024 * 1024;
/// Pending events plus retained receipts. Admission reserves one receipt for
/// each new event so acknowledgement cannot exceed the limit.
pub const MAX_RECEIPTS: i64 = 200_000;
/// Session-to-agent pins retained at once, enforced the same way.
pub const MAX_SESSION_PINS: i64 = 50_000;
const SCHEMA_VERSION: &str = "2";
const IDENTITY: &str = "ai-memory-relay-queue";
#[cfg(test)]
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum OpenPhase {
AfterPreflight,
BeforeJournal,
JournalBusy,
}
const SCHEMA: &str = "
CREATE TABLE IF NOT EXISTS meta(
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS binding(
id INTEGER PRIMARY KEY CHECK(id = 1),
server_url TEXT NOT NULL,
producer TEXT NOT NULL,
actor TEXT NOT NULL,
workspace TEXT NOT NULL,
project TEXT NOT NULL,
created_at_ms INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS pending(
seq INTEGER PRIMARY KEY AUTOINCREMENT,
ingest_key TEXT NOT NULL UNIQUE,
event_id TEXT NOT NULL,
agent TEXT NOT NULL,
event TEXT NOT NULL,
session_id TEXT NOT NULL,
cwd TEXT NOT NULL,
body_json TEXT NOT NULL,
body_sha256 TEXT NOT NULL,
body_bytes INTEGER NOT NULL,
first_seen_ms INTEGER NOT NULL,
first_attempt_ms INTEGER,
attempts INTEGER NOT NULL DEFAULT 0,
last_error TEXT
);
CREATE INDEX IF NOT EXISTS pending_session ON pending(agent, session_id, seq);
CREATE TABLE IF NOT EXISTS receipt(
ingest_key TEXT PRIMARY KEY,
body_sha256 TEXT NOT NULL,
first_attempt_ms INTEGER NOT NULL,
delivered_at_ms INTEGER NOT NULL,
outcome TEXT
);
CREATE INDEX IF NOT EXISTS receipt_first_attempt ON receipt(first_attempt_ms);
CREATE TABLE IF NOT EXISTS session_agent(
session_id TEXT PRIMARY KEY,
agent TEXT NOT NULL,
last_seen_ms INTEGER NOT NULL
);
";
/// What `init` bound this queue directory to.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Binding {
pub server_url: String,
pub producer: String,
pub actor: String,
pub workspace: String,
pub project: String,
}
/// Whether `init` created the binding or recognized an identical one.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BindOutcome {
Created,
Recognized,
}
/// Per-file result of `enqueue`.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct EnqueueReport {
pub accepted: usize,
/// Same id, byte-identical body: already pending or already delivered.
pub recognized: usize,
}
/// One item selected for delivery.
#[derive(Debug, Clone)]
pub struct PendingItem {
pub seq: i64,
pub ingest_key: String,
pub event_id: String,
pub agent: String,
pub event: String,
pub session_id: String,
pub body_json: String,
pub body_bytes: i64,
pub first_seen_ms: i64,
pub first_attempt_ms: Option<i64>,
}
impl PendingItem {
/// Session coordinate: one head per `(agent, session_id)` per batch.
pub fn session(&self) -> (String, String) {
(self.agent.clone(), self.session_id.clone())
}
}
/// A batch plus what was held back while building it.
#[derive(Debug, Default, Clone)]
pub struct Batch {
pub items: Vec<PendingItem>,
/// Sessions whose head is past the retry window. The whole session is held:
/// skipping its head and sending the next event would reorder it.
pub blocked_sessions: usize,
/// Sessions skipped because the byte budget filled first.
pub budget_deferred: usize,
}
/// Counters for `status`. Deliberately payload-free.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct Stats {
pub pending_items: i64,
pub pending_bytes: i64,
pub pending_sessions: i64,
pub attempted_items: i64,
pub expired_items: i64,
pub oldest_pending_age_ms: i64,
pub max_attempts: i64,
pub receipts: i64,
pub known_sessions: i64,
/// Outcomes within the retained first-attempt window, with fixed keys.
pub receipt_outcomes: std::collections::BTreeMap<String, i64>,
}
/// The companion's queue database.
pub struct Queue {
conn: Connection,
path: PathBuf,
}
impl std::fmt::Debug for Queue {
// Path only: never connection internals, never row data.
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Queue").field("path", &self.path).finish()
}
}
impl Queue {
/// Open (or create) the queue database inside an already-prepared directory.
///
/// Check an existing file's identity before schema writes. Schema and
/// metadata commit together, so an interrupted initialization can reopen
/// the empty database left by rollback.
pub fn open(dir: &Path) -> Result<Self> {
#[cfg(test)]
{
Self::open_for_test(dir, Duration::from_secs(10), &mut |_| {})
}
#[cfg(not(test))]
{
Self::open_inner(dir, Duration::from_secs(10))
}
}
#[cfg(test)]
pub(crate) fn open_for_test(
dir: &Path,
timeout: Duration,
hook: &mut dyn FnMut(OpenPhase),
) -> Result<Self> {
Self::open_inner(dir, timeout, hook)
}
fn open_inner(
dir: &Path,
timeout: Duration,
#[cfg(test)] hook: &mut dyn FnMut(OpenPhase),
) -> Result<Self> {
let path = dir.join(fsguard::DB_FILE);
fsguard::check_queue_file(&path)?;
for name in fsguard::SIDECARS {
fsguard::check_queue_file(&dir.join(name))?;
}
if !path.exists() {
// Own the mode before SQLite ever opens the file: the `-wal` and
// `-shm` sidecars inherit the database's permissions, so creating it
// under the ambient umask would make them world-readable.
fsguard::create_private_file(&path)?;
fsguard::check_queue_file(&path)?;
}
let mut conn = Connection::open(&path)
.with_context(|| format!("open relay queue at {}", path.display()))?;
conn.busy_timeout(timeout)?;
classify(&conn, &path)?;
#[cfg(test)]
hook(OpenPhase::AfterPreflight);
// Recheck under the write lock: another opener may have initialized or
// migrated since the read-only preflight.
{
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
match classify(&tx, &path)? {
DbState::Fresh => {
tx.execute_batch(SCHEMA)?;
tx.execute(
"INSERT INTO meta(key, value) VALUES('identity', ?1), ('schema_version', ?2)",
params![IDENTITY, SCHEMA_VERSION],
)?;
}
DbState::V1 => {
tx.execute_batch("ALTER TABLE receipt ADD COLUMN outcome TEXT;")?;
tx.execute(
"UPDATE meta SET value = ?1 WHERE key = 'schema_version'",
params![SCHEMA_VERSION],
)?;
}
DbState::Ready => {}
}
tx.commit()?;
}
#[cfg(test)]
hook(OpenPhase::BeforeJournal);
configure(
&conn,
&path,
timeout,
#[cfg(test)]
hook,
)
.with_context(|| format!("configure relay queue at {}", path.display()))?;
Ok(Self { conn, path })
}
/// Path of the database, for operator-facing messages.
pub fn path(&self) -> &Path {
&self.path
}
/// Bind the queue to a destination, producer, actor and scope.
///
/// Repeated initialization must preserve the destination and identity of
/// queued events, including events whose acknowledgement was lost.
pub fn bind(&mut self, binding: &Binding, now_ms: i64) -> Result<BindOutcome> {
let tx = self
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let outcome = match read_binding(&tx)? {
Some(existing) if existing == *binding => BindOutcome::Recognized,
Some(existing) => {
let mut changed = Vec::new();
let mut note = |label: &str, old: &str, new: &str| {
if old != new {
changed.push(format!("{label}: {old:?} -> {new:?}"));
}
};
note("server-url", &existing.server_url, &binding.server_url);
note("producer", &existing.producer, &binding.producer);
note("actor", &existing.actor, &binding.actor);
note("workspace", &existing.workspace, &binding.workspace);
note("project", &existing.project, &binding.project);
bail!(
"binding mismatch: this queue is already bound ({}). \
Use a separate --queue-dir for a different destination or identity",
changed.join(", ")
);
}
None => {
tx.execute(
"INSERT INTO binding(id, server_url, producer, actor, workspace, project, created_at_ms)
VALUES(1, ?1, ?2, ?3, ?4, ?5, ?6)",
params![
binding.server_url,
binding.producer,
binding.actor,
binding.workspace,
binding.project,
now_ms,
],
)?;
BindOutcome::Created
}
};
tx.commit()?;
Ok(outcome)
}
/// The binding, or a clear error telling the operator to run `init` first.
pub fn binding(&self) -> Result<Binding> {
read_binding(&self.conn)?.ok_or_else(|| {
anyhow::anyhow!("queue is not bound yet; run `ai-memory-relay init` first")
})
}
/// Persist a whole validated file, all-or-nothing.
///
/// `first_seen_ms` is stamped here for reporting. It is *not* the retry
/// clock: that starts at the first durable attempt, recorded before the
/// first HTTP request (see [`Queue::stamp_attempt`]).
pub fn enqueue(&mut self, events: &[ValidEvent], now_ms: i64) -> Result<EnqueueReport> {
let tx = self
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let (mut items, mut bytes): (i64, i64) = tx.query_row(
"SELECT COUNT(*), COALESCE(SUM(body_bytes), 0) FROM pending",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
// Replay protection is bounded, and the bound covers pending *and*
// receipts: every pending item becomes a receipt when it is
// acknowledged, so counting receipts alone would let the queue commit to
// more protection than the cap allows. Pruning stays reserved for
// entries past the 30-day window, so a full queue refuses new ids rather
// than forgetting a recently delivered one. A duplicate is still
// recognized while full, because recognizing it adds nothing.
let receipts: i64 = tx.query_row("SELECT COUNT(*) FROM receipt", [], |r| r.get(0))?;
let mut pins: i64 = tx.query_row("SELECT COUNT(*) FROM session_agent", [], |r| r.get(0))?;
let mut report = EnqueueReport::default();
let mut seen_keys: HashMap<&str, &str> = HashMap::new();
let mut seen_sessions: HashMap<&str, &str> = HashMap::new();
for event in events {
// A session id belongs to exactly one wire agent. Two agents on one
// native session is the producer bug the server can only answer with
// an acknowledged SessionCollision drop.
let bound: Option<String> = tx
.query_row(
"SELECT agent FROM session_agent WHERE session_id = ?1",
params![event.session_id],
|r| r.get(0),
)
.optional()?;
let bound = bound.or_else(|| {
seen_sessions
.get(event.session_id.as_str())
.map(|a| (*a).to_owned())
});
if bound.is_none() {
pins += 1;
if pins > MAX_SESSION_PINS {
bail!(
"session pin table is full: {MAX_SESSION_PINS} session(s) already pinned \
to an agent. Nothing was enqueued and no pin was forgotten; pins are \
released as they pass the 30-day retry window"
);
}
}
if let Some(bound) = bound
&& bound != event.agent
{
bail!(
"session identity conflict: session {:?} is already bound to agent {:?} in \
this queue, but item with event_id {} claims agent {:?}. Nothing was \
enqueued; a native session belongs to one harness",
event.session_id,
bound,
event.event_id,
event.agent
);
}
seen_sessions.insert(&event.session_id, &event.agent);
if let Some(previous) = seen_keys.insert(&event.ingest_key, &event.body_sha256)
&& previous != event.body_sha256
{
bail!(collision_message(
&event.ingest_key,
"another item in this file"
));
}
let known: Option<String> = tx
.query_row(
"SELECT body_sha256 FROM pending WHERE ingest_key = ?1
UNION ALL
SELECT body_sha256 FROM receipt WHERE ingest_key = ?1",
params![event.ingest_key],
|r| r.get(0),
)
.optional()?;
if let Some(known) = known {
if known != event.body_sha256 {
bail!(collision_message(
&event.ingest_key,
"an item already recorded in this queue"
));
}
report.recognized += 1;
continue;
}
let body_bytes = event.body_json.len() as i64;
items += 1;
bytes += body_bytes;
if items > MAX_PENDING_ITEMS || bytes > MAX_PENDING_BYTES {
bail!(
"queue is full: {items} items / {bytes} bytes would exceed the \
{MAX_PENDING_ITEMS}-item / {MAX_PENDING_BYTES}-byte limit. Nothing was \
enqueued and nothing was dropped; flush the queue first"
);
}
// Reserve the receipt this item will need once it is acknowledged.
let protected = receipts + items;
if protected > MAX_RECEIPTS {
bail!(
"replay protection is full: {receipts} acknowledged key(s) plus {items} \
pending would need {protected} slots, over the {MAX_RECEIPTS} limit. \
Nothing was enqueued and nothing was forgotten; slots are released as \
entries pass the 30-day retry window, or start a fresh --queue-dir"
);
}
tx.execute(
"INSERT INTO pending(
ingest_key, event_id, agent, event, session_id, cwd,
body_json, body_sha256, body_bytes, first_seen_ms)
VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
params![
event.ingest_key,
event.event_id,
event.agent,
event.event,
event.session_id,
event.cwd,
event.body_json,
event.body_sha256,
body_bytes,
now_ms,
],
)?;
report.accepted += 1;
}
for (session_id, agent) in seen_sessions {
tx.execute(
"INSERT INTO session_agent(session_id, agent, last_seen_ms) VALUES(?1, ?2, ?3)
ON CONFLICT(session_id) DO UPDATE SET last_seen_ms = excluded.last_seen_ms",
params![session_id, agent, now_ms],
)?;
}
tx.commit()?;
Ok(report)
}
/// Pick at most one pending head per `(agent, session_id)`, oldest first,
/// within both an item count and a wire-byte budget.
///
/// One head per session per batch is what makes a partial acknowledgement
/// safe: two events of one session are never in flight together, so no ack
/// subset can commit event 2 while event 1 is still pending, and a terminal
/// event can never overtake an earlier one. `deferred` holds sessions this
/// flush already stopped on; they are skipped without touching their order.
pub fn select_batch(
&self,
limit: usize,
byte_budget: usize,
now_ms: i64,
deferred: &HashSet<(String, String)>,
cost_of: impl Fn(&PendingItem) -> usize,
) -> Result<Batch> {
let limit = limit.clamp(1, crate::identity::MAX_BATCH_ITEMS);
let mut stmt = self.conn.prepare(
"SELECT seq, ingest_key, event_id, agent, event, session_id, body_json,
body_bytes, first_seen_ms, first_attempt_ms
FROM pending ORDER BY seq ASC",
)?;
let mut rows = stmt.query([])?;
let mut seen: HashSet<(String, String)> = HashSet::new();
let mut batch = Batch::default();
let mut spent = 0usize;
while let Some(row) = rows.next()? {
if batch.items.len() >= limit {
break;
}
let item = PendingItem {
seq: row.get(0)?,
ingest_key: row.get(1)?,
event_id: row.get(2)?,
agent: row.get(3)?,
event: row.get(4)?,
session_id: row.get(5)?,
body_json: row.get(6)?,
body_bytes: row.get(7)?,
first_seen_ms: row.get(8)?,
first_attempt_ms: row.get(9)?,
};
let session = item.session();
if !seen.insert(session.clone()) {
continue;
}
if deferred.contains(&session) {
continue;
}
if is_expired(item.first_attempt_ms, now_ms) {
batch.blocked_sessions += 1;
continue;
}
let cost = cost_of(&item);
if !batch.items.is_empty() && spent + cost > byte_budget {
batch.budget_deferred += 1;
continue;
}
debug_assert!(
!TERMINAL_EVENTS.contains(&item.event.as_str()) || self.session_head(&item)?,
"a terminal event was selected while its session had an older pending item"
);
spent += cost;
batch.items.push(item);
}
Ok(batch)
}
fn session_head(&self, item: &PendingItem) -> Result<bool> {
let older: i64 = self.conn.query_row(
"SELECT COUNT(*) FROM pending WHERE agent = ?1 AND session_id = ?2 AND seq < ?3",
params![item.agent, item.session_id, item.seq],
|r| r.get(0),
)?;
Ok(older == 0)
}
/// Record the durable start of the retry window, **before** the request.
///
/// `COALESCE` is the whole point: a restart, a lost ack, or a later retry
/// never moves the clock forward, so the 30-day guard measures from the
/// first time this event was actually put on the wire.
pub fn stamp_attempt(&mut self, keys: &[String], now_ms: i64) -> Result<()> {
let tx = self
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
for key in keys {
tx.execute(
"UPDATE pending
SET first_attempt_ms = COALESCE(first_attempt_ms, ?2),
attempts = attempts + 1
WHERE ingest_key = ?1",
params![key, now_ms],
)?;
}
tx.commit()?;
Ok(())
}
/// Move acknowledged items to compact receipts, in one transaction.
///
/// An acknowledgement means delivered *or* deliberately dropped by server
/// policy (a capture-protocol drop, a subagent drop, a session-collision
/// drop). It never promises an observation was written.
pub fn confirm(&mut self, keys: &[(String, &str)], now_ms: i64) -> Result<usize> {
if keys.is_empty() {
return Ok(0);
}
let tx = self
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let mut moved = 0;
for (key, outcome) in keys {
let row: Option<(String, Option<i64>, i64)> = tx
.query_row(
"SELECT body_sha256, first_attempt_ms, first_seen_ms
FROM pending WHERE ingest_key = ?1",
params![key],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)
.optional()?;
let Some((body_sha256, first_attempt_ms, first_seen_ms)) = row else {
continue;
};
tx.execute(
"INSERT INTO receipt(ingest_key, body_sha256, first_attempt_ms, delivered_at_ms, outcome)
VALUES(?1, ?2, ?3, ?4, ?5)
ON CONFLICT(ingest_key) DO UPDATE SET delivered_at_ms = excluded.delivered_at_ms,
outcome = COALESCE(NULLIF(receipt.outcome, 'unknown'), excluded.outcome)",
params![
key,
body_sha256,
first_attempt_ms.unwrap_or(first_seen_ms),
now_ms,
crate::ack::outcome(outcome)
],
)?;
tx.execute("DELETE FROM pending WHERE ingest_key = ?1", params![key])?;
moved += 1;
}
tx.commit()?;
Ok(moved)
}
/// Charge one item with a delivery failure. The class is a short label,
/// never a server body and never payload.
pub fn record_failure(&mut self, key: &str, class: &str) -> Result<()> {
let class: String = class.chars().filter(|c| !c.is_control()).take(80).collect();
self.conn.execute(
"UPDATE pending SET last_error = ?2 WHERE ingest_key = ?1",
params![key, class],
)?;
Ok(())
}
/// Release expired receipts and session records without pending events.
pub fn prune(&mut self, now_ms: i64) -> Result<usize> {
let cutoff = now_ms.saturating_sub(RETRY_WINDOW_MS);
let tx = self
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let receipts = tx.execute(
"DELETE FROM receipt WHERE first_attempt_ms < ?1",
params![cutoff],
)?;
let sessions = tx.execute(
"DELETE FROM session_agent WHERE last_seen_ms < ?1
AND session_id NOT IN (SELECT session_id FROM pending)",
params![cutoff],
)?;
tx.commit()?;
Ok(receipts + sessions)
}
/// Payload-free counters.
pub fn stats(&self, now_ms: i64) -> Result<Stats> {
let cutoff = now_ms.saturating_sub(RETRY_WINDOW_MS);
let (pending_items, pending_bytes, oldest, max_attempts, attempted_items) =
self.conn.query_row(
"SELECT COUNT(*), COALESCE(SUM(body_bytes), 0), COALESCE(MIN(first_seen_ms), 0),
COALESCE(MAX(attempts), 0), COUNT(first_attempt_ms)
FROM pending",
[],
|r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, i64>(1)?,
r.get::<_, i64>(2)?,
r.get::<_, i64>(3)?,
r.get::<_, i64>(4)?,
))
},
)?;
let pending_sessions: i64 = self.conn.query_row(
"SELECT COUNT(*) FROM (SELECT DISTINCT agent, session_id FROM pending)",
[],
|r| r.get(0),
)?;
let expired_items: i64 = self.conn.query_row(
"SELECT COUNT(*) FROM pending WHERE first_attempt_ms IS NOT NULL
AND first_attempt_ms <= ?1",
params![cutoff],
|r| r.get(0),
)?;
let receipts: i64 = self
.conn
.query_row("SELECT COUNT(*) FROM receipt", [], |r| r.get(0))?;
let known_sessions: i64 =
self.conn
.query_row("SELECT COUNT(*) FROM session_agent", [], |r| r.get(0))?;
let mut receipt_outcomes = crate::ack::outcome_counts();
let mut stmt = self.conn.prepare(
"SELECT outcome, COUNT(*) FROM receipt WHERE first_attempt_ms >= ?1 GROUP BY outcome",
)?;
for row in stmt.query_map(params![cutoff], |r| {
Ok((r.get::<_, Option<String>>(0)?, r.get::<_, i64>(1)?))
})? {
let (value, count) = row?;
let key = crate::ack::outcome(value.as_deref().unwrap_or("unknown"));
if let Some(total) = receipt_outcomes.get_mut(key) {
*total += count;
}
}
Ok(Stats {
pending_items,
pending_bytes,
pending_sessions,
attempted_items,
expired_items,
oldest_pending_age_ms: if pending_items == 0 {
0
} else {
now_ms.saturating_sub(oldest)
},
max_attempts,
receipts,
known_sessions,
receipt_outcomes,
})
}
}
/// Past the retry window an event may not be auto-resent.
///
/// Refuse the boundary itself. This comparison depends on an accurate host
/// clock; a backwards clock change can extend retries past server key expiry.
pub fn is_expired(first_attempt_ms: Option<i64>, now_ms: i64) -> bool {
match first_attempt_ms {
Some(started) => now_ms.saturating_sub(started) >= RETRY_WINDOW_MS,
None => false,
}
}
fn collision_message(key: &str, other: &str) -> String {
format!(
"identity collision: ingest key {key} was already used by {other} with different content. \
The same producer event id must always carry the same body; nothing was enqueued. \
Body content is deliberately not shown. Per-body limit: {MAX_BODY_BYTES} bytes"
)
}
/// What an existing file turned out to be.
///
/// Schema and identity commit together. Recovery therefore accepts an empty
/// database or an identified queue; generic table names do not establish ownership.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum DbState {
/// A SQLite file with no user objects: empty, or rolled back to empty.
Fresh,
/// A relay queue of this exact schema version.
Ready,
/// Version one is upgraded transactionally.
V1,
}
fn configure(
conn: &Connection,
path: &Path,
timeout: Duration,
#[cfg(test)] hook: &mut dyn FnMut(OpenPhase),
) -> Result<()> {
// WAL needs an exclusive lock outside a transaction. SQLite can return
// BUSY without calling its busy handler when other openers hold read/write
// locks. Own this phase's wait budget instead of restarting a 10s wait.
let deadline = Instant::now() + timeout;
conn.busy_timeout(Duration::ZERO)?;
let result = loop {
let attempt = (|| -> Result<()> {
// Keep the same connection and recheck identity/version metadata
// before every WAL attempt, including after a lock was released.
if classify(conn, path)? != DbState::Ready {
bail!(
"relay queue identity or schema version changed before journal configuration"
);
}
conn.execute_batch(
"PRAGMA journal_mode=WAL;\nPRAGMA synchronous=FULL;\nPRAGMA foreign_keys=ON;",
)?;
Ok(())
})();
match attempt {
Err(error) if is_lock_contention(&error) => {
#[cfg(test)]
hook(OpenPhase::JournalBusy);
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
break Err(error);
}
std::thread::sleep(remaining.min(Duration::from_millis(10)));
if Instant::now() >= deadline {
break Err(error);
}
}
other => break other,
}
};
conn.busy_timeout(timeout)?;
result.map_err(|error| {
if is_lock_contention(&error) {
error.context("relay queue busy; retry opening the same queue")
} else {
error
}
})
}
pub(crate) fn is_lock_contention(error: &anyhow::Error) -> bool {
matches!(error.downcast_ref::<rusqlite::Error>(),
Some(rusqlite::Error::SqliteFailure(error, _))
if matches!(error.code, rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked))
}
/// Classify an existing file before touching it. Never mutates.
fn classify(conn: &Connection, path: &Path) -> Result<DbState> {
let refuse = |detail: String| -> anyhow::Error {
anyhow::anyhow!(
"{} is not a usable relay queue ({detail}); nothing was modified. \
Point --queue-dir at an empty directory or restore the original queue",
path.display()
)
};
// Tables, views, triggers and standalone indexes all count: any user object
// means the file belongs to something, and that something is not this relay
// unless it also carries the identity row.
let objects: Result<i64, rusqlite::Error> = conn.query_row(
"SELECT COUNT(*) FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'",
[],
|r| r.get(0),
);
// A file that is not a SQLite database fails right here, before any write.
let objects = objects.map_err(|e| {
let error = anyhow::Error::new(e);
if is_lock_contention(&error) {
error
} else {
error.context(refuse("cannot read its object list".into()))
}
})?;
if objects == 0 {
return Ok(DbState::Fresh);
}
let rows: Result<Vec<(String, String)>, rusqlite::Error> = (|| {
let mut stmt =
conn.prepare("SELECT key, value FROM meta WHERE key IN ('identity','schema_version')")?;
let mapped = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?;
mapped.collect()
})();
let meta: HashMap<String, String> = rows
.map_err(|e| {
let error = anyhow::Error::new(e);
if is_lock_contention(&error) {
error
} else {
error.context(refuse("it carries no readable relay metadata".into()))
}
})?
.into_iter()
.collect();
match (
meta.get("identity").map(String::as_str),
meta.get("schema_version").map(String::as_str),
) {
(Some(IDENTITY), Some("1")) => Ok(DbState::V1),
(Some(IDENTITY), Some(SCHEMA_VERSION)) => Ok(DbState::Ready),
(Some(IDENTITY), Some(other)) => Err(refuse(format!(
"schema version {other} but this build speaks {SCHEMA_VERSION}"
))),
(Some(other), _) => Err(refuse(format!("identity is {other:?}"))),
_ => Err(refuse(
"it holds user objects but no relay identity row".into(),
)),
}
}
fn read_binding(conn: &Connection) -> Result<Option<Binding>> {
Ok(conn
.query_row(
"SELECT server_url, producer, actor, workspace, project FROM binding WHERE id = 1",
[],
|r| {
Ok(Binding {
server_url: r.get(0)?,
producer: r.get(1)?,
actor: r.get(2)?,
workspace: r.get(3)?,
project: r.get(4)?,
})
},
)
.optional()?)
}