Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
//! Redis-backed PathLock storage. Tree locks must fit within one account or
//! system scope; global roots cannot atomically cover the separate scope HASHes.
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use crate::metrics::duration::DurationMetrics;
use async_trait::async_trait;
use bytes::Bytes;
use opentelemetry::KeyValue;
use crate::cache_runtime::{
CacheError, CacheOperation, CacheRuntime, ScriptDefinition, ScriptRequest, ScriptValue,
};
use super::codec::LockTokenCodec;
use super::path::validate_normalized_logical_path;
use super::provider::{
AcquisitionChange, AtomicAcquisition, AtomicAcquisitionResult, PathLockHandleMode,
PathLockProvider,
};
use super::types::{LockToken, PathLockError, PathLockKind, PathLockRequest, PathLockResult};
const ACQUIRE_BATCH_ID: &str = "pathlock.acquire_batch.v2";
const ROLLBACK_BATCH_ID: &str = "pathlock.rollback_batch.v1";
const COMPARE_WRITE_ID: &str = "pathlock.compare_write.v1";
const COMPARE_REMOVE_ID: &str = "pathlock.compare_remove.v1";
const REFRESH_ID: &str = "pathlock.refresh.v1";
const REFRESH_BATCH_ID: &str = "pathlock.refresh_batch.v1";
// Bound each Lua execution so large leases do not monopolize Redis.
const REFRESH_BATCH_SIZE: usize = 256;
const IS_LOCKED_ID: &str = "pathlock.is_locked.v1";
const SCAN_DESCENDANTS_ID: &str = "pathlock.scan_descendants.v1";
const READ_MANY_ID: &str = "pathlock.read_many.v1";
const CREATE_ID: &str = "pathlock.create.v1";
// Observe requests approaching the default 20ms Redis command timeout.
const SLOW_ACQUIRE_THRESHOLD: Duration = Duration::from_millis(10);
macro_rules! token_script {
($body:literal) => {
concat!(
r#"
local u128_max = "340282366920938463463374607431768211455"
local function normalize_decimal(value)
if not value or not string.match(value, "^%d+$") then
return nil
end
local normalized = string.gsub(value, "^0+", "")
if normalized == "" then
return "0"
end
if string.len(normalized) > string.len(u128_max)
or (string.len(normalized) == string.len(u128_max)
and normalized > u128_max) then
return nil
end
return normalized
end
local function decimal_less(left, right)
left = normalize_decimal(left)
right = normalize_decimal(right)
if not left or not right then
return nil
end
if string.len(left) ~= string.len(right) then
return string.len(left) < string.len(right)
end
return left < right
end
local function decode_token(raw)
local p1 = string.find(raw, ":", 1, true)
local p2 = p1 and string.find(raw, ":", p1 + 1, true)
local p3 = p2 and string.find(raw, ":", p2 + 1, true)
if not p1 or not p2 or p3 then
return nil
end
local token_owner = string.sub(raw, 1, p1 - 1)
local time_ns = string.sub(raw, p1 + 1, p2 - 1)
local kind = string.sub(raw, p2 + 1)
if token_owner == "" or not normalize_decimal(time_ns)
or (kind ~= "E" and kind ~= "T") then
return nil
end
return { owner = token_owner, time_ns = time_ns, kind = kind }
end
local function is_normalized_path(path)
if not path or string.sub(path, 1, 1) ~= "/"
or (path ~= "/" and string.sub(path, -1) == "/")
or string.find(path, "//", 1, true)
or string.find(path, string.char(0), 1, true) then
return false
end
for part in string.gmatch(path, "[^/]+") do
if part == "." or part == ".." then
return false
end
end
return true
end
local function ancestors(path)
local result = {}
if path == "/" then
return result
end
table.insert(result, "/")
local start = 2
while true do
local slash = string.find(path, "/", start, true)
if not slash then
break
end
table.insert(result, string.sub(path, 1, slash - 1))
start = slash + 1
end
return result
end
"#,
$body
)
};
}
const ACQUIRE_BATCH_SCRIPT: &str = token_script!(
r#"
local key = KEYS[1]
local owner = ARGV[1]
local now_ns = ARGV[2]
local stale_before_ns = ARGV[3]
local ttl_ms = tonumber(ARGV[4])
local function is_descendant(path, root)
if root == "/" then
return path ~= "/"
end
return string.sub(path, 1, string.len(root) + 1) == root .. "/"
end
if not owner or owner == "" or string.find(owner, ":", 1, true)
or not normalize_decimal(now_ns) or not normalize_decimal(stale_before_ns)
or not ttl_ms or ttl_ms <= 0 or (#ARGV - 4) % 2 ~= 0 or #ARGV == 4 then
return { "invalid_request", "invalid acquire arguments" }
end
local requests = {}
local includes_tree = false
for index = 5, #ARGV, 2 do
local path = ARGV[index]
local kind = ARGV[index + 1]
if not is_normalized_path(path) or (kind ~= "E" and kind ~= "T") then
return { "invalid_request", "invalid path or lock kind" }
end
table.insert(requests, { path = path, kind = kind })
if kind == "T" then
includes_tree = true
end
end
local records = {}
if includes_tree then
local values = redis.call("HGETALL", key)
for index = 1, #values, 2 do
local path = values[index]
local raw = values[index + 1]
local token = decode_token(raw)
if not token then
return { "invalid_token", path, raw }
end
records[path] = { raw = raw, token = token }
end
else
local fields = {}
local seen = {}
local function add_field(path)
if not seen[path] then
seen[path] = true
table.insert(fields, path)
end
end
for _, request in ipairs(requests) do
add_field(request.path)
for _, ancestor in ipairs(ancestors(request.path)) do
add_field(ancestor)
end
end
local values = redis.call("HMGET", key, unpack(fields))
for index, raw in ipairs(values) do
if raw then
local token = decode_token(raw)
if not token then
return { "invalid_token", fields[index], raw }
end
records[fields[index]] = { raw = raw, token = token }
end
end
end
local stale = {}
local stale_seen = {}
local function is_other_stale(record)
if not record or record.token.owner == owner then
return false
end
return decimal_less(record.token.time_ns, stale_before_ns) == true
end
local function mark_stale(path, record)
if is_other_stale(record) and not stale_seen[path] then
stale_seen[path] = true
table.insert(stale, path)
end
end
for path, record in pairs(records) do
mark_stale(path, record)
end
for _, request in ipairs(requests) do
local target = records[request.path]
if target and target.token.owner ~= owner and not is_other_stale(target) then
return { "conflict", request.path, target.raw }
end
for _, ancestor in ipairs(ancestors(request.path)) do
local record = records[ancestor]
if record and record.token.owner ~= owner and record.token.kind == "T"
and not is_other_stale(record) then
return { "conflict", ancestor, record.raw }
end
end
if request.kind == "T" then
for path, record in pairs(records) do
if is_descendant(path, request.path)
and record.token.owner ~= owner and not is_other_stale(record) then
return { "conflict", path, record.raw }
end
end
end
end
local changes = {}
for _, request in ipairs(requests) do
local existing = records[request.path]
local replacement = owner .. ":" .. now_ns .. ":" .. request.kind
if existing and existing.token.owner == owner then
if request.kind == "T" and existing.token.kind == "E" then
table.insert(changes, {
path = request.path,
change = "upgraded",
previous = existing.raw,
replacement = replacement,
})
else
table.insert(changes, {
path = request.path,
change = "reentrant",
previous = "",
replacement = "",
})
end
else
table.insert(changes, {
path = request.path,
change = "created",
previous = "",
replacement = replacement,
})
end
end
local stale_batch_size = 1000
for start_index = 1, #stale, stale_batch_size do
local end_index = math.min(start_index + stale_batch_size - 1, #stale)
redis.call("HDEL", key, unpack(stale, start_index, end_index))
end
for _, change in ipairs(changes) do
if change.change ~= "reentrant" then
redis.call("HSET", key, change.path, change.replacement)
end
end
if redis.call("HLEN", key) > 0 then
redis.call("PEXPIRE", key, ttl_ms)
end
local result = { "ok", tostring(#stale) }
for _, change in ipairs(changes) do
table.insert(result, change.path)
table.insert(result, change.change)
table.insert(result, change.previous)
table.insert(result, change.replacement)
end
return result
"#
);
const ROLLBACK_BATCH_SCRIPT: &str = r#"
local key = KEYS[1]
local owner = ARGV[1]
local ttl_ms = tonumber(ARGV[2])
if not owner or owner == "" or not ttl_ms or ttl_ms <= 0
or (#ARGV - 2) % 4 ~= 0 then
return { "invalid_request", "invalid rollback arguments" }
end
local changes = {}
for index = 3, #ARGV, 4 do
local path = ARGV[index]
local change = ARGV[index + 1]
local previous = ARGV[index + 2]
local replacement = ARGV[index + 3]
if change ~= "created" and change ~= "reentrant" and change ~= "upgraded" then
return { "invalid_request", "invalid acquisition change" }
end
if change ~= "reentrant" then
local current = redis.call("HGET", key, path)
if current ~= replacement then
return { "changed", path }
end
if string.sub(replacement, 1, string.len(owner) + 1) ~= owner .. ":" then
return { "invalid_request", "replacement owner mismatch" }
end
end
table.insert(changes, {
path = path,
change = change,
previous = previous,
})
end
for index = #changes, 1, -1 do
local change = changes[index]
if change.change == "created" then
redis.call("HDEL", key, change.path)
elseif change.change == "upgraded" then
redis.call("HSET", key, change.path, change.previous)
end
end
if redis.call("HLEN", key) > 0 then
redis.call("PEXPIRE", key, ttl_ms)
end
return { "ok" }
"#;
const COMPARE_WRITE_SCRIPT: &str = r#"
local current = redis.call("HGET", KEYS[1], ARGV[1])
if current ~= ARGV[2] then
return 0
end
redis.call("HSET", KEYS[1], ARGV[1], ARGV[3])
redis.call("PEXPIRE", KEYS[1], ARGV[4])
return 1
"#;
const COMPARE_REMOVE_SCRIPT: &str = r#"
local current = redis.call("HGET", KEYS[1], ARGV[1])
if current ~= ARGV[2] then
return 0
end
redis.call("HDEL", KEYS[1], ARGV[1])
if redis.call("HLEN", KEYS[1]) > 0 then
redis.call("PEXPIRE", KEYS[1], ARGV[3])
end
return 1
"#;
const REFRESH_SCRIPT: &str = token_script!(
r#"
local raw = redis.call("HGET", KEYS[1], ARGV[1])
if not raw then
return { "missing" }
end
local token = decode_token(raw)
if not token then
return { "invalid_token", raw }
end
if token.owner ~= ARGV[2] then
return { "owner_mismatch" }
end
if not normalize_decimal(ARGV[3]) then
return { "invalid_request", "invalid time_ns" }
end
redis.call("HSET", KEYS[1], ARGV[1], token.owner .. ":" .. ARGV[3] .. ":" .. token.kind)
redis.call("PEXPIRE", KEYS[1], ARGV[4])
return { "ok" }
"#
);
const REFRESH_BATCH_SCRIPT: &str = token_script!(
r#"
local owner = ARGV[1]
local time_ns = ARGV[2]
local ttl_ms = ARGV[3]
if not normalize_decimal(time_ns) then
return { "invalid_request", "invalid time_ns" }
end
local results = {}
local refreshed = false
for index = 4, #ARGV do
local path = ARGV[index]
local raw = redis.call("HGET", KEYS[1], path)
if not raw then
table.insert(results, { "missing" })
else
local token = decode_token(raw)
if not token then
table.insert(results, { "invalid_token", raw })
elseif token.owner ~= owner then
table.insert(results, { "owner_mismatch" })
else
redis.call("HSET", KEYS[1], path, token.owner .. ":" .. time_ns .. ":" .. token.kind)
table.insert(results, { "ok" })
refreshed = true
end
end
end
if refreshed then
redis.call("PEXPIRE", KEYS[1], ttl_ms)
end
return results
"#
);
const IS_LOCKED_SCRIPT: &str = token_script!(
r#"
local path = ARGV[1]
local stale_before_ns = ARGV[2]
local ignore_stale = ARGV[3] == "1"
local fields = { path }
for _, ancestor in ipairs(ancestors(path)) do
table.insert(fields, ancestor)
end
local values = redis.call("HMGET", KEYS[1], unpack(fields))
for index, raw in ipairs(values) do
if raw then
local token = decode_token(raw)
if not token then
return { "invalid_token", fields[index], raw }
end
local live = not ignore_stale
or decimal_less(token.time_ns, stale_before_ns) ~= true
if live and (index == 1 or token.kind == "T") then
return { "ok", 1 }
end
end
end
return { "ok", 0 }
"#
);
const SCAN_DESCENDANTS_SCRIPT: &str = r#"
local root = ARGV[1]
local values = redis.call("HGETALL", KEYS[1])
local result = { "ok" }
for index = 1, #values, 2 do
local path = values[index]
local matches = root == "/" or path == root
or string.sub(path, 1, string.len(root) + 1) == root .. "/"
if matches then
table.insert(result, path)
table.insert(result, values[index + 1])
end
end
return result
"#;
const READ_MANY_SCRIPT: &str = r#"
if #ARGV == 0 then
return {}
end
return redis.call("HMGET", KEYS[1], unpack(ARGV))
"#;
const CREATE_SCRIPT: &str = r#"
local current = redis.call("HGET", KEYS[1], ARGV[1])
if current then
return { "exists", current }
end
redis.call("HSET", KEYS[1], ARGV[1], ARGV[2])
redis.call("PEXPIRE", KEYS[1], ARGV[3])
return { "created" }
"#;
macro_rules! script {
($id:expr, $lua:expr) => {
ScriptDefinition {
id: $id,
redis_lua: $lua,
}
};
}
const SCRIPT_DEFINITIONS: &[ScriptDefinition] = &[
script!(ACQUIRE_BATCH_ID, ACQUIRE_BATCH_SCRIPT),
script!(ROLLBACK_BATCH_ID, ROLLBACK_BATCH_SCRIPT),
script!(COMPARE_WRITE_ID, COMPARE_WRITE_SCRIPT),
script!(COMPARE_REMOVE_ID, COMPARE_REMOVE_SCRIPT),
script!(REFRESH_ID, REFRESH_SCRIPT),
script!(REFRESH_BATCH_ID, REFRESH_BATCH_SCRIPT),
script!(IS_LOCKED_ID, IS_LOCKED_SCRIPT),
script!(SCAN_DESCENDANTS_ID, SCAN_DESCENDANTS_SCRIPT),
script!(READ_MANY_ID, READ_MANY_SCRIPT),
script!(CREATE_ID, CREATE_SCRIPT),
];
/// Redis-backed PathLock provider.
pub struct RedisPathLockProvider {
runtime: Arc<CacheRuntime>,
service_name: String,
hash_ttl_ms: i64,
acquire_duration: DurationMetrics,
slow_acquisitions: AtomicU64,
}
impl RedisPathLockProvider {
/// Create Redis-backed PathLock storage and register its Lua scripts.
pub fn new(
runtime: Arc<CacheRuntime>,
service_name: &str,
lock_expire_secs: f64,
) -> PathLockResult<Self> {
if service_name.is_empty()
|| !service_name
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
{
return Err(PathLockError::InvalidRequest(
"pathlock Redis service_name must contain only ASCII letters, digits, '.', '_' or '-'"
.to_string(),
));
}
if !lock_expire_secs.is_finite() || lock_expire_secs < 1.0 {
return Err(PathLockError::InvalidRequest(
"pathlock lock_expire_secs must be finite and >= 1.0".to_string(),
));
}
let hash_ttl_ms = (lock_expire_secs * 2_000.0).ceil();
if hash_ttl_ms > i64::MAX as f64 {
return Err(PathLockError::InvalidRequest(
"pathlock Redis HASH TTL exceeds i64 milliseconds".to_string(),
));
}
runtime
.require_operations(&[CacheOperation::ExecuteScript])
.map_err(Self::cache_error)?;
for definition in SCRIPT_DEFINITIONS {
runtime
.register_script(*definition)
.map_err(Self::cache_error)?;
}
Ok(Self {
runtime,
service_name: service_name.to_string(),
hash_ttl_ms: hash_ttl_ms as i64,
acquire_duration: DurationMetrics::default(),
slow_acquisitions: AtomicU64::new(0),
})
}
/// Return the Redis HASH key responsible for one logical path.
fn tokens_key_for_path(&self, path: &str) -> PathLockResult<String> {
validate_normalized_logical_path(path)?;
let suffix = if path == "/" || path == "/local" {
"global".to_string()
} else {
let scope = path
.strip_prefix("/local/")
.and_then(|path| path.split('/').next())
.filter(|scope| !scope.is_empty())
.ok_or_else(|| {
PathLockError::InvalidRequest(format!(
"cache PathLock path must be '/' or below '/local': {path}"
))
})?;
match scope {
"_system" => "scope:_system".to_string(),
account => format!("scope:account:{account}"),
}
};
Ok(format!(
"{{{}}}:ov:pathlock:{suffix}:tokens",
self.service_name
))
}
/// Reject Tree roots whose descendants span multiple Redis HASHes.
fn validate_lock_scope(path: &str, kind: PathLockKind) -> PathLockResult<()> {
if kind == PathLockKind::Tree && (path == "/" || path == "/local") {
return Err(PathLockError::InvalidRequest(format!(
"cache PathLock Tree lock at '{path}' spans multiple scopes; \
use '/local/<account>' or '/local/_system' as the Tree root"
)));
}
Ok(())
}
/// Return the shared Redis HASH key for one same-scope path batch.
fn tokens_key_for_batch<'a>(
&self,
paths: impl IntoIterator<Item = &'a str>,
) -> PathLockResult<String> {
let mut paths = paths.into_iter();
let first_path = paths.next().ok_or_else(|| {
PathLockError::InvalidRequest("lock request batch must not be empty".to_string())
})?;
let key = self.tokens_key_for_path(first_path)?;
for path in paths {
if self.tokens_key_for_path(path)? != key {
return Err(PathLockError::InvalidRequest(
"cache PathLock batch paths must belong to one scope".to_string(),
));
}
}
Ok(key)
}
/// Execute one registered PathLock script and decode its provider-neutral result.
async fn execute(
&self,
script_id: &str,
key: String,
args: Vec<Bytes>,
) -> PathLockResult<ScriptValue> {
self.runtime
.execute_script(ScriptRequest {
script_id: script_id.to_string(),
keys: vec![key],
args,
})
.await
.and_then(|result| result.decode())
.map_err(Self::cache_error)
}
/// Execute one script and require an array result.
async fn execute_array(
&self,
script_id: &str,
key: String,
args: Vec<Bytes>,
) -> PathLockResult<Vec<ScriptValue>> {
match self.execute(script_id, key, args).await? {
ScriptValue::Array(values) => Ok(values),
other => Err(PathLockError::Internal(format!(
"Redis PathLock {script_id} returned non-array result: {other:?}"
))),
}
}
/// Execute one script and require an integer result.
async fn execute_integer(
&self,
script_id: &str,
key: String,
args: Vec<Bytes>,
) -> PathLockResult<i64> {
match self.execute(script_id, key, args).await? {
ScriptValue::Integer(value) => Ok(value),
other => Err(PathLockError::Internal(format!(
"Redis PathLock {script_id} returned non-integer result: {other:?}"
))),
}
}
/// Convert one cache runtime error into a PathLock I/O error.
fn cache_error(error: CacheError) -> PathLockError {
PathLockError::Io(format!("Redis PathLock operation failed: {error}"))
}
/// Decode one script value as UTF-8 text.
fn result_string<'a>(value: &'a ScriptValue, operation: &str) -> PathLockResult<&'a str> {
match value {
ScriptValue::Bytes(value) => std::str::from_utf8(value).map_err(|error| {
PathLockError::Internal(format!(
"Redis PathLock {operation} returned invalid UTF-8: {error}"
))
}),
other => Err(PathLockError::Internal(format!(
"Redis PathLock {operation} returned non-string value: {other:?}"
))),
}
}
/// Return the leading status string from one script array.
fn result_status<'a>(values: &'a [ScriptValue], operation: &str) -> PathLockResult<&'a str> {
values
.first()
.ok_or_else(|| {
PathLockError::Internal(format!(
"Redis PathLock {operation} returned an empty result"
))
})
.and_then(|value| Self::result_string(value, operation))
}
/// Decode one raw Redis token and include its path in decoding errors.
fn decode_token(path: &str, raw: &str) -> PathLockResult<LockToken> {
LockTokenCodec::decode(raw)
.map_err(|_| PathLockError::InvalidToken(format!("invalid token at '{path}': {raw}")))
}
/// Decode the shared single-token and batch per-path refresh response.
fn decode_refresh_result(lock_path: &str, values: &[ScriptValue]) -> PathLockResult<bool> {
let status = Self::result_status(values, "refresh")?;
match status {
"ok" if values.len() == 1 => Ok(true),
"missing" | "owner_mismatch" if values.len() == 1 => Ok(false),
"invalid_token" if values.len() == 2 => Err(PathLockError::InvalidToken(format!(
"invalid token at '{lock_path}': {}",
Self::result_string(&values[1], "refresh")?
))),
"invalid_request" if values.len() == 2 => Err(PathLockError::InvalidRequest(
Self::result_string(&values[1], "refresh")?.to_string(),
)),
_ => Err(PathLockError::Internal(format!(
"Redis PathLock refresh returned invalid status '{status}'"
))),
}
}
/// Decode a conflict or invalid-token script response.
fn decode_error_result(
operation: &str,
values: &[ScriptValue],
) -> PathLockResult<PathLockError> {
let status = Self::result_status(values, operation)?;
match status {
"conflict" if values.len() == 3 => {
let path = Self::result_string(&values[1], operation)?;
let raw = Self::result_string(&values[2], operation)?;
let token = Self::decode_token(path, raw)?;
Ok(PathLockError::Conflict {
lock_path: path.to_string(),
owner: token.owner_id,
kind: token.lock_type,
})
}
"invalid_token" if values.len() == 3 => {
let path = Self::result_string(&values[1], operation)?;
let raw = Self::result_string(&values[2], operation)?;
Ok(PathLockError::InvalidToken(format!(
"invalid token at '{path}': {raw}"
)))
}
"invalid_request" if values.len() == 2 => Ok(PathLockError::InvalidRequest(
Self::result_string(&values[1], operation)?.to_string(),
)),
"changed" if values.len() == 2 => Ok(PathLockError::Io(format!(
"Redis PathLock rollback value changed at '{}'",
Self::result_string(&values[1], operation)?
))),
_ => Ok(PathLockError::Internal(format!(
"Redis PathLock {operation} returned invalid status '{status}'"
))),
}
}
/// Encode one string-like script argument.
fn bytes(value: impl AsRef<[u8]>) -> Bytes {
Bytes::copy_from_slice(value.as_ref())
}
/// Return the string code used by Redis for one lock kind.
fn kind_code(kind: PathLockKind) -> &'static str {
match kind {
PathLockKind::Exact => "E",
PathLockKind::Tree => "T",
}
}
/// Read and decode multiple token fields while preserving input order.
async fn read_many_tokens(&self, paths: &[String]) -> PathLockResult<Vec<Option<LockToken>>> {
if paths.is_empty() {
return Ok(Vec::new());
}
let key = self.tokens_key_for_batch(paths.iter().map(String::as_str))?;
let values = self
.execute_array(
READ_MANY_ID,
key,
paths.iter().map(Self::bytes).collect::<Vec<_>>(),
)
.await?;
if values.len() != paths.len() {
return Err(PathLockError::Internal(format!(
"Redis PathLock read_many returned {} values for {} paths",
values.len(),
paths.len()
)));
}
values
.iter()
.zip(paths)
.map(|(value, path)| match value {
ScriptValue::Null => Ok(None),
ScriptValue::Bytes(raw) => {
let raw = std::str::from_utf8(raw).map_err(|error| {
PathLockError::InvalidToken(format!(
"invalid UTF-8 token at '{path}': {error}"
))
})?;
Self::decode_token(path, raw).map(Some)
}
other => Err(PathLockError::Internal(format!(
"Redis PathLock read_many returned invalid value for '{path}': {other:?}"
))),
})
.collect()
}
}
#[async_trait]
impl PathLockProvider for RedisPathLockProvider {
fn update_metrics(&self, metrics: &mut super::metrics::LockMetrics) {
metrics.redis_acquire = Some(self.acquire_duration.clone());
metrics.redis_slow_acquisitions = self.slow_acquisitions.load(Ordering::Relaxed);
}
/// Return the provider name used by configuration and diagnostics.
fn name(&self) -> &'static str {
"cache"
}
/// Return logical paths because Redis HASH fields are business paths.
fn handle_mode(&self) -> PathLockHandleMode {
PathLockHandleMode::LogicalPath
}
/// Atomically check conflicts and acquire one normalized request batch.
async fn try_acquire_batch_atomic(
&self,
requests: &[PathLockRequest],
owner_id: &str,
now_ns: u128,
stale_before_ns: u128,
) -> PathLockResult<Option<Vec<AtomicAcquisition>>> {
Ok(self
.try_acquire_batch_atomic_with_stats(requests, owner_id, now_ns, stale_before_ns)
.await?
.map(|result| result.acquisitions))
}
async fn try_acquire_batch_atomic_with_stats(
&self,
requests: &[PathLockRequest],
owner_id: &str,
now_ns: u128,
stale_before_ns: u128,
) -> PathLockResult<Option<AtomicAcquisitionResult>> {
if requests.is_empty() {
return Err(PathLockError::InvalidRequest(
"lock request batch must not be empty".to_string(),
));
}
for request in requests {
Self::validate_lock_scope(&request.path, request.kind)?;
}
let key =
self.tokens_key_for_batch(requests.iter().map(|request| request.path.as_str()))?;
let mut args = vec![
Self::bytes(owner_id),
Self::bytes(now_ns.to_string()),
Self::bytes(stale_before_ns.to_string()),
Self::bytes(self.hash_ttl_ms.to_string()),
];
for request in requests {
args.push(Self::bytes(&request.path));
args.push(Self::bytes(Self::kind_code(request.kind)));
}
let started = Instant::now();
let response = self
.execute_array(ACQUIRE_BATCH_ID, key.clone(), args)
.await;
let elapsed = started.elapsed();
let result = match &response {
Ok(values) => match Self::result_status(values, "acquire_batch") {
Ok("ok") => "ok",
Ok("conflict") => "conflict",
Ok("invalid_token") => "invalid_token",
Ok("invalid_request") => "invalid_request",
_ => "invalid_response",
},
Err(_) => "error",
};
let includes_tree = requests
.iter()
.any(|request| request.kind == PathLockKind::Tree);
self.acquire_duration.record(
elapsed,
&[
KeyValue::new("kind", if includes_tree { "tree" } else { "exact" }),
KeyValue::new("result", result),
],
);
if elapsed > SLOW_ACQUIRE_THRESHOLD {
self.slow_acquisitions.fetch_add(1, Ordering::Relaxed);
let scope = key
.split_once(":ov:pathlock:")
.and_then(|(_, suffix)| suffix.strip_suffix(":tokens"))
.unwrap_or(&key);
tracing::warn!(
script_id = ACQUIRE_BATCH_ID,
service_name = %self.service_name,
scope,
request_count = requests.len(),
includes_tree,
elapsed_ms = elapsed.as_secs_f64() * 1_000.0,
threshold_ms = SLOW_ACQUIRE_THRESHOLD.as_millis() as u64,
result,
error = ?response.as_ref().err(),
"slow Redis pathlock acquire"
);
}
let values = response?;
let status = Self::result_status(&values, "acquire_batch")?;
if status != "ok" {
return Err(Self::decode_error_result("acquire_batch", &values)?);
}
if values.len() < 2 || (values.len() - 2) % 4 != 0 {
return Err(PathLockError::Internal(format!(
"Redis PathLock acquire_batch returned {} trailing values",
values.len().saturating_sub(2)
)));
}
let stale_tokens_removed = Self::result_string(&values[1], "acquire_batch")?
.parse::<usize>()
.map_err(|_| PathLockError::Internal("invalid Redis stale recovery count".into()))?;
let mut acquisitions = Vec::with_capacity((values.len() - 2) / 4);
for values in values[2..].chunks_exact(4) {
let handle = Self::result_string(&values[0], "acquire_batch")?.to_string();
let change = Self::result_string(&values[1], "acquire_batch")?;
let previous = Self::result_string(&values[2], "acquire_batch")?;
let replacement = Self::result_string(&values[3], "acquire_batch")?;
let change = match change {
"created" => AcquisitionChange::Created {
replacement: Self::decode_token(&handle, replacement)?,
},
"reentrant" => AcquisitionChange::Reentrant,
"upgraded" => AcquisitionChange::Upgraded {
previous: Self::decode_token(&handle, previous)?,
replacement: Self::decode_token(&handle, replacement)?,
},
other => {
return Err(PathLockError::Internal(format!(
"Redis PathLock acquire_batch returned unknown change '{other}'"
)));
}
};
acquisitions.push(AtomicAcquisition { handle, change });
}
Ok(Some(AtomicAcquisitionResult {
acquisitions,
stale_tokens_removed,
}))
}
/// Atomically roll back mutations reported by one completed batch.
async fn rollback_acquisitions_atomic(
&self,
acquisitions: &[AtomicAcquisition],
owner_id: &str,
) -> PathLockResult<Option<()>> {
if acquisitions.is_empty() {
return Ok(Some(()));
}
let key =
self.tokens_key_for_batch(acquisitions.iter().map(|item| item.handle.as_str()))?;
let mut args = vec![
Self::bytes(owner_id),
Self::bytes(self.hash_ttl_ms.to_string()),
];
for acquisition in acquisitions {
args.push(Self::bytes(&acquisition.handle));
match &acquisition.change {
AcquisitionChange::Created { replacement } => {
args.push(Self::bytes("created"));
args.push(Self::bytes(""));
args.push(Self::bytes(LockTokenCodec::encode(replacement)));
}
AcquisitionChange::Reentrant => {
args.push(Self::bytes("reentrant"));
args.push(Self::bytes(""));
args.push(Self::bytes(""));
}
AcquisitionChange::Upgraded {
previous,
replacement,
} => {
args.push(Self::bytes("upgraded"));
args.push(Self::bytes(LockTokenCodec::encode(previous)));
args.push(Self::bytes(LockTokenCodec::encode(replacement)));
}
}
}
let values = self.execute_array(ROLLBACK_BATCH_ID, key, args).await?;
let status = Self::result_status(&values, "rollback_batch")?;
if status == "ok" && values.len() == 1 {
return Ok(Some(()));
}
Err(Self::decode_error_result("rollback_batch", &values)?)
}
/// Check whether a logical path is covered by a matching live token.
async fn is_path_locked_atomic(
&self,
path: &str,
_now_ns: u128,
stale_before_ns: u128,
ignore_stale: bool,
) -> PathLockResult<Option<bool>> {
let key = self.tokens_key_for_path(path)?;
let values = self
.execute_array(
IS_LOCKED_ID,
key,
vec![
Self::bytes(path),
Self::bytes(stale_before_ns.to_string()),
Self::bytes(if ignore_stale { "1" } else { "0" }),
],
)
.await?;
let status = Self::result_status(&values, "is_locked")?;
if status != "ok" {
return Err(Self::decode_error_result("is_locked", &values)?);
}
if values.len() != 2 {
return Err(PathLockError::Internal(format!(
"Redis PathLock is_locked returned {} values",
values.len()
)));
}
match values[1] {
ScriptValue::Integer(value) => Ok(Some(value != 0)),
ref other => Err(PathLockError::Internal(format!(
"Redis PathLock is_locked returned non-integer value: {other:?}"
))),
}
}
/// Read and decode one logical path token.
async fn read_token(&self, lock_path: &str) -> PathLockResult<Option<LockToken>> {
self.read_many_tokens(&[lock_path.to_string()])
.await
.map(|mut values| values.pop().flatten())
}
/// Atomically create one token when the path is absent.
async fn try_create_token(&self, lock_path: &str, token: &LockToken) -> PathLockResult<()> {
Self::validate_lock_scope(lock_path, token.lock_type)?;
let key = self.tokens_key_for_path(lock_path)?;
let values = self
.execute_array(
CREATE_ID,
key,
vec![
Self::bytes(lock_path),
Self::bytes(LockTokenCodec::encode(token)),
Self::bytes(self.hash_ttl_ms.to_string()),
],
)
.await?;
let status = Self::result_status(&values, "create")?;
match status {
"created" if values.len() == 1 => Ok(()),
"exists" if values.len() == 2 => {
let raw = Self::result_string(&values[1], "create")?;
let current = Self::decode_token(lock_path, raw)?;
Err(PathLockError::Conflict {
lock_path: lock_path.to_string(),
owner: current.owner_id,
kind: current.lock_type,
})
}
_ => Err(PathLockError::Internal(format!(
"Redis PathLock create returned invalid status '{status}'"
))),
}
}
/// Replace one token only when its complete encoded value still matches.
async fn compare_and_write_token(
&self,
lock_path: &str,
expected: &LockToken,
replacement: &LockToken,
) -> PathLockResult<bool> {
Self::validate_lock_scope(lock_path, replacement.lock_type)?;
let key = self.tokens_key_for_path(lock_path)?;
self.execute_integer(
COMPARE_WRITE_ID,
key,
vec![
Self::bytes(lock_path),
Self::bytes(LockTokenCodec::encode(expected)),
Self::bytes(LockTokenCodec::encode(replacement)),
Self::bytes(self.hash_ttl_ms.to_string()),
],
)
.await
.map(|value| value != 0)
}
/// Refresh one owned token while preserving its lock kind.
async fn refresh_token(
&self,
lock_path: &str,
owner_id: &str,
time_ns: u128,
) -> PathLockResult<bool> {
let key = self.tokens_key_for_path(lock_path)?;
let values = self
.execute_array(
REFRESH_ID,
key,
vec![
Self::bytes(lock_path),
Self::bytes(owner_id),
Self::bytes(time_ns.to_string()),
Self::bytes(self.hash_ttl_ms.to_string()),
],
)
.await?;
Self::decode_refresh_result(lock_path, &values)
}
/// Refresh each scope HASH in bounded batches while preserving input order.
async fn refresh_tokens(
&self,
lock_paths: &[String],
owner_id: &str,
time_ns: u128,
) -> Vec<PathLockResult<bool>> {
let mut results: Vec<_> = lock_paths.iter().map(|_| Ok(false)).collect();
let mut groups: BTreeMap<String, Vec<(usize, &str)>> = BTreeMap::new();
for (index, path) in lock_paths.iter().enumerate() {
match self.tokens_key_for_path(path) {
Ok(key) => groups.entry(key).or_default().push((index, path)),
Err(error) => results[index] = Err(error),
}
}
for (key, paths) in groups {
for batch in paths.chunks(REFRESH_BATCH_SIZE) {
let mut args = vec![
Self::bytes(owner_id),
Self::bytes(time_ns.to_string()),
Self::bytes(self.hash_ttl_ms.to_string()),
];
args.extend(batch.iter().map(|(_, path)| Self::bytes(*path)));
match self
.execute_array(REFRESH_BATCH_ID, key.clone(), args)
.await
{
Ok(values)
if matches!(
Self::result_status(&values, "refresh_batch"),
Ok("invalid_request")
) =>
{
for (index, path) in batch {
results[*index] = Self::decode_refresh_result(path, &values);
}
}
Ok(values) if values.len() == batch.len() => {
for ((index, path), value) in batch.iter().zip(values) {
results[*index] = match value {
ScriptValue::Array(values) => {
Self::decode_refresh_result(path, &values)
}
other => Err(PathLockError::Internal(format!(
"Redis PathLock batch refresh returned non-array result: {other:?}"
))),
};
}
}
Ok(values) => {
for (index, _) in batch {
results[*index] = Err(PathLockError::Internal(format!(
"Redis PathLock batch refresh returned {} results for {} paths",
values.len(),
batch.len()
)));
}
}
Err(error) => {
for (index, _) in batch {
results[*index] =
Err(PathLockError::Io(format!("batch refresh failed: {error}")));
}
}
}
}
}
results
}
/// Remove one owned token using complete-token compare-and-remove.
async fn remove_token(
&self,
lock_path: &str,
owner_id: &str,
_force: bool,
) -> PathLockResult<bool> {
let Some(current) = self.read_token(lock_path).await? else {
return Ok(false);
};
if current.owner_id != owner_id {
return Ok(false);
}
let key = self.tokens_key_for_path(lock_path)?;
self.execute_integer(
COMPARE_REMOVE_ID,
key,
vec![
Self::bytes(lock_path),
Self::bytes(LockTokenCodec::encode(¤t)),
Self::bytes(self.hash_ttl_ms.to_string()),
],
)
.await
.map(|value| value != 0)
}
/// Return logical paths equal to or below one root.
async fn scan_descendant_locks(&self, root: &str) -> PathLockResult<Vec<String>> {
let key = self.tokens_key_for_path(root)?;
let values = self
.execute_array(SCAN_DESCENDANTS_ID, key, vec![Self::bytes(root)])
.await?;
let status = Self::result_status(&values, "scan_descendants")?;
if status != "ok" {
return Err(Self::decode_error_result("scan_descendants", &values)?);
}
if (values.len() - 1) % 2 != 0 {
return Err(PathLockError::Internal(format!(
"Redis PathLock scan_descendants returned {} trailing values",
values.len() - 1
)));
}
let mut paths = Vec::with_capacity((values.len() - 1) / 2);
for values in values[1..].chunks_exact(2) {
let path = Self::result_string(&values[0], "scan_descendants")?;
let raw = Self::result_string(&values[1], "scan_descendants")?;
Self::decode_token(path, raw)?;
paths.push(path.to_string());
}
Ok(paths)
}
}
#[cfg(test)]
#[path = "cache_refresh_tests.rs"]
mod refresh_tests;
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use crate::cache_runtime::{
CacheRuntime, RedisProviderConfig, ScriptDefinition, ScriptRequest, ScriptValue,
};
use crate::core::FileSystem;
use crate::plugins::memfs::MemFileSystem;
use super::super::manager::{PathLockConfig, PathLockManager};
use super::super::provider::{
AcquisitionChange, AtomicAcquisition, PathLockHandleMode, PathLockProvider,
};
use super::super::types::{
LockToken, PathLockError, PathLockHandoffRef, PathLockKind, PathLockRequest,
};
use super::*;
const TEST_PTTL_ID: &str = "pathlock.test.pttl.v1";
const TEST_PTTL_SCRIPT: &str = "return redis.call('PTTL', KEYS[1])";
const TEST_HASH_ID: &str = "pathlock.test.hash.v1";
const TEST_HASH_SCRIPT: &str = r#"
if ARGV[1] == "set" then
redis.call("HSET", KEYS[1], ARGV[2], ARGV[3])
return ARGV[3]
end
return redis.call("HGET", KEYS[1], ARGV[2])
"#;
enum AtomicFault {
WriteThenIo,
Conflict,
}
#[tokio::test]
async fn redis_acquire_metrics_export_outcomes_and_slow_calls() {
let calls = AtomicU64::new(0);
let mock = crate::cache_runtime::MemoryMockProvider::new().with_script_handler(move |_| {
// Inject real response latency so the slow counter follows the timed call.
std::thread::sleep(Duration::from_millis(15));
let values = match calls.fetch_add(1, Ordering::Relaxed) {
0 => vec!["ok", "0", "/local/account-a/file", "reentrant", "", ""],
1 => vec!["conflict", "/local/account-a/file", "other:100:E"],
2 => return Err(CacheError::Unavailable("injected failure".into())),
_ => panic!("unexpected Redis call"),
};
crate::cache_runtime::ScriptResult::encode(&ScriptValue::Array(
values
.into_iter()
.map(|value| ScriptValue::Bytes(value.as_bytes().to_vec()))
.collect(),
))
});
let runtime = CacheRuntime::memory_with_provider(Arc::new(mock));
let provider =
Arc::new(RedisPathLockProvider::new(runtime.clone(), "metrics-test", 30.0).unwrap());
assert!(provider
.try_acquire_batch_atomic(&[], "owner", 100, 0)
.await
.is_err());
for (index, kind) in [PathLockKind::Exact, PathLockKind::Tree, PathLockKind::Exact]
.into_iter()
.enumerate()
{
let result = provider
.try_acquire_batch_atomic(
&[PathLockRequest {
path: "/local/account-a/file".into(),
kind,
}],
"owner",
100,
0,
)
.await;
match index {
0 => assert!(result.is_ok()),
1 => assert!(matches!(result, Err(PathLockError::Conflict { .. }))),
_ => assert!(matches!(result, Err(PathLockError::Io(_)))),
}
}
let manager = PathLockManager::new(
Arc::new(MemFileSystem::new()),
provider,
PathLockConfig::default(),
);
// Repeated exports must retain cumulative observations.
for _ in 0..2 {
let metrics = crate::metrics::lock_metrics(manager.metrics_snapshot().await);
let slow = metrics
.iter()
.find(|metric| metric.name == "lock_redis_slow_acquisitions_total")
.unwrap();
assert!(matches!(
slow.value,
crate::metrics::RagfsMetricValue::Counter { value: 3, .. }
));
let durations = metrics
.iter()
.filter(|metric| metric.name == "lock_redis_acquire_duration_seconds")
.collect::<Vec<_>>();
assert_eq!(durations.len(), 3);
for (kind, result) in [("exact", "ok"), ("tree", "conflict"), ("exact", "error")] {
let metric = durations
.iter()
.find(|metric| {
metric.labels["kind"] == kind && metric.labels["result"] == result
})
.unwrap();
assert_eq!(metric.labels.len(), 2);
match &metric.value {
crate::metrics::RagfsMetricValue::Histogram {
count,
sum,
bucket_counts,
scale,
..
} => {
assert_eq!(*count, 1);
assert_eq!(bucket_counts.iter().sum::<u64>(), 1);
assert!(*sum > 10_000_000);
assert_eq!(*scale, 1e-9);
}
_ => panic!("expected duration histogram"),
}
}
}
runtime.close().await.unwrap();
}
/// Logical test provider for atomic Manager failure paths.
struct FaultProvider {
inner: super::super::provider::MemoryPathLockProvider,
fault: AtomicFault,
remove_calls: AtomicUsize,
}
impl FaultProvider {
/// Create one provider with the selected injected fault.
fn new(fault: AtomicFault) -> Self {
Self {
inner: super::super::provider::MemoryPathLockProvider::new(),
fault,
remove_calls: AtomicUsize::new(0),
}
}
}
#[async_trait]
impl PathLockProvider for FaultProvider {
/// Return the provider name used by test diagnostics.
fn name(&self) -> &'static str {
"fault"
}
/// Return logical handles to exercise the atomic Manager branch.
fn handle_mode(&self) -> PathLockHandleMode {
PathLockHandleMode::LogicalPath
}
/// Inject either an unknown write result or a conflict.
async fn try_acquire_batch_atomic(
&self,
requests: &[PathLockRequest],
owner_id: &str,
now_ns: u128,
_stale_before_ns: u128,
) -> PathLockResult<Option<Vec<AtomicAcquisition>>> {
match self.fault {
AtomicFault::WriteThenIo => {
let request = &requests[0];
self.inner
.try_create_token(&request.path, &token(owner_id, now_ns, request.kind))
.await?;
Err(PathLockError::Io(
"simulated response timeout after write".to_string(),
))
}
AtomicFault::Conflict => Err(PathLockError::Conflict {
lock_path: "/race".to_string(),
owner: "owner-a".to_string(),
kind: PathLockKind::Exact,
}),
}
}
/// Return stored data or the stale conflict snapshot.
async fn read_token(&self, lock_path: &str) -> PathLockResult<Option<LockToken>> {
match self.fault {
AtomicFault::Conflict => Ok(Some(token("owner-a", 1, PathLockKind::Exact))),
AtomicFault::WriteThenIo => self.inner.read_token(lock_path).await,
}
}
/// Delegate fallback token creation to memory storage.
async fn try_create_token(&self, lock_path: &str, token: &LockToken) -> PathLockResult<()> {
self.inner.try_create_token(lock_path, token).await
}
/// Delegate fallback token replacement to memory storage.
async fn compare_and_write_token(
&self,
lock_path: &str,
expected: &LockToken,
replacement: &LockToken,
) -> PathLockResult<bool> {
self.inner
.compare_and_write_token(lock_path, expected, replacement)
.await
}
/// Delegate refresh to memory storage.
async fn refresh_token(
&self,
lock_path: &str,
owner_id: &str,
time_ns: u128,
) -> PathLockResult<bool> {
self.inner.refresh_token(lock_path, owner_id, time_ns).await
}
/// Record cleanup calls and delegate removal.
async fn remove_token(
&self,
lock_path: &str,
owner_id: &str,
force: bool,
) -> PathLockResult<bool> {
self.remove_calls.fetch_add(1, Ordering::SeqCst);
self.inner.remove_token(lock_path, owner_id, force).await
}
/// Delegate descendant scans to memory storage.
async fn scan_descendant_locks(&self, root: &str) -> PathLockResult<Vec<String>> {
self.inner.scan_descendant_locks(root).await
}
}
/// Build a real Redis provider when the integration endpoint is configured.
async fn test_provider(test_name: &str) -> Option<(Arc<CacheRuntime>, RedisPathLockProvider)> {
let endpoint = std::env::var("REDIS_URL").ok()?;
let runtime = CacheRuntime::redis(RedisProviderConfig {
endpoints: vec![endpoint],
command_timeout_ms: 1_000,
..RedisProviderConfig::default()
})
.await
.unwrap();
let service_name = format!("pathlock-test-{}-{test_name}", std::process::id());
let provider = RedisPathLockProvider::new(runtime.clone(), &service_name, 1.0).unwrap();
Some((runtime, provider))
}
/// Build two managers that share one service name.
async fn test_managers(
test_name: &str,
) -> Option<(Arc<CacheRuntime>, PathLockManager, PathLockManager, String)> {
let endpoint = std::env::var("REDIS_URL").ok()?;
let runtime = CacheRuntime::redis(RedisProviderConfig {
endpoints: vec![endpoint],
command_timeout_ms: 1_000,
..RedisProviderConfig::default()
})
.await
.unwrap();
let service_name = format!("pathlock-manager-test-{}-{test_name}", std::process::id());
let config = PathLockConfig {
provider: "cache".to_string(),
service_name: Some(service_name.clone()),
sqlite_path: None,
lock_timeout_secs: 0.0,
lock_expire_secs: 3.0,
};
let first_provider =
Arc::new(RedisPathLockProvider::new(runtime.clone(), &service_name, 3.0).unwrap());
let key = first_provider
.tokens_key_for_path("/local/account-a")
.unwrap();
let first = PathLockManager::new(
Arc::new(MemFileSystem::new()),
first_provider,
config.clone(),
);
let second = PathLockManager::new(
Arc::new(MemFileSystem::new()),
Arc::new(RedisPathLockProvider::new(runtime.clone(), &service_name, 3.0).unwrap()),
config,
);
Some((runtime, first, second, key))
}
/// Read a comma-separated Redis endpoint environment variable.
fn topology_endpoints(variable: &str) -> Option<Vec<String>> {
let endpoints = std::env::var(variable).ok()?;
let endpoints = endpoints
.split(',')
.map(str::trim)
.filter(|endpoint| !endpoint.is_empty())
.map(str::to_string)
.collect::<Vec<_>>();
(!endpoints.is_empty()).then_some(endpoints)
}
/// Exercise one provider against a configured Redis topology.
async fn exercise_topology(
mode: &str,
endpoints: Vec<String>,
master_name: Option<String>,
test_name: &str,
) {
let runtime = CacheRuntime::redis(RedisProviderConfig {
mode: mode.to_string(),
endpoints,
master_name,
connect_timeout_ms: 10_000,
command_timeout_ms: 3_000,
..RedisProviderConfig::default()
})
.await
.unwrap();
let service_name = format!("pathlock-topology-test-{}-{test_name}", std::process::id());
let provider = RedisPathLockProvider::new(runtime.clone(), &service_name, 3.0).unwrap();
provider
.try_acquire_batch_atomic(
&[request("/local/account-a/root", PathLockKind::Tree)],
"owner-a",
100,
0,
)
.await
.unwrap();
assert!(matches!(
provider
.try_acquire_batch_atomic(
&[request("/local/account-a/root/child", PathLockKind::Exact)],
"owner-b",
101,
0,
)
.await,
Err(PathLockError::Conflict { .. })
));
runtime.close().await.unwrap();
}
/// Build one lock token for provider tests.
fn token(owner_id: &str, time_ns: u128, lock_type: PathLockKind) -> LockToken {
LockToken {
owner_id: owner_id.to_string(),
time_ns,
lock_type,
}
}
/// Build one logical lock request.
fn request(path: &str, kind: PathLockKind) -> PathLockRequest {
PathLockRequest {
path: path.to_string(),
kind,
}
}
/// Verify invalid service_name and expiry values are rejected before Redis access.
#[test]
fn provider_rejects_invalid_configuration() {
let runtime = CacheRuntime::memory();
assert!(matches!(
RedisPathLockProvider::new(runtime.clone(), "bad{name}", 1.0),
Err(PathLockError::InvalidRequest(_))
));
assert!(matches!(
RedisPathLockProvider::new(runtime, "prod-a", f64::NAN),
Err(PathLockError::InvalidRequest(_))
));
}
/// Verify global, system, and account paths use separate Redis HASH keys.
#[tokio::test]
async fn provider_routes_tokens_to_scope_hashes() {
let Some((runtime, provider)) = test_provider("scope-keys").await else {
return;
};
runtime
.register_script(ScriptDefinition {
id: TEST_HASH_ID,
redis_lua: TEST_HASH_SCRIPT,
})
.unwrap();
let cases = [
("/", "global"),
("/local", "global"),
("/local/_system/tasks/task.json", "scope:_system"),
("/local/account-a/resources/a.md", "scope:account:account-a"),
("/local/account-b/resources/b.md", "scope:account:account-b"),
];
let service_name = format!("pathlock-test-{}-scope-keys", std::process::id());
for (index, (path, scope)) in cases.iter().enumerate() {
provider
.try_create_token(
path,
&token("owner-a", index as u128 + 1, PathLockKind::Exact),
)
.await
.unwrap();
let raw = runtime
.execute_script(ScriptRequest {
script_id: TEST_HASH_ID.to_string(),
keys: vec![format!("{{{service_name}}}:ov:pathlock:{scope}:tokens")],
args: vec![
RedisPathLockProvider::bytes("get"),
RedisPathLockProvider::bytes(path),
],
})
.await
.unwrap()
.decode()
.unwrap();
assert!(matches!(raw, ScriptValue::Bytes(_)), "{path} missing");
}
assert!(matches!(
provider
.try_acquire_batch_atomic(
&[
request("/local/account-a/a", PathLockKind::Exact),
request("/local/account-b/b", PathLockKind::Exact),
],
"owner-a",
10,
0,
)
.await,
Err(PathLockError::InvalidRequest(_))
));
assert!(matches!(
provider
.try_create_token("/queue", &token("owner-a", 10, PathLockKind::Exact))
.await,
Err(PathLockError::InvalidRequest(_))
));
runtime.close().await.unwrap();
}
/// Real filesystem aliases must not bypass another manager's Redis Exact lock.
#[tokio::test]
async fn redis_path_aliases_cannot_write_a_locked_file() {
let Some((runtime, provider)) = test_provider("path-alias-write").await else {
return;
};
let dir = tempfile::TempDir::new().unwrap();
let fs = Arc::new(
crate::plugins::localfs::LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap(),
);
let path = "/local/account-a/file";
std::fs::create_dir_all(dir.path().join("local/account-a")).unwrap();
fs.write(path, b"original", 0, crate::core::WriteFlag::Create)
.await
.unwrap();
let second_provider =
RedisPathLockProvider::new(runtime.clone(), &provider.service_name, 1.0).unwrap();
let config = PathLockConfig {
lock_expire_secs: 1.0,
..PathLockConfig::default()
};
let first = PathLockManager::new(fs.clone(), Arc::new(provider), config.clone());
let second = Arc::new(PathLockManager::new(
fs.clone(),
Arc::new(second_provider),
config,
));
let wrapped = crate::lock::PathLockWrappedFS::new(second, fs.clone());
let lease = first
.acquire_exact(path, Duration::ZERO, None)
.await
.unwrap();
assert!(matches!(
wrapped
.write(path, b"modified", 0, crate::core::WriteFlag::Truncate)
.await,
Err(crate::core::Error::PathLock(PathLockError::Timeout { .. }))
));
for alias in [
"/local/account-a/./file",
"/local/account-a//file",
"/local/./account-a/file",
] {
assert_eq!(fs.read(alias, 0, 100).await.unwrap(), b"original");
assert!(
wrapped
.write(alias, b"modified", 0, crate::core::WriteFlag::Truncate)
.await
.is_err(),
"write through alias {alias} bypassed the lock"
);
assert_eq!(fs.read(path, 0, 100).await.unwrap(), b"original");
}
first.release(&lease).await.unwrap();
wrapped
.write(path, b"modified", 0, crate::core::WriteFlag::Truncate)
.await
.unwrap();
assert_eq!(fs.read(path, 0, 100).await.unwrap(), b"modified");
runtime.close().await.unwrap();
}
/// Lua validates the entire batch before writing any target or cleaning stale tokens.
#[tokio::test]
async fn redis_lua_rejects_path_aliases_before_mutation() {
let Some((runtime, provider)) = test_provider("path-alias-lua").await else {
return;
};
let valid = "/local/account-a/file";
let stale_path = "/local/account-a/stale";
let stale = token("old-owner", 1, PathLockKind::Exact);
provider.try_create_token(stale_path, &stale).await.unwrap();
for path in [
"",
"relative",
"//",
"/local/account-a/file/",
"/local/account-a//file",
"/local/account-a/./file",
"/local/account-a/../file",
"/local/account-a/file\0",
] {
for kind in ["E", "T"] {
let values = provider
.execute_array(
ACQUIRE_BATCH_ID,
provider.tokens_key_for_path(valid).unwrap(),
["owner", "200", "100", "2000", valid, "E", path, kind]
.iter()
.map(RedisPathLockProvider::bytes)
.collect(),
)
.await
.unwrap();
assert_eq!(
RedisPathLockProvider::result_status(&values, "test").unwrap(),
"invalid_request",
"Lua accepted {path:?}"
);
assert_eq!(provider.read_token(valid).await.unwrap(), None);
assert_eq!(
provider.read_token(stale_path).await.unwrap(),
Some(stale.clone())
);
}
}
runtime.close().await.unwrap();
}
/// Every Redis token entry point rejects noncanonical paths before executing scripts.
#[tokio::test]
async fn redis_provider_rejects_path_aliases_before_script_access() {
let mock = crate::cache_runtime::MemoryMockProvider::new()
.with_script_handler(|_| panic!("invalid path must not execute a Redis script"));
let runtime = CacheRuntime::memory_with_provider(Arc::new(mock));
let provider = RedisPathLockProvider::new(runtime.clone(), "path-validation", 1.0).unwrap();
let original = token("owner", 100, PathLockKind::Exact);
for path in [
"",
"relative",
"//",
"/local/account-a/file/",
"/local/account-a/file//",
"/local/account-a//file",
"/local/account-a/./file",
"/local/account-a/../file",
"/local/account-a/file\0",
] {
for kind in [PathLockKind::Exact, PathLockKind::Tree] {
assert!(matches!(
provider
.try_acquire_batch_atomic(
&[
request("/local/account-a/valid", PathLockKind::Exact),
request(path, kind)
],
"owner",
200,
100,
)
.await,
Err(PathLockError::InvalidRequest(_))
));
}
assert!(matches!(
provider.try_create_token(path, &original).await,
Err(PathLockError::InvalidRequest(_))
));
assert!(matches!(
provider
.compare_and_write_token(path, &original, &original)
.await,
Err(PathLockError::InvalidRequest(_))
));
assert!(matches!(
provider.read_token(path).await,
Err(PathLockError::InvalidRequest(_))
));
assert!(matches!(
provider.refresh_token(path, "owner", 200).await,
Err(PathLockError::InvalidRequest(_))
));
assert!(matches!(
&provider
.refresh_tokens(&[path.to_string()], "owner", 200)
.await[0],
Err(PathLockError::InvalidRequest(_))
));
assert!(matches!(
provider.remove_token(path, "owner", false).await,
Err(PathLockError::InvalidRequest(_))
));
assert!(matches!(
provider.scan_descendant_locks(path).await,
Err(PathLockError::InvalidRequest(_))
));
assert!(matches!(
provider.is_path_locked_atomic(path, 200, 100, true).await,
Err(PathLockError::InvalidRequest(_))
));
}
runtime.close().await.unwrap();
}
/// Reject global Tree requests before accessing Redis, including token upgrades.
#[tokio::test]
async fn provider_rejects_cross_scope_tree_before_redis_access() {
let mock = crate::cache_runtime::MemoryMockProvider::new()
.with_script_handler(|_| panic!("invalid Tree request must not access Redis"));
let runtime = CacheRuntime::memory_with_provider(Arc::new(mock));
let provider = RedisPathLockProvider::new(runtime.clone(), "global-tree", 1.0).unwrap();
for root in ["/", "/local"] {
let exact = token("owner-a", 100, PathLockKind::Exact);
let tree = token("owner-a", 200, PathLockKind::Tree);
for requests in [
vec![request(root, PathLockKind::Tree)],
vec![
request(root, PathLockKind::Exact),
request(root, PathLockKind::Tree),
],
] {
assert!(matches!(
provider
.try_acquire_batch_atomic(&requests, "owner-a", 200, 100)
.await,
Err(PathLockError::InvalidRequest(message))
if message.contains("Tree") && message.contains("scope")
));
}
assert!(matches!(
provider.try_create_token(root, &tree).await,
Err(PathLockError::InvalidRequest(_))
));
assert!(matches!(
provider.compare_and_write_token(root, &exact, &tree).await,
Err(PathLockError::InvalidRequest(_))
));
}
runtime.close().await.unwrap();
}
/// Neither acquisition order can publish a Tree token spanning Redis scope HASHes.
#[tokio::test]
async fn provider_rejects_cross_scope_tree_in_both_acquisition_orders() {
let Some((runtime, provider)) = test_provider("global-tree-orders").await else {
return;
};
for root in ["/", "/local"] {
for scope in ["/local/account-a", "/local/_system"] {
let child = format!("{scope}/file");
for kind in [PathLockKind::Exact, PathLockKind::Tree] {
for child_first in [false, true] {
if child_first {
provider
.try_acquire_batch_atomic(
&[request(&child, kind)],
"owner-b",
100,
0,
)
.await
.unwrap();
}
assert!(matches!(
provider
.try_acquire_batch_atomic(
&[request(root, PathLockKind::Tree)],
"owner-a",
100,
0,
)
.await,
Err(PathLockError::InvalidRequest(_))
));
assert_eq!(provider.read_token(root).await.unwrap(), None);
if !child_first {
provider
.try_acquire_batch_atomic(
&[request(&child, kind)],
"owner-b",
100,
0,
)
.await
.unwrap();
}
assert_eq!(
provider.read_token(&child).await.unwrap(),
Some(token("owner-b", 100, kind))
);
provider
.remove_token(&child, "owner-b", false)
.await
.unwrap();
}
}
}
}
runtime.close().await.unwrap();
}
/// A rejected mixed batch cannot replace or clean up existing global Exact tokens.
#[tokio::test]
async fn provider_rejected_global_tree_batch_preserves_exact_tokens() {
let Some((runtime, provider)) = test_provider("global-tree-no-mutation").await else {
return;
};
let existing = token("owner-a", 1, PathLockKind::Exact);
provider.try_create_token("/", &existing).await.unwrap();
assert!(matches!(
provider
.try_acquire_batch_atomic(
&[
request("/", PathLockKind::Exact),
request("/local", PathLockKind::Tree),
],
"owner-b",
200,
100,
)
.await,
Err(PathLockError::InvalidRequest(_))
));
assert_eq!(
provider.read_token("/").await.unwrap(),
Some(existing.clone())
);
assert_eq!(provider.read_token("/local").await.unwrap(), None);
assert!(matches!(
provider
.compare_and_write_token("/", &existing, &token("owner-a", 200, PathLockKind::Tree))
.await,
Err(PathLockError::InvalidRequest(_))
));
assert_eq!(provider.read_token("/").await.unwrap(), Some(existing));
// Global Exact locks protect only their own path, so child scopes remain available.
provider
.try_acquire_batch_atomic(
&[request("/local/account-a/file", PathLockKind::Exact)],
"owner-b",
200,
0,
)
.await
.unwrap();
runtime.close().await.unwrap();
}
/// Account and system root Trees still exclude descendants in both directions.
#[tokio::test]
async fn provider_scope_root_trees_preserve_mutual_exclusion() {
let Some((runtime, provider)) = test_provider("scope-root-tree").await else {
return;
};
for root in ["/local/account-a", "/local/_system"] {
let child = format!("{root}/file");
for kind in [PathLockKind::Exact, PathLockKind::Tree] {
for child_first in [false, true] {
let (first, second) = if child_first {
(request(&child, kind), request(root, PathLockKind::Tree))
} else {
(request(root, PathLockKind::Tree), request(&child, kind))
};
provider
.try_acquire_batch_atomic(&[first.clone()], "owner-a", 100, 0)
.await
.unwrap();
assert!(matches!(
provider
.try_acquire_batch_atomic(&[second], "owner-b", 100, 0)
.await,
Err(PathLockError::Conflict { .. })
));
provider
.remove_token(&first.path, "owner-a", false)
.await
.unwrap();
}
}
}
runtime.close().await.unwrap();
}
/// Verify CRUD preserves the token format and refreshes the HASH TTL.
#[tokio::test]
async fn provider_crud_preserves_tokens_and_hash_ttl() {
let Some((runtime, provider)) = test_provider("crud").await else {
return;
};
let path = "/local/account-a/a";
let original = token("owner-a", 100, PathLockKind::Exact);
provider.try_create_token(path, &original).await.unwrap();
assert_eq!(
provider.read_token(path).await.unwrap(),
Some(original.clone())
);
assert!(provider.refresh_token(path, "owner-a", 200).await.unwrap());
assert_eq!(
provider.read_token(path).await.unwrap(),
Some(token("owner-a", 200, PathLockKind::Exact))
);
runtime
.register_script(ScriptDefinition {
id: TEST_PTTL_ID,
redis_lua: TEST_PTTL_SCRIPT,
})
.unwrap();
let ttl = runtime
.execute_script(ScriptRequest {
script_id: TEST_PTTL_ID.to_string(),
keys: vec![provider.tokens_key_for_path(path).unwrap()],
args: Vec::new(),
})
.await
.unwrap()
.decode()
.unwrap();
assert!(matches!(ttl, ScriptValue::Integer(value) if value > 0 && value <= 2_000));
assert!(provider.remove_token(path, "owner-a", false).await.unwrap());
assert_eq!(provider.read_token(path).await.unwrap(), None);
runtime.close().await.unwrap();
}
/// Verify Lua rejects conflicting owners without partially writing a batch.
#[tokio::test]
async fn provider_atomic_batch_is_all_or_nothing() {
let Some((runtime, provider)) = test_provider("atomic").await else {
return;
};
let initial = vec![request("/local/account-a/locked", PathLockKind::Tree)];
let acquired = provider
.try_acquire_batch_atomic(&initial, "owner-a", 100, 0)
.await
.unwrap()
.unwrap();
assert!(matches!(
acquired.as_slice(),
[AtomicAcquisition {
handle,
change: AcquisitionChange::Created { .. },
}] if handle == "/local/account-a/locked"
));
let conflicting = vec![
request("/local/account-a/free", PathLockKind::Exact),
request("/local/account-a/locked/child", PathLockKind::Exact),
];
assert!(matches!(
provider
.try_acquire_batch_atomic(&conflicting, "owner-b", 101, 0)
.await,
Err(PathLockError::Conflict { lock_path, .. })
if lock_path == "/local/account-a/locked"
));
assert_eq!(
provider.read_token("/local/account-a/free").await.unwrap(),
None
);
provider
.try_acquire_batch_atomic(
&[request("/local/account-a/a", PathLockKind::Exact)],
"owner-a",
102,
0,
)
.await
.unwrap();
assert!(provider
.try_acquire_batch_atomic(
&[request("/local/account-a/a/b", PathLockKind::Exact)],
"owner-b",
103,
0,
)
.await
.is_ok());
assert!(matches!(
provider
.try_acquire_batch_atomic(&[request("/", PathLockKind::Tree)], "owner-c", 104, 0,)
.await,
Err(PathLockError::InvalidRequest(_))
));
let race = [request("/local/account-a/race", PathLockKind::Exact)];
let (first, second) = tokio::join!(
provider.try_acquire_batch_atomic(&race, "owner-a", 200, 0),
provider.try_acquire_batch_atomic(&race, "owner-b", 200, 0),
);
assert_ne!(first.is_ok(), second.is_ok());
assert!(
matches!(first, Err(PathLockError::Conflict { .. }))
|| matches!(second, Err(PathLockError::Conflict { .. }))
);
runtime.close().await.unwrap();
}
/// Verify rollback removes creates and restores same-owner upgrades.
#[tokio::test]
async fn provider_atomic_rollback_restores_previous_state() {
let Some((runtime, provider)) = test_provider("rollback").await else {
return;
};
let exact = vec![request("/local/account-a/a", PathLockKind::Exact)];
let created = provider
.try_acquire_batch_atomic(&exact, "owner-a", 100, 0)
.await
.unwrap()
.unwrap();
provider
.rollback_acquisitions_atomic(&created, "owner-a")
.await
.unwrap()
.unwrap();
assert_eq!(
provider.read_token("/local/account-a/a").await.unwrap(),
None
);
provider
.try_acquire_batch_atomic(&exact, "owner-a", 200, 0)
.await
.unwrap();
let tree = vec![request("/local/account-a/a", PathLockKind::Tree)];
let upgraded = provider
.try_acquire_batch_atomic(&tree, "owner-a", 300, 0)
.await
.unwrap()
.unwrap();
provider
.rollback_acquisitions_atomic(&upgraded, "owner-a")
.await
.unwrap()
.unwrap();
assert_eq!(
provider.read_token("/local/account-a/a").await.unwrap(),
Some(token("owner-a", 200, PathLockKind::Exact))
);
let changed = provider
.try_acquire_batch_atomic(
&[request("/local/account-a/changed", PathLockKind::Exact)],
"owner-a",
400,
0,
)
.await
.unwrap()
.unwrap();
assert!(provider
.refresh_token("/local/account-a/changed", "owner-a", 500)
.await
.unwrap());
assert!(matches!(
provider
.rollback_acquisitions_atomic(&changed, "owner-a")
.await,
Err(PathLockError::Io(_))
));
assert_eq!(
provider
.read_token("/local/account-a/changed")
.await
.unwrap(),
Some(token("owner-a", 500, PathLockKind::Exact))
);
runtime.close().await.unwrap();
}
/// Classify filesystem migration without authorizing or changing old tokens.
#[tokio::test]
async fn manager_redis_classifies_filesystem_handoffs_without_reading_old_tokens() {
let Some((runtime, redis, _, key)) = test_managers("filesystem-migration").await else {
return;
};
let fs = Arc::new(MemFileSystem::new());
fs.mkdir("/local", 0o755).await.unwrap();
fs.mkdir("/local/account-a", 0o755).await.unwrap();
fs.mkdir("/local/account-a/a", 0o755).await.unwrap();
let old_provider = Arc::new(super::super::provider::FilesystemPathLockProvider::new(
fs.clone(),
30.0,
));
let old = PathLockManager::new(fs, old_provider.clone(), PathLockConfig::default());
let lease = old
.acquire_tree("/local/account-a/a", Duration::ZERO, None)
.await
.unwrap();
let mut handoff = old.to_handoff(&lease);
old.handoff(&lease).await.unwrap();
assert_eq!(handoff.handle_mode, Some(PathLockHandleMode::LockPath));
let old_token = old_provider
.read_token(&handoff.lock_paths[0])
.await
.unwrap();
for historic in [false, true] {
if historic {
handoff.handle_mode = None;
handoff.covered_paths.clear();
}
assert!(matches!(
redis.adopt(&handoff).await,
Err(PathLockError::HandoffRecoveryRequired {
reason: super::super::types::HandoffRecoveryReason::ProviderChanged,
})
));
}
let fresh = redis
.acquire_tree("/local/account-a/a", Duration::ZERO, None)
.await
.unwrap();
assert_ne!(fresh.lease.owner_id, handoff.owner_id);
assert_eq!(
old_provider
.read_token(&handoff.lock_paths[0])
.await
.unwrap(),
old_token
);
redis.release(&fresh).await.unwrap();
runtime.del(&[key]).await.unwrap();
runtime.close().await.unwrap();
}
/// Verify Redis handoff can be adopted by another manager using logical handles.
#[tokio::test]
async fn manager_redis_adopts_handoff_across_managers() {
let Some((runtime, first, second, key)) = test_managers("handoff").await else {
return;
};
let lease = first
.acquire_tree("/local/account-a/a", Duration::ZERO, None)
.await
.unwrap();
let handoff = first.to_handoff(&lease);
first.handoff(&lease).await.unwrap();
let adopted = second.adopt(&handoff).await.unwrap();
assert_eq!(adopted.lease.lock_paths, vec!["/local/account-a/a"]);
second.release(&adopted).await.unwrap();
tokio::time::timeout(Duration::from_secs(2), async {
loop {
if first.observe().await.active_locks == 0 {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("producer must forget a released cross-manager handoff");
let legacy = PathLockHandoffRef {
lease_ref: None,
handle_mode: None,
owner_id: "owner-a".to_string(),
lock_paths: vec!["/local/account-a/a".to_string()],
covered_paths: Vec::new(),
};
assert!(matches!(
second.adopt(&legacy).await,
Err(PathLockError::HandoffRecoveryRequired { .. })
));
runtime.del(&[key]).await.unwrap();
runtime.close().await.unwrap();
}
/// Verify reentrant upgrades, downgrades, refresh, and partial release use logical handles.
#[tokio::test]
async fn manager_redis_preserves_lease_lifecycle_semantics() {
let Some((runtime, first, second, key)) = test_managers("lifecycle").await else {
return;
};
let exact = first
.acquire_exact("/local/account-a/a/", Duration::ZERO, None)
.await
.unwrap();
assert_eq!(exact.lease.lock_paths, vec!["/local/account-a/a"]);
assert!(first.is_locked("/local/account-a/a/", true).await.unwrap());
let tree = first
.acquire_tree(
"/local/account-a/a",
Duration::ZERO,
Some((&exact.lease.lease_ref, &exact.ownership_ref)),
)
.await
.unwrap();
assert_eq!(first.refresh(&exact).await.unwrap(), "refreshed");
first.release(&tree).await.unwrap();
let child = second
.acquire_exact("/local/account-a/a/child", Duration::ZERO, None)
.await
.unwrap();
second.release(&child).await.unwrap();
first.release(&exact).await.unwrap();
assert!(!first.is_locked("/local/account-a/a", true).await.unwrap());
let batch = first
.acquire_exact_batch(
&[
"/local/account-a/x".to_string(),
"/local/account-a/y".to_string(),
],
Duration::ZERO,
None,
)
.await
.unwrap();
first
.release_selected(&batch, &["/local/account-a/x".to_string()])
.await
.unwrap();
let released = second
.acquire_exact("/local/account-a/x", Duration::ZERO, None)
.await
.unwrap();
assert!(second
.acquire_exact("/local/account-a/y", Duration::from_millis(1), None,)
.await
.is_err());
second.release(&released).await.unwrap();
first.release(&batch).await.unwrap();
runtime.del(&[key]).await.unwrap();
runtime.close().await.unwrap();
}
/// Verify stale foreign tokens are replaced while same-owner reentry preserves time.
#[tokio::test]
async fn provider_stale_cleanup_preserves_same_owner_reentry() {
let Some((runtime, provider)) = test_provider("stale").await else {
return;
};
let same_path = "/local/account-a/same";
provider
.try_create_token(same_path, &token("owner-a", 1, PathLockKind::Exact))
.await
.unwrap();
let reentrant = provider
.try_acquire_batch_atomic(
&[request(same_path, PathLockKind::Exact)],
"owner-a",
100,
50,
)
.await
.unwrap()
.unwrap();
assert!(matches!(reentrant[0].change, AcquisitionChange::Reentrant));
assert_eq!(
provider.read_token(same_path).await.unwrap(),
Some(token("owner-a", 1, PathLockKind::Exact))
);
assert_eq!(
provider
.is_path_locked_atomic(same_path, 100, 50, true)
.await
.unwrap(),
Some(false)
);
assert_eq!(
provider
.is_path_locked_atomic(same_path, 100, 50, false)
.await
.unwrap(),
Some(true)
);
let foreign_path = "/local/account-a/foreign";
provider
.try_create_token(foreign_path, &token("owner-b", 1, PathLockKind::Exact))
.await
.unwrap();
provider
.try_acquire_batch_atomic(
&[request(foreign_path, PathLockKind::Exact)],
"owner-a",
100,
50,
)
.await
.unwrap();
assert_eq!(
provider
.read_token(foreign_path)
.await
.unwrap()
.unwrap()
.owner_id,
"owner-a"
);
runtime.close().await.unwrap();
}
/// Verify an out-of-range token is rejected before an atomic upgrade writes.
#[tokio::test]
async fn provider_invalid_u128_token_does_not_modify_hash() {
let Some((runtime, provider)) = test_provider("invalid-u128").await else {
return;
};
runtime
.register_script(ScriptDefinition {
id: TEST_HASH_ID,
redis_lua: TEST_HASH_SCRIPT,
})
.unwrap();
let path = "/local/account-a/invalid";
let key = provider.tokens_key_for_path(path).unwrap();
let invalid = "owner-a:340282366920938463463374607431768211456:E";
runtime
.execute_script(ScriptRequest {
script_id: TEST_HASH_ID.to_string(),
keys: vec![key.clone()],
args: vec![
RedisPathLockProvider::bytes("set"),
RedisPathLockProvider::bytes(path),
RedisPathLockProvider::bytes(invalid),
],
})
.await
.unwrap();
assert!(matches!(
provider
.try_acquire_batch_atomic(&[request(path, PathLockKind::Tree)], "owner-a", 100, 0,)
.await,
Err(PathLockError::InvalidToken(_))
));
let raw = runtime
.execute_script(ScriptRequest {
script_id: TEST_HASH_ID.to_string(),
keys: vec![key],
args: vec![
RedisPathLockProvider::bytes("get"),
RedisPathLockProvider::bytes(path),
],
})
.await
.unwrap()
.decode()
.unwrap();
assert_eq!(raw, ScriptValue::Bytes(invalid.as_bytes().to_vec()));
runtime.close().await.unwrap();
}
/// Verify an unknown atomic write result never publishes a local lease.
#[tokio::test]
async fn manager_unknown_atomic_result_does_not_publish_lease() {
let provider = Arc::new(FaultProvider::new(AtomicFault::WriteThenIo));
let manager = PathLockManager::new(
Arc::new(MemFileSystem::new()),
provider.clone(),
PathLockConfig {
provider: "cache".to_string(),
service_name: Some("timeout-test".to_string()),
..PathLockConfig::default()
},
);
assert!(matches!(
manager
.acquire_exact("/timeout", Duration::ZERO, None)
.await,
Err(PathLockError::Io(_))
));
assert_eq!(manager.observe().await.active_locks, 0);
assert!(provider.read_token("/timeout").await.unwrap().is_some());
}
/// Verify Manager never performs non-atomic stale cleanup after a logical conflict.
#[tokio::test]
async fn manager_logical_conflict_does_not_remove_stale_snapshot() {
let provider = Arc::new(FaultProvider::new(AtomicFault::Conflict));
let manager = PathLockManager::new(
Arc::new(MemFileSystem::new()),
provider.clone(),
PathLockConfig {
provider: "cache".to_string(),
service_name: Some("race-test".to_string()),
lock_expire_secs: 1.0,
..PathLockConfig::default()
},
);
assert!(manager
.acquire_exact("/race", Duration::from_millis(5), None)
.await
.is_err());
assert_eq!(provider.remove_calls.load(Ordering::SeqCst), 0);
}
/// Verify optional cluster and sentinel topology contracts.
#[tokio::test]
async fn topology_cluster_and_sentinel() {
if let Some(endpoints) = topology_endpoints("REDIS_CLUSTER_TEST_URLS") {
exercise_topology("cluster", endpoints, None, "cluster").await;
}
if let (Some(endpoints), Ok(master)) = (
topology_endpoints("REDIS_SENTINEL_TEST_URLS"),
std::env::var("REDIS_SENTINEL_TEST_MASTER"),
) {
exercise_topology("sentinel", endpoints, Some(master), "sentinel").await;
}
}
/// Verify storage-side stale recovery reaches manager metrics.
#[tokio::test]
async fn stale_recovery_count_reaches_manager_observation() {
let Some((runtime, provider)) = test_provider("stale-metrics").await else {
return;
};
let path = "/local/account-a/old";
provider
.try_create_token(path, &token("dead", 1, PathLockKind::Exact))
.await
.unwrap();
let manager = PathLockManager::new(
Arc::new(MemFileSystem::new()),
Arc::new(provider),
PathLockConfig::default(),
);
let lease = manager
.acquire_tree(path, Duration::ZERO, None)
.await
.unwrap();
assert_eq!(manager.observe().await.stale_locks_removed, 1);
manager.release(&lease).await.unwrap();
runtime.close().await.unwrap();
}
/// Measure HASH-only atomic batch latency for representative lock counts.
#[tokio::test]
#[ignore = "manual Redis performance baseline"]
async fn perf_hash_only_batches() {
let Some((runtime, provider)) = test_provider("perf").await else {
return;
};
for count in [100usize, 500, 1_000] {
let requests = (0..count)
.map(|index| PathLockRequest {
path: format!("/local/account-a/perf/{index}"),
kind: PathLockKind::Exact,
})
.collect::<Vec<_>>();
let start = Instant::now();
let acquired = provider
.try_acquire_batch_atomic(&requests, "perf-owner", 100, 0)
.await
.unwrap()
.unwrap();
let elapsed = start.elapsed();
println!(
"pathlock HASH-only count={count} acquire_ms={}",
elapsed.as_secs_f64() * 1_000.0
);
provider
.rollback_acquisitions_atomic(&acquired, "perf-owner")
.await
.unwrap();
}
runtime.close().await.unwrap();
}
}