Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
//! Python bindings for RAGFS - Rust AGFS filesystem
//!
//! Provides `RAGFSBindingClient`, a PyO3 native class that is API-compatible
//! with the existing Go-based `AGFSBindingClient`. This embeds the ragfs
//! filesystem engine directly in the Python process (no HTTP server needed).
use pyo3::exceptions::{PyRuntimeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyDict, PyList, PyType};
use pyo3::wrap_pyfunction;
use std::collections::HashMap;
use std::fs::OpenOptions;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock};
use tracing::Level;
use tracing_appender::non_blocking::WorkerGuard;
use tracing_subscriber::fmt::writer::BoxMakeWriter;
use uuid::Uuid;
/// Cached reference to the Python `openviking.storage.errors.LockAcquisitionError` class,
/// imported at module init so that native and Python code share the same exception type.
static LOCK_ACQUISITION_ERROR_TYPE: OnceLock<Py<PyType>> = OnceLock::new();
static HANDOFF_RECOVERY_ERROR_TYPE: OnceLock<Py<PyType>> = OnceLock::new();
static RUST_TRACING_INIT_RESULT: OnceLock<Result<(), String>> = OnceLock::new();
static RUST_TRACING_GUARD: OnceLock<WorkerGuard> = OnceLock::new();
static RUST_TRACING_FILE_STATE: OnceLock<TracingFileState> = OnceLock::new();
#[derive(Clone)]
struct SharedFileWriter {
file: Arc<Mutex<std::fs::File>>,
}
impl Write for SharedFileWriter {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let mut file = self
.file
.lock()
.map_err(|_| io::Error::other("shared tracing log file lock poisoned"))?;
file.write(buf)
}
fn flush(&mut self) -> io::Result<()> {
let mut file = self
.file
.lock()
.map_err(|_| io::Error::other("shared tracing log file lock poisoned"))?;
file.flush()
}
}
struct TracingFileState {
path: PathBuf,
file: Arc<Mutex<std::fs::File>>,
}
/// Open the configured Rust tracing log file.
fn open_tracing_log_file(path: &Path) -> Result<std::fs::File, String> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| format!("failed to create Rust tracing log dir: {e}"))?;
}
OpenOptions::new()
.create(true)
.append(true)
.open(path)
.map_err(|e| format!("failed to open Rust tracing log file: {e}"))
}
/// Swap the shared Rust tracing file handle to the current active log file.
fn replace_tracing_log_file(shared: &Arc<Mutex<std::fs::File>>, path: &Path) -> Result<(), String> {
let new_file = open_tracing_log_file(path)?;
let mut file = shared
.lock()
.map_err(|_| "shared tracing log file lock poisoned".to_string())?;
*file = new_file;
Ok(())
}
/// Parse the configured OpenViking log level into a Rust tracing level.
fn parse_tracing_level(log_level: &str) -> Result<Level, String> {
match log_level.to_ascii_uppercase().as_str() {
"TRACE" => Ok(Level::TRACE),
"DEBUG" => Ok(Level::DEBUG),
"INFO" => Ok(Level::INFO),
"WARN" | "WARNING" => Ok(Level::WARN),
"CRITICAL" => Ok(Level::ERROR),
"ERROR" => Ok(Level::ERROR),
other => Err(format!("unsupported Rust tracing level: {other}")),
}
}
/// Build the tracing writer from the configured OpenViking log output target.
fn build_tracing_writer(log_output: &str) -> Result<(BoxMakeWriter, WorkerGuard), String> {
match log_output {
"stdout" => {
let (writer, guard) = tracing_appender::non_blocking(std::io::stdout());
Ok((BoxMakeWriter::new(writer), guard))
}
"stderr" => {
let (writer, guard) = tracing_appender::non_blocking(std::io::stderr());
Ok((BoxMakeWriter::new(writer), guard))
}
path => {
let path = Path::new(path);
let shared = Arc::new(Mutex::new(open_tracing_log_file(path)?));
let state = TracingFileState {
path: path.to_path_buf(),
file: shared.clone(),
};
RUST_TRACING_FILE_STATE
.set(state)
.map_err(|_| "Rust tracing file state already initialized".to_string())?;
let (writer, guard) = tracing_appender::non_blocking(SharedFileWriter { file: shared });
Ok((BoxMakeWriter::new(writer), guard))
}
}
}
/// Reopen the Rust tracing file writer after Python log rotation.
fn reopen_tracing_file_impl() -> Result<(), String> {
let Some(state) = RUST_TRACING_FILE_STATE.get() else {
return Ok(());
};
replace_tracing_log_file(&state.file, &state.path)
}
/// Reopen the Rust tracing log file after Python rotates the active log file.
#[pyfunction]
fn reopen_tracing_file() -> PyResult<()> {
reopen_tracing_file_impl().map_err(PyRuntimeError::new_err)
}
/// Initialize Rust tracing once for the embedded ragfs binding process.
fn init_tracing(log_level: &str, log_output: &str) -> PyResult<()> {
let result = RUST_TRACING_INIT_RESULT.get_or_init(|| {
let max_level = parse_tracing_level(log_level)?;
let (writer, guard) = build_tracing_writer(log_output)?;
let subscriber = tracing_subscriber::fmt()
.with_max_level(max_level)
.with_target(true)
.with_file(true)
.with_line_number(true)
.with_ansi(false)
.with_writer(writer)
.finish();
tracing::subscriber::set_global_default(subscriber)
.map_err(|e| format!("failed to install Rust tracing subscriber: {e}"))?;
RUST_TRACING_GUARD
.set(guard)
.map_err(|_| "failed to retain Rust tracing worker guard".to_string())?;
Ok(())
});
result
.as_ref()
.map(|_| ())
.map_err(|err| PyRuntimeError::new_err(err.clone()))
}
/// Map a PathLockError to the appropriate Python exception.
fn pathlock_err_to_py(err: PathLockError) -> PyErr {
match &err {
PathLockError::HandoffRecoveryRequired { reason } => Python::attach(|py| {
let ty = HANDOFF_RECOVERY_ERROR_TYPE
.get()
.expect("HandoffRecoveryRequired not initialized");
PyErr::from_type(ty.bind(py).clone(), reason.to_string())
}),
PathLockError::Conflict { .. }
| PathLockError::Timeout { .. }
| PathLockError::HandoffFailed(_)
| PathLockError::Busy { .. }
| PathLockError::EmptyToken { .. } =>
{
#[allow(deprecated)]
Python::with_gil(|py| {
let ty = LOCK_ACQUISITION_ERROR_TYPE
.get()
.expect("LockAcquisitionError not initialized");
PyErr::from_type(ty.bind(py).clone(), err.to_string())
})
}
PathLockError::InvalidRequest(_) => PyValueError::new_err(err.to_string()),
PathLockError::Io(_) | PathLockError::InvalidToken(_) | PathLockError::Internal(_) => {
new_py_err("AGFSInternalError", err.to_string())
}
}
}
use std::fmt;
use std::fs;
use std::future::Future;
use std::time::{Duration, UNIX_EPOCH};
use tokio::sync::{Mutex as AsyncMutex, RwLock as AsyncRwLock};
/// Validate and convert a wait timeout from Python.
///
/// Must be finite and non-negative. Returns `Duration::from_secs_f64(v)`.
/// NaN, negative, or infinite → `ValueError`.
fn validate_timeout_secs(v: f64) -> PyResult<Duration> {
if v.is_finite() && v >= 0.0 {
Ok(Duration::from_secs_f64(v))
} else {
Err(PyValueError::new_err(
"timeout_secs must be a finite non-negative number",
))
}
}
/// Validate a positive leader lease TTL from Python.
fn validate_leader_ttl_secs(v: f64) -> PyResult<Duration> {
let max_ttl_secs = (i64::MAX as f64) / 1000.0;
if v.is_finite() && (0.001..=max_ttl_secs).contains(&v) {
Ok(Duration::from_secs_f64(v))
} else {
Err(PyValueError::new_err(
"ttl_secs must be a finite number between 0.001 and i64::MAX milliseconds",
))
}
}
use ragfs::cache::{CachePolicy, CacheTraversalMode};
use ragfs::cache_runtime::{CacheError, CacheRuntime, DynamicProviderConfig, RedisProviderConfig};
use ragfs::core::builder::{
CacheFsConfig, CacheRuntimeProviderConfig, CacheStackConfig, EncryptionConfig,
};
use ragfs::core::{
build_configured_stack, ConfigValue, FileInfo, FileSystem, FilesystemStats, FsContext,
FsContextInner, FsOperation, GlobPage, GrepOptions, GrepResult, ListSortBy, MountableFS,
OperationStats, PathLockContext, PluginConfig, RagfsConfig, SortOrder, TreeEntry, WriteFlag,
FS_CTX,
};
use ragfs::leader::{LeaderLease, LeaderManager};
use ragfs::lock::types::PathLockError;
use ragfs::lock::{
BorrowedPathLockLease, OwnedPathLockLease, PathLockConfig, PathLockHandleMode,
PathLockHandoffRef, PathLockKind, PathLockManager, PathLockRequest,
};
use ragfs::metrics::{RagfsMetric, RagfsMetricValue};
/// Parse an optional listing sort field and return the matching RagFS value.
fn parse_list_sort_by(value: Option<&str>) -> PyResult<Option<ListSortBy>> {
match value {
None => Ok(None),
Some("name") => Ok(Some(ListSortBy::Name)),
Some("mtime") => Ok(Some(ListSortBy::Mtime)),
Some(value) => Err(PyValueError::new_err(format!(
"sort_by must be 'name' or 'mtime', got {value:?}"
))),
}
}
/// Parse an optional listing sort direction and return the matching RagFS value.
fn parse_sort_order(value: Option<&str>) -> PyResult<Option<SortOrder>> {
match value {
None | Some("asc") => Ok(Some(SortOrder::Asc)),
Some("desc") => Ok(Some(SortOrder::Desc)),
Some(value) => Err(PyValueError::new_err(format!(
"sort_order must be 'asc' or 'desc', got {value:?}"
))),
}
}
/// Validate `offset` and return it as an unsigned RagFS offset.
fn parse_listing_offset(offset: i64) -> PyResult<usize> {
usize::try_from(offset).map_err(|_| PyValueError::new_err("offset must be non-negative"))
}
/// Validate `limit` and return it as an optional unsigned RagFS limit.
fn parse_listing_limit(limit: Option<i64>) -> PyResult<Option<usize>> {
match limit {
None => Ok(None),
Some(value) if value > 0 => Ok(Some(value as usize)),
Some(_) => Err(PyValueError::new_err("limit must be positive")),
}
}
/// Parse one Python pathlock request without silently defaulting invalid fields.
fn parse_pathlock_request(raw: &HashMap<String, String>) -> PyResult<PathLockRequest> {
let path = raw
.get("path")
.filter(|path| !path.is_empty() && path.starts_with('/'))
.ok_or_else(|| PyValueError::new_err("pathlock request.path must be an absolute path"))?;
let kind = match raw.get("kind").map(String::as_str) {
Some("exact") => PathLockKind::Exact,
Some("tree") => PathLockKind::Tree,
_ => {
return Err(PyValueError::new_err(
"pathlock request.kind must be 'exact' or 'tree'",
));
}
};
Ok(PathLockRequest {
path: path.clone(),
kind,
})
}
/// Parse a non-empty Python pathlock request batch.
fn parse_pathlock_request_batch(
requests: &[HashMap<String, String>],
) -> PyResult<Vec<PathLockRequest>> {
if requests.is_empty() {
return Err(PyValueError::new_err(
"pathlock request batch must not be empty",
));
}
requests.iter().map(parse_pathlock_request).collect()
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum CacheProviderKind {
Redis,
Dynamic,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct RagfsCacheConfig {
enabled: bool,
runtime_enabled: bool,
provider: CacheProviderKind,
service_name: String,
max_file_size_bytes: usize,
traversal_mode: CacheTraversalMode,
bypass_prefixes: Vec<String>,
redis: RedisCacheConfig,
dynamic: DynamicCacheConfig,
}
#[derive(Clone, PartialEq, Eq)]
struct DynamicCacheConfig {
library: String,
params: serde_json::Value,
}
impl fmt::Debug for DynamicCacheConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("DynamicCacheConfig")
.field("library", &self.library)
.field("params", &"<redacted>")
.finish()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct RedisCacheConfig {
mode: String,
endpoints: Vec<String>,
master_name: Option<String>,
username: String,
password_env: String,
password: String,
sentinel_username: String,
sentinel_password_env: String,
sentinel_password: String,
db: i64,
pool_size: usize,
connect_timeout_ms: u64,
command_timeout_ms: u64,
default_ttl_seconds: u64,
tls_insecure_skip_verify: bool,
}
impl Default for RagfsCacheConfig {
fn default() -> Self {
Self {
enabled: false,
runtime_enabled: false,
provider: CacheProviderKind::Redis,
service_name: String::new(),
max_file_size_bytes: CachePolicy::default().max_file_size(),
traversal_mode: CacheTraversalMode::Backend,
bypass_prefixes: Vec::new(),
redis: RedisCacheConfig::default(),
dynamic: DynamicCacheConfig::default(),
}
}
}
impl Default for DynamicCacheConfig {
fn default() -> Self {
Self {
library: String::new(),
params: serde_json::Value::Object(serde_json::Map::new()),
}
}
}
impl Default for RedisCacheConfig {
fn default() -> Self {
Self {
mode: "standalone".to_string(),
endpoints: vec!["redis://127.0.0.1:6379".to_string()],
master_name: None,
username: String::new(),
password_env: String::new(),
password: String::new(),
sentinel_username: String::new(),
sentinel_password_env: String::new(),
sentinel_password: String::new(),
db: 0,
pool_size: 32,
connect_timeout_ms: 1_000,
command_timeout_ms: 20,
default_ttl_seconds: 3_600,
tls_insecure_skip_verify: false,
}
}
}
impl RagfsCacheConfig {
fn stack_config(&self) -> Option<CacheStackConfig> {
if !self.enabled && !self.runtime_enabled {
return None;
}
let provider = match self.provider {
CacheProviderKind::Redis => CacheRuntimeProviderConfig::Redis(RedisProviderConfig {
mode: self.redis.mode.clone(),
endpoints: self.redis.endpoints.clone(),
master_name: self.redis.master_name.clone(),
username: self.redis.username.clone(),
password_env: self.redis.password_env.clone(),
password: self.redis.password.clone(),
sentinel_username: self.redis.sentinel_username.clone(),
sentinel_password_env: self.redis.sentinel_password_env.clone(),
sentinel_password: self.redis.sentinel_password.clone(),
db: self.redis.db,
pool_size: self.redis.pool_size,
connect_timeout_ms: self.redis.connect_timeout_ms,
command_timeout_ms: self.redis.command_timeout_ms,
default_ttl_seconds: self.redis.default_ttl_seconds,
tls_insecure_skip_verify: self.redis.tls_insecure_skip_verify,
}),
CacheProviderKind::Dynamic => {
CacheRuntimeProviderConfig::Dynamic(DynamicProviderConfig {
library_path: PathBuf::from(&self.dynamic.library),
params_json: serde_json::to_string(&self.dynamic.params)
.expect("dynamic provider params are valid JSON"),
})
}
};
Some(CacheStackConfig {
provider: Some(provider),
cachefs: CacheFsConfig {
enabled: self.enabled,
service_name: self.service_name.clone(),
policy: cache_policy_from_config(self),
},
})
}
}
/// Load PathLock configuration from one canonical OpenViking config file.
fn pathlock_config_from_ov_conf(path: &str, service_name: &str) -> Result<PathLockConfig, String> {
let raw = fs::read_to_string(path)
.map_err(|error| format!("failed to read OpenViking config {path}: {error}"))?;
let json: serde_json::Value = serde_json::from_str(&raw)
.map_err(|error| format!("failed to parse OpenViking config {path}: {error}"))?;
pathlock_config_from_canonical_ov_conf(&json, service_name)
}
/// Parse PathLock configuration from the canonical `storage.agfs.pathlock` section.
fn pathlock_config_from_canonical_ov_conf(
json: &serde_json::Value,
service_name: &str,
) -> Result<PathLockConfig, String> {
let pathlock = json
.get("storage")
.and_then(|storage| storage.get("agfs"))
.and_then(|agfs| agfs.get("pathlock"));
match pathlock {
Some(pathlock) => {
let mut pathlock = pathlock.clone();
if pathlock.get("provider").and_then(serde_json::Value::as_str) == Some("sqlite")
&& pathlock
.get("sqlite_path")
.is_none_or(serde_json::Value::is_null)
{
let workspace = json
.get("storage")
.and_then(|storage| storage.get("workspace"))
.and_then(serde_json::Value::as_str)
.filter(|workspace| !workspace.is_empty())
.ok_or_else(|| {
"storage.workspace is required for the default SQLite lock database"
.to_string()
})?;
// The workspace may not exist yet. Resolve relative paths against
// the process working directory without requiring existing parents.
let database = std::path::absolute(workspace)
.map_err(|error| format!("failed to resolve storage.workspace: {error}"))?
.join("_system/pathlock/pathlock.db");
pathlock["sqlite_path"] = serde_json::json!(database);
}
pathlock_config_from_value(&pathlock, service_name)
}
None => Ok(PathLockConfig::default()),
}
}
/// Parse one sectioned PathLock configuration object.
fn pathlock_config_from_value(
value: &serde_json::Value,
service_name: &str,
) -> Result<PathLockConfig, String> {
let pathlock = value
.as_object()
.ok_or_else(|| "pathlock config must be an object".to_string())?;
const FIELDS: &[&str] = &[
"provider",
"sqlite_path",
"lock_timeout_secs",
"lock_expire_secs",
];
if let Some(field) = pathlock
.keys()
.find(|field| !FIELDS.contains(&field.as_str()))
{
return Err(format!("unsupported pathlock field: {field}"));
}
let provider = string_field(pathlock, "provider", "filesystem")?;
if !matches!(
provider.as_str(),
"filesystem" | "memory" | "sqlite" | "cache"
) {
return Err(format!(
"unsupported pathlock.provider: {provider}; expected filesystem, memory, sqlite, or cache"
));
}
let pathlock_service_name =
matches!(provider.as_str(), "cache" | "sqlite").then(|| service_name.to_string());
let sqlite_path = match pathlock.get("sqlite_path") {
None | Some(serde_json::Value::Null) => None,
Some(serde_json::Value::String(path)) => Some(path.clone()),
_ => return Err("pathlock.sqlite_path must be a string".to_string()),
};
if provider == "sqlite"
&& !sqlite_path
.as_deref()
.is_some_and(|path| std::path::Path::new(path).is_absolute())
{
return Err("pathlock.sqlite_path must be an absolute local file path".to_string());
}
let lock_timeout_secs = f64_field(pathlock, "lock_timeout_secs", 0.0)?;
if !lock_timeout_secs.is_finite() || lock_timeout_secs < 0.0 {
return Err("pathlock.lock_timeout_secs must be a finite non-negative number".to_string());
}
let lock_expire_secs = f64_field(pathlock, "lock_expire_secs", 30.0)?;
if !lock_expire_secs.is_finite() || lock_expire_secs < 1.0 {
return Err("pathlock.lock_expire_secs must be finite and >= 1.0".to_string());
}
Ok(PathLockConfig {
provider,
service_name: pathlock_service_name,
sqlite_path,
lock_timeout_secs,
lock_expire_secs,
})
}
fn cache_config_from_ov_conf(path: &str, service_name: &str) -> Result<RagfsCacheConfig, String> {
let raw = fs::read_to_string(path)
.map_err(|error| format!("failed to read OpenViking config {path}: {error}"))?;
let json: serde_json::Value = serde_json::from_str(&raw)
.map_err(|error| format!("failed to parse OpenViking config {path}: {error}"))?;
cache_config_from_canonical_ov_conf(&json, service_name)
}
/// Load canonical cache configuration while forcing a shared Runtime consumer.
fn cache_config_from_ov_conf_with_runtime(
path: &str,
force_runtime: bool,
service_name: &str,
) -> Result<RagfsCacheConfig, String> {
let raw = fs::read_to_string(path)
.map_err(|error| format!("failed to read OpenViking config {path}: {error}"))?;
let json: serde_json::Value = serde_json::from_str(&raw)
.map_err(|error| format!("failed to parse OpenViking config {path}: {error}"))?;
cache_config_from_canonical_ov_conf_with_runtime(&json, force_runtime, service_name)
}
fn cache_config_from_canonical_ov_conf(
json: &serde_json::Value,
service_name: &str,
) -> Result<RagfsCacheConfig, String> {
let pathlock_uses_cache =
pathlock_config_from_canonical_ov_conf(json, service_name)?.provider == "cache";
let backups_use_cache = json
.get("storage")
.and_then(|storage| storage.get("agfs"))
.and_then(|agfs| agfs.get("backups"))
.and_then(|backups| backups.get("provider"))
.and_then(serde_json::Value::as_str)
== Some("cache");
cache_config_from_canonical_ov_conf_with_runtime(
json,
pathlock_uses_cache || backups_use_cache,
service_name,
)
}
/// Parse canonical cache configuration and optionally require its shared Runtime.
fn cache_config_from_canonical_ov_conf_with_runtime(
json: &serde_json::Value,
force_runtime: bool,
service_name: &str,
) -> Result<RagfsCacheConfig, String> {
let agfs = json.get("storage").and_then(|storage| storage.get("agfs"));
if agfs.and_then(|agfs| agfs.get("cache")).is_some() {
return Err(
"storage.agfs.cache has been removed; use cache.provider/cache.params and ".to_string()
+ "storage.agfs.cachefs.backend=cache",
);
}
let queuefs = agfs.and_then(|agfs| agfs.get("queuefs"));
if queuefs.and_then(|queuefs| queuefs.get("redis")).is_some() {
return Err(
"storage.agfs.queuefs.redis has been removed; use queuefs backend=cache with "
.to_string()
+ "cache.provider/cache.params",
);
}
let queuefs_backend = optional_backend(queuefs, "storage.agfs.queuefs", "sqlite")?;
if queuefs_backend == "redis" {
return Err(
"queuefs backend=redis has been removed; use backend=cache with ".to_string()
+ "cache.provider/cache.params",
);
}
let cachefs = agfs.and_then(|agfs| agfs.get("cachefs"));
let cachefs_backend = optional_backend(cachefs, "storage.agfs.cachefs", "local")?;
if !matches!(cachefs_backend.as_str(), "local" | "cache") {
return Err(format!(
"unsupported storage.agfs.cachefs.backend: {cachefs_backend}; expected local or cache"
));
}
let mut config = RagfsCacheConfig::default();
config.service_name = service_name.to_string();
config.enabled = cachefs_backend == "cache";
config.runtime_enabled = config.enabled || queuefs_backend == "cache" || force_runtime;
if let Some(cachefs) = cachefs {
let cachefs = cachefs
.as_object()
.ok_or_else(|| "storage.agfs.cachefs must be an object".to_string())?;
config.max_file_size_bytes =
usize_field(cachefs, "max_file_size_bytes", config.max_file_size_bytes)?;
config.traversal_mode =
traversal_mode(string_field(cachefs, "traversal_mode", "backend")?)?;
config.bypass_prefixes = string_array_field(cachefs, "bypass_prefixes")?;
}
if !config.runtime_enabled {
return Ok(config);
}
let cache = json
.get("cache")
.ok_or_else(|| {
"top-level cache.provider/cache.params is required when AGFS uses cache-backed modules"
.to_string()
})?
.as_object()
.ok_or_else(|| "top-level cache must be an object".to_string())?;
let provider = cache
.get("provider")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| "top-level cache.provider must be a string".to_string())?;
let params = cache
.get("params")
.map(|params| {
params
.as_object()
.ok_or_else(|| "top-level cache.params must be an object".to_string())
})
.transpose()?
.cloned()
.unwrap_or_default();
config.provider = provider_kind(provider.to_string())?;
match config.provider {
CacheProviderKind::Redis => parse_redis_config(&mut config.redis, ¶ms, "cache.params")?,
CacheProviderKind::Dynamic => {
if force_runtime {
return Err(
"cache-backed PathLock requires top-level cache.provider=redis".to_string(),
);
}
parse_dynamic_config(&mut config.dynamic, ¶ms)?;
}
}
validate_cache_runtime_config(&config)?;
Ok(config)
}
fn optional_backend(
section: Option<&serde_json::Value>,
name: &str,
default: &str,
) -> Result<String, String> {
let Some(section) = section else {
return Ok(default.to_string());
};
let section = section
.as_object()
.ok_or_else(|| format!("{name} must be an object"))?;
string_field(section, "backend", default)
}
fn cache_config_from_value(
cache: &serde_json::Value,
service_name: &str,
) -> Result<RagfsCacheConfig, String> {
if cache.is_null() {
let mut config = RagfsCacheConfig::default();
config.service_name = service_name.to_string();
return Ok(config);
}
let cache = cache
.as_object()
.ok_or_else(|| "binding cache config must be an object".to_string())?;
let mut config = RagfsCacheConfig::default();
config.service_name = service_name.to_string();
config.enabled = bool_field(cache, "enabled", config.enabled)?;
config.runtime_enabled = bool_field(cache, "runtime_enabled", config.enabled)?;
let provider = string_field(cache, "provider", "redis")?;
config.provider = provider_kind(provider)?;
config.max_file_size_bytes =
usize_field(cache, "max_file_size_bytes", config.max_file_size_bytes)?;
config.traversal_mode = traversal_mode(string_field(cache, "traversal_mode", "backend")?)?;
config.bypass_prefixes = string_array_field(cache, "bypass_prefixes")?;
if let Some(redis) = cache.get("redis") {
let redis = redis
.as_object()
.ok_or_else(|| "cache.redis must be an object".to_string())?;
parse_redis_config(&mut config.redis, redis, "cache.redis")?;
}
if let Some(dynamic) = cache.get("dynamic") {
let dynamic = dynamic
.as_object()
.ok_or_else(|| "cache.dynamic must be an object".to_string())?;
parse_dynamic_config(&mut config.dynamic, dynamic)?;
}
validate_cache_runtime_config(&config)?;
Ok(config)
}
fn parse_dynamic_config(
config: &mut DynamicCacheConfig,
params: &serde_json::Map<String, serde_json::Value>,
) -> Result<(), String> {
config.library = string_field(params, "library", &config.library)?;
let mut provider_params = params.clone();
provider_params.remove("library");
config.params = serde_json::Value::Object(provider_params);
Ok(())
}
fn parse_redis_config(
config: &mut RedisCacheConfig,
redis: &serde_json::Map<String, serde_json::Value>,
name: &str,
) -> Result<(), String> {
const FIELDS: &[&str] = &[
"mode",
"endpoints",
"master_name",
"username",
"password_env",
"password",
"sentinel_username",
"sentinel_password_env",
"sentinel_password",
"db",
"pool_size",
"connect_timeout_ms",
"command_timeout_ms",
"default_ttl_seconds",
"tls_insecure_skip_verify",
];
if let Some(field) = redis.keys().find(|field| !FIELDS.contains(&field.as_str())) {
return Err(format!("unsupported {name} field: {field}"));
}
config.mode = string_field(redis, "mode", &config.mode)?;
config.endpoints = string_array_field_or_default(redis, "endpoints", &config.endpoints)?;
config.master_name = optional_string_field(redis, "master_name")?;
config.username = string_field(redis, "username", &config.username)?;
config.password_env = string_field(redis, "password_env", &config.password_env)?;
config.password = string_field(redis, "password", &config.password)?;
config.sentinel_username = string_field(redis, "sentinel_username", &config.sentinel_username)?;
config.sentinel_password_env = string_field(
redis,
"sentinel_password_env",
&config.sentinel_password_env,
)?;
config.sentinel_password = string_field(redis, "sentinel_password", &config.sentinel_password)?;
config.db = i64_field(redis, "db", config.db)?;
config.pool_size = usize_field(redis, "pool_size", config.pool_size)?;
config.connect_timeout_ms = u64_field(redis, "connect_timeout_ms", config.connect_timeout_ms)?;
config.command_timeout_ms = u64_field(redis, "command_timeout_ms", config.command_timeout_ms)?;
config.default_ttl_seconds =
u64_field_allow_zero(redis, "default_ttl_seconds", config.default_ttl_seconds)?;
config.tls_insecure_skip_verify = bool_field(
redis,
"tls_insecure_skip_verify",
config.tls_insecure_skip_verify,
)?;
Ok(())
}
fn validate_cache_runtime_config(config: &RagfsCacheConfig) -> Result<(), String> {
if !config.enabled && !config.runtime_enabled {
return Ok(());
}
if matches!(config.provider, CacheProviderKind::Dynamic)
&& config.dynamic.library.trim().is_empty()
{
return Err("cache dynamic provider library must not be empty".to_string());
}
Ok(())
}
fn provider_kind(value: String) -> Result<CacheProviderKind, String> {
match value.as_str() {
"redis" => Ok(CacheProviderKind::Redis),
"dynamic" => Ok(CacheProviderKind::Dynamic),
other => Err(format!(
"unsupported cache.provider: {other}; expected redis or dynamic"
)),
}
}
fn traversal_mode(value: String) -> Result<CacheTraversalMode, String> {
match value.as_str() {
"backend" => Ok(CacheTraversalMode::Backend),
"cached_traversal" => Ok(CacheTraversalMode::CachedTraversal),
other => Err(format!(
"unsupported cachefs.traversal_mode: {other}; expected backend or cached_traversal"
)),
}
}
fn bool_field(
object: &serde_json::Map<String, serde_json::Value>,
key: &str,
default: bool,
) -> Result<bool, String> {
match object.get(key) {
Some(value) => value
.as_bool()
.ok_or_else(|| format!("{key} must be a boolean")),
None => Ok(default),
}
}
fn string_field(
object: &serde_json::Map<String, serde_json::Value>,
key: &str,
default: &str,
) -> Result<String, String> {
match object.get(key) {
Some(value) => value
.as_str()
.map(ToOwned::to_owned)
.ok_or_else(|| format!("{key} must be a string")),
None => Ok(default.to_string()),
}
}
/// Read a floating-point field or return its default.
fn f64_field(
object: &serde_json::Map<String, serde_json::Value>,
key: &str,
default: f64,
) -> Result<f64, String> {
match object.get(key) {
Some(value) => value
.as_f64()
.ok_or_else(|| format!("{key} must be a number")),
None => Ok(default),
}
}
fn optional_string_field(
object: &serde_json::Map<String, serde_json::Value>,
key: &str,
) -> Result<Option<String>, String> {
match object.get(key) {
Some(serde_json::Value::Null) | None => Ok(None),
Some(value) => value
.as_str()
.map(|value| Some(value.to_string()))
.ok_or_else(|| format!("{key} must be a string or null")),
}
}
fn i64_field(
object: &serde_json::Map<String, serde_json::Value>,
key: &str,
default: i64,
) -> Result<i64, String> {
match object.get(key) {
Some(value) => value
.as_i64()
.ok_or_else(|| format!("{key} must be an integer")),
None => Ok(default),
}
}
fn u64_field(
object: &serde_json::Map<String, serde_json::Value>,
key: &str,
default: u64,
) -> Result<u64, String> {
match object.get(key) {
Some(value) => value
.as_u64()
.filter(|value| *value > 0)
.ok_or_else(|| format!("{key} must be a positive integer")),
None => Ok(default),
}
}
fn u64_field_allow_zero(
object: &serde_json::Map<String, serde_json::Value>,
key: &str,
default: u64,
) -> Result<u64, String> {
match object.get(key) {
Some(value) => value
.as_u64()
.ok_or_else(|| format!("{key} must be a non-negative integer")),
None => Ok(default),
}
}
fn usize_field(
object: &serde_json::Map<String, serde_json::Value>,
key: &str,
default: usize,
) -> Result<usize, String> {
let value = u64_field(object, key, default as u64)?;
usize::try_from(value).map_err(|_| format!("{key} must fit in usize"))
}
fn string_array_field(
object: &serde_json::Map<String, serde_json::Value>,
key: &str,
) -> Result<Vec<String>, String> {
match object.get(key) {
Some(value) => value
.as_array()
.ok_or_else(|| format!("{key} must be an array of strings"))?
.iter()
.map(|item| {
item.as_str()
.map(ToOwned::to_owned)
.ok_or_else(|| format!("{key} must be an array of strings"))
})
.collect(),
None => Ok(Vec::new()),
}
}
fn string_array_field_or_default(
object: &serde_json::Map<String, serde_json::Value>,
key: &str,
default: &[String],
) -> Result<Vec<String>, String> {
match object.get(key) {
Some(_) => string_array_field(object, key),
None => Ok(default.to_vec()),
}
}
fn cache_policy_from_config(config: &RagfsCacheConfig) -> CachePolicy {
config.bypass_prefixes.iter().fold(
CachePolicy::new(config.max_file_size_bytes).with_traversal_mode(config.traversal_mode),
|policy, prefix| policy.with_bypass_prefix(prefix),
)
}
mod git;
mod privacy;
mod privacy_extractor;
fn py_detach_blocking<T, F>(py: Python<'_>, f: F) -> T
where
T: Send,
F: Send + FnOnce() -> T,
{
py.detach(f)
}
/// Get a Python exception class from the pyagfs module
fn get_exception<'py>(py: Python<'py>, name: &str) -> PyResult<Bound<'py, PyType>> {
let pyagfs = PyModule::import(py, "openviking.pyagfs")?;
let exc = pyagfs.getattr(name)?;
Ok(exc.cast_into()?)
}
/// Create a PyErr from an exception type name and message
fn new_py_err(name: &str, msg: String) -> PyErr {
Python::attach(|py| {
if let Ok(exc) = get_exception(py, name) {
PyErr::from_type(exc, msg)
} else {
PyRuntimeError::new_err(msg)
}
})
}
/// Convert a ragfs error into the appropriate Python exception
fn to_py_err(e: ragfs::core::Error) -> PyErr {
let msg = e.to_string();
match e {
ragfs::core::Error::NotFound(_) => new_py_err("AGFSNotFoundError", msg),
ragfs::core::Error::AlreadyExists(_) => new_py_err("AGFSAlreadyExistsError", msg),
ragfs::core::Error::PermissionDenied(_) => new_py_err("AGFSPermissionDeniedError", msg),
ragfs::core::Error::InvalidPath(_) => new_py_err("AGFSInvalidPathError", msg),
ragfs::core::Error::NotADirectory(_) => new_py_err("AGFSNotADirectoryError", msg),
ragfs::core::Error::IsADirectory(_) => new_py_err("AGFSIsADirectoryError", msg),
ragfs::core::Error::DirectoryNotEmpty(_) => new_py_err("AGFSDirectoryNotEmptyError", msg),
ragfs::core::Error::InvalidOperation(_) => new_py_err("AGFSInvalidOperationError", msg),
ragfs::core::Error::NotSupported(_) => new_py_err("AGFSNotSupportedError", msg),
ragfs::core::Error::Io(_) => new_py_err("AGFSIoError", msg),
ragfs::core::Error::Plugin(_) => {
// Check if the plugin error message contains known patterns
let err_msg = msg.to_lowercase();
if err_msg.contains("directory not empty") {
new_py_err("AGFSDirectoryNotEmptyError", msg)
} else {
new_py_err("AGFSPluginError", msg)
}
}
ragfs::core::Error::Config(_) => new_py_err("AGFSConfigError", msg),
ragfs::core::Error::MountPointNotFound(_) => new_py_err("AGFSMountPointNotFoundError", msg),
ragfs::core::Error::MountPointExists(_) => new_py_err("AGFSMountPointExistsError", msg),
ragfs::core::Error::Serialization(_) => new_py_err("AGFSSerializationError", msg),
ragfs::core::Error::Network(_) => new_py_err("AGFSNetworkError", msg),
ragfs::core::Error::Timeout(_) => new_py_err("AGFSTimeoutError", msg),
ragfs::core::Error::WouldBlock(_) => new_py_err("AGFSTimeoutError", msg),
ragfs::core::Error::PathLock(error) => pathlock_err_to_py(error),
ragfs::core::Error::SyncWriteQuorum { .. } => new_py_err("AGFSInternalError", msg),
ragfs::core::Error::ContextMissing(_) => new_py_err("AGFSInternalError", msg),
ragfs::core::Error::Internal(_) => new_py_err("AGFSInternalError", msg),
}
}
/// Convert FileInfo to a Python dict matching the Go binding JSON format:
/// {"name": str, "size": int, "mode": int, "modTime": str, "isDir": bool}
fn file_info_to_py_dict(py: Python<'_>, info: &FileInfo) -> PyResult<Py<PyDict>> {
let dict = PyDict::new(py);
dict.set_item("name", &info.name)?;
dict.set_item("size", info.size)?;
dict.set_item("mode", info.mode)?;
// modTime as RFC3339 string (Go binding format)
let secs = info
.mod_time
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let mod_time = format_rfc3339(secs);
dict.set_item("modTime", mod_time)?;
dict.set_item("isDir", info.is_dir)?;
Ok(dict.into())
}
/// Convert TreeEntry to a Python dict:
/// {"path": str, "rel_path": str, "info": dict, "extra": dict}
fn tree_entry_to_py_dict(py: Python<'_>, entry: &TreeEntry) -> PyResult<Py<PyDict>> {
let dict = PyDict::new(py);
dict.set_item("path", &entry.path)?;
dict.set_item("rel_path", &entry.rel_path)?;
dict.set_item("info", file_info_to_py_dict(py, &entry.info)?)?;
let extra_dict = PyDict::new(py);
for (k, v) in &entry.extra {
let py_val: Py<PyAny> = serde_json_to_py(py, v)?;
extra_dict.set_item(k, py_val)?;
}
dict.set_item("extra", extra_dict)?;
Ok(dict.into())
}
/// Convert a serde_json::Value to a Python object.
fn serde_json_to_py(py: Python<'_>, val: &serde_json::Value) -> PyResult<Py<PyAny>> {
match val {
serde_json::Value::Null => Ok(py.None()),
serde_json::Value::Bool(b) => {
let val: bool = *b;
let bound = val
.into_pyobject(py)
.map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
Ok(bound.as_any().clone().unbind())
}
serde_json::Value::Number(n) => {
if let Some(i) = n.as_i64() {
Ok(i.into_pyobject(py)?.into_any().unbind())
} else if let Some(f) = n.as_f64() {
Ok(f.into_pyobject(py)?.into_any().unbind())
} else {
Ok(py.None())
}
}
serde_json::Value::String(s) => Ok(s.into_pyobject(py)?.into_any().unbind()),
serde_json::Value::Array(arr) => {
let list = PyList::empty(py);
for item in arr {
list.append(serde_json_to_py(py, item)?)?;
}
Ok(list.into())
}
serde_json::Value::Object(obj) => {
let d = PyDict::new(py);
for (k, v) in obj {
d.set_item(k, serde_json_to_py(py, v)?)?;
}
Ok(d.into())
}
}
}
/// Convert GrepResult to a Python dict matching the Go binding JSON format:
/// {"matches": [{"file": str, "line": int, "content": str, ...}, ...], "count": int}
fn grep_result_to_py_dict(py: Python<'_>, result: &GrepResult) -> PyResult<Py<PyDict>> {
let dict = PyDict::new(py);
let matches_list = PyList::empty(py);
for m in &result.matches {
let match_dict = PyDict::new(py);
match_dict.set_item("file", &m.file)?;
match_dict.set_item("line", m.line)?;
match_dict.set_item("content", &m.content)?;
if let Some(lines) = &m.before_context {
let context = PyList::empty(py);
for line in lines {
let item = PyDict::new(py);
item.set_item("line", line.line)?;
item.set_item("content", &line.content)?;
context.append(item)?;
}
match_dict.set_item("before_context", context)?;
}
if let Some(lines) = &m.after_context {
let context = PyList::empty(py);
for line in lines {
let item = PyDict::new(py);
item.set_item("line", line.line)?;
item.set_item("content", &line.content)?;
context.append(item)?;
}
match_dict.set_item("after_context", context)?;
}
matches_list.append(match_dict)?;
}
dict.set_item("matches", matches_list)?;
dict.set_item("count", result.count)?;
Ok(dict.into())
}
/// Convert the supplied native statistics into the legacy microsecond Python dict.
fn operation_stats_to_py_dict(py: Python<'_>, stats: &OperationStats) -> PyResult<Py<PyDict>> {
let dict = PyDict::new(py);
dict.set_item("count", stats.count)?;
dict.set_item("total_time_us", stats.total_time_ns / 1_000)?;
dict.set_item(
"min_time_us",
if stats.count == 0 {
u64::MAX
} else {
stats.min_time_ns / 1_000
},
)?;
dict.set_item("max_time_us", stats.max_time_ns / 1_000)?;
dict.set_item("avg_time_us", stats.avg_time_ns() / 1_000.0)?;
Ok(dict.into())
}
/// Convert the supplied native metric into a flat Python dict, preserving integer values.
fn metric_to_py_dict(py: Python<'_>, metric: &RagfsMetric) -> PyResult<Py<PyDict>> {
let dict = PyDict::new(py);
dict.set_item("name", &metric.name)?;
dict.set_item("labels", &metric.labels)?;
match &metric.value {
RagfsMetricValue::Counter { value, scale } => {
dict.set_item("type", "counter")?;
dict.set_item("value", value)?;
dict.set_item("scale", scale)?;
}
RagfsMetricValue::Gauge(value) => {
dict.set_item("type", "gauge")?;
dict.set_item("value", value)?;
}
RagfsMetricValue::Histogram {
bucket_bounds,
bucket_counts,
count,
sum,
scale,
} => {
dict.set_item("type", "histogram")?;
dict.set_item("bucket_bounds", bucket_bounds)?;
dict.set_item("bucket_counts", bucket_counts)?;
dict.set_item("count", count)?;
dict.set_item("sum", sum)?;
dict.set_item("scale", scale)?;
}
}
Ok(dict.into())
}
/// Convert FilesystemStats to a Python dict.
fn filesystem_stats_to_py_dict(py: Python<'_>, stats: &FilesystemStats) -> PyResult<Py<PyDict>> {
let dict = PyDict::new(py);
let ops_dict = PyDict::new(py);
for op in FsOperation::all() {
let op_stats = stats.get(*op);
let op_dict = operation_stats_to_py_dict(py, op_stats)?;
ops_dict.set_item(op.as_str(), op_dict)?;
}
dict.set_item("operations", ops_dict)?;
Ok(dict.into())
}
/// Format unix timestamp as RFC3339 string (simplified, UTC)
fn format_rfc3339(secs: u64) -> String {
let s = secs;
let days = s / 86400;
let time_of_day = s % 86400;
let h = time_of_day / 3600;
let m = (time_of_day % 3600) / 60;
let sec = time_of_day % 60;
// Calculate date from days since epoch (simplified)
let (year, month, day) = days_to_ymd(days);
format!(
"{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z",
year, month, day, h, m, sec
)
}
/// Convert days since Unix epoch to (year, month, day)
fn days_to_ymd(days: u64) -> (u64, u64, u64) {
// Algorithm from http://howardhinnant.github.io/date_algorithms.html
let z = days + 719468;
let era = z / 146097;
let doe = z - era * 146097;
let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
let y = yoe + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let y = if m <= 2 { y + 1 } else { y };
(y, m, d)
}
/// Convert a Python dict to HashMap<String, ConfigValue>, with recursive nesting support.
fn py_dict_to_config(dict: &Bound<'_, PyDict>) -> PyResult<HashMap<String, ConfigValue>> {
let mut params = HashMap::new();
for (k, v) in dict.iter() {
let key: String = k.extract()?;
let value = py_any_to_config_value(&v)?;
params.insert(key, value);
}
Ok(params)
}
/// Convert a single Python value to ConfigValue, recursing into dicts/lists.
fn py_any_to_config_value(v: &Bound<'_, PyAny>) -> PyResult<ConfigValue> {
if let Ok(s) = v.extract::<String>() {
Ok(ConfigValue::String(s))
} else if let Ok(b) = v.extract::<bool>() {
Ok(ConfigValue::Bool(b))
} else if let Ok(i) = v.extract::<i64>() {
Ok(ConfigValue::Int(i))
} else if let Ok(list) = v.downcast::<PyList>() {
let mut strings = Vec::new();
let mut has_nested = false;
for item in list.iter() {
if let Ok(s) = item.extract::<String>() {
strings.push(s);
} else {
has_nested = true;
break;
}
}
if has_nested || strings.is_empty() {
// Convert to serde_json::Value for complex lists
let json_val = py_to_json_value(v)?;
Ok(ConfigValue::Json(json_val))
} else {
Ok(ConfigValue::StringList(strings))
}
} else if v.is_instance_of::<PyDict>() {
let json_val = py_to_json_value(v)?;
Ok(ConfigValue::Json(json_val))
} else {
// Fallback: try string representation
Ok(ConfigValue::String(v.str()?.to_string()))
}
}
/// Convert a Python value to serde_json::Value recursively.
fn py_to_json_value(v: &Bound<'_, PyAny>) -> PyResult<serde_json::Value> {
if v.is_none() {
Ok(serde_json::Value::Null)
} else if let Ok(b) = v.extract::<bool>() {
Ok(serde_json::Value::Bool(b))
} else if let Ok(i) = v.extract::<i64>() {
Ok(serde_json::Value::Number(i.into()))
} else if let Ok(f) = v.extract::<f64>() {
if let Some(n) = serde_json::Number::from_f64(f) {
Ok(serde_json::Value::Number(n))
} else {
Ok(serde_json::Value::String(f.to_string()))
}
} else if let Ok(s) = v.extract::<String>() {
Ok(serde_json::Value::String(s))
} else if let Ok(list) = v.downcast::<PyList>() {
let mut arr = Vec::new();
for item in list.iter() {
arr.push(py_to_json_value(&item)?);
}
Ok(serde_json::Value::Array(arr))
} else if let Ok(dict) = v.downcast::<PyDict>() {
let mut map = serde_json::Map::new();
for (k, val) in dict.iter() {
let key: String = k.extract()?;
map.insert(key, py_to_json_value(&val)?);
}
Ok(serde_json::Value::Object(map))
} else {
Ok(serde_json::Value::String(v.str()?.to_string()))
}
}
/// Build an immutable `FsContext` from an optional Python `{"account_id": str}` dict.
///
/// A context is always constructed (even when the dict is absent or lacks `account_id`), so the
/// "every ragfs call carries a ctx" invariant holds. Missing `account_id` is represented as an
/// empty field; `FsContextView::account_id()` treats it as absent, so encrypted content operations
/// fail fast instead of using an accidental empty tenant key.
///
/// When the dict contains `"disable_auto_pathlock": "true"`, the `PathLockWrappedFS` auto-lock
/// layer is suppressed so callers that already hold a pathlock lease can safely delegate to the
/// inner filesystem without double-locking.
///
/// When the dict contains `"bypass_cache": "true"`, plugin-local metadata caches (e.g. the S3FS
/// stat cache) are bypassed so the operation observes fresh backend state.
fn build_fs_context(ctx: Option<HashMap<String, String>>) -> FsContext {
let mut account_id = String::new();
let mut disable_auto_pathlock = false;
let mut lease_ref: Option<String> = None;
let mut bypass_cache = false;
if let Some(m) = &ctx {
account_id = m.get("account_id").cloned().unwrap_or_default();
disable_auto_pathlock = m
.get("disable_auto_pathlock")
.map(|s| s == "true")
.unwrap_or(false);
lease_ref = m.get("lease_ref").cloned();
bypass_cache = m.get("bypass_cache").map(|s| s == "true").unwrap_or(false);
}
let pathlock = if disable_auto_pathlock || lease_ref.is_some() {
Some(PathLockContext {
lease_ref,
disable_auto_pathlock,
})
} else {
None
};
Arc::new(
FsContextInner::with_pathlock(account_id, pathlock.unwrap_or_default())
.with_bypass_cache(bypass_cache),
)
}
/// Convert an OwnedPathLockLease to a Python dict.
fn owned_lease_to_py_dict(py: Python<'_>, lease: &OwnedPathLockLease) -> PyResult<Py<PyAny>> {
let dict = PyDict::new(py);
dict.set_item("lease_ref", &lease.lease.lease_ref)?;
dict.set_item("owner_id", &lease.lease.owner_id)?;
dict.set_item("lock_paths", &lease.lease.lock_paths)?;
dict.set_item("ownership_ref", &lease.ownership_ref)?;
dict.set_item("owned", true)?;
Ok(dict.into())
}
/// Convert a BorrowedPathLockLease to a Python dict.
fn borrowed_lease_to_py_dict(py: Python<'_>, lease: &BorrowedPathLockLease) -> PyResult<Py<PyAny>> {
let dict = PyDict::new(py);
dict.set_item("lease_ref", &lease.lease.lease_ref)?;
dict.set_item("owner_id", &lease.lease.owner_id)?;
dict.set_item("lock_paths", &lease.lease.lock_paths)?;
dict.set_item("owned", false)?;
Ok(dict.into())
}
/// Require an owned marker and return its opaque lease and lifecycle capability refs.
fn require_owned_lease_ref(
py: Python<'_>,
ref_dict: &HashMap<String, Py<PyAny>>,
) -> PyResult<(String, String)> {
let owned: bool = ref_dict
.get("owned")
.map(|v| v.extract(py))
.transpose()?
.unwrap_or(false);
if !owned {
return Err(PyValueError::new_err(
"lease ref is not owned; release/refresh/handoff require an owned lease",
));
}
let lease_ref: String = ref_dict
.get("lease_ref")
.ok_or_else(|| PyValueError::new_err("owned_lease_ref.lease_ref required"))?
.extract(py)?;
let ownership_ref: String = ref_dict
.get("ownership_ref")
.ok_or_else(|| PyValueError::new_err("owned_lease_ref.ownership_ref required"))?
.extract(py)?;
if lease_ref.is_empty() || ownership_ref.is_empty() {
return Err(PyValueError::new_err(
"owned lease and ownership refs must not be empty",
));
}
Ok((lease_ref, ownership_ref))
}
/// Extract an opaque PathLock lease ref without granting lifecycle permissions.
fn extract_lease_ref(py: Python<'_>, lease_ref: &Py<PyAny>) -> PyResult<String> {
if let Ok(raw) = lease_ref.extract::<String>(py) {
if !raw.is_empty() {
return Ok(raw);
}
}
let ref_dict: HashMap<String, Py<PyAny>> = lease_ref.extract(py)?;
ref_dict
.get("lease_ref")
.ok_or_else(|| PyValueError::new_err("lease_ref.lease_ref required"))?
.extract(py)
}
/// Extract an owned PathLock lease and lifecycle capability from a typed dictionary.
fn extract_owned_lease_ref(py: Python<'_>, lease_ref: &Py<PyAny>) -> PyResult<(String, String)> {
let ref_dict: HashMap<String, Py<PyAny>> = lease_ref.extract(py)?;
require_owned_lease_ref(py, &ref_dict)
}
/// Extract an optional owned lease capability used for reentrant acquisition.
fn extract_optional_owned_lease_ref(
py: Python<'_>,
lease_ref: Option<&Py<PyAny>>,
) -> PyResult<Option<(String, String)>> {
lease_ref
.map(|value| extract_owned_lease_ref(py, value))
.transpose()
}
/// Convert a PathLockHandoffRef to a Python dict.
fn handoff_ref_to_py_dict(py: Python<'_>, handoff: &PathLockHandoffRef) -> PyResult<Py<PyAny>> {
let dict = PyDict::new(py);
if let Some(lease_ref) = &handoff.lease_ref {
dict.set_item("lease_ref", lease_ref)?;
}
dict.set_item("owner_id", &handoff.owner_id)?;
if let Some(mode) = handoff.handle_mode {
dict.set_item(
"handle_mode",
match mode {
PathLockHandleMode::LockPath => "lock_path",
PathLockHandleMode::LogicalPath => "logical_path",
},
)?;
}
dict.set_item("lock_paths", &handoff.lock_paths)?;
let covered = PyList::empty(py);
for req in &handoff.covered_paths {
let item = PyDict::new(py);
item.set_item("path", &req.path)?;
item.set_item(
"kind",
match req.kind {
PathLockKind::Exact => "exact",
PathLockKind::Tree => "tree",
},
)?;
covered.append(item)?;
}
dict.set_item("covered_paths", covered)?;
Ok(dict.into())
}
#[derive(serde::Deserialize)]
struct BindingConfig {
#[serde(default)]
git: Option<ragfs::git::GitConfig>,
}
fn build_git_from_cfg(
git_cfg: ragfs::git::GitConfig,
fs: &Arc<MountableFS>,
rt: &tokio::runtime::Runtime,
) -> PyResult<(Option<Arc<ragfs::git::GitService>>, Option<String>)> {
let backend = git_cfg.backend.clone();
let vfs = fs.clone() as Arc<dyn ragfs::core::FileSystem>;
let _guard = rt.enter();
let svc = git::build_git_service(&git_cfg, vfs)?;
Ok((svc, Some(backend)))
}
fn load_git_from_config(
path: &str,
fs: &Arc<MountableFS>,
rt: &tokio::runtime::Runtime,
) -> PyResult<(Option<Arc<ragfs::git::GitService>>, Option<String>)> {
let body = std::fs::read_to_string(path)
.map_err(|e| PyRuntimeError::new_err(format!("read config_path {}: {}", path, e)))?;
let cfg: BindingConfig = toml::from_str(&body)
.map_err(|e| PyRuntimeError::new_err(format!("parse config_path: {}", e)))?;
match cfg.git {
Some(git_cfg) => build_git_from_cfg(git_cfg, fs, rt),
None => Ok((None, None)),
}
}
/// RAGFS Python Binding Client.
///
/// Embeds the ragfs filesystem engine directly in the Python process.
/// API-compatible with the Go-based AGFSBindingClient.
#[pyclass]
struct RAGFSBindingClient {
/// Mount manager: mount/unmount/list/get_*_stats/register_plugin.
mountable: Arc<MountableFS>,
/// Data entry point: `Stats(Encryption(Mountable))` when encrypted, else `Stats(Mountable)`.
top: Arc<dyn FileSystem>,
rt: tokio::runtime::Runtime,
git_service: Option<Arc<ragfs::git::GitService>>,
git_backend: Option<String>,
/// PathLock manager. OpenViking always builds ragfs with PathLock enabled.
pathlock_manager: Arc<PathLockManager>,
/// Shared cache runtime. Closed only after mounted QueueFS instances shut down.
cache_runtime: Option<Arc<CacheRuntime>>,
/// Service namespace used for shared leader keys.
service_name: String,
/// Opaque public references to hidden leader ownership tokens.
leader_leases: Arc<AsyncMutex<HashMap<String, LeaderLease>>>,
/// Blocks leader operations during close and waits in-flight operations out.
leader_closing: Arc<AsyncRwLock<bool>>,
}
impl RAGFSBindingClient {
/// Run an async filesystem op on the runtime, releasing the GIL, inside the FS_CTX scope.
///
/// Centralizes the GIL-detach + `FS_CTX.scope` + `block_on` pattern so every data method
/// shares one implementation (no per-method duplication).
fn run_scoped<T, Fut, F>(&self, py: Python<'_>, ctx: FsContext, f: F) -> T
where
T: Send,
Fut: Future<Output = T>,
F: Send + FnOnce() -> Fut,
{
py_detach_blocking(py, move || {
self.rt
.block_on(FS_CTX.scope(ctx, async move { f().await }))
})
}
/// Clone the PathLock manager shared by this binding client.
fn clone_pathlock_manager(&self) -> Arc<PathLockManager> {
self.pathlock_manager.clone()
}
/// Build the leader manager lazily so providers without scripting still construct normally.
fn clone_leader_manager(&self) -> PyResult<Arc<LeaderManager>> {
let runtime = self.cache_runtime.clone().ok_or_else(|| {
PyRuntimeError::new_err("distributed leadership requires an AGFS CacheRuntime")
})?;
LeaderManager::new(runtime, &self.service_name)
.map(Arc::new)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))
}
}
#[pymethods]
impl RAGFSBindingClient {
/// Create a new RAGFS binding client.
///
/// `config_path` loads the canonical ov.conf schema. `config.service_name` must be supplied
/// by the Python runtime.
/// `config` is an optional sectioned dict produced by the Python runtime. The `encryption`
/// section, when present, carries `root_key` (32 bytes) + `provider_type` (int) and causes the
/// stack to include an `EncryptionWrappedFS` layer. Runtime `cache` configuration takes
/// precedence over `config_path`.
#[new]
#[pyo3(signature = (config_path=None, config=None, git_config_path=None))]
fn new(
py: Python<'_>,
config_path: Option<&str>,
config: Option<HashMap<String, Py<PyAny>>>,
git_config_path: Option<&str>,
) -> PyResult<Self> {
let rt = tokio::runtime::Runtime::new()
.map_err(|e| PyRuntimeError::new_err(format!("Failed to create runtime: {}", e)))?;
// Phase A (holding GIL): parse the sectioned config into an owned RagfsConfig.
let mut ragfs_cfg = RagfsConfig::default();
let service_name = match config.as_ref() {
Some(cfg) => Some(
cfg.get("service_name")
.ok_or_else(|| PyValueError::new_err("service_name is required"))?
.extract::<String>(py)?,
),
None => None,
};
let inline_pathlock_configured = config
.as_ref()
.is_some_and(|config| config.contains_key("pathlock"));
if !inline_pathlock_configured {
if let Some(path) = config_path {
let service_name = service_name
.as_deref()
.ok_or_else(|| PyValueError::new_err("service_name is required"))?;
ragfs_cfg.pathlock =
pathlock_config_from_ov_conf(path, service_name).map_err(|error| {
PyRuntimeError::new_err(format!("Invalid PathLock config: {error}"))
})?;
}
}
let mut runtime_cache_config = None;
let mut inline_git_cfg: Option<ragfs::git::GitConfig> = None;
if let Some(cfg) = config {
let service_name = service_name
.as_deref()
.ok_or_else(|| PyValueError::new_err("service_name is required"))?;
if let Some(log_obj) = cfg.get("log") {
let log_cfg: HashMap<String, Py<PyAny>> = log_obj.extract(py)?;
let log_level = log_cfg
.get("level")
.map(|value| value.extract::<String>(py))
.transpose()?
.unwrap_or_else(|| "INFO".to_string());
let log_output = log_cfg
.get("output")
.map(|value| value.extract::<String>(py))
.transpose()?
.unwrap_or_else(|| "stderr".to_string());
init_tracing(&log_level, &log_output)?;
}
if let Some(enc_obj) = cfg.get("encryption") {
let enc: HashMap<String, Py<PyAny>> = enc_obj.extract(py)?;
let rk: Vec<u8> = enc
.get("root_key")
.ok_or_else(|| PyValueError::new_err("encryption.root_key required"))?
.extract(py)?;
let root_key: [u8; 32] = rk
.as_slice()
.try_into()
.map_err(|_| PyValueError::new_err("root_key must be 32 bytes"))?;
let provider_type: u8 = enc
.get("provider_type")
.ok_or_else(|| PyValueError::new_err("encryption.provider_type required"))?
.extract(py)?;
ragfs_cfg.encryption = Some(EncryptionConfig {
root_key,
provider_type,
});
}
if let Some(cache_obj) = cfg.get("cache") {
let cache_value = py_to_json_value(cache_obj.bind(py))?;
runtime_cache_config = Some(
cache_config_from_value(&cache_value, service_name).map_err(|error| {
PyRuntimeError::new_err(format!("Invalid cache config: {error}"))
})?,
);
}
if let Some(git_obj) = cfg.get("git") {
let git_value = py_to_json_value(git_obj.bind(py))?;
let git_cfg: ragfs::git::GitConfig = serde_json::from_value(git_value)
.map_err(|e| PyRuntimeError::new_err(format!("Invalid git config: {e}")))?;
inline_git_cfg = Some(git_cfg);
}
if let Some(pl_obj) = cfg.get("pathlock") {
let pl_value = py_to_json_value(pl_obj.bind(py))?;
ragfs_cfg.pathlock = pathlock_config_from_value(&pl_value, service_name)
.map_err(PyValueError::new_err)?;
}
}
let inline_cache_configured = runtime_cache_config.is_some();
let pathlock_uses_cache = ragfs_cfg.pathlock.provider == "cache";
let file_cache_config = config_path
.map(|path| {
let service_name = service_name
.as_deref()
.ok_or_else(|| "service_name is required".to_string())?;
if inline_cache_configured {
cache_config_from_ov_conf_with_runtime(path, false, service_name)
} else if inline_pathlock_configured {
cache_config_from_ov_conf_with_runtime(path, pathlock_uses_cache, service_name)
} else {
cache_config_from_ov_conf(path, service_name)
}
})
.transpose()
.map_err(|error| PyRuntimeError::new_err(format!("Invalid cache config: {error}")))?;
let mut cache_config = runtime_cache_config
.or(file_cache_config)
.unwrap_or_default();
if pathlock_uses_cache {
if cache_config.provider != CacheProviderKind::Redis {
return Err(PyValueError::new_err(
"cache-backed PathLock requires cache.provider=redis",
));
}
if inline_cache_configured {
cache_config.runtime_enabled = true;
}
if !cache_config.runtime_enabled {
return Err(PyValueError::new_err(
"top-level cache config is required for cache-backed PathLock",
));
}
}
// Phase B: RAGFS owns Runtime construction and injects the same instance
// into CacheFS and QueueFS. The binding only translates configuration.
let stack = rt
.block_on(build_configured_stack(
ragfs_cfg,
cache_config.stack_config(),
))
.map_err(|error| {
if matches!(error, ragfs::core::Error::NotSupported(_)) {
to_py_err(error)
} else {
PyRuntimeError::new_err(format!("Failed to build RAGFS stack: {error}"))
}
})?;
// Build the git service from inline config when present; otherwise fall
// back to loading the [git] section from a config file path.
let (git_service, git_backend) = match inline_git_cfg {
Some(git_cfg) => build_git_from_cfg(git_cfg, &stack.mountable, &rt)?,
None => match git_config_path {
Some(path) => load_git_from_config(path, &stack.mountable, &rt)?,
None => (None, None),
},
};
Ok(Self {
mountable: stack.mountable,
top: stack.top,
rt,
git_service,
git_backend,
pathlock_manager: stack.pathlock_manager,
cache_runtime: stack.cache_runtime,
service_name: service_name.unwrap_or_else(|| "ragfs".to_string()),
leader_leases: Arc::new(AsyncMutex::new(HashMap::new())),
leader_closing: Arc::new(AsyncRwLock::new(false)),
})
}
/// Stop mounts requiring explicit cleanup, then close the shared CacheRuntime.
fn close(&self, py: Python<'_>) -> PyResult<()> {
let mountable = Arc::clone(&self.mountable);
let cache_runtime = self.cache_runtime.clone();
let service_name = self.service_name.clone();
let leader_leases = Arc::clone(&self.leader_leases);
let leader_closing = Arc::clone(&self.leader_closing);
py_detach_blocking(py, || {
self.rt.block_on(async move {
let mut closing = leader_closing.write().await;
*closing = true;
let mount_result = mountable.shutdown().await;
let cache_result = match cache_runtime {
Some(runtime) => {
if let Ok(manager) = LeaderManager::new(runtime.clone(), &service_name) {
let leases = std::mem::take(&mut *leader_leases.lock().await);
for lease in leases.into_values() {
let _ = manager.release(&lease).await;
}
}
runtime.close().await.map_err(|error| error.to_string())
}
None => Ok(()),
};
drop(closing);
mount_result.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
cache_result.map_err(PyRuntimeError::new_err)
})
})
}
/// Check client health.
fn health(&self) -> PyResult<HashMap<String, String>> {
let mut m = HashMap::new();
m.insert("status".to_string(), "healthy".to_string());
m.insert(
"git_enabled".to_string(),
if self.git_service.is_some() {
"true".into()
} else {
"false".into()
},
);
if let Some(b) = &self.git_backend {
m.insert("git_backend".to_string(), b.clone());
}
Ok(m)
}
/// Commit a snapshot of the account's tree.
#[pyo3(signature = (**kwargs))]
fn git_commit(
&self,
py: Python<'_>,
kwargs: Option<&Bound<'_, PyDict>>,
) -> PyResult<Py<PyAny>> {
let svc = self.git_service.clone().ok_or_else(|| {
git::new_py_err_pub(py, "AGFSNotSupportedError", "git feature disabled".into())
})?;
let empty = PyDict::new(py);
let kw = kwargs.unwrap_or(&empty);
let req = git::parse_commit_request(kw)?;
let resp = py_detach_blocking(py, move || self.rt.block_on(svc.commit(req)))
.map_err(|e| git::map_git_error(py, e))?;
git::commit_response_to_pydict(py, resp)
}
/// Restore a project_dir subtree to the state at source_commit.
#[pyo3(signature = (*, ctx=None, **kwargs))]
fn git_restore(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
kwargs: Option<&Bound<'_, PyDict>>,
) -> PyResult<Py<PyAny>> {
let svc = self.git_service.clone().ok_or_else(|| {
git::new_py_err_pub(py, "AGFSNotSupportedError", "git feature disabled".into())
})?;
let empty = PyDict::new(py);
let kw = kwargs.unwrap_or(&empty);
let req = git::parse_restore_request(kw)?;
let resp = self
.run_scoped(py, build_fs_context(ctx), || svc.restore(req))
.map_err(|e| git::map_git_error(py, e))?;
git::restore_response_to_pydict(py, resp)
}
/// Read a commit's metadata or a blob's bytes at a path.
#[pyo3(signature = (**kwargs))]
fn git_show(&self, py: Python<'_>, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<Py<PyAny>> {
let svc = self.git_service.clone().ok_or_else(|| {
git::new_py_err_pub(py, "AGFSNotSupportedError", "git feature disabled".into())
})?;
let empty = PyDict::new(py);
let kw = kwargs.unwrap_or(&empty);
let req = git::parse_show_request(kw)?;
let max_blob_bytes = git::parse_show_max_blob_bytes(kw)?;
let resp = py_detach_blocking(py, move || {
self.rt.block_on(svc.show_with_limit(req, max_blob_bytes))
})
.map_err(|e| git::map_git_error(py, e))?;
git::show_response_to_pydict(py, resp)
}
/// Build a bounded unified text diff outside the Python interpreter.
#[pyo3(signature = (**kwargs))]
fn git_diff_text(
&self,
py: Python<'_>,
kwargs: Option<&Bound<'_, PyDict>>,
) -> PyResult<String> {
let empty = PyDict::new(py);
let kw = kwargs.unwrap_or(&empty);
let req = git::parse_diff_text_request(kw)?;
let result = py_detach_blocking(py, move || {
git::build_bounded_unified_diff(
&req.before,
&req.after,
&req.fromfile,
&req.tofile,
req.timeout_ms,
req.max_output_bytes,
)
});
result.map_err(|err| match err {
git::BoundedDiffError::OutputTooLarge { limit } => git::new_py_err_pub(
py,
"AGFSResourceExhaustedError",
format!("snapshot diff output size limit exceeded ({limit} bytes)"),
),
git::BoundedDiffError::Render(message) => {
git::new_py_err_pub(py, "AGFSInternalError", message)
}
})
}
/// List changed files under directories between two commits.
#[pyo3(signature = (**kwargs))]
fn git_diff_paths(
&self,
py: Python<'_>,
kwargs: Option<&Bound<'_, PyDict>>,
) -> PyResult<Vec<(String, String)>> {
let svc = self.git_service.clone().ok_or_else(|| {
git::new_py_err_pub(py, "AGFSNotSupportedError", "git feature disabled".into())
})?;
let empty = PyDict::new(py);
let kw = kwargs.unwrap_or(&empty);
let req = git::parse_diff_paths_request(kw)?;
let resp = py_detach_blocking(py, move || {
self.rt.block_on(svc.diff_paths(
&req.account,
req.base.as_deref(),
&req.head,
&req.paths,
))
})
.map_err(|e| git::map_git_error(py, e))?;
Ok(resp
.into_iter()
.map(|(path, change)| (path, change.to_string()))
.collect())
}
/// Walk snapshot history, optionally filtered to commits touching paths.
#[pyo3(signature = (**kwargs))]
fn git_log(&self, py: Python<'_>, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<Py<PyAny>> {
let svc = self.git_service.clone().ok_or_else(|| {
git::new_py_err_pub(py, "AGFSNotSupportedError", "git feature disabled".into())
})?;
let empty = PyDict::new(py);
let kw = kwargs.unwrap_or(&empty);
let req = git::parse_log_request(kw)?;
let resp = py_detach_blocking(py, move || self.rt.block_on(svc.log(req)))
.map_err(|e| git::map_git_error(py, e))?;
git::log_entries_to_pylist(py, resp)
}
/// Get client capabilities.
fn get_capabilities(&self) -> PyResult<HashMap<String, Py<PyAny>>> {
Python::attach(|py| {
let mut m = HashMap::new();
m.insert(
"version".to_string(),
"ragfs-python".into_pyobject(py)?.into_any().unbind(),
);
let mut features = vec![
"memfs",
"kvfs",
"queuefs",
"sqlfs",
"localfs",
"serverinfofs",
];
#[cfg(feature = "s3")]
features.push("s3fs");
m.insert(
"features".to_string(),
features.into_pyobject(py)?.into_any().unbind(),
);
Ok(m)
})
}
/// List directory contents.
///
/// Returns a list of file info dicts with keys:
/// name, size, mode, modTime, isDir
#[pyo3(signature = (path, ctx=None, *, offset=0, limit=None, sort_by=None, sort_order=None))]
fn ls(
&self,
py: Python<'_>,
path: String,
ctx: Option<HashMap<String, String>>,
offset: i64,
limit: Option<i64>,
sort_by: Option<&str>,
sort_order: Option<&str>,
) -> PyResult<Py<PyAny>> {
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
let offset = parse_listing_offset(offset)?;
let limit = parse_listing_limit(limit)?;
let sort_by = parse_list_sort_by(sort_by)?;
let sort_order = parse_sort_order(sort_order)?;
let entries = self
.run_scoped(py, fs_ctx, move || async move {
top.read_dir(&path, Some(offset), limit, sort_by, sort_order)
.await
})
.map_err(to_py_err)?;
Python::attach(|py| {
let list = PyList::empty(py);
for entry in &entries {
let dict = file_info_to_py_dict(py, entry)?;
list.append(dict)?;
}
Ok(list.into())
})
}
/// Read file content.
///
/// Args:
/// path: File path
/// offset: Starting position (default: 0)
/// size: Number of bytes to read (default: -1, read all)
/// stream: Not supported in binding mode
/// ctx: Optional FsContext dict (e.g. {"account_id": ...})
#[pyo3(signature = (path, offset=0, size=-1, stream=false, ctx=None))]
fn read(
&self,
py: Python<'_>,
path: String,
offset: i64,
size: i64,
stream: bool,
ctx: Option<HashMap<String, String>>,
) -> PyResult<Py<PyAny>> {
if stream {
return Err(PyRuntimeError::new_err(
"Streaming not supported in binding mode",
));
}
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
let off = if offset < 0 { 0u64 } else { offset as u64 };
let sz = if size < 0 { 0u64 } else { size as u64 };
let data = self
.run_scoped(
py,
fs_ctx,
move || async move { top.read(&path, off, sz).await },
)
.map_err(to_py_err)?;
Python::attach(|py| Ok(PyBytes::new(py, &data).into()))
}
/// Read file content (alias for read).
#[pyo3(signature = (path, offset=0, size=-1, stream=false, ctx=None))]
fn cat(
&self,
py: Python<'_>,
path: String,
offset: i64,
size: i64,
stream: bool,
ctx: Option<HashMap<String, String>>,
) -> PyResult<Py<PyAny>> {
self.read(py, path, offset, size, stream, ctx)
}
/// Write data to file.
///
/// Args:
/// path: File path
/// data: File content as bytes
/// ctx: Optional FsContext dict (e.g. {"account_id": ...})
#[pyo3(signature = (path, data, max_retries=3, ctx=None))]
fn write(
&self,
py: Python<'_>,
path: String,
data: Vec<u8>,
max_retries: i32,
ctx: Option<HashMap<String, String>>,
) -> PyResult<String> {
let _ = max_retries; // not applicable for local binding
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
let len = data.len();
self.run_scoped(py, fs_ctx, move || async move {
top.write(&path, &data, 0, WriteFlag::Create).await
})
.map_err(to_py_err)?;
Ok(format!("Written {} bytes", len))
}
/// Create a new empty file.
#[pyo3(signature = (path, ctx=None))]
fn create(
&self,
py: Python<'_>,
path: String,
ctx: Option<HashMap<String, String>>,
) -> PyResult<HashMap<String, String>> {
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
self.run_scoped(py, fs_ctx, move || async move { top.create(&path).await })
.map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), "created".to_string());
Ok(m)
}
/// Create a directory.
#[pyo3(signature = (path, mode="755", ctx=None))]
fn mkdir(
&self,
py: Python<'_>,
path: String,
mode: &str,
ctx: Option<HashMap<String, String>>,
) -> PyResult<HashMap<String, String>> {
let mode_int = u32::from_str_radix(mode, 8)
.map_err(|e| PyRuntimeError::new_err(format!("Invalid mode '{}': {}", mode, e)))?;
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
self.run_scoped(py, fs_ctx, move || async move {
top.mkdir(&path, mode_int).await
})
.map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), "created".to_string());
Ok(m)
}
/// Ensure all parent directories exist for the given path.
#[pyo3(signature = (path, mode="755", ctx=None))]
fn ensure_parent_dirs(
&self,
py: Python<'_>,
path: String,
mode: &str,
ctx: Option<HashMap<String, String>>,
) -> PyResult<HashMap<String, String>> {
let mode_int = u32::from_str_radix(mode, 8)
.map_err(|e| PyRuntimeError::new_err(format!("Invalid mode '{}': {}", mode, e)))?;
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
self.run_scoped(py, fs_ctx, move || async move {
top.ensure_parent_dirs(&path, mode_int).await
})
.map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert(
"message".to_string(),
"parent directories ensured".to_string(),
);
Ok(m)
}
/// Remove a file or directory.
#[pyo3(signature = (path, recursive=false, ctx=None))]
fn rm(
&self,
py: Python<'_>,
path: String,
recursive: bool,
ctx: Option<HashMap<String, String>>,
) -> PyResult<HashMap<String, String>> {
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
self.run_scoped(py, fs_ctx, move || async move {
if recursive {
top.remove_all(&path).await
} else {
top.remove(&path).await
}
})
.map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), "deleted".to_string());
Ok(m)
}
/// Get file/directory information.
#[pyo3(signature = (path, ctx=None))]
fn stat(
&self,
py: Python<'_>,
path: String,
ctx: Option<HashMap<String, String>>,
) -> PyResult<Py<PyAny>> {
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
let info = self
.run_scoped(py, fs_ctx, move || async move { top.stat(&path).await })
.map_err(to_py_err)?;
Python::attach(|py| {
let dict = file_info_to_py_dict(py, &info)?;
Ok(dict.into())
})
}
/// Rename/move a file or directory.
#[pyo3(signature = (old_path, new_path, ctx=None))]
fn mv(
&self,
py: Python<'_>,
old_path: String,
new_path: String,
ctx: Option<HashMap<String, String>>,
) -> PyResult<HashMap<String, String>> {
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
self.run_scoped(py, fs_ctx, move || async move {
top.rename(&old_path, &new_path).await
})
.map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), "renamed".to_string());
Ok(m)
}
/// Attempt a same-mount verbatim copy and report whether the fast-path was used.
#[pyo3(signature = (src_path, dst_path, ctx=None))]
fn copy_within_mount(
&self,
py: Python<'_>,
src_path: String,
dst_path: String,
ctx: Option<HashMap<String, String>>,
) -> PyResult<HashMap<String, bool>> {
let fs_ctx = build_fs_context(ctx);
let mountable = self.mountable.clone();
let performed = self
.run_scoped(py, fs_ctx, move || async move {
mountable.copy_within_mount(&src_path, &dst_path).await
})
.map_err(to_py_err)?;
let mut result = HashMap::new();
result.insert("performed".to_string(), performed);
Ok(result)
}
/// Change file permissions.
#[pyo3(signature = (path, mode, ctx=None))]
fn chmod(
&self,
py: Python<'_>,
path: String,
mode: u32,
ctx: Option<HashMap<String, String>>,
) -> PyResult<HashMap<String, String>> {
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
self.run_scoped(
py,
fs_ctx,
move || async move { top.chmod(&path, mode).await },
)
.map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), "chmod ok".to_string());
Ok(m)
}
/// Touch a file (create if not exists, or update timestamp).
#[pyo3(signature = (path, ctx=None))]
fn touch(
&self,
py: Python<'_>,
path: String,
ctx: Option<HashMap<String, String>>,
) -> PyResult<HashMap<String, String>> {
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
self.run_scoped(py, fs_ctx, move || async move {
// Try create; if already exists, write empty to update mtime
match top.create(&path).await {
Ok(_) => Ok(()),
Err(_) => {
let existing = top.read(&path, 0, 0).await?;
top.write(&path, &existing, 0, WriteFlag::Create).await?;
Ok(())
}
}
})
.map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), "touched".to_string());
Ok(m)
}
/// List all mounted plugins.
fn mounts(&self, py: Python<'_>) -> PyResult<Vec<HashMap<String, String>>> {
let mountable = self.mountable.clone();
let mount_list = py_detach_blocking(py, move || {
self.rt
.block_on(async move { mountable.list_mounts().await })
});
let result: Vec<HashMap<String, String>> = mount_list
.into_iter()
.map(|(path, fstype)| {
let mut m = HashMap::new();
m.insert("path".to_string(), path);
m.insert("fstype".to_string(), fstype);
m
})
.collect();
Ok(result)
}
/// Mount a plugin dynamically.
///
/// Args:
/// fstype: Filesystem type (e.g., "memfs", "sqlfs", "kvfs", "queuefs")
/// path: Mount path
/// config: Plugin configuration as dict. For multi-write, include:
/// - "backups": nested dict matching BackendsConfig schema
/// - "server_encryption_enabled": bool
/// - "primary_encryption_enabled": bool
#[pyo3(signature = (fstype, path, config=None))]
fn mount(
&self,
py: Python<'_>,
fstype: String,
path: String,
config: Option<&Bound<'_, PyDict>>,
) -> PyResult<HashMap<String, String>> {
let params = match config {
Some(dict) => py_dict_to_config(dict)?,
None => HashMap::new(),
};
let plugin_config = PluginConfig::from_raw_parts(fstype.clone(), path.clone(), params)
.map_err(to_py_err)?;
let mountable = self.mountable.clone();
py_detach_blocking(py, move || {
self.rt
.block_on(async move { mountable.mount(plugin_config).await })
})
.map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert(
"message".to_string(),
format!("mounted {} at {}", fstype, path),
);
Ok(m)
}
/// Unmount a plugin.
fn unmount(&self, py: Python<'_>, path: String) -> PyResult<HashMap<String, String>> {
let mountable = self.mountable.clone();
let path_clone = path.clone();
py_detach_blocking(py, move || {
self.rt
.block_on(async move { mountable.unmount(&path_clone).await })
})
.map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), format!("unmounted {}", path));
Ok(m)
}
/// List all registered plugin names.
fn list_plugins(&self) -> PyResult<Vec<String>> {
// Return names of built-in plugins
let mut plugins = vec![
"memfs".to_string(),
"kvfs".to_string(),
"queuefs".to_string(),
"sqlfs".to_string(),
"localfs".to_string(),
"serverinfofs".to_string(),
];
#[cfg(feature = "s3")]
plugins.push("s3fs".to_string());
Ok(plugins)
}
/// Get detailed plugin information.
fn get_plugins_info(&self) -> PyResult<Vec<String>> {
self.list_plugins()
}
/// Load an external plugin (not supported in Rust binding).
fn load_plugin(&self, _library_path: String) -> PyResult<HashMap<String, String>> {
Err(PyRuntimeError::new_err(
"External plugin loading not supported in ragfs-python binding",
))
}
/// Unload an external plugin (not supported in Rust binding).
fn unload_plugin(&self, _library_path: String) -> PyResult<HashMap<String, String>> {
Err(PyRuntimeError::new_err(
"External plugin unloading not supported in ragfs-python binding",
))
}
/// Search for pattern in files using regular expressions.
///
/// Args:
/// path: File or directory path to search
/// pattern: Regular expression pattern to search for
/// recursive: Whether to search recursively in subdirectories (default: false)
/// case_insensitive: Whether to perform case-insensitive matching (default: false)
/// stream: Not supported in binding mode
/// node_limit: Maximum number of matches to return (default: None, no limit)
/// exclude_path: Optional path prefix to exclude from search (default: None)
/// level_limit: Optional maximum depth relative to query root (default: None)
/// ctx: Optional FsContext dict (e.g. {"account_id": ...})
/// before_context: Number of lines to include before each match (default: 0)
/// after_context: Number of lines to include after each match (default: 0)
///
/// Returns:
/// A dict with "matches" (list of match dicts) and "count" (total matches)
#[pyo3(signature = (path, pattern, recursive=false, case_insensitive=false, stream=false, node_limit=None, exclude_path=None, level_limit=None, ctx=None, before_context=0, after_context=0))]
#[allow(clippy::too_many_arguments)]
fn grep(
&self,
py: Python<'_>,
path: String,
pattern: String,
recursive: bool,
case_insensitive: bool,
stream: bool,
node_limit: Option<i32>,
exclude_path: Option<String>,
level_limit: Option<i32>,
ctx: Option<HashMap<String, String>>,
before_context: i32,
after_context: i32,
) -> PyResult<Py<PyAny>> {
if stream {
return Err(PyRuntimeError::new_err(
"Streaming not supported in binding mode",
));
}
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
let limit = node_limit.map(|n| if n < 0 { 0 } else { n as usize });
let level_limit_usize = level_limit.map(|n| if n < 0 { 0 } else { n as usize });
let before_context = before_context.max(0) as usize;
let after_context = after_context.max(0) as usize;
let result = self
.run_scoped(py, fs_ctx, move || async move {
top.grep(
&path,
&pattern,
GrepOptions {
recursive,
case_insensitive,
node_limit: limit,
exclude_path: exclude_path.as_deref(),
level_limit: level_limit_usize,
before_context,
after_context,
},
)
.await
})
.map_err(to_py_err)?;
Python::attach(|py| {
let dict = grep_result_to_py_dict(py, &result)?;
Ok(dict.into())
})
}
/// Recursively traverse a directory tree.
///
/// Args:
/// path: The root path of the traversal
/// show_hidden: Whether to include hidden files (default: False)
/// node_limit: Maximum number of nodes to return (default: None, no limit)
/// level_limit: Maximum depth relative to query root (default: None, no limit)
/// directories_only: Whether to return only directory entries (default: False)
/// ctx: Optional FsContext dict (e.g. {"account_id": ...})
///
/// Returns:
/// A list of dicts, each with keys: path, rel_path, info, extra
#[pyo3(signature = (path, show_hidden=false, node_limit=None, level_limit=None, ctx=None, *, offset=0, sort_by=None, sort_order=None, directories_only=false))]
fn tree_directory(
&self,
py: Python<'_>,
path: String,
show_hidden: bool,
node_limit: Option<i32>,
level_limit: Option<i32>,
ctx: Option<HashMap<String, String>>,
offset: i64,
sort_by: Option<&str>,
sort_order: Option<&str>,
directories_only: bool,
) -> PyResult<Py<PyAny>> {
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
let limit = node_limit.map(|n| if n < 0 { 0 } else { n as usize });
let level_limit_usize = level_limit.map(|n| if n < 0 { 0 } else { n as usize });
let offset = parse_listing_offset(offset)?;
let sort_by = parse_list_sort_by(sort_by)?;
let sort_order = parse_sort_order(sort_order)?;
let entries = self
.run_scoped(py, fs_ctx, move || async move {
top.tree_directory(
&path,
show_hidden,
limit,
level_limit_usize,
Some(offset),
sort_by,
sort_order,
directories_only,
)
.await
})
.map_err(to_py_err)?;
Python::attach(|py| {
let list = PyList::empty(py);
for entry in &entries {
let dict = tree_entry_to_py_dict(py, entry)?;
list.append(dict)?;
}
Ok(list.into())
})
}
/// Return one page of flat glob results.
///
/// Args:
/// path: The root path of the traversal
/// pattern: Glob pattern matched against query-root-relative paths
/// show_hidden: Whether to include hidden files (default: False)
/// page_size: Maximum number of matched entries returned in this page
/// level_limit: Maximum depth relative to query root (default: None)
/// continuation_token: Opaque token returned by the previous page
/// ctx: Optional FsContext dict (e.g. {"account_id": ...})
///
/// Returns:
/// A dict with keys: entries (list[GlobEntry]), next_token (str | None)
#[pyo3(signature = (path, pattern, show_hidden=false, page_size=None, level_limit=None, continuation_token=None, ctx=None))]
fn glob_directory(
&self,
py: Python<'_>,
path: String,
pattern: String,
show_hidden: bool,
page_size: Option<i32>,
level_limit: Option<i32>,
continuation_token: Option<String>,
ctx: Option<HashMap<String, String>>,
) -> PyResult<Py<PyAny>> {
let fs_ctx = build_fs_context(ctx);
let top = self.top.clone();
let page_size = page_size.map(|n| if n < 0 { 0 } else { n as usize });
let level_limit_usize = level_limit.map(|n| if n < 0 { 0 } else { n as usize });
let page: GlobPage = self
.run_scoped(py, fs_ctx, move || async move {
top.glob_directory(
&path,
&pattern,
show_hidden,
page_size,
level_limit_usize,
continuation_token,
)
.await
})
.map_err(to_py_err)?;
Python::attach(|py| {
let value = serde_json::to_value(&page)
.map_err(|err| PyRuntimeError::new_err(err.to_string()))?;
serde_json_to_py(py, &value)
})
}
/// Calculate file digest (not yet implemented in ragfs).
#[pyo3(signature = (path, algorithm="xxh3"))]
fn digest(&self, path: String, algorithm: &str) -> PyResult<HashMap<String, String>> {
let _ = (path, algorithm);
Err(PyRuntimeError::new_err(
"digest not yet implemented in ragfs-python",
))
}
// ── PathLock API ──
/// Validate that distributed leadership is available on this client.
fn leader_validate(&self) -> PyResult<()> {
self.clone_leader_manager().map(|_| ())
}
/// Try to acquire a distributed leader lease and return an opaque local reference.
#[pyo3(signature = (key, holder_id, ttl_secs))]
fn leader_try_acquire_async<'py>(
&self,
py: Python<'py>,
key: String,
holder_id: String,
ttl_secs: f64,
) -> PyResult<Bound<'py, PyAny>> {
let ttl = validate_leader_ttl_secs(ttl_secs)?;
let manager = self.clone_leader_manager()?;
let leases = Arc::clone(&self.leader_leases);
let leader_closing = Arc::clone(&self.leader_closing);
let task = self.rt.spawn(async move {
let closing = leader_closing.read().await;
if *closing {
return Err(CacheError::Closed);
}
let lease = manager.try_acquire(&key, &holder_id, ttl).await?;
let Some(lease) = lease else {
return Ok(None);
};
let lease_ref = Uuid::new_v4().to_string();
leases.lock().await.insert(lease_ref.clone(), lease);
Ok::<Option<String>, CacheError>(Some(lease_ref))
});
pyo3_async_runtimes::tokio::future_into_py(py, async move {
task.await
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?
.map_err(|error| PyRuntimeError::new_err(error.to_string()))
})
}
/// Renew an opaque leader lease. False means ownership has been lost.
#[pyo3(signature = (lease_ref, ttl_secs))]
fn leader_renew_async<'py>(
&self,
py: Python<'py>,
lease_ref: String,
ttl_secs: f64,
) -> PyResult<Bound<'py, PyAny>> {
let ttl = validate_leader_ttl_secs(ttl_secs)?;
let manager = self.clone_leader_manager()?;
let leases = Arc::clone(&self.leader_leases);
let leader_closing = Arc::clone(&self.leader_closing);
let task = self.rt.spawn(async move {
let closing = leader_closing.read().await;
if *closing {
return Err(CacheError::Closed);
}
let lease = {
let leases = leases.lock().await;
leases.get(&lease_ref).cloned()
};
let Some(lease) = lease else {
return Ok(false);
};
let renewed = manager.renew(&lease, ttl).await?;
if !renewed {
leases.lock().await.remove(&lease_ref);
}
Ok::<bool, ragfs::cache_runtime::CacheError>(renewed)
});
pyo3_async_runtimes::tokio::future_into_py(py, async move {
task.await
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?
.map_err(|error| PyRuntimeError::new_err(error.to_string()))
})
}
/// Release an opaque leader lease if it is still owned.
#[pyo3(signature = (lease_ref))]
fn leader_release_async<'py>(
&self,
py: Python<'py>,
lease_ref: String,
) -> PyResult<Bound<'py, PyAny>> {
let manager = self.clone_leader_manager()?;
let leases = Arc::clone(&self.leader_leases);
let leader_closing = Arc::clone(&self.leader_closing);
let task = self.rt.spawn(async move {
let closing = leader_closing.read().await;
if *closing {
return Err(CacheError::Closed);
}
let lease = {
let leases = leases.lock().await;
leases.get(&lease_ref).cloned()
};
if let Some(lease) = lease {
manager.release(&lease).await?;
leases.lock().await.remove(&lease_ref);
}
Ok::<(), ragfs::cache_runtime::CacheError>(())
});
pyo3_async_runtimes::tokio::future_into_py(py, async move {
task.await
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?
.map_err(|error| PyRuntimeError::new_err(error.to_string()))
})
}
/// Acquire an exact lock on a single path.
#[pyo3(signature = (ctx, path, timeout_secs=0.0, owner_lease_ref=None))]
fn pathlock_acquire_exact(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
path: String,
timeout_secs: f64,
owner_lease_ref: Option<Py<PyAny>>,
) -> PyResult<Py<PyAny>> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let timeout = validate_timeout_secs(timeout_secs)?;
let owner_capability = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?;
let lease = self
.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let path = path.clone();
let capability = owner_capability.clone();
async move {
let capability = capability.as_ref().map(|(lease_ref, ownership_ref)| {
(lease_ref.as_str(), ownership_ref.as_str())
});
mgr.acquire_exact(&path, timeout, capability).await
}
})
.map_err(pathlock_err_to_py)?;
Python::attach(|py| owned_lease_to_py_dict(py, &lease))
}
/// Acquire exact locks on multiple paths.
#[pyo3(signature = (ctx, paths, timeout_secs=0.0, owner_lease_ref=None))]
fn pathlock_acquire_exact_batch(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
paths: Vec<String>,
timeout_secs: f64,
owner_lease_ref: Option<Py<PyAny>>,
) -> PyResult<Py<PyAny>> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let timeout = validate_timeout_secs(timeout_secs)?;
let owner_capability = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?;
let lease = self
.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let paths = paths.clone();
let capability = owner_capability.clone();
async move {
let capability = capability.as_ref().map(|(lease_ref, ownership_ref)| {
(lease_ref.as_str(), ownership_ref.as_str())
});
mgr.acquire_exact_batch(&paths, timeout, capability).await
}
})
.map_err(pathlock_err_to_py)?;
Python::attach(|py| owned_lease_to_py_dict(py, &lease))
}
/// Acquire a tree lock on a single path.
#[pyo3(signature = (ctx, path, timeout_secs=0.0, owner_lease_ref=None))]
fn pathlock_acquire_tree(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
path: String,
timeout_secs: f64,
owner_lease_ref: Option<Py<PyAny>>,
) -> PyResult<Py<PyAny>> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let timeout = validate_timeout_secs(timeout_secs)?;
let owner_capability = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?;
let lease = self
.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let path = path.clone();
let capability = owner_capability.clone();
async move {
let capability = capability.as_ref().map(|(lease_ref, ownership_ref)| {
(lease_ref.as_str(), ownership_ref.as_str())
});
mgr.acquire_tree(&path, timeout, capability).await
}
})
.map_err(pathlock_err_to_py)?;
Python::attach(|py| owned_lease_to_py_dict(py, &lease))
}
/// Acquire tree locks on multiple paths.
#[pyo3(signature = (ctx, paths, timeout_secs=0.0, owner_lease_ref=None))]
fn pathlock_acquire_tree_batch(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
paths: Vec<String>,
timeout_secs: f64,
owner_lease_ref: Option<Py<PyAny>>,
) -> PyResult<Py<PyAny>> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let timeout = validate_timeout_secs(timeout_secs)?;
let owner_capability = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?;
let lease = self
.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let paths = paths.clone();
let capability = owner_capability.clone();
async move {
let capability = capability.as_ref().map(|(lease_ref, ownership_ref)| {
(lease_ref.as_str(), ownership_ref.as_str())
});
mgr.acquire_tree_batch(&paths, timeout, capability).await
}
})
.map_err(pathlock_err_to_py)?;
Python::attach(|py| owned_lease_to_py_dict(py, &lease))
}
/// Acquire a mixed batch of exact and tree locks.
#[pyo3(signature = (ctx, exact_paths, tree_paths, timeout_secs=0.0, owner_lease_ref=None))]
fn pathlock_acquire_exact_tree_batch(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
exact_paths: Vec<String>,
tree_paths: Vec<String>,
timeout_secs: f64,
owner_lease_ref: Option<Py<PyAny>>,
) -> PyResult<Py<PyAny>> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let timeout = validate_timeout_secs(timeout_secs)?;
let owner_capability = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?;
let lease = self
.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let exact = exact_paths.clone();
let tree = tree_paths.clone();
let capability = owner_capability.clone();
async move {
let capability = capability.as_ref().map(|(lease_ref, ownership_ref)| {
(lease_ref.as_str(), ownership_ref.as_str())
});
mgr.acquire_exact_tree_batch(&exact, &tree, timeout, capability)
.await
}
})
.map_err(pathlock_err_to_py)?;
Python::attach(|py| owned_lease_to_py_dict(py, &lease))
}
/// Acquire a batch of locks from a list of request dicts.
/// Each request dict: {"path": str, "kind": "exact"|"tree"}
#[pyo3(signature = (ctx, requests, timeout_secs=0.0, owner_lease_ref=None))]
fn pathlock_acquire_batch(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
requests: Vec<HashMap<String, String>>,
timeout_secs: f64,
owner_lease_ref: Option<Py<PyAny>>,
) -> PyResult<Py<PyAny>> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let timeout = validate_timeout_secs(timeout_secs)?;
let owner_capability = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?;
let lock_requests = parse_pathlock_request_batch(&requests)?;
let lease = self
.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let reqs = lock_requests.clone();
let capability = owner_capability.clone();
async move {
let capability = capability.as_ref().map(|(lease_ref, ownership_ref)| {
(lease_ref.as_str(), ownership_ref.as_str())
});
mgr.acquire_batch(&reqs, timeout, capability).await
}
})
.map_err(pathlock_err_to_py)?;
Python::attach(|py| owned_lease_to_py_dict(py, &lease))
}
/// Acquire a batch on Tokio; the Python adapter settles cancellation before releasing it.
#[pyo3(signature = (ctx, requests, timeout_secs=0.0, owner_lease_ref=None))]
fn pathlock_acquire_batch_async<'py>(
&self,
py: Python<'py>,
ctx: Option<HashMap<String, String>>,
requests: Vec<HashMap<String, String>>,
timeout_secs: f64,
owner_lease_ref: Option<Py<PyAny>>,
) -> PyResult<Bound<'py, PyAny>> {
let fs_ctx = build_fs_context(ctx);
let timeout = validate_timeout_secs(timeout_secs)?;
let owner = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?;
let requests = parse_pathlock_request_batch(&requests)?;
let manager = self.clone_pathlock_manager();
let task = self.rt.spawn(FS_CTX.scope(fs_ctx, async move {
let capability = owner
.as_ref()
.map(|(lease_ref, ownership_ref)| (lease_ref.as_str(), ownership_ref.as_str()));
manager.acquire_batch(&requests, timeout, capability).await
}));
pyo3_async_runtimes::tokio::future_into_py(py, async move {
let lease = task
.await
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?
.map_err(pathlock_err_to_py)?;
Python::attach(|py| owned_lease_to_py_dict(py, &lease))
})
}
/// Create a borrowed view of an owned lease.
#[pyo3(signature = (ctx, owned_lease_ref))]
fn pathlock_as_borrowed(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
owned_lease_ref: Py<PyAny>,
) -> PyResult<Py<PyAny>> {
let mgr = self.clone_pathlock_manager();
let mgr2 = mgr.clone();
let fs_ctx = build_fs_context(ctx);
let lease_ref = extract_lease_ref(py, &owned_lease_ref)?;
let lease = self
.run_scoped(py, fs_ctx, move || {
let mgr = mgr2.clone();
let lr = lease_ref.clone();
async move {
mgr.get_owned_lease_by_ref(&lr).await.ok_or_else(|| {
PathLockError::Internal(format!("lease not found for ref '{lr}'"))
})
}
})
.map_err(|e: PathLockError| PyRuntimeError::new_err(e.to_string()))?;
let borrowed = mgr.as_borrowed(&lease);
Python::attach(|py| borrowed_lease_to_py_dict(py, &borrowed))
}
/// Refresh an owned lease. Returns "refreshed", "lost", or "failed".
#[pyo3(signature = (ctx, owned_lease_ref))]
fn pathlock_refresh(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
owned_lease_ref: Py<PyAny>,
) -> PyResult<String> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let (lease_ref, ownership_ref) = extract_owned_lease_ref(py, &owned_lease_ref)?;
let status = self
.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let lr = lease_ref.clone();
let ownership = ownership_ref.clone();
async move {
let lease = mgr
.get_owned_lease_by_capability(&lr, &ownership)
.await
.ok_or_else(|| {
PathLockError::InvalidRequest(format!(
"owned lease capability does not match ref '{lr}'"
))
})?;
mgr.refresh(&lease).await
}
})
.map_err(pathlock_err_to_py)?;
Ok(status)
}
/// Release an owned lease.
#[pyo3(signature = (ctx, owned_lease_ref))]
fn pathlock_release(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
owned_lease_ref: Py<PyAny>,
) -> PyResult<()> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let (lease_ref, ownership_ref) = extract_owned_lease_ref(py, &owned_lease_ref)?;
self.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let lr = lease_ref.clone();
let ownership = ownership_ref.clone();
async move {
let lease = mgr
.get_releasable_lease_by_capability(&lr, &ownership)
.await
.ok_or_else(|| {
PathLockError::InvalidRequest(format!(
"owned lease capability does not match ref '{lr}'"
))
})?;
mgr.release(&lease).await
}
})
.map_err(pathlock_err_to_py)?;
Ok(())
}
/// Release selected lock paths from an owned lease.
#[pyo3(signature = (ctx, owned_lease_ref, lock_paths))]
fn pathlock_release_selected(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
owned_lease_ref: Py<PyAny>,
lock_paths: Vec<String>,
) -> PyResult<()> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let (lease_ref, ownership_ref) = extract_owned_lease_ref(py, &owned_lease_ref)?;
self.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let lr = lease_ref.clone();
let ownership = ownership_ref.clone();
let paths = lock_paths.clone();
async move {
let lease = mgr
.get_releasable_lease_by_capability(&lr, &ownership)
.await
.ok_or_else(|| {
PathLockError::InvalidRequest(format!(
"owned lease capability does not match ref '{lr}'"
))
})?;
mgr.release_selected(&lease, &paths).await
}
})
.map_err(pathlock_err_to_py)?;
Ok(())
}
/// Export a handoff ref from an owned lease.
#[pyo3(signature = (ctx, owned_lease_ref))]
fn pathlock_to_handoff(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
owned_lease_ref: Py<PyAny>,
) -> PyResult<Py<PyAny>> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let (lease_ref, ownership_ref) = extract_owned_lease_ref(py, &owned_lease_ref)?;
let handoff = self
.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let lr = lease_ref.clone();
let ownership = ownership_ref.clone();
async move {
let lease = mgr
.get_owned_lease_by_capability(&lr, &ownership)
.await
.ok_or_else(|| {
PathLockError::InvalidRequest(format!(
"owned lease capability does not match ref '{lr}'"
))
})?;
Ok(mgr.to_handoff(&lease))
}
})
.map_err(pathlock_err_to_py)?;
Python::attach(|py| handoff_ref_to_py_dict(py, &handoff))
}
/// Mark an owned lease as handed off.
#[pyo3(signature = (ctx, owned_lease_ref))]
fn pathlock_handoff(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
owned_lease_ref: Py<PyAny>,
) -> PyResult<()> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let (lease_ref, ownership_ref) = extract_owned_lease_ref(py, &owned_lease_ref)?;
self.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let lr = lease_ref.clone();
let ownership = ownership_ref.clone();
async move {
let lease = mgr
.get_owned_lease_by_capability(&lr, &ownership)
.await
.ok_or_else(|| {
PathLockError::InvalidRequest(format!(
"owned lease capability does not match ref '{lr}'"
))
})?;
mgr.handoff(&lease).await
}
})
.map_err(pathlock_err_to_py)?;
Ok(())
}
/// Adopt a handoff ref, returning a new owned lease.
#[pyo3(signature = (ctx, handoff_ref))]
fn pathlock_adopt(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
handoff_ref: HashMap<String, Py<PyAny>>,
) -> PyResult<Py<PyAny>> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let owner_id: String = handoff_ref
.get("owner_id")
.or_else(|| handoff_ref.get("handle_id"))
.ok_or_else(|| {
PyValueError::new_err("handoff_ref.owner_id or handoff_ref.handle_id required")
})?
.extract(py)?;
let lock_paths: Vec<String> = handoff_ref
.get("lock_paths")
.ok_or_else(|| PyValueError::new_err("handoff_ref.lock_paths required"))?
.extract(py)?;
if owner_id.is_empty()
|| lock_paths.is_empty()
|| lock_paths.iter().any(|path| !path.starts_with('/'))
{
return Err(PyValueError::new_err(
"handoff_ref owner_id/handle_id and absolute lock_paths are required",
));
}
let covered_paths: Vec<PathLockRequest> = handoff_ref
.get("covered_paths")
.map(|v| {
let raw: Vec<HashMap<String, String>> = v.extract(py)?;
raw.iter().map(parse_pathlock_request).collect()
})
.transpose()?
.unwrap_or_default();
let lease_ref: Option<String> = handoff_ref
.get("lease_ref")
.map(|v| v.extract(py))
.transpose()?
.filter(|s: &String| !s.is_empty());
let handle_mode = handoff_ref
.get("handle_mode")
.filter(|v| !v.bind(py).is_none())
.map(|v| -> PyResult<PathLockHandleMode> {
match v.extract::<String>(py)?.as_str() {
"lock_path" => Ok(PathLockHandleMode::LockPath),
"logical_path" => Ok(PathLockHandleMode::LogicalPath),
_ => Err(PyValueError::new_err("invalid handoff handle_mode")),
}
})
.transpose()?;
let handoff = PathLockHandoffRef {
lease_ref,
handle_mode,
owner_id,
lock_paths,
covered_paths,
};
let lease = self
.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let h = handoff.clone();
async move { mgr.adopt(&h).await }
})
.map_err(pathlock_err_to_py)?;
Python::attach(|py| owned_lease_to_py_dict(py, &lease))
}
/// Check if a path is locked.
#[pyo3(signature = (ctx, path, ignore_stale=true))]
fn pathlock_is_locked(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
path: String,
ignore_stale: bool,
) -> PyResult<bool> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
self.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
let p = path.clone();
async move { mgr.is_locked(&p, ignore_stale).await }
})
.map_err(|e| PyRuntimeError::new_err(e.to_string()))
}
/// Return an observability snapshot of current lock state.
#[pyo3(signature = (ctx=None))]
fn pathlock_observe(
&self,
py: Python<'_>,
ctx: Option<HashMap<String, String>>,
) -> PyResult<Py<PyAny>> {
let mgr = self.clone_pathlock_manager();
let fs_ctx = build_fs_context(ctx);
let snapshot = self.run_scoped(py, fs_ctx, move || {
let mgr = mgr.clone();
async move { mgr.observe().await }
});
Python::attach(|py| {
let dict = PyDict::new(py);
dict.set_item("active_locks", snapshot.active_locks)?;
dict.set_item("waiting_locks", snapshot.waiting_locks)?;
dict.set_item("stale_locks_removed", snapshot.stale_locks_removed)?;
let conflicts = PyList::empty(py);
for c in &snapshot.conflicts {
let cd = PyDict::new(py);
cd.set_item("lock_path", &c.lock_path)?;
cd.set_item("conflicting_owner", &c.conflicting_owner)?;
cd.set_item("conflicting_kind", format!("{:?}", c.conflicting_kind))?;
conflicts.append(cd)?;
}
dict.set_item("conflicts", conflicts)?;
Ok(dict.into())
})
}
/// Read all native metrics with no arguments and return a list of flat Python dicts.
fn metrics(&self, py: Python<'_>) -> PyResult<Py<PyList>> {
let fs = self.mountable.clone();
let metrics = py_detach_blocking(py, move || {
self.rt.block_on(async move { fs.metrics().await })
})
.map_err(to_py_err)?;
let result = PyList::empty(py);
for metric in &metrics {
result.append(metric_to_py_dict(py, metric)?)?;
}
Ok(result.into())
}
/// Get filesystem statistics.
///
/// Args:
/// path: Optional mount path to get stats for. If None, get stats for all mounts.
///
/// Returns:
/// Statistics data as a dict.
#[pyo3(signature = (path=None))]
fn get_stats(&self, py: Python<'_>, path: Option<String>) -> PyResult<Py<PyAny>> {
let fs = self.mountable.clone();
if let Some(mount_path) = path {
// Get stats for a specific mount
let fs2 = fs.clone();
let mount_path_clone = mount_path.clone();
let stats = py_detach_blocking(py, move || {
self.rt
.block_on(async move { fs.get_mount_stats(&mount_path_clone).await })
})
.map_err(to_py_err)?;
let mounts = py_detach_blocking(py, move || {
self.rt.block_on(async move { fs2.list_mounts().await })
});
let plugin_name = mounts
.into_iter()
.find(|(p, _)| p == &mount_path)
.map(|(_, plugin)| plugin)
.unwrap_or_default();
Python::attach(|py| {
let result = PyDict::new(py);
result.set_item("path", mount_path)?;
result.set_item("plugin", plugin_name)?;
let stats_dict = filesystem_stats_to_py_dict(py, &stats)?;
result.set_item("stats", stats_dict)?;
Ok(result.into())
})
} else {
// Get stats for all mounts
let all_stats = py_detach_blocking(py, move || {
self.rt.block_on(async move { fs.get_all_stats().await })
});
Python::attach(|py| {
let mounts_list = PyList::empty(py);
for (path, (plugin, stats)) in all_stats {
let mount_dict = PyDict::new(py);
mount_dict.set_item("path", path)?;
mount_dict.set_item("plugin", plugin)?;
let stats_dict = filesystem_stats_to_py_dict(py, &stats)?;
mount_dict.set_item("stats", stats_dict)?;
mounts_list.append(mount_dict)?;
}
let result = PyDict::new(py);
result.set_item("mounts", mounts_list)?;
Ok(result.into())
})
}
}
}
/// Python module definition
#[pymodule]
fn ragfs_python(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<RAGFSBindingClient>()?;
m.add_class::<privacy::PyPrivacyMatcher>()?;
m.add_class::<privacy_extractor::PyPrivacyExtractor>()?;
m.add_function(wrap_pyfunction!(reopen_tracing_file, m)?)?;
// Import the Python business exception so native code raises the same type
// that `openviking.storage.errors` defines, enabling `except LockAcquisitionError`.
let py = m.py();
let errors_mod = py.import("openviking.storage.errors")?;
let exc_type: Py<PyType> = errors_mod.getattr("LockAcquisitionError")?.extract()?;
LOCK_ACQUISITION_ERROR_TYPE
.set(exc_type)
.map_err(|_| PyRuntimeError::new_err("LockAcquisitionError already initialized"))?;
HANDOFF_RECOVERY_ERROR_TYPE
.set(errors_mod.getattr("HandoffRecoveryRequired")?.extract()?)
.map_err(|_| PyRuntimeError::new_err("HandoffRecoveryRequired already initialized"))?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use pyo3::types::PyDict;
use std::fs;
#[test]
fn detach_blocking_helper_runs_without_python_objects() {
Python::attach(|py| {
let value: i32 = py_detach_blocking(py, || 40 + 2);
assert_eq!(value, 42);
});
}
#[test]
fn pathlock_failures_remain_internal_through_filesystem_wrappers() {
Python::initialize();
Python::attach(|py| {
let errors_mod = py.import("openviking.storage.errors").unwrap();
let lock_error_type: Py<PyType> = errors_mod
.getattr("LockAcquisitionError")
.unwrap()
.extract()
.unwrap();
let _ = LOCK_ACQUISITION_ERROR_TYPE.set(lock_error_type.clone_ref(py));
let internal_error_type = get_exception(py, "AGFSInternalError").unwrap();
for wrapped in [false, true] {
for failure in [
PathLockError::Io("failed to create lock dir".to_string()),
PathLockError::InvalidToken("missing ':' in token".to_string()),
] {
let error = if wrapped {
to_py_err(failure.into())
} else {
pathlock_err_to_py(failure)
};
assert!(error.is_instance(py, &internal_error_type));
}
}
});
}
#[test]
fn pathlock_contention_survives_filesystem_wrappers() {
Python::initialize();
Python::attach(|py| {
let errors_mod = py.import("openviking.storage.errors").unwrap();
let lock_error_type: Py<PyType> = errors_mod
.getattr("LockAcquisitionError")
.unwrap()
.extract()
.unwrap();
let _ = LOCK_ACQUISITION_ERROR_TYPE.set(lock_error_type.clone_ref(py));
for wrapped in [false, true] {
for contention in [
PathLockError::Busy {
lock_path: "/data/.path.ovlock".to_string(),
operation: "remove".to_string(),
},
PathLockError::Timeout { elapsed_ms: 1000 },
] {
let error = if wrapped {
to_py_err(contention.into())
} else {
pathlock_err_to_py(contention)
};
assert!(error.is_instance(py, lock_error_type.bind(py)));
}
}
});
}
#[test]
fn constructor_accepts_canonical_config_path_with_runtime_service_name() {
let path = std::env::temp_dir().join(format!(
"openviking-canonical-config-path-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{"service_name": "test-service", "storage": {"agfs": {"backend": "local"}}}"#,
)
.unwrap();
Python::initialize();
Python::attach(|py| {
let ty = py.get_type::<RAGFSBindingClient>();
let config = PyDict::new(py);
config.set_item("service_name", "test-service").unwrap();
let kwargs = PyDict::new(py);
kwargs
.set_item("config_path", path.to_str().unwrap())
.unwrap();
kwargs.set_item("config", config).unwrap();
let instance = ty.call((), Some(&kwargs));
assert!(instance.is_ok());
});
fs::remove_file(path).unwrap();
}
#[test]
fn constructor_inline_pathlock_overrides_canonical_pathlock() {
Python::initialize();
let path = std::env::temp_dir().join(format!(
"openviking-inline-pathlock-override-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{
"service_name": "test-service",
"storage": {
"agfs": {
"pathlock": {"provider": "cache"}
}
}
}"#,
)
.unwrap();
Python::attach(|py| {
let ty = py.get_type::<RAGFSBindingClient>();
let pathlock = PyDict::new(py);
pathlock.set_item("provider", "filesystem").unwrap();
let config = PyDict::new(py);
config.set_item("service_name", "inline-prod").unwrap();
config.set_item("pathlock", pathlock).unwrap();
let kwargs = PyDict::new(py);
kwargs
.set_item("config_path", path.to_str().unwrap())
.unwrap();
kwargs.set_item("config", config).unwrap();
let instance = ty.call((), Some(&kwargs));
assert!(instance.is_ok());
});
fs::remove_file(path).unwrap();
}
#[test]
fn constructor_inline_redis_pathlock_uses_inline_cache() {
let Ok(endpoint) = std::env::var("REDIS_URL") else {
return;
};
Python::initialize();
let path = std::env::temp_dir().join(format!(
"openviking-inline-redis-pathlock-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{"service_name": "test-service", "storage": {"agfs": {"backend": "local"}}}"#,
)
.unwrap();
Python::attach(|py| {
let ty = py.get_type::<RAGFSBindingClient>();
let redis = PyDict::new(py);
redis.set_item("endpoints", vec![endpoint]).unwrap();
redis.set_item("command_timeout_ms", 1_000).unwrap();
let cache = PyDict::new(py);
cache.set_item("enabled", false).unwrap();
cache.set_item("runtime_enabled", false).unwrap();
cache.set_item("provider", "redis").unwrap();
cache.set_item("redis", redis).unwrap();
let pathlock = PyDict::new(py);
pathlock.set_item("provider", "cache").unwrap();
pathlock.set_item("lock_expire_secs", 3.0).unwrap();
let config = PyDict::new(py);
config.set_item("service_name", "inline-prod").unwrap();
config.set_item("cache", cache).unwrap();
config.set_item("pathlock", pathlock).unwrap();
let kwargs = PyDict::new(py);
kwargs
.set_item("config_path", path.to_str().unwrap())
.unwrap();
kwargs.set_item("config", config).unwrap();
let instance = ty.call((), Some(&kwargs));
assert!(instance.is_ok());
});
fs::remove_file(path).unwrap();
}
#[test]
fn parse_tracing_level_accepts_critical_as_error() {
assert_eq!(parse_tracing_level("CRITICAL").unwrap(), Level::ERROR);
}
#[test]
fn replace_tracing_log_file_switches_shared_writer_to_new_active_file() {
let active = std::env::temp_dir().join(format!(
"openviking-rust-tracing-active-{}.log",
std::process::id()
));
let rotated = std::env::temp_dir().join(format!(
"openviking-rust-tracing-rotated-{}.log",
std::process::id()
));
let _ = fs::remove_file(&active);
let _ = fs::remove_file(&rotated);
let shared = Arc::new(Mutex::new(open_tracing_log_file(&active).unwrap()));
let mut writer = SharedFileWriter {
file: shared.clone(),
};
writer.write_all(b"before-rotate\n").unwrap();
writer.flush().unwrap();
fs::rename(&active, &rotated).unwrap();
replace_tracing_log_file(&shared, &active).unwrap();
writer.write_all(b"after-rotate\n").unwrap();
writer.flush().unwrap();
assert!(fs::read_to_string(&rotated)
.unwrap()
.contains("before-rotate"));
assert!(fs::read_to_string(&active)
.unwrap()
.contains("after-rotate"));
let _ = fs::remove_file(&active);
let _ = fs::remove_file(&rotated);
}
#[test]
fn missing_cache_config_defaults_to_disabled_redis_config() {
let path = std::env::temp_dir().join(format!(
"openviking-no-cache-config-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{"service_name": "test-service", "storage": {"agfs": {"backend": "local"}}}"#,
)
.unwrap();
let cache_config =
cache_config_from_ov_conf(path.to_str().unwrap(), "test-service").unwrap();
assert!(!cache_config.enabled);
assert_eq!(cache_config.provider, CacheProviderKind::Redis);
assert_eq!(cache_config.service_name, "test-service");
assert_eq!(cache_config.traversal_mode, CacheTraversalMode::Backend);
fs::remove_file(path).unwrap();
}
#[test]
fn canonical_redis_pathlock_config_is_parsed_and_enables_runtime() {
let path = std::env::temp_dir().join(format!(
"openviking-redis-pathlock-config-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{
"service_name": "prod-a",
"cache": {
"provider": "redis",
"params": {"endpoints": ["redis://redis:6379"]}
},
"storage": {
"agfs": {
"pathlock": {
"provider": "cache",
"lock_expire_secs": 60.0
}
}
}
}"#,
)
.unwrap();
let pathlock = pathlock_config_from_ov_conf(path.to_str().unwrap(), "prod-a").unwrap();
let cache = cache_config_from_ov_conf(path.to_str().unwrap(), "prod-a").unwrap();
assert_eq!(pathlock.provider, "cache");
assert_eq!(pathlock.service_name.as_deref(), Some("prod-a"));
assert_eq!(pathlock.lock_expire_secs, 60.0);
assert!(cache.runtime_enabled);
assert_eq!(cache.provider, CacheProviderKind::Redis);
assert_eq!(cache.redis.endpoints, vec!["redis://redis:6379"]);
fs::remove_file(path).unwrap();
}
#[test]
fn redis_pathlock_config_uses_runtime_service_name() {
let pathlock = pathlock_config_from_canonical_ov_conf(
&serde_json::json!({
"service_name": "prod-a",
"storage": {
"agfs": {
"pathlock": {"provider": "cache"}
}
}
}),
"prod-a",
)
.unwrap();
assert_eq!(pathlock.service_name.as_deref(), Some("prod-a"));
}
#[test]
fn constructor_sqlite_config_path_resolves_defaults_and_creates_parent() {
let root = std::env::temp_dir().join(format!(
"openviking-sqlite-config-path-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
fs::create_dir_all(&root).unwrap();
Python::initialize();
Python::attach(|py| {
for mode in ["omitted", "null", "explicit"] {
let workspace = root.join(mode).join("workspace");
let database = if mode == "explicit" {
root.join(mode).join("custom").join("locks.db")
} else {
workspace.join("_system/pathlock/pathlock.db")
};
let mut pathlock = serde_json::json!({"provider": "sqlite"});
if mode == "null" {
pathlock["sqlite_path"] = serde_json::Value::Null;
} else if mode == "explicit" {
pathlock["sqlite_path"] = serde_json::json!(database);
}
let path = root.join(format!("{mode}.json"));
fs::write(
&path,
serde_json::json!({
"storage": {
"workspace": workspace,
"agfs": {"pathlock": pathlock}
}
})
.to_string(),
)
.unwrap();
assert!(!database.parent().unwrap().exists());
let config = PyDict::new(py);
config.set_item("service_name", "local-a").unwrap();
let kwargs = PyDict::new(py);
kwargs
.set_item("config_path", path.to_str().unwrap())
.unwrap();
kwargs.set_item("config", config).unwrap();
let ty = py.get_type::<RAGFSBindingClient>();
let first = ty.call((), Some(&kwargs)).unwrap();
let second = ty.call((), Some(&kwargs)).unwrap();
assert!(database.is_file());
if mode == "explicit" {
assert!(!workspace.join("_system/pathlock/pathlock.db").exists());
}
let first = first.extract::<PyRef<'_, RAGFSBindingClient>>().unwrap();
let second = second.extract::<PyRef<'_, RAGFSBindingClient>>().unwrap();
let lease = first
.rt
.block_on(first.pathlock_manager.acquire_tree(
"/missing",
std::time::Duration::ZERO,
None,
))
.unwrap();
assert!(second
.rt
.block_on(second.pathlock_manager.acquire_exact(
"/missing/child",
std::time::Duration::ZERO,
None,
))
.is_err());
first
.rt
.block_on(first.pathlock_manager.release(&lease))
.unwrap();
let lease = second
.rt
.block_on(second.pathlock_manager.acquire_exact(
"/missing/child",
std::time::Duration::ZERO,
None,
))
.unwrap();
second
.rt
.block_on(second.pathlock_manager.release(&lease))
.unwrap();
}
});
fs::remove_dir_all(root).unwrap();
}
#[test]
fn canonical_sqlite_default_requires_workspace_and_preserves_explicit_path() {
for pathlock in [
serde_json::json!({"provider": "sqlite"}),
serde_json::json!({"provider": "sqlite", "sqlite_path": null}),
] {
let json = serde_json::json!({
"storage": {"workspace": "relative/workspace", "agfs": {"pathlock": pathlock}}
});
let config = pathlock_config_from_canonical_ov_conf(&json, "local-a").unwrap();
let expected = std::path::absolute("relative/workspace")
.unwrap()
.join("_system/pathlock/pathlock.db");
assert_eq!(config.sqlite_path.as_deref(), expected.to_str());
assert_eq!(config.service_name.as_deref(), Some("local-a"));
}
for workspace in [
serde_json::Value::Null,
serde_json::json!(""),
serde_json::json!(42),
] {
let json = serde_json::json!({
"storage": {"workspace": workspace, "agfs": {"pathlock": {"provider": "sqlite"}}}
});
let error = pathlock_config_from_canonical_ov_conf(&json, "local-a").unwrap_err();
assert!(error.contains("storage.workspace"), "{error}");
}
let path = std::env::temp_dir().join("explicit-pathlock.db");
let json = serde_json::json!({
"storage": {"agfs": {"pathlock": {"provider": "sqlite", "sqlite_path": path}}}
});
assert_eq!(
pathlock_config_from_canonical_ov_conf(&json, "local-a")
.unwrap()
.sqlite_path
.as_deref(),
path.to_str()
);
assert_eq!(
pathlock_config_from_canonical_ov_conf(&serde_json::json!({}), "local-a")
.unwrap()
.provider,
"filesystem"
);
}
#[test]
fn sqlite_pathlock_config_parses_shared_path_and_namespace() {
let path = std::env::temp_dir().join("openviking-locks.sqlite3");
let config = pathlock_config_from_value(
&serde_json::json!({"provider": "sqlite", "sqlite_path": path}),
"local-a",
)
.unwrap();
assert_eq!(config.sqlite_path.as_deref(), path.to_str());
assert_eq!(config.service_name.as_deref(), Some("local-a"));
for value in [
serde_json::json!({"provider": "sqlite"}),
serde_json::json!({"provider": "sqlite", "sqlite_path": "relative.db"}),
serde_json::json!({"provider": "sqlite", "sqlite_path": 42}),
] {
assert!(pathlock_config_from_value(&value, "local-a").is_err());
}
}
#[test]
fn canonical_cache_config_is_parsed_from_ov_conf() {
let path = std::env::temp_dir().join(format!(
"openviking-canonical-cache-config-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{
"service_name": "tenant-a",
"cache": {
"provider": "redis",
"params": {
"mode": "sentinel",
"endpoints": ["redis://sentinel:26379"],
"master_name": "mymaster",
"command_timeout_ms": 750
}
},
"storage": {
"agfs": {
"cachefs": {
"backend": "cache",
"traversal_mode": "cached_traversal"
},
"queuefs": {"backend": "cache"}
}
}
}"#,
)
.unwrap();
let cache_config = cache_config_from_ov_conf(path.to_str().unwrap(), "tenant-a").unwrap();
assert!(cache_config.enabled);
assert!(cache_config.runtime_enabled);
assert_eq!(cache_config.provider, CacheProviderKind::Redis);
assert_eq!(cache_config.service_name, "tenant-a");
assert_eq!(
cache_config.traversal_mode,
CacheTraversalMode::CachedTraversal
);
assert_eq!(cache_config.redis.mode, "sentinel");
assert_eq!(cache_config.redis.master_name.as_deref(), Some("mymaster"));
assert_eq!(cache_config.redis.command_timeout_ms, 750);
fs::remove_file(path).unwrap();
}
#[test]
fn config_path_rejects_removed_nested_cache_schema() {
let path = std::env::temp_dir().join(format!(
"openviking-removed-cache-config-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{"service_name":"test-service","storage":{"agfs":{"cache":{"enabled":true}}}}"#,
)
.unwrap();
let error = cache_config_from_ov_conf(path.to_str().unwrap(), "test-service").unwrap_err();
assert!(error.contains("storage.agfs.cache has been removed"));
assert!(error.contains("cache.provider"));
assert!(error.contains("storage.agfs.cachefs.backend"));
fs::remove_file(path).unwrap();
}
#[test]
fn config_path_rejects_removed_queuefs_redis_schema() {
let path = std::env::temp_dir().join(format!(
"openviking-removed-queue-redis-config-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{"storage":{"agfs":{"queuefs":{"backend":"redis"}}}}"#,
)
.unwrap();
let error = cache_config_from_ov_conf(path.to_str().unwrap(), "test-service").unwrap_err();
assert!(error.contains("queuefs backend=redis has been removed"));
assert!(error.contains("backend=cache"));
assert!(error.contains("cache.params"));
fs::remove_file(path).unwrap();
}
#[test]
fn queuefs_cache_runtime_does_not_enable_cachefs() {
let path = std::env::temp_dir().join(format!(
"openviking-queue-cache-config-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{
"service_name": "test-service",
"cache": {"provider": "redis", "params": {}},
"storage": {
"agfs": {
"queuefs": {"backend": "cache"}
}
}
}"#,
)
.unwrap();
let cache_config =
cache_config_from_ov_conf(path.to_str().unwrap(), "test-service").unwrap();
let stack_config = cache_config.stack_config().unwrap();
assert!(cache_config.runtime_enabled);
assert!(!stack_config.cachefs.enabled);
fs::remove_file(path).unwrap();
}
#[test]
fn queuefs_cache_runtime_rejects_redis_provider_key_prefix() {
let path = std::env::temp_dir().join(format!(
"openviking-queue-cache-prefix-config-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{
"service_name": "test-service",
"cache": {
"provider": "redis",
"params": {"key_prefix": "provider-prefix"}
},
"storage": {
"agfs": {
"queuefs": {"backend": "cache"}
}
}
}"#,
)
.unwrap();
let error = cache_config_from_ov_conf(path.to_str().unwrap(), "test-service").unwrap_err();
assert!(error.contains("key_prefix"));
fs::remove_file(path).unwrap();
}
#[test]
fn queuefs_cache_runtime_rejects_unsupported_provider_name() {
let path = std::env::temp_dir().join(format!(
"openviking-queue-cache-legacy-provider-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{
"service_name": "test-service",
"cache": {"provider": "mooncake", "params": {}},
"storage": {
"agfs": {
"queuefs": {"backend": "cache"}
}
}
}"#,
)
.unwrap();
let error = cache_config_from_ov_conf(path.to_str().unwrap(), "test-service").unwrap_err();
assert!(error.contains("mooncake"));
fs::remove_file(path).unwrap();
}
#[test]
fn disabled_binding_cache_rejects_removed_provider_names() {
for provider in ["memory", "yuanrong", "mooncake"] {
let config = serde_json::json!({
"enabled": false,
"runtime_enabled": false,
"provider": provider,
});
let error = cache_config_from_value(&config, "runtime-prod").unwrap_err();
assert!(error.contains(provider));
}
}
#[test]
fn cache_config_rejects_invalid_traversal_mode() {
let path = std::env::temp_dir().join(format!(
"openviking-invalid-cache-traversal-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{
"service_name": "test-service",
"cache": {"provider": "redis", "params": {}},
"storage": {"agfs": {"cachefs": {
"backend": "cache",
"traversal_mode": "bogus"
}}}
}"#,
)
.unwrap();
let error = cache_config_from_ov_conf(path.to_str().unwrap(), "test-service").unwrap_err();
assert!(error.contains("traversal_mode"));
fs::remove_file(path).unwrap();
}
#[test]
fn constructor_runtime_cache_overrides_config_path() {
Python::initialize();
let path = std::env::temp_dir().join(format!(
"openviking-runtime-cache-override-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{
"service_name": "test-service",
"cache": {"provider": "redis", "params": {}},
"storage": {"agfs": {"cachefs": {"backend": "cache"}}}
}"#,
)
.unwrap();
Python::attach(|py| {
let ty = py.get_type::<RAGFSBindingClient>();
let cache = PyDict::new(py);
cache.set_item("enabled", false).unwrap();
cache.set_item("provider", "redis").unwrap();
let config = PyDict::new(py);
config.set_item("service_name", "runtime-cache").unwrap();
config.set_item("cache", cache).unwrap();
let kwargs = PyDict::new(py);
kwargs
.set_item("config_path", path.to_str().unwrap())
.unwrap();
kwargs.set_item("config", config).unwrap();
let instance = ty.call((), Some(&kwargs));
assert!(instance.is_ok());
});
fs::remove_file(path).unwrap();
}
#[test]
fn constructor_rejects_removed_config_path_schema_with_runtime_config() {
Python::initialize();
let path = std::env::temp_dir().join(format!(
"openviking-removed-runtime-cache-override-{}.json",
std::process::id()
));
fs::write(
&path,
r#"{"service_name": "test-service", "storage": {"agfs": {"cache": {"enabled": false}}}}"#,
)
.unwrap();
Python::attach(|py| {
let ty = py.get_type::<RAGFSBindingClient>();
let cache = PyDict::new(py);
cache.set_item("enabled", false).unwrap();
cache.set_item("runtime_enabled", false).unwrap();
cache.set_item("provider", "redis").unwrap();
let config = PyDict::new(py);
config.set_item("service_name", "runtime-cache").unwrap();
config.set_item("cache", cache).unwrap();
let kwargs = PyDict::new(py);
kwargs
.set_item("config_path", path.to_str().unwrap())
.unwrap();
kwargs.set_item("config", config).unwrap();
let error = ty.call((), Some(&kwargs)).unwrap_err().to_string();
assert!(error.contains("storage.agfs.cache has been removed"));
});
fs::remove_file(path).unwrap();
}
#[test]
fn canonical_config_rejects_removed_redis_fields() {
for field in ["key_prefix", "read_from_replica", "tls_enabled"] {
let path = std::env::temp_dir().join(format!(
"openviking-removed-redis-field-{field}-{}.json",
std::process::id()
));
fs::write(
&path,
format!(
r#"{{
"service_name": "test-service",
"cache": {{"provider": "redis", "params": {{"{field}": true}}}},
"storage": {{"agfs": {{"cachefs": {{"backend": "cache"}}}}}}
}}"#
),
)
.unwrap();
let error =
cache_config_from_ov_conf(path.to_str().unwrap(), "test-service").unwrap_err();
assert!(error.contains(field));
fs::remove_file(path).unwrap();
}
}
#[test]
fn dynamic_cache_debug_output_redacts_provider_params() {
let config = DynamicCacheConfig {
library: "/opt/openviking/libprovider.so".to_string(),
params: serde_json::json!({"password": "binding-secret"}),
};
let output = format!("{config:?}");
assert!(output.contains("/opt/openviking/libprovider.so"));
assert!(!output.contains("binding-secret"));
assert!(output.contains("<redacted>"));
}
}