Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(cache): add DynamicProvider support through a versioned C ABI refactor (#4568)
txfange7f58639e383
diff --git a/Cargo.lock b/Cargo.lockindex 028e06eeb..8b2079660 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3066,6 +3066,16 @@ dependencies = ["cc",]+[[package]]+name = "libloading"+version = "0.8.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"+dependencies = [+ "cfg-if",+ "windows-link",+]+[[package]]name = "libm"version = "0.2.16"@@ -4066,6 +4076,7 @@ dependencies = ["grep-searcher","hkdf","ignore",+ "libloading","lru","mime_guess","native-tls",diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rsindex 5f375ef93..f1bc7e842 100644--- a/crates/ragfs-python/src/lib.rs+++ b/crates/ragfs-python/src/lib.rs@@ -178,6 +178,7 @@ fn pathlock_err_to_py(err: PathLockError) -> PyErr {}}}+use std::fmt;use std::fs;use std::future::Future;use std::time::{Duration, UNIX_EPOCH};@@ -277,12 +278,22 @@ struct RagfsCacheConfig {dynamic: DynamicCacheConfig,}-#[derive(Debug, Clone, PartialEq, Eq)]+#[derive(Clone, PartialEq, Eq)]struct DynamicCacheConfig {library: String,params: serde_json::Value,}+impl fmt::Debug for DynamicCacheConfig {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ formatter+ .debug_struct("DynamicCacheConfig")+ .field("library", &self.library)+ .field("params", &"<redacted>")+ .finish()+ }+}+#[derive(Debug, Clone, PartialEq, Eq)]struct RedisCacheConfig {mode: String,@@ -478,11 +489,7 @@ fn cache_config_from_canonical_ov_conf(match config.provider {CacheProviderKind::Redis => parse_redis_config(&mut config.redis, ¶ms, "cache.params")?,CacheProviderKind::Dynamic => {- config.dynamic.library = string_field(¶ms, "library", "")?;- config.dynamic.params = params- .get("params")- .cloned()- .unwrap_or_else(|| serde_json::Value::Object(serde_json::Map::new()));+ parse_dynamic_config(&mut config.dynamic, ¶ms)?;}}validate_cache_runtime_config(&config)?;@@ -536,16 +543,23 @@ fn cache_config_from_value(cache: &serde_json::Value) -> Result<RagfsCacheConfiglet dynamic = dynamic.as_object().ok_or_else(|| "cache.dynamic must be an object".to_string())?;- config.dynamic.library = string_field(dynamic, "library", &config.dynamic.library)?;- config.dynamic.params = dynamic- .get("params")- .cloned()- .unwrap_or_else(|| serde_json::Value::Object(serde_json::Map::new()));+ parse_dynamic_config(&mut config.dynamic, dynamic)?;}validate_cache_runtime_config(&config)?;Ok(config)}+fn parse_dynamic_config(+ config: &mut DynamicCacheConfig,+ params: &serde_json::Map<String, serde_json::Value>,+) -> Result<(), String> {+ config.library = string_field(params, "library", &config.library)?;+ let mut provider_params = params.clone();+ provider_params.remove("library");+ config.params = serde_json::Value::Object(provider_params);+ Ok(())+}+fn parse_redis_config(config: &mut RedisCacheConfig,redis: &serde_json::Map<String, serde_json::Value>,@@ -2907,51 +2921,6 @@ mod tests {let _ = fs::remove_file(&rotated);}- #[test]- fn dynamic_cache_config_is_parsed_from_ov_conf() {- let path = std::env::temp_dir().join(format!(- "openviking-cache-config-{}.json",- std::process::id()- ));- fs::write(- &path,- r#"{- "cache": {- "provider": "dynamic",- "params": {- "library": "/opt/openviking/libprovider.so",- "params": {"endpoint": "provider:1234"}- }- },- "storage": {- "agfs": {- "cachefs": {- "backend": "cache",- "namespace": "ov-test",- "traversal_mode": "cached_traversal"- }- }- }- }"#,- )- .unwrap();-- let cache_config = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap();- assert!(cache_config.enabled);- assert_eq!(cache_config.provider, CacheProviderKind::Dynamic);- assert_eq!(cache_config.namespace, "ov-test");- assert_eq!(- cache_config.traversal_mode,- CacheTraversalMode::CachedTraversal- );- assert_eq!(- cache_config.dynamic.library,- "/opt/openviking/libprovider.so"- );-- fs::remove_file(path).unwrap();- }-#[test]fn missing_cache_config_defaults_to_disabled_redis_config() {let path = std::env::temp_dir().join(format!(@@ -3273,4 +3242,18 @@ mod tests {fs::remove_file(path).unwrap();}}++ #[test]+ fn dynamic_cache_debug_output_redacts_provider_params() {+ let config = DynamicCacheConfig {+ library: "/opt/openviking/libprovider.so".to_string(),+ params: serde_json::json!({"password": "binding-secret"}),+ };++ let output = format!("{config:?}");++ assert!(output.contains("/opt/openviking/libprovider.so"));+ assert!(!output.contains("binding-secret"));+ assert!(output.contains("<redacted>"));+ }}diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.tomlindex 97f28a8cc..c60c3e72f 100644--- a/crates/ragfs/Cargo.toml+++ b/crates/ragfs/Cargo.toml@@ -84,6 +84,7 @@ bytes = "1.5"# Databaserusqlite = { version = "0.32", features = ["bundled"] }fred = { version = "10.1.0", default-features = false, features = ["i-keys", "i-lists", "i-scripts", "i-server", "i-sets", "sentinel-auth", "sha-1", "vendored-openssl"], optional = true }+libloading = { version = "0.8", optional = true }native-tls = { version = "0.2", optional = true }sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite", "mysql"], optional = true }@@ -136,7 +137,7 @@ uuid = { version = "1.0", features = ["v4"] }[features]default = []-cache = ["dep:fred", "dep:native-tls"]+cache = ["dep:fred", "dep:libloading", "dep:native-tls"]test-utils = []s3 = ["aws-sdk-s3", "aws-config", "aws-types"]full = ["s3", "cache"]diff --git a/crates/ragfs/include/openviking_cache_provider_v1.h b/crates/ragfs/include/openviking_cache_provider_v1.hnew file mode 100644index 000000000..c30504011--- /dev/null+++ b/crates/ragfs/include/openviking_cache_provider_v1.h@@ -0,0 +1,345 @@+#ifndef OPENVIKING_CACHE_PROVIDER_V1_H+#define OPENVIKING_CACHE_PROVIDER_V1_H++#include <stddef.h>+#include <stdint.h>++#if defined(_WIN32)+#define OV_CACHE_PROVIDER_EXPORT __declspec(dllexport)+#elif defined(__GNUC__) || defined(__clang__)+#define OV_CACHE_PROVIDER_EXPORT __attribute__((visibility("default")))+#else+#define OV_CACHE_PROVIDER_EXPORT+#endif++#ifdef __cplusplus+extern "C" {+#define OV_CACHE_ALIGNOF(type) alignof(type)+#else+#define OV_CACHE_ALIGNOF(type) _Alignof(type)+#endif++#define OV_CACHE_PROVIDER_ABI_V1 1u++#define OV_CACHE_STATUS_OK 0+#define OV_CACHE_STATUS_TIMEOUT 2+#define OV_CACHE_STATUS_UNAVAILABLE 3+#define OV_CACHE_STATUS_AUTHENTICATION 4+#define OV_CACHE_STATUS_PERMISSION_DENIED 5+#define OV_CACHE_STATUS_INVALID_ARGUMENT 6+#define OV_CACHE_STATUS_INVALID_DATA 7+#define OV_CACHE_STATUS_CROSS_SLOT 8+#define OV_CACHE_STATUS_READ_ONLY 9+#define OV_CACHE_STATUS_UNSUPPORTED_OPERATION 10+#define OV_CACHE_STATUS_INTERNAL 11++#define OV_CACHE_SET_CONDITION_NONE 0u+#define OV_CACHE_SET_CONDITION_NX 1u+#define OV_CACHE_SET_CONDITION_XX 2u++#define OV_CACHE_SET_APPLIED 0u+#define OV_CACHE_SET_CONDITION_NOT_MET 1u++#define OV_CACHE_LIST_BEFORE 0u+#define OV_CACHE_LIST_AFTER 1u+#define OV_CACHE_LIST_LEFT 0u+#define OV_CACHE_LIST_RIGHT 1u++#define OV_CACHE_SCRIPT_NULL 0u+#define OV_CACHE_SCRIPT_INTEGER 1u+#define OV_CACHE_SCRIPT_BYTES 2u+#define OV_CACHE_SCRIPT_ARRAY 3u+#define OV_CACHE_SCRIPT_BOOLEAN 4u++typedef struct {+ const uint8_t *data;+ size_t len;+} OvCacheByteSliceV1;++typedef struct {+ uint8_t *data;+ size_t len;+} OvCacheOwnedBufferV1;++typedef struct {+ OvCacheOwnedBufferV1 value;+ uint8_t present;+} OvCacheOptionalOwnedBufferV1;++typedef struct {+ OvCacheOwnedBufferV1 *items;+ size_t len;+} OvCacheOwnedBufferArrayV1;++typedef struct {+ OvCacheOptionalOwnedBufferV1 *items;+ size_t len;+} OvCacheOptionalOwnedBufferArrayV1;++typedef struct {+ OvCacheByteSliceV1 key;+ OvCacheByteSliceV1 value;+} OvCacheKeyValueV1;++typedef struct {+ /* -1 means no expiration; otherwise milliseconds. */+ int64_t expiration_ms;+ uint32_t condition;+ uint8_t keep_ttl;+} OvCacheSetOptionsV1;++typedef struct OvCacheScriptValueV1 OvCacheScriptValueV1;++struct OvCacheScriptValueV1 {+ OvCacheOwnedBufferV1 bytes;+ OvCacheScriptValueV1 *items;+ size_t items_len;+ int64_t integer;+ uint32_t kind;+ uint8_t boolean;+};++typedef uint8_t *(*OvCacheHostAllocV1)(size_t size, size_t alignment);+typedef void (*OvCacheHostDeallocV1)(uint8_t *data, size_t size, size_t alignment);++typedef struct {+ uint32_t abi_version;+ size_t struct_size;+ OvCacheHostAllocV1 alloc;+ OvCacheHostDeallocV1 dealloc;+} OvCacheHostApiV1;++/*+ * Provider contract:+ *+ * - OpenViking consumes cache.params.library and passes the remaining fields+ * to create as opaque JSON.+ * - Input slices and arrays are borrowed only for the duration of the callback.+ * The provider must not retain their pointers after the callback returns.+ * - All callbacks use the C ABI. Exceptions and language-runtime panics must+ * be caught inside the provider library.+ * - A successfully created provider handle must support concurrent calls.+ * - Every returned buffer, array, and nested item must be a separate,+ * non-aliasing allocation created with host->alloc using its exact C size and+ * alignment. Extra capacity, interior pointers, shared child buffers, and+ * splitting one allocation across multiple returned objects are forbidden.+ * - Empty buffers and arrays must use the canonical NULL pointer plus zero+ * length representation. Unused fields in optional and tagged outputs must+ * also use their zero representation.+ * - On OV_CACHE_STATUS_OK, the error buffer must be empty and every output must+ * be fully initialized. On any error status, normal outputs must remain in+ * their zero representation and only the error buffer may be returned.+ * - The provider owns and must release all partially constructed output before+ * returning an error. Once a successful output is returned, OpenViking owns+ * every allocation and releases it through the matching host allocator.+ * - host->alloc may reject unreasonable sizes or invalid alignments by+ * returning NULL. The provider must handle allocation failure.+ * - create, ping and close are required. Other callbacks are optional; NULL+ * means that operation is unsupported.+ * - close is terminal and consumes the provider handle even when it returns an+ * error status. It must stop and join provider-owned background work before+ * returning. OpenViking never retries close and unloads the library only+ * after close returns.+ */++typedef int32_t (*OvCacheProviderCreateV1)(+ const OvCacheHostApiV1 *host,+ OvCacheByteSliceV1 params_json,+ void **provider,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderPingV1)(+ void *provider,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderCloseV1)(+ void *provider,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderGetV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ OvCacheOptionalOwnedBufferV1 *output,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderSetV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ OvCacheByteSliceV1 value,+ OvCacheSetOptionsV1 options,+ uint8_t *result,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderDelV1)(+ void *provider,+ const OvCacheByteSliceV1 *keys,+ size_t keys_len,+ uint64_t *removed,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderMgetV1)(+ void *provider,+ const OvCacheByteSliceV1 *keys,+ size_t keys_len,+ OvCacheOptionalOwnedBufferArrayV1 *output,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderMsetV1)(+ void *provider,+ const OvCacheKeyValueV1 *entries,+ size_t entries_len,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderIncrbyV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ int64_t delta,+ int64_t *value,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderSismemberV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ OvCacheByteSliceV1 member,+ uint8_t *present,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderSmembersV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ OvCacheOwnedBufferArrayV1 *output,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderScardV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ uint64_t *count,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderListPushV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ const OvCacheByteSliceV1 *values,+ size_t values_len,+ uint64_t *length,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderListPopV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ uint8_t has_count,+ uint64_t count,+ OvCacheOwnedBufferArrayV1 *output,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderLlenV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ uint64_t *length,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderLrangeV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ int64_t start,+ int64_t stop,+ OvCacheOwnedBufferArrayV1 *output,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderLindexV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ int64_t index,+ OvCacheOptionalOwnedBufferV1 *output,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderLsetV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ int64_t index,+ OvCacheByteSliceV1 value,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderLtrimV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ int64_t start,+ int64_t stop,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderLremV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ int64_t count,+ OvCacheByteSliceV1 value,+ uint64_t *removed,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderLinsertV1)(+ void *provider,+ OvCacheByteSliceV1 key,+ uint32_t position,+ OvCacheByteSliceV1 pivot,+ OvCacheByteSliceV1 value,+ int64_t *length,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderLmoveV1)(+ void *provider,+ OvCacheByteSliceV1 source,+ OvCacheByteSliceV1 destination,+ uint32_t source_direction,+ uint32_t destination_direction,+ OvCacheOptionalOwnedBufferV1 *output,+ OvCacheOwnedBufferV1 *error);++typedef int32_t (*OvCacheProviderExecuteScriptV1)(+ void *provider,+ OvCacheByteSliceV1 script_id,+ OvCacheByteSliceV1 script_source,+ const OvCacheByteSliceV1 *keys,+ size_t keys_len,+ const OvCacheByteSliceV1 *args,+ size_t args_len,+ OvCacheScriptValueV1 *output,+ OvCacheOwnedBufferV1 *error);++typedef struct {+ uint32_t abi_version;+ size_t struct_size;+ OvCacheProviderCreateV1 create;+ OvCacheProviderPingV1 ping;+ OvCacheProviderCloseV1 close;+ OvCacheProviderGetV1 get;+ OvCacheProviderSetV1 set;+ OvCacheProviderDelV1 del;+ OvCacheProviderMgetV1 mget;+ OvCacheProviderMsetV1 mset;+ OvCacheProviderIncrbyV1 incrby;+ OvCacheProviderSismemberV1 sismember;+ OvCacheProviderSmembersV1 smembers;+ OvCacheProviderScardV1 scard;+ OvCacheProviderListPushV1 lpush;+ OvCacheProviderListPushV1 rpush;+ OvCacheProviderListPopV1 lpop;+ OvCacheProviderListPopV1 rpop;+ OvCacheProviderLlenV1 llen;+ OvCacheProviderLrangeV1 lrange;+ OvCacheProviderLindexV1 lindex;+ OvCacheProviderLsetV1 lset;+ OvCacheProviderLtrimV1 ltrim;+ OvCacheProviderLremV1 lrem;+ OvCacheProviderLinsertV1 linsert;+ OvCacheProviderLmoveV1 lmove;+ OvCacheProviderExecuteScriptV1 execute_script;+} OvCacheProviderApiV1;++/* The only symbol resolved by DynamicProvider. */+OV_CACHE_PROVIDER_EXPORT+const OvCacheProviderApiV1 *openviking_cache_provider_v1(void);++#ifdef __cplusplus+}+#endif++#endifdiff --git a/crates/ragfs/src/cache_runtime/dynamic/abi.rs b/crates/ragfs/src/cache_runtime/dynamic/abi.rsnew file mode 100644index 000000000..cf75aab86--- /dev/null+++ b/crates/ragfs/src/cache_runtime/dynamic/abi.rs@@ -0,0 +1,297 @@+use std::ffi::c_void;++pub(super) const ABI_VERSION_V1: u32 = 1;+pub(super) const ENTRY_SYMBOL_V1: &[u8] = b"openviking_cache_provider_v1\0";++pub(super) const STATUS_OK: i32 = 0;+pub(super) const STATUS_TIMEOUT: i32 = 2;+pub(super) const STATUS_UNAVAILABLE: i32 = 3;+pub(super) const STATUS_AUTHENTICATION: i32 = 4;+pub(super) const STATUS_PERMISSION_DENIED: i32 = 5;+pub(super) const STATUS_INVALID_ARGUMENT: i32 = 6;+pub(super) const STATUS_INVALID_DATA: i32 = 7;+pub(super) const STATUS_CROSS_SLOT: i32 = 8;+pub(super) const STATUS_READ_ONLY: i32 = 9;+pub(super) const STATUS_UNSUPPORTED_OPERATION: i32 = 10;+pub(super) const SET_CONDITION_NONE: u32 = 0;+pub(super) const SET_CONDITION_NX: u32 = 1;+pub(super) const SET_CONDITION_XX: u32 = 2;+pub(super) const SET_APPLIED: u8 = 0;+pub(super) const SET_CONDITION_NOT_MET: u8 = 1;+pub(super) const LIST_BEFORE: u32 = 0;+pub(super) const LIST_AFTER: u32 = 1;+pub(super) const LIST_LEFT: u32 = 0;+pub(super) const LIST_RIGHT: u32 = 1;+pub(super) const SCRIPT_NULL: u32 = 0;+pub(super) const SCRIPT_INTEGER: u32 = 1;+pub(super) const SCRIPT_BYTES: u32 = 2;+pub(super) const SCRIPT_ARRAY: u32 = 3;+pub(super) const SCRIPT_BOOLEAN: u32 = 4;++#[repr(C)]+#[derive(Clone, Copy)]+pub(super) struct ByteSliceV1 {+ pub(super) data: *const u8,+ pub(super) len: usize,+}++impl ByteSliceV1 {+ pub(super) fn from_slice(value: &[u8]) -> Self {+ Self {+ data: value.as_ptr(),+ len: value.len(),+ }+ }+}++#[repr(C)]+#[derive(Default)]+pub(super) struct OwnedBufferV1 {+ pub(super) data: *mut u8,+ pub(super) len: usize,+}++#[repr(C)]+#[derive(Default)]+pub(super) struct OptionalOwnedBufferV1 {+ pub(super) value: OwnedBufferV1,+ pub(super) present: u8,+}++#[repr(C)]+#[derive(Default)]+pub(super) struct OwnedBufferArrayV1 {+ pub(super) items: *mut OwnedBufferV1,+ pub(super) len: usize,+}++#[repr(C)]+#[derive(Default)]+pub(super) struct OptionalOwnedBufferArrayV1 {+ pub(super) items: *mut OptionalOwnedBufferV1,+ pub(super) len: usize,+}++#[repr(C)]+#[derive(Clone, Copy)]+pub(super) struct KeyValueV1 {+ pub(super) key: ByteSliceV1,+ pub(super) value: ByteSliceV1,+}++#[repr(C)]+#[derive(Clone, Copy)]+pub(super) struct SetOptionsV1 {+ pub(super) expiration_ms: i64,+ pub(super) condition: u32,+ pub(super) keep_ttl: u8,+}++#[repr(C)]+#[derive(Default)]+pub(super) struct ScriptValueV1 {+ pub(super) bytes: OwnedBufferV1,+ pub(super) items: *mut ScriptValueV1,+ pub(super) items_len: usize,+ pub(super) integer: i64,+ pub(super) kind: u32,+ pub(super) boolean: u8,+}++pub(super) type HostAllocV1 = unsafe extern "C" fn(size: usize, alignment: usize) -> *mut u8;+pub(super) type HostDeallocV1 = unsafe extern "C" fn(data: *mut u8, size: usize, alignment: usize);++#[repr(C)]+pub(super) struct HostApiV1 {+ pub(super) abi_version: u32,+ pub(super) struct_size: usize,+ pub(super) alloc: HostAllocV1,+ pub(super) dealloc: HostDeallocV1,+}++pub(super) type ProviderCreateV1 = unsafe extern "C" fn(+ host: *const HostApiV1,+ params_json: ByteSliceV1,+ provider: *mut *mut c_void,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderPingV1 =+ unsafe extern "C" fn(provider: *mut c_void, error: *mut OwnedBufferV1) -> i32;+pub(super) type ProviderCloseV1 = ProviderPingV1;+pub(super) type ProviderGetV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ output: *mut OptionalOwnedBufferV1,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderSetV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ value: ByteSliceV1,+ options: SetOptionsV1,+ result: *mut u8,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderDelV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ keys: *const ByteSliceV1,+ keys_len: usize,+ removed: *mut u64,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderMgetV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ keys: *const ByteSliceV1,+ keys_len: usize,+ output: *mut OptionalOwnedBufferArrayV1,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderMsetV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ entries: *const KeyValueV1,+ entries_len: usize,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderIncrbyV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ delta: i64,+ value: *mut i64,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderSismemberV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ member: ByteSliceV1,+ present: *mut u8,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderSmembersV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ output: *mut OwnedBufferArrayV1,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderScardV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ count: *mut u64,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderListPushV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ values: *const ByteSliceV1,+ values_len: usize,+ length: *mut u64,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderListPopV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ has_count: u8,+ count: u64,+ output: *mut OwnedBufferArrayV1,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderLlenV1 = ProviderScardV1;+pub(super) type ProviderLrangeV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ start: i64,+ stop: i64,+ output: *mut OwnedBufferArrayV1,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderLindexV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ index: i64,+ output: *mut OptionalOwnedBufferV1,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderLsetV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ index: i64,+ value: ByteSliceV1,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderLtrimV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ start: i64,+ stop: i64,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderLremV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ count: i64,+ value: ByteSliceV1,+ removed: *mut u64,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderLinsertV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ key: ByteSliceV1,+ position: u32,+ pivot: ByteSliceV1,+ value: ByteSliceV1,+ length: *mut i64,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderLmoveV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ source: ByteSliceV1,+ destination: ByteSliceV1,+ source_direction: u32,+ destination_direction: u32,+ output: *mut OptionalOwnedBufferV1,+ error: *mut OwnedBufferV1,+) -> i32;+pub(super) type ProviderExecuteScriptV1 = unsafe extern "C" fn(+ provider: *mut c_void,+ script_id: ByteSliceV1,+ script_source: ByteSliceV1,+ keys: *const ByteSliceV1,+ keys_len: usize,+ args: *const ByteSliceV1,+ args_len: usize,+ output: *mut ScriptValueV1,+ error: *mut OwnedBufferV1,+) -> i32;++#[repr(C)]+#[derive(Clone, Copy)]+pub(super) struct ProviderApiV1 {+ pub(super) abi_version: u32,+ pub(super) struct_size: usize,+ pub(super) create: Option<ProviderCreateV1>,+ pub(super) ping: Option<ProviderPingV1>,+ pub(super) close: Option<ProviderCloseV1>,+ pub(super) get: Option<ProviderGetV1>,+ pub(super) set: Option<ProviderSetV1>,+ pub(super) del: Option<ProviderDelV1>,+ pub(super) mget: Option<ProviderMgetV1>,+ pub(super) mset: Option<ProviderMsetV1>,+ pub(super) incrby: Option<ProviderIncrbyV1>,+ pub(super) sismember: Option<ProviderSismemberV1>,+ pub(super) smembers: Option<ProviderSmembersV1>,+ pub(super) scard: Option<ProviderScardV1>,+ pub(super) lpush: Option<ProviderListPushV1>,+ pub(super) rpush: Option<ProviderListPushV1>,+ pub(super) lpop: Option<ProviderListPopV1>,+ pub(super) rpop: Option<ProviderListPopV1>,+ pub(super) llen: Option<ProviderLlenV1>,+ pub(super) lrange: Option<ProviderLrangeV1>,+ pub(super) lindex: Option<ProviderLindexV1>,+ pub(super) lset: Option<ProviderLsetV1>,+ pub(super) ltrim: Option<ProviderLtrimV1>,+ pub(super) lrem: Option<ProviderLremV1>,+ pub(super) linsert: Option<ProviderLinsertV1>,+ pub(super) lmove: Option<ProviderLmoveV1>,+ pub(super) execute_script: Option<ProviderExecuteScriptV1>,+}++pub(super) type ProviderEntryV1 = unsafe extern "C" fn() -> *const ProviderApiV1;diff --git a/crates/ragfs/src/cache_runtime/dynamic/config.rs b/crates/ragfs/src/cache_runtime/dynamic/config.rsindex ca58d7c83..392ef82d5 100644--- a/crates/ragfs/src/cache_runtime/dynamic/config.rs+++ b/crates/ragfs/src/cache_runtime/dynamic/config.rs@@ -1,10 +1,41 @@+use std::fmt;use std::path::PathBuf;/// Configuration for one dynamically loaded cache provider.-#[derive(Debug, Clone, PartialEq, Eq)]+#[derive(Clone, PartialEq, Eq)]pub struct DynamicProviderConfig {- /// Absolute or process-resolvable path to the provider library.+ /// Absolute path to the provider library.pub library_path: PathBuf,- /// Provider-owned JSON configuration passed unchanged to `init`.+ /// Provider-owned JSON configuration passed unchanged to `create`.pub params_json: String,}++impl fmt::Debug for DynamicProviderConfig {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ formatter+ .debug_struct("DynamicProviderConfig")+ .field("library_path", &self.library_path)+ .field("params_json", &"<redacted>")+ .field("params_length", &self.params_json.len())+ .finish()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn debug_output_redacts_provider_params() {+ let config = DynamicProviderConfig {+ library_path: PathBuf::from("/opt/openviking/libprovider.so"),+ params_json: r#"{"password":"secret-value"}"#.to_string(),+ };++ let output = format!("{config:?}");++ assert!(output.contains("/opt/openviking/libprovider.so"));+ assert!(!output.contains("secret-value"));+ assert!(output.contains("<redacted>"));+ }+}diff --git a/crates/ragfs/src/cache_runtime/dynamic/loader.rs b/crates/ragfs/src/cache_runtime/dynamic/loader.rsnew file mode 100644index 000000000..7379f3bf6--- /dev/null+++ b/crates/ragfs/src/cache_runtime/dynamic/loader.rs@@ -0,0 +1,94 @@+use super::abi::{ProviderApiV1, ProviderEntryV1, ABI_VERSION_V1, ENTRY_SYMBOL_V1};+use crate::cache_runtime::{CacheError, CacheResult};+use libloading::{Library, Symbol};+use std::mem::{align_of, size_of};+use std::path::Path;+use std::ptr;++#[repr(C)]+#[derive(Clone, Copy)]+struct ProviderApiHeaderV1 {+ abi_version: u32,+ struct_size: usize,+}++pub(super) struct LoadedProviderApi {+ pub(super) library: Library,+ pub(super) api: ProviderApiV1,+}++pub(super) fn load(path: &Path) -> CacheResult<LoadedProviderApi> {+ if !path.is_absolute() {+ return Err(CacheError::InvalidArgument(format!(+ "dynamic provider library path must be absolute: {}",+ path.display()+ )));+ }+ let library = unsafe { Library::new(path) }.map_err(|error| {+ CacheError::Unavailable(format!(+ "failed to load dynamic provider {}: {error}",+ path.display()+ ))+ })?;+ let entry: Symbol<ProviderEntryV1> =+ unsafe { library.get(ENTRY_SYMBOL_V1) }.map_err(|error| {+ CacheError::AbiMismatch(format!(+ "dynamic provider {} does not export openviking_cache_provider_v1: {error}",+ path.display()+ ))+ })?;+ let api_ptr = unsafe { entry() };+ if api_ptr.is_null() {+ return Err(CacheError::AbiMismatch(format!(+ "dynamic provider {} returned a null API table",+ path.display()+ )));+ }+ if !(api_ptr as usize).is_multiple_of(align_of::<ProviderApiV1>()) {+ return Err(CacheError::AbiMismatch(format!(+ "dynamic provider {} returned a misaligned API table",+ path.display()+ )));+ }+ let header = unsafe { ptr::read(api_ptr.cast::<ProviderApiHeaderV1>()) };+ validate_header(header, path)?;+ let api = unsafe { ptr::read(api_ptr) };+ validate_callbacks(&api, path)?;+ Ok(LoadedProviderApi { library, api })+}++fn validate_header(header: ProviderApiHeaderV1, path: &Path) -> CacheResult<()> {+ if header.abi_version != ABI_VERSION_V1 {+ return Err(CacheError::AbiMismatch(format!(+ "dynamic provider {} uses ABI {}, expected {}",+ path.display(),+ header.abi_version,+ ABI_VERSION_V1+ )));+ }+ if header.struct_size < size_of::<ProviderApiV1>() {+ return Err(CacheError::AbiMismatch(format!(+ "dynamic provider {} API table is {} bytes, expected at least {}",+ path.display(),+ header.struct_size,+ size_of::<ProviderApiV1>()+ )));+ }+ Ok(())+}++fn validate_callbacks(api: &ProviderApiV1, path: &Path) -> CacheResult<()> {+ for (name, present) in [+ ("create", api.create.is_some()),+ ("ping", api.ping.is_some()),+ ("close", api.close.is_some()),+ ] {+ if !present {+ return Err(CacheError::AbiMismatch(format!(+ "dynamic provider {} is missing required {name} callback",+ path.display()+ )));+ }+ }+ Ok(())+}diff --git a/crates/ragfs/src/cache_runtime/dynamic/mod.rs b/crates/ragfs/src/cache_runtime/dynamic/mod.rsindex 7e9f1b5d7..86c55c151 100644--- a/crates/ragfs/src/cache_runtime/dynamic/mod.rs+++ b/crates/ragfs/src/cache_runtime/dynamic/mod.rs@@ -1,3 +1,7 @@+mod abi;mod config;+mod loader;+mod provider;pub use config::DynamicProviderConfig;+pub(super) use provider::DynamicProvider;diff --git a/crates/ragfs/src/cache_runtime/dynamic/provider.rs b/crates/ragfs/src/cache_runtime/dynamic/provider.rsnew file mode 100644index 000000000..26359408f--- /dev/null+++ b/crates/ragfs/src/cache_runtime/dynamic/provider.rs@@ -0,0 +1,1658 @@+use super::abi::*;+use super::config::DynamicProviderConfig;+use super::loader;+use crate::cache_runtime::provider::{CacheOperation, CacheProvider};+use crate::cache_runtime::{+ CacheError, CacheResult, Expiration, ListDirection, ListInsertPosition, ListInsertRequest,+ ListMoveRequest, ScriptRegistry, ScriptRequest, ScriptResult, ScriptValue, SetCondition,+ SetOptions, SetResult,+};+use async_trait::async_trait;+use bytes::Bytes;+use libloading::Library;+use std::alloc::{alloc, dealloc, Layout};+use std::collections::HashMap;+use std::ffi::c_void;+use std::mem::size_of;+use std::ptr;+use std::slice;+use std::sync::atomic::{AtomicU8, Ordering};+use std::sync::{Arc, Mutex, OnceLock, RwLock};++const LIFECYCLE_OPEN: u8 = 0;+const LIFECYCLE_CLOSING: u8 = 1;+const LIFECYCLE_CLOSED: u8 = 2;+const MAX_HOST_ALLOCATION_BYTES: usize = 128 * 1024 * 1024;+const MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024;+const MAX_ERROR_BYTES: usize = 64 * 1024;+const MAX_ARRAY_ITEMS: usize = 100_000;+const MAX_AGGREGATE_BYTES: usize = 128 * 1024 * 1024;+const MAX_SCRIPT_DEPTH: usize = 64;+const MAX_SCRIPT_NODES: usize = 100_000;++static HOST_API: HostApiV1 = HostApiV1 {+ abi_version: ABI_VERSION_V1,+ struct_size: size_of::<HostApiV1>(),+ alloc: host_alloc,+ dealloc: host_dealloc,+};++pub(crate) struct DynamicProvider {+ state: Arc<DynamicState>,+ scripts: Arc<ScriptRegistry>,+}++struct DynamicState {+ _library: Library,+ api: ProviderApiV1,+ handle: *mut c_void,+ calls: RwLock<()>,+ lifecycle: AtomicU8,+}++// The ABI requires the provider handle to support concurrent calls. A provider+// backed by a non-thread-safe SDK must serialize or pool internally.+unsafe impl Send for DynamicState {}+unsafe impl Sync for DynamicState {}++impl DynamicProvider {+ pub(crate) async fn connect(+ config: DynamicProviderConfig,+ scripts: Arc<ScriptRegistry>,+ ) -> CacheResult<Self> {+ let state = tokio::task::spawn_blocking(move || connect_state(config))+ .await+ .map_err(|error| {+ CacheError::Internal(format!("dynamic provider create task failed: {error}"))+ })??;+ let provider = Self { state, scripts };+ if let Err(error) = provider.ping().await {+ let _ = provider.close().await;+ return Err(error);+ }+ Ok(provider)+ }++ async fn call<T, F>(&self, operation: &'static str, call: F) -> CacheResult<T>+ where+ T: Send + 'static,+ F: FnOnce(&DynamicState) -> CacheResult<T> + Send + 'static,+ {+ let state = Arc::clone(&self.state);+ tokio::task::spawn_blocking(move || call(&state))+ .await+ .map_err(|error| {+ CacheError::Internal(format!("dynamic provider {operation} task failed: {error}"))+ })?+ }++ async fn list_push(+ &self,+ operation: CacheOperation,+ key: &str,+ values: Vec<Bytes>,+ ) -> CacheResult<u64> {+ if values.is_empty() {+ return Err(CacheError::InvalidArgument(format!(+ "{} requires at least one value",+ operation.name()+ )));+ }+ let key = key.to_string();+ self.call(operation.name(), move |state| {+ state.with_call(operation.name(), |api, handle| {+ let callback = match operation {+ CacheOperation::Lpush => required(api.lpush, operation)?,+ CacheOperation::Rpush => required(api.rpush, operation)?,+ _ => unreachable!("list_push only accepts push operations"),+ };+ let raw_values = byte_slices(&values);+ let mut length = 0;+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ raw_values.as_ptr(),+ raw_values.len(),+ &mut length,+ &mut error,+ )+ };+ status_result(status, error, operation.name())?;+ Ok(length)+ })+ })+ .await+ }++ async fn list_pop(+ &self,+ operation: CacheOperation,+ key: &str,+ count: Option<u64>,+ ) -> CacheResult<Vec<Bytes>> {+ let key = key.to_string();+ self.call(operation.name(), move |state| {+ state.with_call(operation.name(), |api, handle| {+ let callback = match operation {+ CacheOperation::Lpop => required(api.lpop, operation)?,+ CacheOperation::Rpop => required(api.rpop, operation)?,+ _ => unreachable!("list_pop only accepts pop operations"),+ };+ let mut output = OwnedBufferArrayV1::default();+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ u8::from(count.is_some()),+ count.unwrap_or_default(),+ &mut output,+ &mut error,+ )+ };+ take_output(+ status,+ error,+ operation.name(),+ output,+ take_buffer_array,+ discard_buffer_array,+ )+ })+ })+ .await+ }+}++fn connect_state(config: DynamicProviderConfig) -> CacheResult<Arc<DynamicState>> {+ let loaded = loader::load(&config.library_path)?;+ let create = loaded.api.create.expect("validated create callback");+ let mut handle = ptr::null_mut();+ let mut error = OwnedBufferV1::default();+ let params = ByteSliceV1::from_slice(config.params_json.as_bytes());+ let status = unsafe { create(&HOST_API, params, &mut handle, &mut error) };+ if let Err(error) = status_result(status, error, "create") {+ if !handle.is_null() {+ close_handle(loaded.api, handle);+ }+ return Err(error);+ }+ if handle.is_null() {+ return Err(CacheError::InvalidData(+ "dynamic provider create returned a null handle".into(),+ ));+ }++ Ok(Arc::new(DynamicState {+ _library: loaded.library,+ api: loaded.api,+ handle,+ calls: RwLock::new(()),+ lifecycle: AtomicU8::new(LIFECYCLE_OPEN),+ }))+}++impl DynamicState {+ fn with_call<T>(+ &self,+ operation: &str,+ call: impl FnOnce(&ProviderApiV1, *mut c_void) -> CacheResult<T>,+ ) -> CacheResult<T> {+ let _guard = self+ .calls+ .read()+ .map_err(|_| CacheError::Internal("dynamic provider call lock poisoned".into()))?;+ if self.lifecycle.load(Ordering::Acquire) != LIFECYCLE_OPEN {+ return Err(CacheError::Closed);+ }+ call(&self.api, self.handle).map_err(|error| match error {+ CacheError::UnsupportedOperation(_) => {+ CacheError::UnsupportedOperation(operation.to_string())+ }+ other => other,+ })+ }++ fn supports(&self, operation: CacheOperation) -> bool {+ match operation {+ CacheOperation::Get => self.api.get.is_some(),+ CacheOperation::Set => self.api.set.is_some(),+ CacheOperation::Del => self.api.del.is_some(),+ CacheOperation::Mget => self.api.mget.is_some(),+ CacheOperation::Mset => self.api.mset.is_some(),+ CacheOperation::IncrBy => self.api.incrby.is_some(),+ CacheOperation::Sismember => self.api.sismember.is_some(),+ CacheOperation::Smembers => self.api.smembers.is_some(),+ CacheOperation::Scard => self.api.scard.is_some(),+ CacheOperation::Lpush => self.api.lpush.is_some(),+ CacheOperation::Rpush => self.api.rpush.is_some(),+ CacheOperation::Lpop => self.api.lpop.is_some(),+ CacheOperation::Rpop => self.api.rpop.is_some(),+ CacheOperation::Llen => self.api.llen.is_some(),+ CacheOperation::Lrange => self.api.lrange.is_some(),+ CacheOperation::Lindex => self.api.lindex.is_some(),+ CacheOperation::Lset => self.api.lset.is_some(),+ CacheOperation::Ltrim => self.api.ltrim.is_some(),+ CacheOperation::Lrem => self.api.lrem.is_some(),+ CacheOperation::Linsert => self.api.linsert.is_some(),+ CacheOperation::Lmove => self.api.lmove.is_some(),+ CacheOperation::ExecuteScript => self.api.execute_script.is_some(),+ }+ }++ fn ping(&self) -> CacheResult<()> {+ self.with_call("ping", |api, handle| {+ let callback = api.ping.expect("validated ping callback");+ let mut error = OwnedBufferV1::default();+ let status = unsafe { callback(handle, &mut error) };+ status_result(status, error, "ping")+ })+ }++ fn close(&self) -> CacheResult<()> {+ match self.lifecycle.compare_exchange(+ LIFECYCLE_OPEN,+ LIFECYCLE_CLOSING,+ Ordering::AcqRel,+ Ordering::Acquire,+ ) {+ Ok(_) => {}+ Err(state) if matches!(state, LIFECYCLE_CLOSING | LIFECYCLE_CLOSED) => return Ok(()),+ Err(other) => {+ return Err(CacheError::Internal(format!(+ "dynamic provider has invalid lifecycle state {other}"+ )))+ }+ }+ let _guard = self+ .calls+ .write()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ let callback = self.api.close.expect("validated close callback");+ let mut error = OwnedBufferV1::default();+ let status = unsafe { callback(self.handle, &mut error) };+ self.lifecycle.store(LIFECYCLE_CLOSED, Ordering::Release);+ status_result(status, error, "close")+ }+}++impl Drop for DynamicState {+ fn drop(&mut self) {+ if self+ .lifecycle+ .compare_exchange(+ LIFECYCLE_OPEN,+ LIFECYCLE_CLOSING,+ Ordering::AcqRel,+ Ordering::Acquire,+ )+ .is_ok()+ {+ let _guard = self+ .calls+ .write()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ close_handle(self.api, self.handle);+ self.lifecycle.store(LIFECYCLE_CLOSED, Ordering::Release);+ }+ }+}++#[async_trait]+impl CacheProvider for DynamicProvider {+ fn validate_operations(&self, operations: &[CacheOperation]) -> CacheResult<()> {+ if let Some(operation) = operations+ .iter()+ .copied()+ .find(|operation| !self.state.supports(*operation))+ {+ return Err(CacheError::UnsupportedOperation(+ operation.name().to_string(),+ ));+ }+ Ok(())+ }++ async fn get(&self, key: &str) -> CacheResult<Option<Bytes>> {+ let key = key.to_string();+ self.call("get", move |state| {+ state.with_call("get", |api, handle| {+ let callback = required(api.get, CacheOperation::Get)?;+ let mut output = OptionalOwnedBufferV1::default();+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ &mut output,+ &mut error,+ )+ };+ take_output(+ status,+ error,+ "get",+ output,+ take_optional_buffer,+ discard_optional_buffer,+ )+ })+ })+ .await+ }++ async fn set(&self, key: &str, value: Bytes, options: SetOptions) -> CacheResult<SetResult> {+ if options.keep_ttl && options.expiration.is_some() {+ return Err(CacheError::InvalidArgument(+ "set cannot combine expiration with keep_ttl".into(),+ ));+ }+ let options = set_options(options)?;+ let key = key.to_string();+ self.call("set", move |state| {+ state.with_call("set", |api, handle| {+ let callback = required(api.set, CacheOperation::Set)?;+ let mut result = SET_APPLIED;+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ ByteSliceV1::from_slice(&value),+ options,+ &mut result,+ &mut error,+ )+ };+ status_result(status, error, "set")?;+ match result {+ SET_APPLIED => Ok(SetResult::Applied),+ SET_CONDITION_NOT_MET => Ok(SetResult::ConditionNotMet),+ other => Err(CacheError::InvalidData(format!(+ "dynamic provider set returned invalid result {other}"+ ))),+ }+ })+ })+ .await+ }++ async fn del(&self, keys: &[String]) -> CacheResult<u64> {+ if keys.is_empty() {+ return Ok(0);+ }+ let keys = keys.to_vec();+ self.call("del", move |state| {+ state.with_call("del", |api, handle| {+ let callback = required(api.del, CacheOperation::Del)?;+ let raw_keys = string_slices(&keys);+ let mut removed = 0;+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ raw_keys.as_ptr(),+ raw_keys.len(),+ &mut removed,+ &mut error,+ )+ };+ status_result(status, error, "del")?;+ Ok(removed)+ })+ })+ .await+ }++ async fn mget(&self, keys: &[String]) -> CacheResult<Vec<Option<Bytes>>> {+ if keys.is_empty() {+ return Ok(Vec::new());+ }+ let keys = keys.to_vec();+ self.call("mget", move |state| {+ state.with_call("mget", |api, handle| {+ let callback = required(api.mget, CacheOperation::Mget)?;+ let raw_keys = string_slices(&keys);+ let mut output = OptionalOwnedBufferArrayV1::default();+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ raw_keys.as_ptr(),+ raw_keys.len(),+ &mut output,+ &mut error,+ )+ };+ let values = take_output(+ status,+ error,+ "mget",+ output,+ take_optional_buffer_array,+ discard_optional_buffer_array,+ )?;+ if values.len() != keys.len() {+ return Err(CacheError::InvalidData(format!(+ "dynamic provider mget returned {} values for {} keys",+ values.len(),+ keys.len()+ )));+ }+ Ok(values)+ })+ })+ .await+ }++ async fn mset(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> {+ if entries.is_empty() {+ return Ok(());+ }+ self.call("mset", move |state| {+ state.with_call("mset", |api, handle| {+ let callback = required(api.mset, CacheOperation::Mset)?;+ let raw_entries = entries+ .iter()+ .map(|(key, value)| KeyValueV1 {+ key: ByteSliceV1::from_slice(key.as_bytes()),+ value: ByteSliceV1::from_slice(value),+ })+ .collect::<Vec<_>>();+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(handle, raw_entries.as_ptr(), raw_entries.len(), &mut error)+ };+ status_result(status, error, "mset")+ })+ })+ .await+ }++ async fn incr_by(&self, key: &str, delta: i64) -> CacheResult<i64> {+ let key = key.to_string();+ self.call("incrby", move |state| {+ state.with_call("incrby", |api, handle| {+ let callback = required(api.incrby, CacheOperation::IncrBy)?;+ let mut value = 0;+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ delta,+ &mut value,+ &mut error,+ )+ };+ status_result(status, error, "incrby")?;+ Ok(value)+ })+ })+ .await+ }++ async fn sismember(&self, key: &str, member: &[u8]) -> CacheResult<bool> {+ let key = key.to_string();+ let member = member.to_vec();+ self.call("sismember", move |state| {+ state.with_call("sismember", |api, handle| {+ let callback = required(api.sismember, CacheOperation::Sismember)?;+ let mut present = 0;+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ ByteSliceV1::from_slice(&member),+ &mut present,+ &mut error,+ )+ };+ status_result(status, error, "sismember")?;+ take_bool(present, "sismember")+ })+ })+ .await+ }++ async fn smembers(&self, key: &str) -> CacheResult<Vec<Bytes>> {+ let key = key.to_string();+ self.call("smembers", move |state| {+ state.with_call("smembers", |api, handle| {+ let callback = required(api.smembers, CacheOperation::Smembers)?;+ let mut output = OwnedBufferArrayV1::default();+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ &mut output,+ &mut error,+ )+ };+ take_output(+ status,+ error,+ "smembers",+ output,+ take_buffer_array,+ discard_buffer_array,+ )+ })+ })+ .await+ }++ async fn scard(&self, key: &str) -> CacheResult<u64> {+ scalar_u64(self, CacheOperation::Scard, key).await+ }++ async fn lpush(&self, key: &str, values: Vec<Bytes>) -> CacheResult<u64> {+ self.list_push(CacheOperation::Lpush, key, values).await+ }++ async fn rpush(&self, key: &str, values: Vec<Bytes>) -> CacheResult<u64> {+ self.list_push(CacheOperation::Rpush, key, values).await+ }++ async fn lpop(&self, key: &str, count: Option<u64>) -> CacheResult<Vec<Bytes>> {+ self.list_pop(CacheOperation::Lpop, key, count).await+ }++ async fn rpop(&self, key: &str, count: Option<u64>) -> CacheResult<Vec<Bytes>> {+ self.list_pop(CacheOperation::Rpop, key, count).await+ }++ async fn llen(&self, key: &str) -> CacheResult<u64> {+ scalar_u64(self, CacheOperation::Llen, key).await+ }++ async fn lrange(&self, key: &str, start: i64, stop: i64) -> CacheResult<Vec<Bytes>> {+ let key = key.to_string();+ self.call("lrange", move |state| {+ state.with_call("lrange", |api, handle| {+ let callback = required(api.lrange, CacheOperation::Lrange)?;+ let mut output = OwnedBufferArrayV1::default();+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ start,+ stop,+ &mut output,+ &mut error,+ )+ };+ take_output(+ status,+ error,+ "lrange",+ output,+ take_buffer_array,+ discard_buffer_array,+ )+ })+ })+ .await+ }++ async fn lindex(&self, key: &str, index: i64) -> CacheResult<Option<Bytes>> {+ let key = key.to_string();+ self.call("lindex", move |state| {+ state.with_call("lindex", |api, handle| {+ let callback = required(api.lindex, CacheOperation::Lindex)?;+ let mut output = OptionalOwnedBufferV1::default();+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ index,+ &mut output,+ &mut error,+ )+ };+ take_output(+ status,+ error,+ "lindex",+ output,+ take_optional_buffer,+ discard_optional_buffer,+ )+ })+ })+ .await+ }++ async fn lset(&self, key: &str, index: i64, value: Bytes) -> CacheResult<()> {+ let key = key.to_string();+ self.call("lset", move |state| {+ state.with_call("lset", |api, handle| {+ let callback = required(api.lset, CacheOperation::Lset)?;+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ index,+ ByteSliceV1::from_slice(&value),+ &mut error,+ )+ };+ status_result(status, error, "lset")+ })+ })+ .await+ }++ async fn ltrim(&self, key: &str, start: i64, stop: i64) -> CacheResult<()> {+ let key = key.to_string();+ self.call("ltrim", move |state| {+ state.with_call("ltrim", |api, handle| {+ let callback = required(api.ltrim, CacheOperation::Ltrim)?;+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ start,+ stop,+ &mut error,+ )+ };+ status_result(status, error, "ltrim")+ })+ })+ .await+ }++ async fn lrem(&self, key: &str, count: i64, value: Bytes) -> CacheResult<u64> {+ let key = key.to_string();+ self.call("lrem", move |state| {+ state.with_call("lrem", |api, handle| {+ let callback = required(api.lrem, CacheOperation::Lrem)?;+ let mut removed = 0;+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ count,+ ByteSliceV1::from_slice(&value),+ &mut removed,+ &mut error,+ )+ };+ status_result(status, error, "lrem")?;+ Ok(removed)+ })+ })+ .await+ }++ async fn linsert(&self, request: ListInsertRequest) -> CacheResult<i64> {+ self.call("linsert", move |state| {+ state.with_call("linsert", |api, handle| {+ let callback = required(api.linsert, CacheOperation::Linsert)?;+ let position = match request.position {+ ListInsertPosition::Before => LIST_BEFORE,+ ListInsertPosition::After => LIST_AFTER,+ };+ let mut length = 0;+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(request.key.as_bytes()),+ position,+ ByteSliceV1::from_slice(&request.pivot),+ ByteSliceV1::from_slice(&request.value),+ &mut length,+ &mut error,+ )+ };+ status_result(status, error, "linsert")?;+ Ok(length)+ })+ })+ .await+ }++ async fn lmove(&self, request: ListMoveRequest) -> CacheResult<Option<Bytes>> {+ self.call("lmove", move |state| {+ state.with_call("lmove", |api, handle| {+ let callback = required(api.lmove, CacheOperation::Lmove)?;+ let mut output = OptionalOwnedBufferV1::default();+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(request.source.as_bytes()),+ ByteSliceV1::from_slice(request.destination.as_bytes()),+ direction(request.source_direction),+ direction(request.destination_direction),+ &mut output,+ &mut error,+ )+ };+ take_output(+ status,+ error,+ "lmove",+ output,+ take_optional_buffer,+ discard_optional_buffer,+ )+ })+ })+ .await+ }++ async fn execute_script(&self, request: ScriptRequest) -> CacheResult<ScriptResult> {+ let script_source = self.scripts.resolve(&request.script_id)?.to_string();+ self.call("execute_script", move |state| {+ state.with_call("execute_script", |api, handle| {+ let callback = required(api.execute_script, CacheOperation::ExecuteScript)?;+ let raw_keys = string_slices(&request.keys);+ let raw_args = byte_slices(&request.args);+ let mut output = ScriptValueV1::default();+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(request.script_id.as_bytes()),+ ByteSliceV1::from_slice(script_source.as_bytes()),+ raw_keys.as_ptr(),+ raw_keys.len(),+ raw_args.as_ptr(),+ raw_args.len(),+ &mut output,+ &mut error,+ )+ };+ let value = take_output(+ status,+ error,+ "execute_script",+ output,+ take_script_value,+ discard_script_value,+ )?;+ let result = ScriptResult::encode(&value)?;+ if result.payload.len() > MAX_AGGREGATE_BYTES {+ return Err(invalid_output(format!(+ "encoded script result byte count {} exceeds limit {}",+ result.payload.len(),+ MAX_AGGREGATE_BYTES+ )));+ }+ Ok(result)+ })+ })+ .await+ }++ async fn ping(&self) -> CacheResult<()> {+ self.call("ping", DynamicState::ping).await+ }++ async fn close(&self) -> CacheResult<()> {+ self.call("close", DynamicState::close).await+ }+}++async fn scalar_u64(+ provider: &DynamicProvider,+ operation: CacheOperation,+ key: &str,+) -> CacheResult<u64> {+ let key = key.to_string();+ provider+ .call(operation.name(), move |state| {+ state.with_call(operation.name(), |api, handle| {+ let callback = match operation {+ CacheOperation::Scard => required(api.scard, operation)?,+ CacheOperation::Llen => required(api.llen, operation)?,+ _ => unreachable!("scalar_u64 only accepts scalar operations"),+ };+ let mut value = 0;+ let mut error = OwnedBufferV1::default();+ let status = unsafe {+ callback(+ handle,+ ByteSliceV1::from_slice(key.as_bytes()),+ &mut value,+ &mut error,+ )+ };+ status_result(status, error, operation.name())?;+ Ok(value)+ })+ })+ .await+}++fn required<T: Copy>(callback: Option<T>, operation: CacheOperation) -> CacheResult<T> {+ callback.ok_or_else(|| CacheError::UnsupportedOperation(operation.name().to_string()))+}++fn set_options(options: SetOptions) -> CacheResult<SetOptionsV1> {+ let expiration_ms = match options.expiration {+ None => -1,+ Some(Expiration::After(duration)) => i64::try_from(duration.as_millis())+ .map_err(|_| CacheError::InvalidArgument("set expiration is too large".to_string()))?,+ };+ let condition = match options.condition {+ SetCondition::None => SET_CONDITION_NONE,+ SetCondition::Nx => SET_CONDITION_NX,+ SetCondition::Xx => SET_CONDITION_XX,+ };+ Ok(SetOptionsV1 {+ expiration_ms,+ condition,+ keep_ttl: u8::from(options.keep_ttl),+ })+}++fn direction(direction: ListDirection) -> u32 {+ match direction {+ ListDirection::Left => LIST_LEFT,+ ListDirection::Right => LIST_RIGHT,+ }+}++fn string_slices(values: &[String]) -> Vec<ByteSliceV1> {+ values+ .iter()+ .map(|value| ByteSliceV1::from_slice(value.as_bytes()))+ .collect()+}++fn byte_slices(values: &[Bytes]) -> Vec<ByteSliceV1> {+ values+ .iter()+ .map(|value| ByteSliceV1::from_slice(value))+ .collect()+}++#[derive(Default)]+struct DecodeBudget {+ aggregate_bytes: usize,+ script_nodes: usize,+}++impl DecodeBudget {+ fn charge_bytes(&mut self, bytes: usize, label: &str) -> CacheResult<()> {+ self.aggregate_bytes = self+ .aggregate_bytes+ .checked_add(bytes)+ .ok_or_else(|| invalid_output(format!("{label} aggregate byte count overflowed")))?;+ if self.aggregate_bytes > MAX_AGGREGATE_BYTES {+ return Err(invalid_output(format!(+ "{label} aggregate byte count {} exceeds limit {}",+ self.aggregate_bytes, MAX_AGGREGATE_BYTES+ )));+ }+ Ok(())+ }++ fn charge_script_node(&mut self) -> CacheResult<()> {+ self.script_nodes = self+ .script_nodes+ .checked_add(1)+ .ok_or_else(|| invalid_output("script node count overflowed"))?;+ if self.script_nodes > MAX_SCRIPT_NODES {+ return Err(invalid_output(format!(+ "script node count {} exceeds limit {}",+ self.script_nodes, MAX_SCRIPT_NODES+ )));+ }+ Ok(())+ }+}++#[derive(Debug)]+struct RegisteredAllocation {+ data: *mut u8,+ layout: Layout,+}++impl Drop for RegisteredAllocation {+ fn drop(&mut self) {+ unsafe { dealloc(self.data, self.layout) };+ }+}++struct PendingBuffers {+ items: std::vec::IntoIter<OwnedBufferV1>,+}++impl Drop for PendingBuffers {+ fn drop(&mut self) {+ for item in self.items.by_ref() {+ discard_buffer(item);+ }+ }+}++struct PendingOptionalBuffers {+ items: std::vec::IntoIter<OptionalOwnedBufferV1>,+}++impl Drop for PendingOptionalBuffers {+ fn drop(&mut self) {+ for item in self.items.by_ref() {+ discard_buffer(item.value);+ }+ }+}++struct PendingScriptValues {+ items: std::vec::IntoIter<ScriptValueV1>,+}++impl Drop for PendingScriptValues {+ fn drop(&mut self) {+ for item in self.items.by_ref() {+ discard_script_value(item);+ }+ }+}++fn take_output<T, O>(+ status: i32,+ error: OwnedBufferV1,+ operation: &str,+ output: O,+ take: impl FnOnce(O) -> CacheResult<T>,+ discard: impl FnOnce(O),+) -> CacheResult<T> {+ match status_result(status, error, operation) {+ Ok(()) => take(output),+ Err(error) => {+ discard(output);+ Err(error)+ }+ }+}++fn status_result(status: i32, error: OwnedBufferV1, operation: &str) -> CacheResult<()> {+ let message = take_buffer_with_limit(error, MAX_ERROR_BYTES, "error buffer")?;+ if status == STATUS_OK {+ if message.is_empty() {+ return Ok(());+ }+ return Err(invalid_output(format!(+ "dynamic provider {operation} returned an error payload with success status"+ )));+ }+ let message = String::from_utf8_lossy(&message);+ let details = if message.is_empty() {+ format!("dynamic provider {operation} failed with status {status}")+ } else {+ format!("dynamic provider {operation}: {message}")+ };+ Err(match status {+ STATUS_TIMEOUT => CacheError::Timeout(details),+ STATUS_UNAVAILABLE => CacheError::Unavailable(details),+ STATUS_AUTHENTICATION => CacheError::Authentication(details),+ STATUS_PERMISSION_DENIED => CacheError::PermissionDenied(details),+ STATUS_INVALID_ARGUMENT => CacheError::InvalidArgument(details),+ STATUS_INVALID_DATA => CacheError::InvalidData(details),+ STATUS_CROSS_SLOT => CacheError::CrossSlot(details),+ STATUS_READ_ONLY => CacheError::ReadOnly(details),+ STATUS_UNSUPPORTED_OPERATION => CacheError::UnsupportedOperation(details),+ _ => CacheError::Internal(details),+ })+}++fn take_bool(value: u8, operation: &str) -> CacheResult<bool> {+ match value {+ 0 => Ok(false),+ 1 => Ok(true),+ other => Err(CacheError::InvalidData(format!(+ "dynamic provider {operation} returned invalid boolean {other}"+ ))),+ }+}++fn take_optional_buffer(value: OptionalOwnedBufferV1) -> CacheResult<Option<Bytes>> {+ match value.present {+ 0 if is_empty_buffer(&value.value) => Ok(None),+ 0 => {+ discard_buffer(value.value);+ Err(invalid_output(+ "absent optional buffer returned a non-empty value",+ ))+ }+ 1 => take_buffer(value.value).map(Some),+ other => {+ discard_buffer(value.value);+ Err(invalid_output(format!(+ "dynamic provider returned invalid optional flag {other}"+ )))+ }+ }+}++fn take_buffer(buffer: OwnedBufferV1) -> CacheResult<Bytes> {+ take_buffer_with_limit(buffer, MAX_BUFFER_BYTES, "buffer")+}++fn take_buffer_with_limit(+ buffer: OwnedBufferV1,+ max_bytes: usize,+ label: &str,+) -> CacheResult<Bytes> {+ let mut budget = DecodeBudget::default();+ take_buffer_with_budget(buffer, max_bytes, label, &mut budget)+}++fn take_buffer_with_budget(+ buffer: OwnedBufferV1,+ max_bytes: usize,+ label: &str,+ budget: &mut DecodeBudget,+) -> CacheResult<Bytes> {+ if buffer.len == 0 {+ if buffer.data.is_null() {+ return Ok(Bytes::new());+ }+ release_registered(buffer.data);+ return Err(invalid_output(format!(+ "dynamic provider returned a non-null empty {label}"+ )));+ }+ if buffer.data.is_null() {+ return Err(invalid_output(format!(+ "dynamic provider returned a null {label}"+ )));+ }+ let allocation = claim_allocation::<u8>(buffer.data, buffer.len, max_bytes, label, budget)?;+ let value = Bytes::copy_from_slice(unsafe { slice::from_raw_parts(buffer.data, buffer.len) });+ drop(allocation);+ Ok(value)+}++fn take_buffer_array(array: OwnedBufferArrayV1) -> CacheResult<Vec<Bytes>> {+ let mut budget = DecodeBudget::default();+ take_buffer_array_with_budget(array, &mut budget)+}++fn take_buffer_array_with_budget(+ array: OwnedBufferArrayV1,+ budget: &mut DecodeBudget,+) -> CacheResult<Vec<Bytes>> {+ if array.len == 0 {+ if array.items.is_null() {+ return Ok(Vec::new());+ }+ release_registered(array.items.cast::<u8>());+ return Err(invalid_output(+ "dynamic provider returned a non-null empty buffer array",+ ));+ }+ if array.items.is_null() {+ return Err(invalid_output(+ "dynamic provider returned a null buffer array",+ ));+ }+ let items = take_raw_array(array.items, array.len, "buffer array", budget)?;+ let mut pending = PendingBuffers {+ items: items.into_iter(),+ };+ let mut values = Vec::with_capacity(array.len);+ while let Some(item) = pending.items.next() {+ values.push(take_buffer_with_budget(+ item,+ MAX_BUFFER_BYTES,+ "buffer array item",+ budget,+ )?);+ }+ Ok(values)+}++fn take_optional_buffer_array(+ array: OptionalOwnedBufferArrayV1,+) -> CacheResult<Vec<Option<Bytes>>> {+ let mut budget = DecodeBudget::default();+ if array.len == 0 {+ if array.items.is_null() {+ return Ok(Vec::new());+ }+ release_registered(array.items.cast::<u8>());+ return Err(invalid_output(+ "dynamic provider returned a non-null empty optional buffer array",+ ));+ }+ if array.items.is_null() {+ return Err(invalid_output(+ "dynamic provider returned a null optional buffer array",+ ));+ }+ let items = take_raw_array(array.items, array.len, "optional buffer array", &mut budget)?;+ let mut pending = PendingOptionalBuffers {+ items: items.into_iter(),+ };+ let mut values = Vec::with_capacity(array.len);+ while let Some(item) = pending.items.next() {+ values.push(take_optional_buffer_with_budget(item, &mut budget)?);+ }+ Ok(values)+}++fn take_script_value(value: ScriptValueV1) -> CacheResult<ScriptValue> {+ let mut budget = DecodeBudget::default();+ take_script_value_with_budget(value, 0, &mut budget)+}++fn take_script_value_with_budget(+ value: ScriptValueV1,+ depth: usize,+ budget: &mut DecodeBudget,+) -> CacheResult<ScriptValue> {+ if depth > MAX_SCRIPT_DEPTH {+ discard_script_value(value);+ return Err(invalid_output(format!(+ "script result depth {depth} exceeds limit {MAX_SCRIPT_DEPTH}"+ )));+ }+ if let Err(error) = budget.charge_script_node() {+ discard_script_value(value);+ return Err(error);+ }+ let ScriptValueV1 {+ bytes,+ items,+ items_len,+ integer,+ kind,+ boolean,+ } = value;+ match kind {+ SCRIPT_NULL => {+ ensure_unused_script_storage(bytes, items, items_len, "null")?;+ Ok(ScriptValue::Null)+ }+ SCRIPT_INTEGER => {+ ensure_unused_script_storage(bytes, items, items_len, "integer")?;+ Ok(ScriptValue::Integer(integer))+ }+ SCRIPT_BYTES => {+ ensure_empty_script_items(items, items_len, "bytes")?;+ take_buffer_with_budget(bytes, MAX_BUFFER_BYTES, "script bytes", budget)+ .map(|value| ScriptValue::Bytes(value.to_vec()))+ }+ SCRIPT_BOOLEAN => {+ ensure_unused_script_storage(bytes, items, items_len, "boolean")?;+ take_bool(boolean, "execute_script").map(ScriptValue::Boolean)+ }+ SCRIPT_ARRAY => {+ ensure_empty_script_buffer(bytes, "array")?;+ if items_len == 0 {+ if items.is_null() {+ return Ok(ScriptValue::Array(Vec::new()));+ }+ release_registered(items.cast::<u8>());+ return Err(invalid_output(+ "dynamic provider returned a non-null empty script array",+ ));+ }+ if items.is_null() {+ return Err(invalid_output(+ "dynamic provider returned a null script array",+ ));+ }+ let raw_items = take_raw_array(items, items_len, "script array", budget)?;+ let mut pending = PendingScriptValues {+ items: raw_items.into_iter(),+ };+ let mut values = Vec::with_capacity(items_len);+ while let Some(item) = pending.items.next() {+ values.push(take_script_value_with_budget(item, depth + 1, budget)?);+ }+ Ok(ScriptValue::Array(values))+ }+ other => {+ discard_buffer(bytes);+ discard_script_items(items, items_len);+ Err(invalid_output(format!(+ "dynamic provider returned invalid script value kind {other}"+ )))+ }+ }+}++fn take_optional_buffer_with_budget(+ value: OptionalOwnedBufferV1,+ budget: &mut DecodeBudget,+) -> CacheResult<Option<Bytes>> {+ match value.present {+ 0 if is_empty_buffer(&value.value) => Ok(None),+ 0 => {+ discard_buffer(value.value);+ Err(invalid_output(+ "absent optional buffer returned a non-empty value",+ ))+ }+ 1 => take_buffer_with_budget(value.value, MAX_BUFFER_BYTES, "optional buffer", budget)+ .map(Some),+ other => {+ discard_buffer(value.value);+ Err(invalid_output(format!(+ "dynamic provider returned invalid optional flag {other}"+ )))+ }+ }+}++fn take_raw_array<T>(+ items: *mut T,+ len: usize,+ label: &str,+ budget: &mut DecodeBudget,+) -> CacheResult<Vec<T>> {+ if len > MAX_ARRAY_ITEMS {+ release_registered(items.cast::<u8>());+ return Err(invalid_output(format!(+ "{label} item count {len} exceeds limit {MAX_ARRAY_ITEMS}"+ )));+ }+ let allocation = claim_allocation::<T>(items, len, MAX_HOST_ALLOCATION_BYTES, label, budget)?;+ let mut values = Vec::with_capacity(len);+ for index in 0..len {+ values.push(unsafe { ptr::read(items.add(index)) });+ }+ drop(allocation);+ Ok(values)+}++fn claim_allocation<T>(+ data: *mut T,+ len: usize,+ max_bytes: usize,+ label: &str,+ budget: &mut DecodeBudget,+) -> CacheResult<RegisteredAllocation> {+ let layout = Layout::array::<T>(len)+ .map_err(|_| invalid_output(format!("{label} allocation size overflowed")))?;+ if layout.size() > isize::MAX as usize {+ release_registered(data.cast::<u8>());+ return Err(invalid_output(format!(+ "{label} allocation exceeds isize::MAX"+ )));+ }+ if layout.size() > max_bytes {+ release_registered(data.cast::<u8>());+ return Err(invalid_output(format!(+ "{label} byte count {} exceeds limit {max_bytes}",+ layout.size()+ )));+ }+ if (data as usize) % layout.align() != 0 {+ release_registered(data.cast::<u8>());+ return Err(invalid_output(format!(+ "{label} pointer is not aligned to {} bytes",+ layout.align()+ )));+ }+ if let Err(error) = budget.charge_bytes(layout.size(), label) {+ release_registered(data.cast::<u8>());+ return Err(error);+ }+ let actual = allocation_registry()+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ .remove(&(data as usize))+ .ok_or_else(|| invalid_output(format!("{label} was not allocated by host->alloc")))?;+ if actual != layout {+ unsafe { dealloc(data.cast::<u8>(), actual) };+ return Err(invalid_output(format!(+ "{label} allocation layout does not match returned length/alignment"+ )));+ }+ Ok(RegisteredAllocation {+ data: data.cast::<u8>(),+ layout,+ })+}++fn allocation_registry() -> &'static Mutex<HashMap<usize, Layout>> {+ static ALLOCATIONS: OnceLock<Mutex<HashMap<usize, Layout>>> = OnceLock::new();+ ALLOCATIONS.get_or_init(|| Mutex::new(HashMap::new()))+}++#[cfg(test)]+fn host_allocation_is_registered(data: *mut u8) -> bool {+ allocation_registry()+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ .contains_key(&(data as usize))+}++fn release_registered(data: *mut u8) {+ if data.is_null() {+ return;+ }+ let layout = allocation_registry()+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ .remove(&(data as usize));+ if let Some(layout) = layout {+ unsafe { dealloc(data, layout) };+ }+}++fn discard_buffer(buffer: OwnedBufferV1) {+ release_registered(buffer.data);+}++fn discard_optional_buffer(buffer: OptionalOwnedBufferV1) {+ discard_buffer(buffer.value);+}++fn discard_buffer_array(array: OwnedBufferArrayV1) {+ if array.items.is_null() || array.len == 0 || array.len > MAX_ARRAY_ITEMS {+ release_registered(array.items.cast::<u8>());+ return;+ }+ let mut budget = DecodeBudget::default();+ match take_raw_array(+ array.items,+ array.len,+ "discarded buffer array",+ &mut budget,+ ) {+ Ok(items) => {+ for item in items {+ discard_buffer(item);+ }+ }+ Err(_) => release_registered(array.items.cast::<u8>()),+ }+}++fn discard_optional_buffer_array(array: OptionalOwnedBufferArrayV1) {+ if array.items.is_null() || array.len == 0 || array.len > MAX_ARRAY_ITEMS {+ release_registered(array.items.cast::<u8>());+ return;+ }+ let mut budget = DecodeBudget::default();+ match take_raw_array(+ array.items,+ array.len,+ "discarded optional buffer array",+ &mut budget,+ ) {+ Ok(items) => {+ for item in items {+ discard_optional_buffer(item);+ }+ }+ Err(_) => release_registered(array.items.cast::<u8>()),+ }+}++fn discard_script_items(items: *mut ScriptValueV1, len: usize) {+ if items.is_null() {+ return;+ }+ if len == 0 || len > MAX_ARRAY_ITEMS {+ release_registered(items.cast::<u8>());+ return;+ }+ let mut budget = DecodeBudget::default();+ match take_raw_array(items, len, "discarded script array", &mut budget) {+ Ok(values) => {+ for value in values {+ discard_script_value(value);+ }+ }+ Err(_) => release_registered(items.cast::<u8>()),+ }+}++fn discard_script_value(value: ScriptValueV1) {+ let mut pending = vec![value];+ let mut visited = 0usize;+ while let Some(value) = pending.pop() {+ visited += 1;+ let ScriptValueV1 {+ bytes,+ items,+ items_len,+ ..+ } = value;+ discard_buffer(bytes);+ if items.is_null() {+ continue;+ }+ if visited > MAX_SCRIPT_NODES || items_len == 0 || items_len > MAX_ARRAY_ITEMS {+ release_registered(items.cast::<u8>());+ continue;+ }+ let mut budget = DecodeBudget::default();+ match take_raw_array(items, items_len, "discarded script array", &mut budget) {+ Ok(mut values) => pending.append(&mut values),+ Err(_) => release_registered(items.cast::<u8>()),+ }+ }+}++fn ensure_unused_script_storage(+ bytes: OwnedBufferV1,+ items: *mut ScriptValueV1,+ items_len: usize,+ kind: &str,+) -> CacheResult<()> {+ ensure_empty_script_buffer(bytes, kind)?;+ ensure_empty_script_items(items, items_len, kind)+}++fn ensure_empty_script_buffer(buffer: OwnedBufferV1, kind: &str) -> CacheResult<()> {+ if is_empty_buffer(&buffer) {+ return Ok(());+ }+ discard_buffer(buffer);+ Err(invalid_output(format!(+ "script {kind} value returned an unexpected byte buffer"+ )))+}++fn ensure_empty_script_items(+ items: *mut ScriptValueV1,+ items_len: usize,+ kind: &str,+) -> CacheResult<()> {+ if items.is_null() && items_len == 0 {+ return Ok(());+ }+ discard_script_items(items, items_len);+ Err(invalid_output(format!(+ "script {kind} value returned unexpected child items"+ )))+}++fn is_empty_buffer(buffer: &OwnedBufferV1) -> bool {+ buffer.data.is_null() && buffer.len == 0+}++fn invalid_output(message: impl Into<String>) -> CacheError {+ CacheError::InvalidData(message.into())+}++fn close_handle(api: ProviderApiV1, handle: *mut c_void) {+ if handle.is_null() {+ return;+ }+ let mut error = OwnedBufferV1::default();+ if let Some(close) = api.close {+ unsafe {+ close(handle, &mut error);+ }+ let _ = take_buffer(error);+ }+}++unsafe extern "C" fn host_alloc(size: usize, alignment: usize) -> *mut u8 {+ if size == 0 || size > MAX_HOST_ALLOCATION_BYTES {+ return ptr::null_mut();+ }+ match Layout::from_size_align(size, alignment) {+ Ok(layout) => {+ let data = unsafe { alloc(layout) };+ if !data.is_null() {+ allocation_registry()+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ .insert(data as usize, layout);+ }+ data+ }+ Err(_) => ptr::null_mut(),+ }+}++unsafe extern "C" fn host_dealloc(data: *mut u8, size: usize, alignment: usize) {+ if data.is_null() || size == 0 {+ return;+ }+ if let Ok(expected) = Layout::from_size_align(size, alignment) {+ let mut allocations = allocation_registry()+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ if allocations.get(&(data as usize)) == Some(&expected) {+ allocations.remove(&(data as usize));+ drop(allocations);+ unsafe { dealloc(data, expected) };+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn script_result_rejects_excessive_nesting() {+ let mut value = ScriptValueV1 {+ kind: SCRIPT_NULL,+ ..ScriptValueV1::default()+ };+ for _ in 0..80 {+ let layout = Layout::array::<ScriptValueV1>(1).unwrap();+ let items =+ unsafe { host_alloc(layout.size(), layout.align()) }.cast::<ScriptValueV1>();+ assert!(!items.is_null());+ unsafe { ptr::write(items, value) };+ value = ScriptValueV1 {+ items,+ items_len: 1,+ kind: SCRIPT_ARRAY,+ ..ScriptValueV1::default()+ };+ }++ let error = take_script_value(value).unwrap_err();++ assert!(matches!(error, CacheError::InvalidData(message) if message.contains("depth")));+ }++ #[test]+ fn buffer_array_rejects_excessive_item_count() {+ const ITEM_COUNT: usize = 100_001;+ let layout = Layout::array::<OwnedBufferV1>(ITEM_COUNT).unwrap();+ let items = unsafe { host_alloc(layout.size(), layout.align()) }.cast::<OwnedBufferV1>();+ assert!(!items.is_null());+ for index in 0..ITEM_COUNT {+ unsafe { ptr::write(items.add(index), OwnedBufferV1::default()) };+ }++ let error = take_buffer_array(OwnedBufferArrayV1 {+ items,+ len: ITEM_COUNT,+ })+ .unwrap_err();++ assert!(+ matches!(error, CacheError::InvalidData(message) if message.contains("item count"))+ );+ }++ #[test]+ fn optional_array_failure_releases_all_host_allocations() {+ let first_data = unsafe { host_alloc(1, align_of::<u8>()) };+ let second_data = unsafe { host_alloc(1, align_of::<u8>()) };+ assert!(!first_data.is_null());+ assert!(!second_data.is_null());+ unsafe {+ *first_data = b'a';+ *second_data = b'b';+ }+ let layout = Layout::array::<OptionalOwnedBufferV1>(2).unwrap();+ let items =+ unsafe { host_alloc(layout.size(), layout.align()) }.cast::<OptionalOwnedBufferV1>();+ unsafe {+ ptr::write(+ items,+ OptionalOwnedBufferV1 {+ value: OwnedBufferV1 {+ data: first_data,+ len: 1,+ },+ present: 2,+ },+ );+ ptr::write(+ items.add(1),+ OptionalOwnedBufferV1 {+ value: OwnedBufferV1 {+ data: second_data,+ len: 1,+ },+ present: 1,+ },+ );+ }++ let result = take_optional_buffer_array(OptionalOwnedBufferArrayV1 { items, len: 2 });++ assert!(matches!(result, Err(CacheError::InvalidData(_))));+ assert!(!host_allocation_is_registered(first_data));+ assert!(!host_allocation_is_registered(second_data));+ assert!(!host_allocation_is_registered(items.cast::<u8>()));+ }++ #[test]+ fn checked_reader_rejects_overflow_and_misalignment() {+ let overflow = claim_allocation::<ScriptValueV1>(+ std::ptr::NonNull::dangling().as_ptr(),+ usize::MAX,+ MAX_HOST_ALLOCATION_BYTES,+ "overflow array",+ &mut DecodeBudget::default(),+ )+ .unwrap_err();+ assert!(+ matches!(overflow, CacheError::InvalidData(message) if message.contains("overflow"))+ );++ let misaligned = claim_allocation::<ScriptValueV1>(+ 1usize as *mut ScriptValueV1,+ 1,+ MAX_HOST_ALLOCATION_BYTES,+ "misaligned array",+ &mut DecodeBudget::default(),+ )+ .unwrap_err();+ assert!(+ matches!(misaligned, CacheError::InvalidData(message) if message.contains("aligned"))+ );+ }+}diff --git a/crates/ragfs/src/cache_runtime/error.rs b/crates/ragfs/src/cache_runtime/error.rsindex d4ba510b1..2c8f9a70c 100644--- a/crates/ragfs/src/cache_runtime/error.rs+++ b/crates/ragfs/src/cache_runtime/error.rs@@ -36,6 +36,9 @@ pub enum CacheError {/// The selected provider does not implement a named script.#[error("unsupported cache script: {0}")]UnsupportedScript(String),+ /// The selected provider does not implement a cache operation.+ #[error("unsupported cache operation: {0}")]+ UnsupportedOperation(String),/// A dynamic provider uses an incompatible ABI.#[error("cache provider ABI mismatch: {0}")]AbiMismatch(String),diff --git a/crates/ragfs/src/cache_runtime/mod.rs b/crates/ragfs/src/cache_runtime/mod.rsindex 7ed07060a..ad27ab46c 100644--- a/crates/ragfs/src/cache_runtime/mod.rs+++ b/crates/ragfs/src/cache_runtime/mod.rs@@ -22,6 +22,7 @@ pub use redis::{RedisDeploymentMode, RedisProviderConfig};use bytes::Bytes;use executor::RuntimeExecutor;+pub(crate) use provider::CacheOperation;use provider::CacheProvider;pub(crate) use script::{ScriptDefinition, ScriptRegistry, ScriptValue};use std::sync::atomic::{AtomicBool, Ordering};@@ -120,12 +121,19 @@ impl CacheRuntime {}/// Load one provider through the versioned dynamic C ABI.- pub async fn dynamic(_config: DynamicProviderConfig) -> CacheResult<Arc<Self>> {- Err(CacheError::UnsupportedProvider(- "DynamicProvider is planned for a later release".into(),+ pub async fn dynamic(config: DynamicProviderConfig) -> CacheResult<Arc<Self>> {+ let scripts = Arc::new(ScriptRegistry::default());+ let provider = dynamic::DynamicProvider::connect(config, Arc::clone(&scripts)).await?;+ Ok(Self::from_provider_with_scripts(+ Arc::new(provider),+ scripts,))}+ pub(crate) fn require_operations(&self, operations: &[CacheOperation]) -> CacheResult<()> {+ self.provider.validate_operations(operations)+ }+pub(crate) fn register_script(&self, definition: ScriptDefinition) -> CacheResult<()> {self.scripts.register(definition)}diff --git a/crates/ragfs/src/cache_runtime/provider.rs b/crates/ragfs/src/cache_runtime/provider.rsindex 3b0fd50e6..1c6abad29 100644--- a/crates/ragfs/src/cache_runtime/provider.rs+++ b/crates/ragfs/src/cache_runtime/provider.rs@@ -7,8 +7,67 @@ use super::{use async_trait::async_trait;use bytes::Bytes;+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(crate) enum CacheOperation {+ Get,+ Set,+ Del,+ Mget,+ Mset,+ IncrBy,+ Sismember,+ Smembers,+ Scard,+ Lpush,+ Rpush,+ Lpop,+ Rpop,+ Llen,+ Lrange,+ Lindex,+ Lset,+ Ltrim,+ Lrem,+ Linsert,+ Lmove,+ ExecuteScript,+}++impl CacheOperation {+ pub(crate) const fn name(self) -> &'static str {+ match self {+ Self::Get => "get",+ Self::Set => "set",+ Self::Del => "del",+ Self::Mget => "mget",+ Self::Mset => "mset",+ Self::IncrBy => "incrby",+ Self::Sismember => "sismember",+ Self::Smembers => "smembers",+ Self::Scard => "scard",+ Self::Lpush => "lpush",+ Self::Rpush => "rpush",+ Self::Lpop => "lpop",+ Self::Rpop => "rpop",+ Self::Llen => "llen",+ Self::Lrange => "lrange",+ Self::Lindex => "lindex",+ Self::Lset => "lset",+ Self::Ltrim => "ltrim",+ Self::Lrem => "lrem",+ Self::Linsert => "linsert",+ Self::Lmove => "lmove",+ Self::ExecuteScript => "execute_script",+ }+ }+}+#[async_trait]pub(crate) trait CacheProvider: Send + Sync {+ fn validate_operations(&self, _operations: &[CacheOperation]) -> CacheResult<()> {+ Ok(())+ }+async fn get(&self, key: &str) -> CacheResult<Option<Bytes>>;async fn set(&self, key: &str, value: Bytes, options: SetOptions) -> CacheResult<SetResult>;async fn del(&self, keys: &[String]) -> CacheResult<u64>;diff --git a/crates/ragfs/src/core/builder.rs b/crates/ragfs/src/core/builder.rsindex 22fcfbb60..ebf9f546b 100644--- a/crates/ragfs/src/core/builder.rs+++ b/crates/ragfs/src/core/builder.rs@@ -19,7 +19,9 @@ use super::errors::{Error, Result};#[cfg(feature = "cache")]use crate::cache::{CacheNamespace, CachePolicy};#[cfg(feature = "cache")]-use crate::cache_runtime::{CacheRuntime, DynamicProviderConfig, RedisProviderConfig};+use crate::cache_runtime::{+ CacheOperation, CacheRuntime, DynamicProviderConfig, RedisProviderConfig,+};use crate::lock::{FilesystemPathLockProvider, MemoryPathLockProvider, PathLockConfig, PathLockManager,@@ -164,6 +166,18 @@ pub async fn build_configured_stack(),None => None,};+ if cache.cachefs.enabled {+ runtime+ .as_ref()+ .expect("cache provider is validated")+ .require_operations(&[+ CacheOperation::Get,+ CacheOperation::Set,+ CacheOperation::Del,+ CacheOperation::Mget,+ ])+ .map_err(|error| Error::config(format!("CacheFS provider is incompatible: {error}")))?;+ }let mountable = if cache.cachefs.enabled {Arc::new(MountableFS::with_cache_runtime(runtimediff --git a/crates/ragfs/src/plugins/queuefs/cache_backend.rs b/crates/ragfs/src/plugins/queuefs/cache_backend.rsindex ef39f7f97..5c89aafff 100644--- a/crates/ragfs/src/plugins/queuefs/cache_backend.rs+++ b/crates/ragfs/src/plugins/queuefs/cache_backend.rs@@ -7,7 +7,8 @@ use super::cache_protocol::{STARTUP_RECOVERY_SWEEPS,};use crate::cache_runtime::{- CacheError, CacheRuntime, Expiration, ScriptDefinition, ScriptRequest, ScriptValue, SetOptions,+ CacheError, CacheOperation, CacheRuntime, Expiration, ScriptDefinition, ScriptRequest,+ ScriptValue, SetOptions,};use crate::core::errors::{Error, Result};use bytes::Bytes;@@ -81,6 +82,18 @@ pub(super) struct CacheQueueStorage {impl CacheQueueStorage {pub(super) async fn open(runtime: Arc<CacheRuntime>, key_prefix: String) -> Result<Self> {+ runtime+ .require_operations(&[+ CacheOperation::Set,+ CacheOperation::Del,+ CacheOperation::Mget,+ CacheOperation::Sismember,+ CacheOperation::Smembers,+ CacheOperation::Llen,+ CacheOperation::Lrange,+ CacheOperation::ExecuteScript,+ ])+ .map_err(|error| cache_error("validate provider operations", error))?;for definition in SCRIPT_DEFINITIONS {runtime.register_script(*definition)diff --git a/crates/ragfs/tests/dynamic_provider.rs b/crates/ragfs/tests/dynamic_provider.rsindex 650ec0ddb..291340fb0 100644--- a/crates/ragfs/tests/dynamic_provider.rs+++ b/crates/ragfs/tests/dynamic_provider.rs@@ -1,15 +1,121 @@+use bytes::Bytes;+use libloading::{Library, Symbol};use ragfs::cache_runtime::{CacheError, CacheRuntime, DynamicProviderConfig};-use std::path::PathBuf;+use std::path::{Path, PathBuf};+use std::process::Command;+use std::sync::OnceLock;#[tokio::test]-async fn dynamic_provider_is_explicitly_unsupported_in_this_release() {- let result = CacheRuntime::dynamic(DynamicProviderConfig {- library_path: PathBuf::from("libcache_provider.so"),- params_json: "{}".into(),- })- .await;+async fn dynamic_provider_loads_shared_library_and_dispatches_get() {+ let runtime = CacheRuntime::dynamic(config(fixture_library()))+ .await+ .unwrap();+ assert_eq!(+ runtime.get("key").await.unwrap(),+ Some(Bytes::from_static(b"fixture-value"))+ );+ runtime.close().await.unwrap();+}++#[tokio::test]+async fn close_error_consumes_handle_and_is_not_retried_by_drop() {+ let library_path = close_error_fixture_library();+ let probe = unsafe { Library::new(library_path) }.unwrap();+ let close_count: Symbol<unsafe extern "C" fn() -> u32> =+ unsafe { probe.get(b"ov_fixture_close_count\0") }.unwrap();+ let runtime = CacheRuntime::dynamic(config(library_path)).await.unwrap();++ assert!(matches!(+ runtime.close().await,+ Err(CacheError::Internal(_))+ ));+ assert_eq!(unsafe { close_count() }, 1);+ drop(runtime);+ assert_eq!(unsafe { close_count() }, 1);+}++#[tokio::test]+async fn incompatible_abi_is_rejected_before_create() {+ let result = CacheRuntime::dynamic(config(bad_abi_fixture_library())).await;++ assert!(matches!(result, Err(CacheError::AbiMismatch(_))));+}++#[tokio::test]+async fn mismatched_output_length_is_rejected_without_unsafe_read() {+ let runtime = CacheRuntime::dynamic(config(bad_length_fixture_library()))+ .await+ .unwrap();++ let result = runtime.get("key").await;++ assert!(matches!(result, Err(CacheError::InvalidData(_))));+ runtime.close().await.unwrap();+}++fn config(library_path: &Path) -> DynamicProviderConfig {+ DynamicProviderConfig {+ library_path: library_path.to_path_buf(),+ params_json: "{}".to_string(),+ }+}++fn fixture_library() -> &'static PathBuf {+ static LIBRARY: OnceLock<PathBuf> = OnceLock::new();+ LIBRARY.get_or_init(|| compile_fixture("normal", &[]))+}++fn close_error_fixture_library() -> &'static PathBuf {+ static LIBRARY: OnceLock<PathBuf> = OnceLock::new();+ LIBRARY.get_or_init(|| compile_fixture("close-error", &["OV_FIXTURE_CLOSE_ERROR"]))+}++fn bad_abi_fixture_library() -> &'static PathBuf {+ static LIBRARY: OnceLock<PathBuf> = OnceLock::new();+ LIBRARY.get_or_init(|| compile_fixture("bad-abi", &["OV_FIXTURE_BAD_ABI"]))+}++fn bad_length_fixture_library() -> &'static PathBuf {+ static LIBRARY: OnceLock<PathBuf> = OnceLock::new();+ LIBRARY.get_or_init(|| compile_fixture("bad-length", &["OV_FIXTURE_BAD_LENGTH"]))+}++fn compile_fixture(name: &str, defines: &[&str]) -> PathBuf {+ let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR"));+ let output_dir = manifest.join("../../target/dynamic-provider-fixture");+ std::fs::create_dir_all(&output_dir).unwrap();+ let extension = if cfg!(target_os = "windows") {+ "dll"+ } else if cfg!(target_os = "macos") {+ "dylib"+ } else {+ "so"+ };+ let library = output_dir.join(format!("libopenviking_cache_{name}.{extension}"));+ let mut command = Command::new("cc");+ if cfg!(target_os = "macos") {+ command.arg("-dynamiclib");+ } else {+ command.arg("-shared");+ }+ if !cfg!(target_os = "windows") {+ command.arg("-fPIC");+ }+ for define in defines {+ command.arg(format!("-D{define}"));+ }+ let status = command+ .arg("-I")+ .arg(manifest.join("include"))+ .arg(manifest.join("tests/fixtures/dynamic_provider_fixture.c"))+ .arg("-o")+ .arg(&library)+ .status()+ .expect("C compiler must be available for the dynamic provider ABI test");assert!(- matches!(result, Err(CacheError::UnsupportedProvider(message)) if message.contains("DynamicProvider"))+ status.success(),+ "failed to compile dynamic provider fixture");+ library.canonicalize().unwrap()}diff --git a/crates/ragfs/tests/fixtures/dynamic_provider_fixture.c b/crates/ragfs/tests/fixtures/dynamic_provider_fixture.cnew file mode 100644index 000000000..df0418f1d--- /dev/null+++ b/crates/ragfs/tests/fixtures/dynamic_provider_fixture.c@@ -0,0 +1,102 @@+#include "openviking_cache_provider_v1.h"++#include <stdatomic.h>+#include <stdlib.h>+#include <string.h>++typedef struct {+ const OvCacheHostApiV1 *host;+} FixtureProvider;++static const OvCacheHostApiV1 *fixture_host = NULL;+static _Atomic uint32_t fixture_close_calls = 0;++static int32_t fixture_copy(+ const uint8_t *data,+ size_t len,+ OvCacheOwnedBufferV1 *output) {+ output->data = fixture_host->alloc(len, OV_CACHE_ALIGNOF(uint8_t));+ if (len > 0 && output->data == NULL) {+ return OV_CACHE_STATUS_INTERNAL;+ }+ if (len > 0) {+ memcpy(output->data, data, len);+ }+ output->len = len;+ return OV_CACHE_STATUS_OK;+}++static int32_t fixture_create(+ const OvCacheHostApiV1 *host,+ OvCacheByteSliceV1 params_json,+ void **provider,+ OvCacheOwnedBufferV1 *error) {+ (void)params_json;+ (void)error;+ FixtureProvider *state = (FixtureProvider *)malloc(sizeof(FixtureProvider));+ if (state == NULL) {+ return OV_CACHE_STATUS_INTERNAL;+ }+ state->host = host;+ fixture_host = host;+ *provider = state;+ return OV_CACHE_STATUS_OK;+}++static int32_t fixture_ping(void *provider, OvCacheOwnedBufferV1 *error) {+ (void)provider;+ (void)error;+ return OV_CACHE_STATUS_OK;+}++static int32_t fixture_close(void *provider, OvCacheOwnedBufferV1 *error) {+ (void)error;+ atomic_fetch_add(&fixture_close_calls, 1);+ free(provider);+#ifdef OV_FIXTURE_CLOSE_ERROR+ return OV_CACHE_STATUS_INTERNAL;+#else+ return OV_CACHE_STATUS_OK;+#endif+}++static int32_t fixture_get(+ void *provider,+ OvCacheByteSliceV1 key,+ OvCacheOptionalOwnedBufferV1 *output,+ OvCacheOwnedBufferV1 *error) {+ (void)provider;+ (void)key;+ (void)error;+ static const uint8_t value[] = "fixture-value";+ output->present = 1;+ int32_t status = fixture_copy(value, sizeof(value) - 1, &output->value);+#ifdef OV_FIXTURE_BAD_LENGTH+ if (status == OV_CACHE_STATUS_OK) {+ output->value.len += 1;+ }+#endif+ return status;+}++OV_CACHE_PROVIDER_EXPORT uint32_t ov_fixture_close_count(void) {+ return atomic_load(&fixture_close_calls);+}++static const OvCacheProviderApiV1 FIXTURE_API = {+#ifdef OV_FIXTURE_BAD_ABI+ .abi_version = OV_CACHE_PROVIDER_ABI_V1 + 1,+#else+ .abi_version = OV_CACHE_PROVIDER_ABI_V1,+#endif+ .struct_size = sizeof(OvCacheProviderApiV1),+ .create = fixture_create,+ .ping = fixture_ping,+ .close = fixture_close,+ .get = fixture_get,+};++OV_CACHE_PROVIDER_EXPORT+const OvCacheProviderApiV1 *openviking_cache_provider_v1(void) {+ return &FIXTURE_API;+}diff --git a/docs/en/guides/14-ragfs-cache.md b/docs/en/guides/14-ragfs-cache.mdindex b1ef406e5..60089669a 100644--- a/docs/en/guides/14-ragfs-cache.md+++ b/docs/en/guides/14-ragfs-cache.md@@ -66,7 +66,7 @@ Available Providers:| Provider | Best for | Notes ||----------|----------|-------|| `redis` | Default delivery on standard networks | Built into RAGFS; supports standalone, Cluster, and Sentinel |-| `dynamic` | YuanRong, Mooncake, or closed-source cache systems | Not implemented in this release; returns UnsupportedProvider |+| `dynamic` | YuanRong, Mooncake, or closed-source cache systems | Loaded from an external shared library through the versioned C ABI |`MemoryMockProvider` is only used by unit and smoke tests; it is not a production configuration option.@@ -85,9 +85,9 @@ Legacy cache configuration is no longer accepted. Migrate before upgrading:OpenViking rejects removed fields with a migration error instead of silently translating them.-## Future DynamicProvider+## DynamicProvider-This release only ships the built-in RedisProvider. DynamicProvider, `.so` loading, and the versioned C ABI are deferred; configuring `provider=dynamic` currently returns UnsupportedProvider during startup.+OpenViking ships the DynamicProvider loader and a versioned C ABI. The default wheel does not bundle third-party SDKs or Provider libraries. Deploy the Provider shared library separately and configure `provider=dynamic` when an external cache system is required.A dynamic library must export this versioned entry point:@@ -95,7 +95,9 @@ A dynamic library must export this versioned entry point:openviking_cache_provider_v1```-Provider artifacts should declare the ABI version, target OS and CPU, minimum glibc version, external SDK version, dynamic dependencies, and SHA256. When a Provider depends on native libraries, its publisher must make them discoverable through RPATH, `LD_LIBRARY_PATH`, or deployment instructions.+The C contract is defined by `crates/ragfs/include/openviking_cache_provider_v1.h`. The Provider must use the Host allocator for returned data, obey the documented ownership and close semantics, catch exceptions before they cross the C ABI, and make its handle safe for concurrent calls.++Provider artifacts should declare the ABI version, target OS and CPU, minimum runtime version, external SDK version, dynamic dependencies, and SHA256. When a Provider depends on native libraries, its publisher must make them discoverable through RPATH, `LD_LIBRARY_PATH`, or deployment instructions.External Providers can be upgraded independently without rebuilding the default OpenViking wheel. OpenViking only needs a coordinated upgrade when the DynamicProvider ABI becomes incompatible.@@ -105,7 +107,7 @@ The top-level `cache` section is a sibling of `storage`:| Option | Type | Default | Description ||--------|------|---------|-------------|-| `provider` | str | none | Provider name; this release supports `redis` |+| `provider` | str | none | Provider name; supports `redis` and `dynamic` || `params` | object | `{}` | Provider-owned parameters |`storage.agfs.cachefs` controls CacheFS behavior:@@ -134,9 +136,9 @@ Redis configuration:All Redis reads are sent to the primary node so QueueFS does not observe stale queue state. CacheRuntime relies on Redis transport security when required and does not add a separate application-layer value encryption format.-Future DynamicProvider configuration shape:+DynamicProvider configuration:-`cache.params` is entirely Provider-owned. The fields below only illustrate configuration forwarding; use the schema documented by the Provider publisher.+OpenViking uses `cache.params.library` to load the dynamic library. All remaining fields are Provider-owned and passed to `create` as JSON. Use the schema documented by the Provider publisher.```json{@@ -156,7 +158,7 @@ Future DynamicProvider configuration shape:RAGFS splits caching into two layers:- `CachedFileSystem`: implements filesystem semantics, including cache hit/miss handling, backend fallback, cache fill, invalidation, generation checks, and metrics.-- `CacheRuntime`: exposes common primitive operations and binds the built-in RedisProvider in this release; DynamicProvider remains a future extension.+- `CacheRuntime`: exposes common primitive operations and binds either the built-in RedisProvider or an external DynamicProvider during startup.Call flow:@@ -165,7 +167,7 @@ OpenViking-> RAGFS / MountableFS-> CachedFileSystem|-> CacheRuntime -> RedisProvider- | `-> DynamicProvider (future)+ | `-> DynamicProvider -> external shared library`-> Backend FileSystem```diff --git a/docs/zh/guides/14-ragfs-cache.md b/docs/zh/guides/14-ragfs-cache.mdindex 7572649a0..a4a2a50c5 100644--- a/docs/zh/guides/14-ragfs-cache.md+++ b/docs/zh/guides/14-ragfs-cache.md@@ -66,7 +66,7 @@ openviking-server| Provider | 适用场景 | 备注 ||----------|----------|------|| `redis` | 默认交付、普通网络环境 | 内置于 RAGFS,支持 standalone、Cluster 和 Sentinel |-| `dynamic` | YuanRong、Mooncake 或闭源缓存系统 | 本期未实现,配置后返回 UnsupportedProvider |+| `dynamic` | YuanRong、Mooncake 或闭源缓存系统 | 通过版本化 C ABI 从外部动态库加载 |`MemoryMockProvider` 只用于单元测试和 smoke test,不是生产配置项。@@ -85,9 +85,9 @@ openviking-serverOpenViking 会对已删除字段直接返回迁移错误,不再静默转换。-## 后续 DynamicProvider+## DynamicProvider-本期标准 OpenViking wheel 只内置 RedisProvider。DynamicProvider、`.so` 加载器和版本化 C ABI 放在后续阶段实现;当前配置 `provider=dynamic` 会在启动阶段返回 UnsupportedProvider。+OpenViking 已内置 DynamicProvider 加载器和版本化 C ABI。默认 wheel 不携带第三方 SDK 或 Provider 动态库;需要接入外部缓存系统时,独立部署 Provider 动态库并配置 `provider=dynamic`。动态库必须导出以下版本化入口:@@ -95,7 +95,9 @@ OpenViking 会对已删除字段直接返回迁移错误,不再静默转换。openviking_cache_provider_v1```-Provider 发布物应注明 ABI 版本、目标 OS/CPU、最低 glibc 版本、外部 SDK 版本、动态依赖和 SHA256。依赖外部原生库时,由 Provider 发布方通过 RPATH、`LD_LIBRARY_PATH` 或部署说明保证动态链接器能够找到依赖。+C 接口契约定义在 `crates/ragfs/include/openviking_cache_provider_v1.h`。Provider 返回的数据必须使用 Host allocator 分配,遵守文档中的内存所有权和关闭语义,禁止异常穿过 C ABI,并保证 handle 可以安全并发调用。++Provider 发布物应注明 ABI 版本、目标 OS/CPU、最低运行时版本、外部 SDK 版本、动态依赖和 SHA256。依赖外部原生库时,由 Provider 发布方通过 RPATH、`LD_LIBRARY_PATH` 或部署说明保证动态链接器能够找到依赖。外部 Provider 可以独立升级,不需要重新构建默认 OpenViking wheel;只有 DynamicProvider ABI 不兼容时,才需要同步升级 OpenViking。@@ -105,7 +107,7 @@ Provider 发布物应注明 ABI 版本、目标 OS/CPU、最低 glibc 版本、| 参数 | 类型 | 默认值 | 说明 ||------|------|--------|------|-| `provider` | str | 无 | Provider 名称,本期支持 `redis` |+| `provider` | str | 无 | Provider 名称,支持 `redis` 和 `dynamic` || `params` | object | `{}` | Provider 自有参数 |`storage.agfs.cachefs` 支持以下业务配置:@@ -134,9 +136,9 @@ Redis 配置:所有 Redis 读命令均发送到主节点,避免 QueueFS 读取到延迟的队列状态。需要传输加密时使用 Redis TLS,CacheRuntime 不额外增加应用层数据加密格式。-未来 DynamicProvider 配置结构:+DynamicProvider 配置:-`cache.params` 完全由外部 Provider 定义,下面的字段只用于说明配置传递方式,实际配置以 Provider 发布说明为准。+`cache.params.library` 由 OpenViking 用于加载动态库,其余字段由外部 Provider 定义并作为 JSON 传入 `create`。实际参数以 Provider 发布说明为准。```json{@@ -156,7 +158,7 @@ Redis 配置:RAGFS 将缓存拆成两层:- `CachedFileSystem`:实现文件系统语义,包括 cache hit/miss、backend 回源、回填、失效、generation 校验和指标。-- `CacheRuntime`:向业务层提供统一基础操作,本期在启动时绑定内置 RedisProvider;DynamicProvider 为后续扩展。+- `CacheRuntime`:向业务层提供统一基础操作,启动时绑定内置 RedisProvider 或外部 DynamicProvider。调用关系:@@ -165,7 +167,7 @@ OpenViking-> RAGFS / MountableFS-> CachedFileSystem|-> CacheRuntime -> RedisProvider- | `-> DynamicProvider(后续)+ | `-> DynamicProvider -> 外部动态库`-> Backend FileSystem```