Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
//! Database backend abstraction for SQLFS
//!
//! This module provides an abstraction over different database backends
//! (SQLite, MySQL/TiDB) to allow SQLFS to work with multiple databases.
use crate::core::{ConfigValue, Error, Result};
use rusqlite::{params, Connection};
use std::collections::HashMap;
use std::sync::Mutex;
/// Maximum file size in bytes (5MB, same as Go version)
pub const MAX_FILE_SIZE: usize = 5 * 1024 * 1024;
/// Maximum file size in MB (for display)
pub const MAX_FILE_SIZE_MB: usize = 5;
/// Database backend trait
///
/// All database backends must implement this trait to provide
/// uniform access to different database systems.
pub trait DatabaseBackend: Send + Sync {
/// Get the driver name for logging and metadata
fn driver_name(&self) -> &'static str;
/// Check if this path exists
fn path_exists(&self, path: &str) -> Result<bool>;
/// Check if a path is a directory
fn is_directory(&self, path: &str) -> Result<bool>;
/// Create a new file entry
fn create_file(&self, path: &str, mode: u32, data: &[u8]) -> Result<()>;
/// Create a new directory entry
fn create_directory(&self, path: &str, mode: u32) -> Result<()>;
/// Delete a file or directory entry
fn delete_entry(&self, path: &str) -> Result<()>;
/// Delete entries matching a pattern (for recursive delete)
fn delete_entries_by_pattern(&self, pattern: &str, exclude_path: Option<&str>)
-> Result<usize>;
/// Read file data
fn read_file(&self, path: &str) -> Result<Option<(bool, Vec<u8>)>>;
/// Update file data
fn update_file(&self, path: &str, data: &[u8]) -> Result<()>;
/// Update file data only when the current data equals `expected`.
fn compare_and_update_file(&self, path: &str, expected: &[u8], data: &[u8]) -> Result<bool>;
/// Delete a file only when the current data equals `expected`.
fn compare_and_delete_file(&self, path: &str, expected: &[u8]) -> Result<bool>;
/// Get file metadata
fn get_metadata(&self, path: &str) -> Result<Option<FileMetadata>>;
/// Update file mode
fn update_mode(&self, path: &str, mode: u32) -> Result<()>;
/// Rename a path (file or directory)
fn rename_path(&self, old_path: &str, new_path: &str) -> Result<()>;
/// Rename all children under a path (for directory rename)
fn rename_children(&self, old_path: &str, new_path: &str) -> Result<()>;
/// List directory contents (direct children only)
fn list_directory(&self, path: &str) -> Result<Vec<FileMetadata>>;
/// Count entries matching a pattern
fn count_by_pattern(&self, pattern: &str) -> Result<i64>;
/// Get parent path
fn parent_path(&self, path: &str) -> String;
}
/// File metadata from database
#[derive(Debug, Clone)]
pub struct FileMetadata {
/// Full path of the file or directory
pub path: String,
/// Whether this entry is a directory
pub is_dir: bool,
/// Unix-style file permissions
pub mode: u32,
/// File size in bytes
pub size: i64,
/// Last modification time as Unix timestamp
pub mod_time: i64,
/// File content data (None for metadata-only queries)
pub data: Option<Vec<u8>>,
}
/// SQLite backend implementation
///
/// Uses `Mutex<Connection>` to satisfy `Send + Sync` requirements.
/// rusqlite's `Connection` is not `Sync` due to internal `RefCell` usage,
/// so we wrap it in a `Mutex` for thread-safe access.
pub struct SQLiteBackend {
conn: Mutex<Connection>,
}
impl SQLiteBackend {
/// Create a new SQLite backend
///
/// Initializes the database schema and applies optimizations (WAL mode, etc.)
pub fn new(db_path: Option<&str>) -> Result<Self> {
let path = db_path.unwrap_or(":memory:");
let conn = Connection::open(path)
.map_err(|e| Error::internal(format!("sqlite connection error: {}", e)))?;
// Initialize schema
conn.execute_batch(
r#"
CREATE TABLE IF NOT EXISTS files (
path TEXT PRIMARY KEY,
is_dir INTEGER NOT NULL,
mode INTEGER NOT NULL,
size INTEGER NOT NULL,
mod_time INTEGER NOT NULL,
data BLOB
);
CREATE INDEX IF NOT EXISTS idx_parent ON files(path);
"#,
)
.map_err(|e| Error::internal(format!("schema init error: {}", e)))?;
// Apply optimizations
conn.execute_batch(
r#"
PRAGMA journal_mode=WAL;
PRAGMA synchronous=NORMAL;
PRAGMA cache_size=-64000;
"#,
)
.map_err(|e| Error::internal(format!("optimization error: {}", e)))?;
// Ensure root directory exists
let now = chrono::Utc::now().timestamp();
conn.execute(
"INSERT OR IGNORE INTO files (path, is_dir, mode, size, mod_time, data) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params!["/", 1, 0o755, 0i64, now, None::<Vec<u8>>],
)
.map_err(|e| Error::internal(format!("root init error: {}", e)))?;
Ok(Self {
conn: Mutex::new(conn),
})
}
}
impl DatabaseBackend for SQLiteBackend {
fn driver_name(&self) -> &'static str {
"sqlite3"
}
fn path_exists(&self, path: &str) -> Result<bool> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let mut stmt = conn
.prepare_cached("SELECT COUNT(*) FROM files WHERE path = ?1")
.map_err(|e| Error::internal(format!("prepare error: {}", e)))?;
let count: i64 = match stmt.query_row(params![path], |row| row.get(0)) {
Ok(count) => count,
Err(rusqlite::Error::QueryReturnedNoRows) => 0,
Err(e) => return Err(Error::internal(format!("query error: {}", e))),
};
Ok(count > 0)
}
fn is_directory(&self, path: &str) -> Result<bool> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let mut stmt = conn
.prepare_cached("SELECT is_dir FROM files WHERE path = ?1")
.map_err(|e| Error::internal(format!("prepare error: {}", e)))?;
match stmt.query_row(params![path], |row| row.get::<_, i32>(0)) {
Ok(is_dir) => Ok(is_dir == 1),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(false),
Err(e) => Err(Error::internal(format!("query error: {}", e))),
}
}
fn create_file(&self, path: &str, mode: u32, data: &[u8]) -> Result<()> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let now = chrono::Utc::now().timestamp();
conn.execute(
"INSERT INTO files (path, is_dir, mode, size, mod_time, data) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![path, 0, mode, data.len() as i64, now, data],
)
.map_err(|e| Error::internal(format!("insert error: {}", e)))?;
Ok(())
}
fn create_directory(&self, path: &str, mode: u32) -> Result<()> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let now = chrono::Utc::now().timestamp();
conn.execute(
"INSERT INTO files (path, is_dir, mode, size, mod_time, data) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![path, 1, mode, 0i64, now, None::<Vec<u8>>],
)
.map_err(|e| Error::internal(format!("insert error: {}", e)))?;
Ok(())
}
fn delete_entry(&self, path: &str) -> Result<()> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
conn.execute("DELETE FROM files WHERE path = ?1", params![path])
.map_err(|e| Error::internal(format!("delete error: {}", e)))?;
Ok(())
}
fn delete_entries_by_pattern(
&self,
pattern: &str,
exclude_path: Option<&str>,
) -> Result<usize> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let result = if let Some(exclude) = exclude_path {
conn.execute(
"DELETE FROM files WHERE path LIKE ?1 AND path != ?2",
params![pattern, exclude],
)
.map_err(|e| Error::internal(format!("delete error: {}", e)))?
} else {
conn.execute("DELETE FROM files WHERE path LIKE ?1", params![pattern])
.map_err(|e| Error::internal(format!("delete error: {}", e)))?
};
Ok(result)
}
fn read_file(&self, path: &str) -> Result<Option<(bool, Vec<u8>)>> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let mut stmt = conn
.prepare_cached("SELECT is_dir, data FROM files WHERE path = ?1")
.map_err(|e| Error::internal(format!("prepare error: {}", e)))?;
match stmt.query_row(params![path], |row| {
let is_dir: i32 = row.get(0)?;
let data: Option<Vec<u8>> = row.get(1)?;
Ok((is_dir == 1, data.unwrap_or_default()))
}) {
Ok(result) => Ok(Some(result)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(Error::internal(format!("query error: {}", e))),
}
}
fn update_file(&self, path: &str, data: &[u8]) -> Result<()> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let now = chrono::Utc::now().timestamp();
conn.execute(
"UPDATE files SET data = ?1, size = ?2, mod_time = ?3 WHERE path = ?4",
params![data, data.len() as i64, now, path],
)
.map_err(|e| Error::internal(format!("update error: {}", e)))?;
Ok(())
}
fn compare_and_update_file(&self, path: &str, expected: &[u8], data: &[u8]) -> Result<bool> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let now = chrono::Utc::now().timestamp();
let changed = conn
.execute(
"UPDATE files SET data = ?1, size = ?2, mod_time = ?3 WHERE path = ?4 AND is_dir = 0 AND data = ?5",
params![data, data.len() as i64, now, path, expected],
)
.map_err(|e| Error::internal(format!("conditional update error: {}", e)))?;
Ok(changed == 1)
}
fn compare_and_delete_file(&self, path: &str, expected: &[u8]) -> Result<bool> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let changed = conn
.execute(
"DELETE FROM files WHERE path = ?1 AND is_dir = 0 AND data = ?2",
params![path, expected],
)
.map_err(|e| Error::internal(format!("conditional delete error: {}", e)))?;
Ok(changed == 1)
}
fn get_metadata(&self, path: &str) -> Result<Option<FileMetadata>> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let mut stmt = conn
.prepare_cached("SELECT path, is_dir, mode, size, mod_time FROM files WHERE path = ?1")
.map_err(|e| Error::internal(format!("prepare error: {}", e)))?;
match stmt.query_row(params![path], |row| {
Ok(FileMetadata {
path: row.get(0)?,
is_dir: row.get::<_, i32>(1)? == 1,
mode: row.get(2)?,
size: row.get(3)?,
mod_time: row.get(4)?,
data: None,
})
}) {
Ok(meta) => Ok(Some(meta)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(Error::internal(format!("query error: {}", e))),
}
}
fn update_mode(&self, path: &str, mode: u32) -> Result<()> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let now = chrono::Utc::now().timestamp();
conn.execute(
"UPDATE files SET mode = ?1, mod_time = ?2 WHERE path = ?3",
params![mode, now, path],
)
.map_err(|e| Error::internal(format!("update error: {}", e)))?;
Ok(())
}
fn rename_path(&self, old_path: &str, new_path: &str) -> Result<()> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
conn.execute(
"UPDATE files SET path = ?1 WHERE path = ?2",
params![new_path, old_path],
)
.map_err(|e| Error::internal(format!("rename error: {}", e)))?;
Ok(())
}
fn rename_children(&self, old_path: &str, new_path: &str) -> Result<()> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let old_pattern = format!("{}/%", old_path);
let old_len = (old_path.len() + 1) as i32;
let sql = "UPDATE files SET path = ?1 || SUBSTR(path, ?2) WHERE path LIKE ?3";
conn.execute(sql, params![new_path, old_len, old_pattern])
.map_err(|e| Error::internal(format!("rename children error: {}", e)))?;
Ok(())
}
fn list_directory(&self, path: &str) -> Result<Vec<FileMetadata>> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
// Build pattern for direct children only
// For root "/": children are like "/<name>" (no further slashes)
// For "/dir": children are like "/dir/<name>" (no further slashes)
let prefix = if path == "/" {
"/".to_string()
} else {
format!("{}/", path)
};
// Query all entries that start with the prefix,
// excluding the directory itself
let sql = "SELECT path, is_dir, mode, size, mod_time FROM files WHERE path LIKE ?1 AND path != ?2 ORDER BY path";
let like_pattern = format!("{}%", prefix);
let mut stmt = conn
.prepare_cached(sql)
.map_err(|e| Error::internal(format!("prepare error: {}", e)))?;
let mut results = Vec::new();
let prefix_len = prefix.len();
let rows = stmt
.query_map(params![like_pattern, path], |row| {
Ok(FileMetadata {
path: row.get(0)?,
is_dir: row.get::<_, i32>(1)? == 1,
mode: row.get(2)?,
size: row.get(3)?,
mod_time: row.get(4)?,
data: None,
})
})
.map_err(|e| Error::internal(format!("query error: {}", e)))?;
for row_result in rows {
let meta = row_result.map_err(|e| Error::internal(format!("row error: {}", e)))?;
// Only include direct children (no further '/' after the prefix)
let remainder = &meta.path[prefix_len..];
if !remainder.contains('/') {
results.push(meta);
}
}
Ok(results)
}
fn count_by_pattern(&self, pattern: &str) -> Result<i64> {
let conn = self
.conn
.lock()
.map_err(|e| Error::internal(e.to_string()))?;
let mut stmt = conn
.prepare_cached("SELECT COUNT(*) FROM files WHERE path LIKE ?1")
.map_err(|e| Error::internal(format!("prepare error: {}", e)))?;
let count: i64 = stmt
.query_row(params![pattern], |row| row.get(0))
.map_err(|e| Error::internal(format!("query error: {}", e)))?;
Ok(count)
}
fn parent_path(&self, path: &str) -> String {
if path == "/" {
return "/".to_string();
}
// Remove trailing slash
let trimmed = path.trim_end_matches('/');
if trimmed.is_empty() {
return "/".to_string();
}
// Find last slash
if let Some(pos) = trimmed.rfind('/') {
if pos == 0 {
return "/".to_string();
}
return trimmed[..pos].to_string();
}
"/".to_string()
}
}
/// Create a database backend from configuration
pub fn create_backend(config: &HashMap<String, ConfigValue>) -> Result<Box<dyn DatabaseBackend>> {
let backend_type = config
.get("backend")
.and_then(|v| v.as_string())
.unwrap_or("sqlite");
match backend_type {
"sqlite" | "sqlite3" => {
let db_path = config.get("db_path").and_then(|v| v.as_string());
let backend = SQLiteBackend::new(db_path)?;
Ok(Box::new(backend))
}
"mysql" | "tidb" => {
// TODO: Implement MySQL/TiDB backend
Err(Error::internal("MySQL/TiDB backend not yet implemented"))
}
_ => Err(Error::config(format!(
"unsupported database backend: {} (valid options: sqlite, sqlite3)",
backend_type
))),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parent_path() {
let backend = SQLiteBackend::new(Some(":memory:")).unwrap();
assert_eq!(backend.parent_path("/"), "/");
assert_eq!(backend.parent_path("/file.txt"), "/");
assert_eq!(backend.parent_path("/dir/"), "/");
assert_eq!(backend.parent_path("/dir/file.txt"), "/dir");
assert_eq!(backend.parent_path("/a/b/c/file.txt"), "/a/b/c");
}
#[test]
fn test_sqlite_backend_basic() {
let backend = SQLiteBackend::new(Some(":memory:")).unwrap();
// Root should already exist
assert!(backend.path_exists("/").unwrap());
assert!(backend.is_directory("/").unwrap());
// Create a directory
backend.create_directory("/testdir", 0o755).unwrap();
assert!(backend.path_exists("/testdir").unwrap());
assert!(backend.is_directory("/testdir").unwrap());
// Create a file
backend
.create_file("/testdir/file.txt", 0o644, b"hello")
.unwrap();
assert!(backend.path_exists("/testdir/file.txt").unwrap());
assert!(!backend.is_directory("/testdir/file.txt").unwrap());
// Read file
let result = backend.read_file("/testdir/file.txt").unwrap();
assert!(result.is_some());
let (is_dir, data) = result.unwrap();
assert!(!is_dir);
assert_eq!(data, b"hello");
// List directory - should return only direct children
let entries = backend.list_directory("/testdir").unwrap();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].path, "/testdir/file.txt");
}
#[test]
fn test_list_directory_direct_children() {
let backend = SQLiteBackend::new(Some(":memory:")).unwrap();
// Create nested structure: /a/b/c
backend.create_directory("/a", 0o755).unwrap();
backend.create_directory("/a/b", 0o755).unwrap();
backend.create_directory("/a/b/c", 0o755).unwrap();
backend.create_file("/a/file1.txt", 0o644, b"").unwrap();
backend.create_file("/a/b/file2.txt", 0o644, b"").unwrap();
// List /a - should only return /a/b and /a/file1.txt
let entries = backend.list_directory("/a").unwrap();
assert_eq!(entries.len(), 2);
let paths: Vec<&str> = entries.iter().map(|e| e.path.as_str()).collect();
assert!(paths.contains(&"/a/b"));
assert!(paths.contains(&"/a/file1.txt"));
// List / - should only return /a
let entries = backend.list_directory("/").unwrap();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].path, "/a");
}
}