Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(encrypt): Protect encrypted writes with temp-file publish and dual-path locking (#2894)

baojun-zhang843a4a3a39f7
diff --git a/crates/ragfs/src/cache/wrapper.rs b/crates/ragfs/src/cache/wrapper.rs
index b0d738b18..67f8587af 100644
--- a/crates/ragfs/src/cache/wrapper.rs
+++ b/crates/ragfs/src/cache/wrapper.rs
@@ -1134,6 +1134,18 @@ impl FileSystem for CachedFileSystem {
Ok(())
}
+ async fn replace(&self, src_path: &str, dst_path: &str) -> Result<()> {
+ let _guard = self.operation_lock.write().await;
+ self.backend.replace(src_path, dst_path).await?;
+ self.bump_generation(src_path).await;
+ self.bump_generation(dst_path).await;
+ self.invalidate_path_objects(src_path).await;
+ self.invalidate_path_objects(dst_path).await;
+ self.invalidate_parent_directory(src_path).await;
+ self.invalidate_parent_directory(dst_path).await;
+ Ok(())
+ }
+
async fn chmod(&self, path: &str, mode: u32) -> Result<()> {
let _guard = self.operation_lock.write().await;
self.backend.chmod(path, mode).await?;
diff --git a/crates/ragfs/src/core/encryption_wrapper.rs b/crates/ragfs/src/core/encryption_wrapper.rs
index 98841d663..a4f9d5236 100644
--- a/crates/ragfs/src/core/encryption_wrapper.rs
+++ b/crates/ragfs/src/core/encryption_wrapper.rs
@@ -12,6 +12,7 @@ use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use async_trait::async_trait;
+use sha2::{Digest, Sha256};
use tracing::warn;
use crate::crypto;
@@ -131,6 +132,23 @@ impl EncryptionWrappedFS {
}
Ok(())
}
+
+ /// Build a deterministic internal temp-file path for a final file path.
+ fn encrypted_temp_path(path: &str) -> String {
+ let normalized = if path.starts_with('/') {
+ path.to_string()
+ } else {
+ format!("/{}", path)
+ };
+ let parts: Vec<&str> = normalized.split('/').filter(|part| !part.is_empty()).collect();
+ let temp_root = match parts.as_slice() {
+ ["local", account_id, ..] => format!("/local/{}/temp/.encrypt_stage", account_id),
+ [mount_root, _, ..] => format!("/{}/temp/.encrypt_stage", mount_root),
+ _ => "/temp/.encrypt_stage".to_string(),
+ };
+ let digest = Sha256::digest(normalized.as_bytes());
+ format!("{}/{:x}.encrypt", temp_root, digest)
+ }
}
/// Slice `data` by `(offset, size)` with `size == 0` meaning "to end", matching plugin read semantics.
@@ -211,7 +229,28 @@ impl FileSystem for EncryptionWrappedFS {
let ct = crypto::aes_gcm_encrypt(&file_key, &data_iv, data)?;
let enc_key = crypto::aes_gcm_encrypt(&account_key, &key_iv, &file_key)?;
let envelope = crypto::build_envelope(self.provider_type, &enc_key, &key_iv, &data_iv, &ct);
- self.inner.write(path, &envelope, 0, flags).await
+ let temp_path = Self::encrypted_temp_path(path);
+ self.inner.ensure_parent_dirs(&temp_path, 0o755).await?;
+
+ match self.inner.remove(&temp_path).await {
+ Ok(()) | Err(Error::NotFound(_)) => {}
+ Err(err) => return Err(err),
+ }
+
+ let written = self
+ .inner
+ .write(&temp_path, &envelope, 0, WriteFlag::Create)
+ .await?;
+ if written != envelope.len() as u64 {
+ return Err(Error::internal(format!(
+ "encrypted temp write incomplete: wrote {} of {} bytes",
+ written,
+ envelope.len()
+ )));
+ }
+
+ self.inner.replace(&temp_path, path).await?;
+ Ok(written)
}
async fn grep(
@@ -553,4 +592,63 @@ mod tests {
.await;
assert!(result.is_err());
}
+
+ #[tokio::test]
+ async fn write_replaces_stale_internal_temp_file_before_publish() {
+ let inner = memfs_stack().await;
+ let enc = EncryptionWrappedFS::new(inner.clone(), [6u8; 32], crypto::PROVIDER_LOCAL);
+ let temp_path =
+ "/mem/temp/.encrypt_stage/18d3740a61555bfecb941730d4b708dd3090a1b0ba2fcafe8c696d55c5e4b67a.encrypt";
+
+ inner.ensure_parent_dirs(temp_path, 0o755).await.unwrap();
+ inner
+ .write(temp_path, b"stale-temp", 0, WriteFlag::Create)
+ .await
+ .unwrap();
+
+ FS_CTX
+ .scope(ctx("tenant-1"), async {
+ enc.write("/mem/a.txt", b"fresh", 0, WriteFlag::Create)
+ .await
+ .unwrap();
+ let out = enc.read("/mem/a.txt", 0, 0).await.unwrap();
+ assert_eq!(out, b"fresh");
+ })
+ .await;
+
+ assert!(
+ inner
+ .stat(temp_path)
+ .await
+ .is_err()
+ );
+ let raw = inner.read("/mem/a.txt", 0, 0).await.unwrap();
+ assert!(crypto::is_encrypted(&raw));
+ }
+
+ #[test]
+ fn encrypted_temp_path_mapping_matches_python() {
+ let cases = [
+ (
+ "/local/default/resources/note.md",
+ "/local/default/temp/.encrypt_stage/823e7fa1698ab91f00481e4960d38764c6d77fa9230a86a0cca178bf87becc93.encrypt",
+ ),
+ (
+ "/local/default/resources/.abstract.md",
+ "/local/default/temp/.encrypt_stage/86cb67683e7cee2a07dce5921d450af2b6d5757bec1f05f5c1738b966cdc8edd.encrypt",
+ ),
+ (
+ "/note.md",
+ "/temp/.encrypt_stage/71f3eca3f3ad54df082026dd6b40f2f3b2c2ba67b51bb5d24fda5632044c3228.encrypt",
+ ),
+ (
+ ".abstract.md",
+ "/temp/.encrypt_stage/2f3429bdbec8a484aec67cd1584e5dd64d8cf139d2fc4b4cca97d9cdc9b66ad3.encrypt",
+ ),
+ ];
+
+ for (final_path, temp_path) in cases {
+ assert_eq!(EncryptionWrappedFS::encrypted_temp_path(final_path), temp_path);
+ }
+ }
}
diff --git a/crates/ragfs/src/core/filesystem.rs b/crates/ragfs/src/core/filesystem.rs
index d7fa57132..3b2dc6da3 100644
--- a/crates/ragfs/src/core/filesystem.rs
+++ b/crates/ragfs/src/core/filesystem.rs
@@ -203,6 +203,19 @@ pub trait FileSystem: Send + Sync + Any {
/// * `Error::AlreadyExists` - If new_path already exists
async fn rename(&self, old_path: &str, new_path: &str) -> Result<()>;
+ /// Replace a file by moving `src_path` onto `dst_path`, consuming the source.
+ ///
+ /// Unlike `rename`, this operation allows `dst_path` to already exist.
+ /// Backends that cannot provide replace semantics should override this
+ /// method only when support is available; the default returns
+ /// `Error::InvalidOperation`.
+ async fn replace(&self, src_path: &str, dst_path: &str) -> Result<()> {
+ Err(Error::invalid_operation(format!(
+ "replace is not supported: {} -> {}",
+ src_path, dst_path
+ )))
+ }
+
/// Change file permissions
///
/// # Arguments
diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rs
index 15015279b..6cc493e0d 100644
--- a/crates/ragfs/src/core/mountable.rs
+++ b/crates/ragfs/src/core/mountable.rs
@@ -86,6 +86,15 @@ struct MountCacheConfig {
}
impl MountableFS {
+ /// Return whether one plugin may be wrapped by EncryptionWrappedFS.
+ ///
+ /// Encrypted whole-file publish now depends on overwrite-on-publish
+ /// semantics (`replace(temp, final)`). Keep this gate at the wrapper
+ /// creation entrypoint so unsupported backends fail fast before mounting.
+ fn supports_encrypted_publish(plugin_name: &str) -> bool {
+ matches!(plugin_name, "localfs" | "s3fs" | "memfs")
+ }
+
/// Return the raw backend for one mounted path when raw access is supported.
fn raw_backend_for_mount<'a>(
mount_info: &'a MountInfo,
@@ -301,6 +310,12 @@ impl MountableFS {
} else {
match (enc_root_key, enc_provider_type) {
(Some(rk), Some(pt)) => {
+ if !Self::supports_encrypted_publish(&config.name) {
+ return Err(Error::config(format!(
+ "encrypted backend '{}' must support replace() semantics",
+ config.name
+ )));
+ }
Arc::new(EncryptionWrappedFS::new(storage_fs, rk, pt))
}
_ => storage_fs,
@@ -669,6 +684,10 @@ impl FileSystem for ArcFileSystem {
self.0.rename(old_path, new_path).await
}
+ async fn replace(&self, src_path: &str, dst_path: &str) -> Result<()> {
+ self.0.replace(src_path, dst_path).await
+ }
+
async fn chmod(&self, path: &str, mode: u32) -> Result<()> {
self.0.chmod(path, mode).await
}
@@ -761,6 +780,19 @@ impl FileSystem for MountableFS {
mount_info_old.fs.rename(&rel_old, &rel_new).await
}
+ async fn replace(&self, src_path: &str, dst_path: &str) -> Result<()> {
+ let (mount_info_src, rel_src) = self.find_mount(src_path).await?;
+ let (mount_info_dst, rel_dst) = self.find_mount(dst_path).await?;
+
+ if mount_info_src.path != mount_info_dst.path {
+ return Err(Error::InvalidOperation(
+ "Cannot replace across different mount points".to_string(),
+ ));
+ }
+
+ mount_info_src.fs.replace(&rel_src, &rel_dst).await
+ }
+
async fn chmod(&self, path: &str, mode: u32) -> Result<()> {
let (mount_info, rel_path) = self.find_mount(path).await?;
mount_info.fs.chmod(&rel_path, mode).await
diff --git a/crates/ragfs/src/core/stats_wrapper.rs b/crates/ragfs/src/core/stats_wrapper.rs
index 0a32380ea..cfc76f254 100644
--- a/crates/ragfs/src/core/stats_wrapper.rs
+++ b/crates/ragfs/src/core/stats_wrapper.rs
@@ -127,6 +127,13 @@ impl FileSystem for StatsWrappedFS {
result
}
+ async fn replace(&self, src_path: &str, dst_path: &str) -> Result<()> {
+ let timer = OperationTimer::start(FsOperation::Rename, Arc::clone(&self.stats));
+ let result = self.inner.replace(src_path, dst_path).await;
+ timer.finish().await;
+ result
+ }
+
async fn chmod(&self, path: &str, mode: u32) -> Result<()> {
let timer = OperationTimer::start(FsOperation::Chmod, Arc::clone(&self.stats));
let result = self.inner.chmod(path, mode).await;
diff --git a/crates/ragfs/src/git/enumerate.rs b/crates/ragfs/src/git/enumerate.rs
index 088d877e5..fc2a287a6 100644
--- a/crates/ragfs/src/git/enumerate.rs
+++ b/crates/ragfs/src/git/enumerate.rs
@@ -262,6 +262,10 @@ mod tests {
("/local/acct/resources/x.faiss", false),
("/local/acct/resources/x.index", false),
("/local/acct/resources/embedding_cache/v.bin", false),
+ (
+ "/local/acct/temp/.encrypt_stage/823e7fa1698ab91f00481e4960d38764c6d77fa9230a86a0cca178bf87becc93.encrypt",
+ false,
+ ),
],
);
let fs: Arc<dyn FileSystem> = Arc::new(mock);
diff --git a/crates/ragfs/src/multibackend/factory.rs b/crates/ragfs/src/multibackend/factory.rs
index efaec6824..0ac98d581 100644
--- a/crates/ragfs/src/multibackend/factory.rs
+++ b/crates/ragfs/src/multibackend/factory.rs
@@ -20,6 +20,16 @@ use crate::multibackend::meta::MountRootFsContextResolver;
use crate::multibackend::types::MultiBackendBuildContext;
use crate::shape::validate::ensure_backend_shape;
+/// Return whether one backend may be wrapped by EncryptionWrappedFS.
+///
+/// Keep the check local to the wrapper creation entrypoint for this minimal
+/// fix: encrypted publish requires overwrite-on-publish semantics
+/// (`replace(temp, final)`), so unsupported backends must be rejected before
+/// the encrypted wrapper is built.
+fn supports_encrypted_publish(backend_name: &str) -> bool {
+ matches!(backend_name, "localfs" | "s3fs" | "memfs")
+}
+
/// Initialize one backend plugin instance from config params.
pub async fn init_backend_plugin(
registry: &Arc<RwLock<HashMap<String, Arc<dyn ServicePlugin>>>>,
@@ -61,6 +71,12 @@ pub async fn build_multi_write_fs(
)
.await?;
let primary_backend: Arc<dyn FileSystem> = if global_encryption_enabled {
+ if !supports_encrypted_publish(&config.name) {
+ return Err(Error::config(format!(
+ "encrypted backend '{}' must support replace() semantics",
+ config.name
+ )));
+ }
Arc::new(EncryptionWrappedFS::new(
primary_raw.clone(),
build_ctx
@@ -116,6 +132,12 @@ pub async fn build_multi_write_fs(
.await?;
let backup_backend: Arc<dyn FileSystem> = if backup_encrypted {
+ if !supports_encrypted_publish(&item.backend) {
+ return Err(Error::config(format!(
+ "encrypted backend '{}' must support replace() semantics",
+ item.backend
+ )));
+ }
Arc::new(EncryptionWrappedFS::new(
backup_raw,
build_ctx
diff --git a/crates/ragfs/src/plugins/localfs/mod.rs b/crates/ragfs/src/plugins/localfs/mod.rs
index 08cd2d6d3..8ab96b6c7 100644
--- a/crates/ragfs/src/plugins/localfs/mod.rs
+++ b/crates/ragfs/src/plugins/localfs/mod.rs
@@ -774,6 +774,86 @@ impl FileSystem for LocalFileSystem {
Ok(())
}
+ async fn replace(&self, src_path: &str, dst_path: &str) -> Result<()> {
+ let src_local = self.resolve_path(src_path);
+ let dst_local = self.resolve_path(dst_path);
+
+ if !src_local.exists() {
+ return Err(Error::NotFound(src_path.to_string()));
+ }
+
+ if src_local.is_dir() {
+ return Err(Error::invalid_operation(
+ "replace only supports files in localfs",
+ ));
+ }
+
+ if dst_local.is_dir() {
+ return Err(Error::IsADirectory(dst_path.to_string()));
+ }
+
+ if let Some(parent) = dst_local.parent() {
+ if !parent.exists() {
+ return Err(Error::NotFound(parent.to_string_lossy().to_string()));
+ }
+ }
+
+ // Keep replace semantics explicit instead of inheriting rename behavior:
+ // encrypted publish needs an overwrite-on-destination move.
+ #[cfg(windows)]
+ {
+ use std::ffi::OsStr;
+ use std::io;
+ use std::os::windows::ffi::OsStrExt;
+
+ type BOOL = i32;
+ type DWORD = u32;
+ type LPCWSTR = *const u16;
+
+ const MOVEFILE_REPLACE_EXISTING: DWORD = 0x1;
+ const MOVEFILE_WRITE_THROUGH: DWORD = 0x8;
+
+ unsafe extern "system" {
+ fn MoveFileExW(
+ lpExistingFileName: LPCWSTR,
+ lpNewFileName: LPCWSTR,
+ dwFlags: DWORD,
+ ) -> BOOL;
+ }
+
+ let src_wide: Vec<u16> = OsStr::new(src_local.as_os_str())
+ .encode_wide()
+ .chain(std::iter::once(0))
+ .collect();
+ let dst_wide: Vec<u16> = OsStr::new(dst_local.as_os_str())
+ .encode_wide()
+ .chain(std::iter::once(0))
+ .collect();
+
+ let replaced = unsafe {
+ MoveFileExW(
+ src_wide.as_ptr(),
+ dst_wide.as_ptr(),
+ MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH,
+ )
+ };
+ if replaced == 0 {
+ return Err(Error::plugin(format!(
+ "failed to replace: {}",
+ io::Error::last_os_error()
+ )));
+ }
+ }
+
+ #[cfg(not(windows))]
+ {
+ fs::rename(&src_local, &dst_local)
+ .map_err(|e| Error::plugin(format!("failed to replace: {}", e)))?;
+ }
+
+ Ok(())
+ }
+
async fn chmod(&self, path: &str, _mode: u32) -> Result<()> {
let local_path = self.resolve_path(path);
@@ -951,6 +1031,24 @@ mod tests {
assert_eq!(result.matches[0].file, "a.txt");
}
+ #[tokio::test]
+ async fn test_localfs_replace_overwrites_existing_file() {
+ let (_dir, fs) = fallback_localfs();
+
+ fs.write("/src.txt", b"fresh", 0, WriteFlag::Create)
+ .await
+ .unwrap();
+ fs.write("/dst.txt", b"stale", 0, WriteFlag::Create)
+ .await
+ .unwrap();
+
+ fs.replace("/src.txt", "/dst.txt").await.unwrap();
+
+ assert!(fs.stat("/src.txt").await.is_err());
+ let data = fs.read("/dst.txt", 0, 0).await.unwrap();
+ assert_eq!(data, b"fresh");
+ }
+
#[tokio::test]
async fn test_localfs_grep_case_insensitive() {
let (dir, fs) = fallback_localfs();
diff --git a/crates/ragfs/src/plugins/memfs/mod.rs b/crates/ragfs/src/plugins/memfs/mod.rs
index 444fa36c2..af85f9c19 100644
--- a/crates/ragfs/src/plugins/memfs/mod.rs
+++ b/crates/ragfs/src/plugins/memfs/mod.rs
@@ -435,6 +435,40 @@ impl FileSystem for MemFileSystem {
Ok(())
}
+ async fn replace(&self, src_path: &str, dst_path: &str) -> Result<()> {
+ let src_normalized = Self::normalize_path(src_path);
+ let dst_normalized = Self::normalize_path(dst_path);
+ let mut entries = self.entries.write().await;
+
+ let src_entry = entries
+ .get(&src_normalized)
+ .ok_or_else(|| Error::not_found(&src_normalized))?
+ .clone();
+
+ if src_entry.is_dir {
+ return Err(Error::invalid_operation(
+ "replace only supports files in memfs",
+ ));
+ }
+
+ if let Some(parent) = Self::parent_path(&dst_normalized) {
+ match entries.get(&parent) {
+ Some(e) if e.is_dir => {}
+ Some(_) => return Err(Error::NotADirectory(parent)),
+ None => return Err(Error::not_found(&parent)),
+ }
+ }
+
+ if entries.get(&dst_normalized).is_some_and(|entry| entry.is_dir) {
+ return Err(Error::IsADirectory(dst_normalized));
+ }
+
+ entries.remove(&dst_normalized);
+ entries.remove(&src_normalized);
+ entries.insert(dst_normalized, src_entry);
+ Ok(())
+ }
+
async fn chmod(&self, path: &str, mode: u32) -> Result<()> {
let normalized = Self::normalize_path(path);
let mut entries = self.entries.write().await;
@@ -605,6 +639,24 @@ mod tests {
assert_eq!(data, b"data");
}
+ #[tokio::test]
+ async fn test_replace_overwrites_existing_file() {
+ let fs = MemFileSystem::new();
+
+ fs.write("/src.txt", b"fresh", 0, WriteFlag::Create)
+ .await
+ .unwrap();
+ fs.write("/dst.txt", b"stale", 0, WriteFlag::Create)
+ .await
+ .unwrap();
+
+ fs.replace("/src.txt", "/dst.txt").await.unwrap();
+
+ assert!(fs.stat("/src.txt").await.is_err());
+ let data = fs.read("/dst.txt", 0, 0).await.unwrap();
+ assert_eq!(data, b"fresh");
+ }
+
#[tokio::test]
async fn test_write_flags() {
let fs = MemFileSystem::new();
diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rs
index 0d3646687..b91b0838e 100644
--- a/crates/ragfs/src/plugins/s3fs/mod.rs
+++ b/crates/ragfs/src/plugins/s3fs/mod.rs
@@ -682,6 +682,38 @@ impl FileSystem for S3FileSystem {
Err(Error::not_found(&old_normalized))
}
+
+ async fn replace(&self, src_path: &str, dst_path: &str) -> Result<()> {
+ let src_normalized = Self::normalize_path(src_path);
+ let dst_normalized = Self::normalize_path(dst_path);
+
+ if src_normalized == "/" || dst_normalized == "/" {
+ return Err(Error::invalid_operation("cannot replace root directory"));
+ }
+
+ let src_key = self.client.build_key(&src_normalized);
+ let src_meta = self
+ .client
+ .head_object(&src_key)
+ .await?
+ .ok_or_else(|| Error::not_found(&src_normalized))?;
+ if src_meta.is_dir_marker {
+ return Err(Error::invalid_operation(
+ "replace only supports files in s3fs",
+ ));
+ }
+
+ let dst_key = self.client.build_key(&dst_normalized);
+ self.client.copy_object(&src_key, &dst_key).await?;
+ self.client.delete_object(&src_key).await?;
+
+ self.dir_cache.invalidate_parent(&src_normalized).await;
+ self.dir_cache.invalidate_parent(&dst_normalized).await;
+ self.stat_cache.invalidate(&src_normalized).await;
+ self.stat_cache.invalidate(&dst_normalized).await;
+
+ Ok(())
+ }
async fn chmod(&self, _path: &str, _mode: u32) -> Result<()> {
// S3 doesn't support Unix permissions - no-op
Ok(())
diff --git a/openviking/storage/content_write.py b/openviking/storage/content_write.py
index 987ae2583..90b117a4a 100644
--- a/openviking/storage/content_write.py
+++ b/openviking/storage/content_write.py
@@ -5,7 +5,7 @@
from __future__ import annotations
import os
-from typing import Any, Dict, Optional
+from typing import TYPE_CHECKING, Any, Dict, Optional
from openviking.core.namespace import NamespaceShapeError, canonicalize_uri, context_type_for_uri
from openviking.resource.watch_storage import is_watch_task_control_uri
@@ -35,6 +35,9 @@ from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
+if TYPE_CHECKING:
+ from openviking.storage.transaction.lock_handle import LockHandle
+
_DERIVED_FILENAMES = frozenset({".abstract.md", ".overview.md", ".relations.json"})
_CREATE_ALLOWED_EXTENSIONS = frozenset(
{".md", ".txt", ".json", ".yaml", ".yml", ".toml", ".py", ".js", ".ts"}
@@ -257,7 +260,13 @@ class ContentWriteCoordinator:
previous_content = await self._viking_fs.read_file(uri, ctx=ctx)
if wait and telemetry_id:
get_request_wait_tracker().register_request(telemetry_id)
- await self._write_in_place(uri, content, mode=mode, ctx=ctx)
+ await self._write_in_place(
+ uri,
+ content,
+ mode=mode,
+ ctx=ctx,
+ lock_handle=handle,
+ )
content_written = True
await self._enqueue_semantic_refresh(
root_uri=root_uri,
@@ -313,7 +322,12 @@ class ContentWriteCoordinator:
await self._viking_fs.rm(uri, ctx=ctx, lock_handle=lock_handle)
return
if previous_content is not None:
- await self._viking_fs.write_file(uri, previous_content, ctx=ctx)
+ await self._viking_fs.write_file(
+ uri,
+ previous_content,
+ ctx=ctx,
+ lock_handle=lock_handle,
+ )
except Exception:
logger.error("Failed to rollback direct content write for %s", uri, exc_info=True)
@@ -422,6 +436,7 @@ class ContentWriteCoordinator:
*,
mode: str,
ctx: RequestContext,
+ lock_handle: Optional["LockHandle"] = None,
) -> None:
if context_type_for_uri(uri) == "memory":
if mode == "replace":
@@ -435,7 +450,12 @@ class ContentWriteCoordinator:
else:
mf = MemoryFileUtils.read(content, uri=uri)
sync_memory_resource_refs(mf, source=RESOURCE_REF_SOURCE_CONTENT_WRITE)
- await self._viking_fs.write_file(uri, MemoryFileUtils.write(mf), ctx=ctx)
+ await self._viking_fs.write_file(
+ uri,
+ MemoryFileUtils.write(mf),
+ ctx=ctx,
+ lock_handle=lock_handle,
+ )
return
if mode == "append":
@@ -443,9 +463,14 @@ class ContentWriteCoordinator:
mf = MemoryFileUtils.read(existing_raw, uri=uri)
mf.content = mf.content + content
updated_raw = MemoryFileUtils.write(mf)
- await self._viking_fs.write_file(uri, updated_raw, ctx=ctx)
+ await self._viking_fs.write_file(
+ uri,
+ updated_raw,
+ ctx=ctx,
+ lock_handle=lock_handle,
+ )
return
- await self._viking_fs.write_file(uri, content, ctx=ctx)
+ await self._viking_fs.write_file(uri, content, ctx=ctx, lock_handle=lock_handle)
async def _enqueue_semantic_refresh(
self,
@@ -539,7 +564,13 @@ class ContentWriteCoordinator:
released = False
request_registered = False
try:
- await self._write_in_place(uri, content, mode=mode, ctx=ctx)
+ await self._write_in_place(
+ uri,
+ content,
+ mode=mode,
+ ctx=ctx,
+ lock_handle=handle,
+ )
await lock_manager.release(handle)
released = True
if wait and telemetry_id and self._vikingdb_has_queue():
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index e632fbdfa..7bb9eb174 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -23,7 +23,7 @@ from contextlib import contextmanager
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import PurePath
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, TypeVar, Union
from openviking.core.context import ContextLevel
from openviking.core.namespace import (
@@ -78,6 +78,7 @@ logger = get_logger(__name__)
# materializes its first 1000 subdirectories. Pass this explicitly at those
# call sites.
LS_ALL_NODES = 2**31 - 1
+_T = TypeVar("_T")
def _ensure_non_empty_search_query(query: str) -> None:
@@ -336,6 +337,54 @@ class VikingFS:
return normalized, parts
+ # TODO: Once pathlock moves down into ragfs, stop reconstructing the
+ # encrypted mount-relative path in Python and derive the lock target from
+ # the same backend-side source of truth.
+ def _encrypted_mount_relative_path(self, path: str) -> tuple[str, str]:
+ """Return the mount prefix and mount-relative path used by Rust encrypted writes."""
+ normalized = path if path.startswith("/") else f"/{path}"
+ parts = [part for part in normalized.split("/") if part]
+ if len(parts) < 2:
+ return "", normalized
+ return f"/{parts[0]}", f"/{'/'.join(parts[1:])}"
+
+ def _encrypted_temp_path(self, path: str) -> str:
+ """Build the deterministic internal `.encrypt` temp-file path for a final file path."""
+ mount_prefix, relative_path = self._encrypted_mount_relative_path(path)
+ relative_parts = [part for part in relative_path.split("/") if part]
+ if len(relative_parts) >= 2 and relative_parts[0] == "local":
+ temp_root = f"/local/{relative_parts[1]}/temp/.encrypt_stage"
+ elif len(relative_parts) >= 2:
+ temp_root = f"/{relative_parts[0]}/temp/.encrypt_stage"
+ else:
+ temp_root = "/temp/.encrypt_stage"
+ digest = hashlib.sha256(relative_path.encode("utf-8")).hexdigest()
+ return f"{mount_prefix}{temp_root}/{digest}.encrypt"
+
+ def _encrypted_write_lock_paths(self, path: str) -> List[str]:
+ """Return the final and temp paths protected by the encrypted write protocol."""
+ return [path, self._encrypted_temp_path(path)]
+
+ async def _run_with_encrypted_write_lock(
+ self,
+ path: str,
+ operation: Callable[[], Awaitable[_T]],
+ lock_handle: Optional["LockHandle"] = None,
+ ) -> _T:
+ """Run a write operation under a dual-path exact lock when encryption is enabled."""
+ if self._encryptor is None:
+ return await operation()
+
+ from openviking.storage.transaction import LockContext, get_lock_manager
+
+ async with LockContext(
+ get_lock_manager(),
+ self._encrypted_write_lock_paths(path),
+ lock_mode="exact",
+ handle=lock_handle,
+ ):
+ return await operation()
+
def _ensure_access(self, uri: str, ctx: Optional[RequestContext]) -> None:
real_ctx = self._ctx_or_default(ctx)
normalized_uri, _ = self._normalized_uri_parts(uri)
@@ -2973,6 +3022,7 @@ class VikingFS:
uri: str,
content: Union[str, bytes],
ctx: Optional[RequestContext] = None,
+ lock_handle: Optional["LockHandle"] = None,
) -> None:
"""Write file directly."""
self._ensure_mutable_access(uri, ctx)
@@ -2982,7 +3032,11 @@ class VikingFS:
if isinstance(content, str):
content = content.encode("utf-8")
- await self._async_agfs.write(path, content)
+ await self._run_with_encrypted_write_lock(
+ path,
+ lambda: self._async_agfs.write(path, content),
+ lock_handle=lock_handle,
+ )
async def read_file(
self,
@@ -3083,40 +3137,55 @@ class VikingFS:
uri: str,
content: bytes,
ctx: Optional[RequestContext] = None,
+ lock_handle: Optional["LockHandle"] = None,
) -> None:
"""Write single binary file."""
self._ensure_mutable_access(uri, ctx)
path = self._uri_to_path(uri, ctx=ctx)
await self._ensure_parent_dirs(path, ctx=ctx)
- await self._async_agfs.write(path, content)
+ await self._run_with_encrypted_write_lock(
+ path,
+ lambda: self._async_agfs.write(path, content),
+ lock_handle=lock_handle,
+ )
async def append_file(
self,
uri: str,
content: str,
ctx: Optional[RequestContext] = None,
+ lock_handle: Optional["LockHandle"] = None,
) -> None:
"""Append content to file."""
self._ensure_mutable_access(uri, ctx)
path = self._uri_to_path(uri, ctx=ctx)
try:
- existing = ""
- try:
- existing_bytes = self._handle_agfs_read(await self._async_agfs.read(path))
- existing = self._decode_bytes(existing_bytes)
- except FileNotFoundError:
- pass
- except AGFSHTTPError as e:
- if e.status_code != 404:
+ await self._ensure_parent_dirs(path, ctx=ctx)
+
+ async def _append_under_lock() -> None:
+ """Read old content and rewrite the whole file under lock to avoid lost updates."""
+ existing = ""
+ try:
+ existing_bytes = self._handle_agfs_read(await self._async_agfs.read(path))
+ existing = self._decode_bytes(existing_bytes)
+ except FileNotFoundError:
+ pass
+ except AGFSHTTPError as e:
+ if e.status_code != 404:
+ raise
+ except AGFSClientError:
raise
- except AGFSClientError:
- raise
- await self._ensure_parent_dirs(path, ctx=ctx)
- final_content = (existing + content).encode("utf-8")
- await self._async_agfs.write(path, final_content)
+ final_content = (existing + content).encode("utf-8")
+ await self._async_agfs.write(path, final_content)
+
+ await self._run_with_encrypted_write_lock(
+ path,
+ _append_under_lock,
+ lock_handle=lock_handle,
+ )
except Exception as e:
logger.error(f"[VikingFS] Failed to append to file {uri}: {e}")
@@ -3387,7 +3456,14 @@ class VikingFS:
# callers fail fast in Python with a clear error rather than passing a
# path that the Rust side will silently drop.
_GIT_INTERNAL_FIRST_SEGMENTS = frozenset(
- {"_system", "tasks", "temp", "queue", "upload", ".path.ovlock"}
+ {
+ "_system",
+ "tasks",
+ "temp",
+ "queue",
+ "upload",
+ ".path.ovlock",
+ }
)
_DEFAULT_GIT_AUTHOR_NAME = "viking-bot"
diff --git a/tests/server/test_content_write_service.py b/tests/server/test_content_write_service.py
index fb81ac433..c21bc07cf 100644
--- a/tests/server/test_content_write_service.py
+++ b/tests/server/test_content_write_service.py
@@ -346,9 +346,9 @@ class _FakeVikingFS:
del ctx
return self.content[uri]
- async def write_file(self, uri: str, content: str, ctx=None):
+ async def write_file(self, uri: str, content: str, ctx=None, lock_handle=None):
del ctx
- self.write_file_calls.append((uri, content))
+ self.write_file_calls.append((uri, content, lock_handle))
self.content[uri] = content
async def rm(self, uri: str, ctx=None, lock_handle=None):
@@ -462,6 +462,42 @@ async def test_write_timeout_after_enqueue_releases_resource_lock(monkeypatch):
assert viking_fs.content[file_uri] == "updated"
+async def test_write_direct_reuses_outer_lock_handle_for_viking_fs(monkeypatch):
+ file_uri = "viking://resources/demo/doc.md"
+ root_uri = "viking://resources/demo"
+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
+ viking_fs = _FakeVikingFS(file_uri=file_uri, root_uri=root_uri)
+ coordinator = ContentWriteCoordinator(viking_fs=viking_fs)
+ lock_manager = _FakeLockManager()
+
+ async def _fake_enqueue_semantic_refresh(**kwargs):
+ del kwargs
+ return None
+
+ monkeypatch.setattr(
+ "openviking.storage.content_write.get_lock_manager",
+ lambda: lock_manager,
+ )
+ monkeypatch.setattr(coordinator, "_enqueue_semantic_refresh", _fake_enqueue_semantic_refresh)
+
+ result = await coordinator._write_direct_with_refresh(
+ uri=file_uri,
+ root_uri=root_uri,
+ content="updated",
+ mode="replace",
+ context_type="resource",
+ wait=False,
+ timeout=None,
+ ctx=ctx,
+ written_bytes=len("updated".encode("utf-8")),
+ telemetry_id="",
+ )
+
+ assert result["uri"] == file_uri
+ assert viking_fs.write_file_calls == [(file_uri, "updated", lock_manager.handle)]
+
+
@pytest.mark.asyncio
async def test_resource_write_updates_target_and_queues_refresh_before_return(monkeypatch):
file_uri = "viking://resources/demo/doc.md"
@@ -530,6 +566,10 @@ async def test_resource_write_rolls_back_replace_when_enqueue_fails(monkeypatch)
)
assert viking_fs.content[file_uri] == "original"
+ assert viking_fs.write_file_calls == [
+ (file_uri, "updated", lock_manager.handle),
+ (file_uri, "original", lock_manager.handle),
+ ]
assert lock_manager.release_calls == ["lock-1"]
@@ -580,8 +620,8 @@ async def test_memory_write_wait_skips_semantic_queue_and_releases_write_lock(mo
lambda: lock_manager,
)
- async def _fake_write_in_place(uri, content, *, mode, ctx):
- del uri, content, mode, ctx
+ async def _fake_write_in_place(uri, content, *, mode, ctx, lock_handle=None):
+ del uri, content, mode, ctx, lock_handle
return None
async def _fail_wait_for_request(*, telemetry_id, timeout):
@@ -659,9 +699,9 @@ class _FakeVikingFSForCreate:
del ctx
self.delete_temp_calls.append(temp_uri)
- async def write_file(self, uri: str, content: str, *, ctx=None):
+ async def write_file(self, uri: str, content: str, *, ctx=None, lock_handle=None):
del ctx
- self.write_file_calls.append((uri, content))
+ self.write_file_calls.append((uri, content, lock_handle))
self.content[uri] = content
parent = uri.rsplit("/", 1)[0]
while parent.startswith("viking://") and parent not in self.existing_dirs:
@@ -692,8 +732,8 @@ async def test_create_mode_new_file_success(monkeypatch):
write_calls = []
- async def _fake_write_in_place(uri, content, *, mode, ctx):
- del mode, ctx
+ async def _fake_write_in_place(uri, content, *, mode, ctx, lock_handle=None):
+ del mode, ctx, lock_handle
write_calls.append((uri, content))
return content
@@ -731,8 +771,8 @@ async def test_create_mode_canonicalizes_user_shorthand_memory_uri(monkeypatch):
write_calls = []
refresh_calls = []
- async def _fake_write_in_place(uri, content, *, mode, ctx):
- del mode, ctx
+ async def _fake_write_in_place(uri, content, *, mode, ctx, lock_handle=None):
+ del mode, ctx, lock_handle
write_calls.append((uri, content))
return content
@@ -770,8 +810,8 @@ async def test_create_mode_existing_file_raises_409(monkeypatch):
viking_fs = _FakeVikingFSForCreate(file_uri=file_uri, root_uri=root_uri, file_exists=True)
coordinator = ContentWriteCoordinator(viking_fs=viking_fs)
- async def _fake_write_in_place(uri, content, *, mode, ctx):
- del uri, content, mode, ctx
+ async def _fake_write_in_place(uri, content, *, mode, ctx, lock_handle=None):
+ del uri, content, mode, ctx, lock_handle
return None
async def _fake_wait_for_queues(*, timeout):
@@ -793,8 +833,8 @@ async def test_create_mode_invalid_extension_raises_400(monkeypatch):
viking_fs = _FakeVikingFSForCreate(file_uri=file_uri, root_uri=root_uri, file_exists=False)
coordinator = ContentWriteCoordinator(viking_fs=viking_fs)
- async def _fake_write_in_place(uri, content, *, mode, ctx):
- del uri, content, mode, ctx
+ async def _fake_write_in_place(uri, content, *, mode, ctx, lock_handle=None):
+ del uri, content, mode, ctx, lock_handle
return None
async def _fake_wait_for_queues(*, timeout):
@@ -821,8 +861,8 @@ async def test_create_mode_parent_dirs_auto_created(monkeypatch):
write_calls = []
- async def _fake_write_in_place(uri, content, *, mode, ctx):
- del mode, ctx
+ async def _fake_write_in_place(uri, content, *, mode, ctx, lock_handle=None):
+ del mode, ctx, lock_handle
write_calls.append((uri, content))
return content
@@ -861,8 +901,8 @@ async def test_create_mode_valid_extensions_pass(monkeypatch):
"openviking.storage.content_write.get_lock_manager", lambda _l=_captured_lock: _l
)
- async def _fake_write_in_place(uri, content, *, mode, ctx):
- del uri, mode, ctx
+ async def _fake_write_in_place(uri, content, *, mode, ctx, lock_handle=None):
+ del uri, mode, ctx, lock_handle
return content
async def _fake_wait_for_queues(*, timeout):
@@ -889,8 +929,8 @@ async def test_create_mode_memory_scope(monkeypatch):
monkeypatch.setattr("openviking.storage.content_write.get_lock_manager", lambda: lock_manager)
- async def _fake_write_in_place(uri, content, *, mode, ctx):
- del uri, mode, ctx
+ async def _fake_write_in_place(uri, content, *, mode, ctx, lock_handle=None):
+ del uri, mode, ctx, lock_handle
return content
refresh_calls = []
@@ -994,10 +1034,10 @@ async def test_create_mode_regression_replace_unchanged(monkeypatch):
monkeypatch.setattr("openviking.storage.content_write.get_lock_manager", lambda: lock_manager)
- async def _fake_write_in_place(uri, content, *, mode, ctx):
+ async def _fake_write_in_place(uri, content, *, mode, ctx, lock_handle=None):
# Verify mode="replace" still works
assert mode == "replace"
- del uri, content, ctx
+ del uri, content, ctx, lock_handle
return None
async def _fake_wait_for_queues(*, timeout):
@@ -1025,10 +1065,10 @@ async def test_create_mode_regression_append_unchanged(monkeypatch):
monkeypatch.setattr("openviking.storage.content_write.get_lock_manager", lambda: lock_manager)
- async def _fake_write_in_place(uri, content, *, mode, ctx):
+ async def _fake_write_in_place(uri, content, *, mode, ctx, lock_handle=None):
# Verify mode="append" still works
assert mode == "append"
- del uri, content, ctx
+ del uri, content, ctx, lock_handle
return None
async def _fake_wait_for_queues(*, timeout):
@@ -1305,7 +1345,10 @@ async def test_set_tags_recursive_directory_updates_descendants(monkeypatch):
nested_file_uri,
}
assert sorted(fake_store.update_calls) == sorted(
- [(file_uri, ["env=prod"], "append"), (nested_file_uri, ["env=prod"], "append")]
+ [
+ (file_uri, ["env=prod"], "append"),
+ (nested_file_uri, ["env=prod"], "append"),
+ ]
)
assert sorted(fake_store.directory_update_calls) == sorted(
[
diff --git a/tests/storage/test_viking_fs_write_locking.py b/tests/storage/test_viking_fs_write_locking.py
new file mode 100644
index 000000000..ae8abe2f6
--- /dev/null
+++ b/tests/storage/test_viking_fs_write_locking.py
@@ -0,0 +1,115 @@
+import pytest
+
+from openviking.storage.viking_fs import VikingFS
+
+
+class _FakeAGFS:
+ """Minimal synchronous AGFS stub for VikingFS write-path tests."""
+
+ def __init__(self):
+ self.storage = {}
+
+ def ensure_parent_dirs(self, path, ctx=None):
+ """Pretend parent directories already exist."""
+ return None
+
+ def write(self, path, data, ctx=None):
+ """Record writes and simulate a successful backend write."""
+ self.storage[path] = data
+ return path
+
+ def read(self, path, ctx=None):
+ """Return stored data or raise FileNotFoundError when missing."""
+ if path not in self.storage:
+ raise FileNotFoundError(path)
+ return self.storage[path]
+
+
+class _RecordingLockContext:
+ """Record LockContext arguments so tests can assert dual-path locking."""
+
+ calls = []
+
+ def __init__(self, lock_manager, paths, lock_mode="exact", **kwargs):
+ """Store lock paths and mode while ignoring unrelated details."""
+ self.lock_manager = lock_manager
+ self.paths = list(paths)
+ self.lock_mode = lock_mode
+ self.kwargs = kwargs
+
+ async def __aenter__(self):
+ """Record the call on context entry."""
+ type(self).calls.append((self.paths, self.lock_mode, self.kwargs))
+ return self
+
+ async def __aexit__(self, exc_type, exc, tb):
+ """Do not swallow exceptions on exit."""
+ return False
+
+
+ ("method_name", "args", "expected_bytes"),
+ [
+ ("write_file", ("hello",), b"hello"),
+ ("append_file", (" world",), b"hello world"),
+ ],
+)
+async def test_encrypted_writes_lock_final_and_temp_paths(
+ monkeypatch, method_name, args, expected_bytes
+):
+ """Encrypted writes must lock both the final file path and the hidden temp path."""
+ import openviking.storage.transaction as transaction_module
+
+ _RecordingLockContext.calls.clear()
+ monkeypatch.setattr(transaction_module, "LockContext", _RecordingLockContext)
+ monkeypatch.setattr(transaction_module, "get_lock_manager", lambda: object())
+
+ agfs = _FakeAGFS()
+ agfs.storage["/local/default/resources/note.md"] = b"hello"
+ fs = VikingFS(agfs=agfs, encryptor=object())
+
+ await getattr(fs, method_name)("viking://resources/note.md", *args)
+
+ assert _RecordingLockContext.calls == [
+ (
+ [
+ "/local/default/resources/note.md",
+ "/local/default/temp/.encrypt_stage/571a25aab6e6bca05a60a6e4aec646389a9ac38237daf55ceda4f72f3d1b4afe.encrypt",
+ ],
+ "exact",
+ {"handle": None},
+ )
+ ]
+ assert agfs.storage["/local/default/resources/note.md"] == expected_bytes
+
+
+ ("final_path", "temp_path"),
+ [
+ (
+ "/local/default/resources/.abstract.md",
+ "/local/default/temp/.encrypt_stage/1ea2c4fea9d85474a57fd03cac44d3bbe7c85fd6eb3c678c54044c4f49ecdbf7.encrypt",
+ ),
+ (
+ "/local/default/resources/note.md",
+ "/local/default/temp/.encrypt_stage/571a25aab6e6bca05a60a6e4aec646389a9ac38237daf55ceda4f72f3d1b4afe.encrypt",
+ ),
+ (
+ "/s3/bucket/docs/a.md",
+ "/s3/bucket/temp/.encrypt_stage/2a95445c0cdc88c4efdf32724651d61f72990a95d445fe0ce146150e39e5630d.encrypt",
+ ),
+ (
+ "/note.md",
+ "/temp/.encrypt_stage/71f3eca3f3ad54df082026dd6b40f2f3b2c2ba67b51bb5d24fda5632044c3228.encrypt",
+ ),
+ (
+ ".abstract.md",
+ "/temp/.encrypt_stage/2f3429bdbec8a484aec67cd1584e5dd64d8cf139d2fc4b4cca97d9cdc9b66ad3.encrypt",
+ ),
+ ],
+)
+def test_encrypted_temp_path_mapping_matches_rust(final_path, temp_path):
+ """Keep Python temp-path mapping aligned with the Rust encryption wrapper."""
+ fs = VikingFS(agfs=_FakeAGFS(), encryptor=object())
+ assert fs._encrypted_temp_path(final_path) == temp_path