Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
//! S3 Client wrapper
//!
//! Provides a filesystem-oriented abstraction over the AWS S3 SDK.
//! Supports AWS S3 and S3-compatible services (MinIO, LocalStack, TOS).
use crate::core::{ConfigValue, Error, Result};
use aws_sdk_s3::config::http::HttpResponse;
use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::operation::{RequestId, RequestIdExt};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::Client;
use std::collections::HashMap;
use std::fmt::Write as _;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
const ENCODED_SEGMENT_PREFIX: char = '!';
const HEX_UPPER: &[u8; 16] = b"0123456789ABCDEF";
fn partial_delete_error(bucket: &str, errors: &[aws_sdk_s3::types::Error]) -> Option<Error> {
if errors.is_empty() {
return None;
}
let details = errors
.iter()
.map(|error| {
format!(
"key={} code={} message={}",
error.key().unwrap_or("<unknown>"),
error.code().unwrap_or("<unknown>"),
error.message().unwrap_or("<unknown>")
)
})
.collect::<Vec<_>>()
.join("; ");
Some(Error::internal(format!(
"S3 DeleteObjects partial failure: bucket={bucket} errors=[{details}]"
)))
}
fn build_s3_error_message(
op: &str,
scope: &str,
raw_error: &str,
code: Option<&str>,
message: Option<&str>,
request_id: Option<&str>,
extended_request_id: Option<&str>,
) -> String {
let mut output = format!("S3 {op} error: {scope}");
if let Some(code) = code {
let _ = write!(output, " code={code}");
}
if let Some(message) = message {
let _ = write!(output, " message={message}");
}
if let Some(request_id) = request_id {
let _ = write!(output, " request_id={request_id}");
}
if let Some(extended_request_id) = extended_request_id {
let _ = write!(output, " extended_request_id={extended_request_id}");
}
let _ = write!(output, " raw={raw_error}");
output
}
fn format_s3_service_error<E>(op: &str, scope: &str, err: &E) -> Error
where
E: std::fmt::Display + ProvideErrorMetadata + RequestId + RequestIdExt,
{
Error::internal(build_s3_error_message(
op,
scope,
&err.to_string(),
err.code(),
err.message(),
err.request_id(),
err.extended_request_id(),
))
}
fn is_s3_compatible_not_found_code<E>(err: &E) -> bool
where
E: ProvideErrorMetadata,
{
// AWS S3 uses the modeled NoSuchKey error; some TOS direct backends return NotFound.
err.code()
.map(|code| code == "NotFound")
.unwrap_or(false)
}
fn format_sdk_s3_error<E, R>(op: &str, scope: &str, sdk_err: &SdkError<E, R>) -> Error
where
E: std::fmt::Display + ProvideErrorMetadata + RequestId + RequestIdExt,
R: std::fmt::Debug + Send + Sync + 'static,
{
use std::any::Any;
match sdk_err {
SdkError::ServiceError(se) => {
if let Some(resp) = (se.raw() as &dyn Any).downcast_ref::<HttpResponse>() {
let status = resp.status().as_u16();
let headers = resp.headers();
Error::internal(build_s3_error_message(
op,
scope,
&format!("HTTP {}: {}", status, se.err()),
se.err().code(),
se.err().message(),
se.err()
.request_id()
.or_else(|| headers.get("x-amz-request-id")),
se.err()
.extended_request_id()
.or_else(|| headers.get("x-amz-id-2")),
))
} else {
Error::internal(build_s3_error_message(
op,
scope,
&format!("service error: {}", se.err()),
se.err().code(),
se.err().message(),
se.err().request_id(),
se.err().extended_request_id(),
))
}
}
_ => Error::internal(build_s3_error_message(
op,
scope,
&sdk_err.to_string(),
None,
None,
None,
None,
)),
}
}
/// Return true when an S3 SDK service error represents a failed conditional request.
fn is_s3_conditional_failure<E, R>(sdk_err: &SdkError<E, R>) -> bool
where
E: ProvideErrorMetadata,
R: 'static,
{
use std::any::Any;
let SdkError::ServiceError(service_err) = sdk_err else {
return false;
};
if matches!(
service_err.err().code(),
Some("PreconditionFailed" | "ConditionalRequestConflict")
) {
return true;
}
(service_err.raw() as &dyn Any)
.downcast_ref::<HttpResponse>()
.is_some_and(|response| matches!(response.status().as_u16(), 409 | 412))
}
fn format_generic_s3_error(
op: &str,
bucket: &str,
key: &str,
raw_error: impl std::fmt::Display,
) -> Error {
Error::internal(build_s3_error_message(
op,
&format!("bucket={bucket} key={key}"),
&raw_error.to_string(),
None,
None,
None,
None,
))
}
fn format_bucket_s3_error(op: &str, bucket: &str, raw_error: impl std::fmt::Display) -> Error {
Error::internal(build_s3_error_message(
op,
&format!("bucket={bucket}"),
&raw_error.to_string(),
None,
None,
None,
None,
))
}
fn encode_path(path: &str, normalize_encoding_chars: &str) -> String {
if normalize_encoding_chars.is_empty() {
return path.to_string();
}
let target_chars: Vec<char> = normalize_encoding_chars.chars().collect();
if !path
.chars()
.any(|ch| ch != '/' && ch.is_ascii() && target_chars.contains(&ch))
{
return path.to_string();
}
let extra = path
.chars()
.filter(|ch| *ch != '/' && ch.is_ascii() && target_chars.contains(ch))
.count()
* 2;
let mut encoded = String::with_capacity(path.len() + extra);
for ch in path.chars() {
if ch == '/' || !ch.is_ascii() || !target_chars.contains(&ch) {
encoded.push(ch);
} else {
push_encoded_byte(&mut encoded, ch as u8);
}
}
encoded
}
fn decode_path(path: &str, normalize_encoding_chars: &str) -> String {
if normalize_encoding_chars.is_empty() {
return path.to_string();
}
decode_segment(path)
}
fn decode_segment(segment: &str) -> String {
if segment.is_empty() {
return String::new();
}
let bytes = segment.as_bytes();
let mut decoded = Vec::with_capacity(bytes.len());
let mut idx = 0usize;
while idx < bytes.len() {
if bytes[idx] == ENCODED_SEGMENT_PREFIX as u8 && idx + 2 < bytes.len() {
let hi = decode_hex_nibble(bytes[idx + 1]);
let lo = decode_hex_nibble(bytes[idx + 2]);
if let (Some(hi), Some(lo)) = (hi, lo) {
decoded.push((hi << 4) | lo);
idx += 3;
continue;
}
}
decoded.push(bytes[idx]);
idx += 1;
}
String::from_utf8(decoded).unwrap_or_else(|_| segment.to_string())
}
fn push_encoded_byte(output: &mut String, byte: u8) {
output.push(ENCODED_SEGMENT_PREFIX);
output.push(HEX_UPPER[(byte >> 4) as usize] as char);
output.push(HEX_UPPER[(byte & 0x0f) as usize] as char);
}
fn decode_hex_nibble(byte: u8) -> Option<u8> {
match byte {
b'0'..=b'9' => Some(byte - b'0'),
b'A'..=b'F' => Some(byte - b'A' + 10),
b'a'..=b'f' => Some(byte - b'a' + 10),
_ => None,
}
}
/// Directory marker mode
#[derive(Debug, Clone, PartialEq)]
pub enum DirectoryMarkerMode {
/// No directory markers (pure prefix-based)
None,
/// Zero-byte marker objects (default, works with AWS S3 and MinIO)
Empty,
/// Single-byte newline marker (for services that reject zero-byte objects like TOS)
NonEmpty,
}
impl DirectoryMarkerMode {
/// Parse from string
pub fn from_str(s: &str) -> Self {
match s {
"none" => Self::None,
"nonempty" => Self::NonEmpty,
_ => Self::Empty, // default
}
}
/// Get the marker data to write for directory creation
pub fn marker_data(&self) -> Option<Vec<u8>> {
match self {
Self::None => Option::None,
Self::Empty => Some(Vec::new()),
Self::NonEmpty => Some(b"\n".to_vec()),
}
}
}
/// Object metadata from HeadObject
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct ObjectMeta {
/// Object key
pub key: String,
/// Object size in bytes
pub size: i64,
/// Last modified time
pub last_modified: SystemTime,
/// Whether this is a directory marker
pub is_dir_marker: bool,
}
/// One flat `ListObjectsV2` page used by S3-backed tree/glob traversal.
#[derive(Debug, Clone)]
pub struct ListTreePage {
pub objects: Vec<ObjectMeta>,
pub next_continuation_token: Option<String>,
}
/// Result of a ListObjects operation
#[derive(Debug)]
pub struct ListResult {
/// Files (non-directory objects)
pub files: Vec<ObjectMeta>,
/// Directory prefixes (common prefixes)
pub directories: Vec<String>,
}
/// Convert AWS DateTime to SystemTime
fn aws_datetime_to_systemtime(dt: &aws_sdk_s3::primitives::DateTime) -> SystemTime {
let secs = dt.secs();
if secs >= 0 {
UNIX_EPOCH + Duration::from_secs(secs as u64)
} else {
UNIX_EPOCH
}
}
/// The Content-Type to be written to S3 is inferred from the filename suffix of the object key.
fn detect_content_type_for_key(key: &str) -> Option<String> {
if key.ends_with('/') {
return None;
}
Some(
mime_guess::from_path(key)
.first_or_octet_stream()
.essence_str()
.to_string(),
)
}
/// S3 Client wrapper
pub struct S3Client {
client: Client,
bucket: String,
prefix: String,
normalize_encoding_chars: String,
marker_mode: DirectoryMarkerMode,
disable_batch_delete: bool,
auto_detect_content_type: bool,
}
impl S3Client {
fn strip_prefix_with_codec(prefix: &str, normalize_encoding_chars: &str, key: &str) -> String {
let stripped = if prefix.is_empty() {
key
} else {
let prefix = format!("{}/", prefix.trim_end_matches('/'));
key.strip_prefix(&prefix).unwrap_or(key)
};
decode_path(stripped, normalize_encoding_chars)
}
/// Create a new S3 client from configuration
pub async fn new(config: &HashMap<String, ConfigValue>) -> Result<Self> {
let bucket = config
.get("bucket")
.and_then(|v| v.as_string())
.ok_or_else(|| Error::config("bucket is required for S3FS"))?
.to_string();
let region = config
.get("region")
.and_then(|v| v.as_string())
.unwrap_or("us-east-1")
.to_string();
let raw_endpoint = config.get("endpoint").and_then(|v| v.as_string());
let use_ssl = if let Some(v) = config.get("use_ssl").and_then(|v| v.as_bool()) {
v
} else if let Some(v) = config.get("disable_ssl").and_then(|v| v.as_bool()) {
!v
} else {
true
};
let endpoint = raw_endpoint.map(|ep| {
if ep.starts_with("https://") || ep.starts_with("http://") {
ep.to_string()
} else if use_ssl {
format!("https://{}", ep)
} else {
format!("http://{}", ep)
}
});
let access_key = config
.get("access_key_id")
.and_then(|v| v.as_string())
.map(|s| s.to_string());
let secret_key = config
.get("secret_access_key")
.and_then(|v| v.as_string())
.map(|s| s.to_string());
let use_path_style = config
.get("use_path_style")
.and_then(|v| v.as_bool())
.unwrap_or(true);
let prefix = config
.get("prefix")
.and_then(|v| v.as_string())
.unwrap_or("")
.to_string();
let normalize_encoding_chars = config
.get("normalize_encoding_chars")
.and_then(|v| v.as_string())
.unwrap_or("?#%+@")
.to_string();
let marker_mode = config
.get("directory_marker_mode")
.and_then(|v| v.as_string())
.map(|s| DirectoryMarkerMode::from_str(s))
.unwrap_or(DirectoryMarkerMode::Empty);
let disable_batch_delete = config
.get("disable_batch_delete")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let auto_detect_content_type = config
.get("auto_detect_content_type")
.and_then(|v| v.as_bool())
.unwrap_or(false);
// Build S3 config
let mut s3_config_builder = aws_sdk_s3::Config::builder()
.behavior_version(BehaviorVersion::latest())
.region(Region::new(region))
.force_path_style(use_path_style);
// Set endpoint if provided (MinIO, LocalStack, TOS)
if let Some(ep) = endpoint {
s3_config_builder = s3_config_builder.endpoint_url(ep.to_string());
}
// Set credentials if provided, otherwise SDK uses default chain
if let (Some(ak), Some(sk)) = (access_key, secret_key) {
let creds = Credentials::new(ak, sk, None, None, "ragfs-s3fs");
s3_config_builder = s3_config_builder.credentials_provider(creds);
}
let s3_config = s3_config_builder.build();
let client = Client::from_conf(s3_config);
Ok(Self {
client,
bucket,
prefix,
normalize_encoding_chars,
marker_mode,
disable_batch_delete,
auto_detect_content_type,
})
}
/// Build the full S3 key from a filesystem path
pub fn build_key(&self, path: &str) -> String {
let clean = encode_path(path.trim_start_matches('/'), &self.normalize_encoding_chars);
if self.prefix.is_empty() {
clean
} else {
let prefix = self.prefix.trim_end_matches('/');
if clean.is_empty() {
format!("{}/", prefix)
} else {
format!("{}/{}", prefix, clean)
}
}
}
/// Strip the prefix from an S3 key to get the filesystem path
pub fn strip_prefix(&self, key: &str) -> String {
Self::strip_prefix_with_codec(&self.prefix, &self.normalize_encoding_chars, key)
}
/// Get an object's contents
pub async fn get_object(&self, key: &str) -> Result<Vec<u8>> {
let resp = match self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
{
Ok(resp) => resp,
Err(sdk_err) => {
let service_err = sdk_err.into_service_error();
if service_err.is_no_such_key() || is_s3_compatible_not_found_code(&service_err) {
return Err(Error::NotFound(key.to_string()));
}
return Err(format_s3_service_error(
"GetObject",
&format!("bucket={} key={}", self.bucket, key),
&service_err,
));
}
};
let bytes = resp
.body
.collect()
.await
.map_err(|e| format_generic_s3_error("ReadBody", &self.bucket, key, e))?;
Ok(bytes.to_vec())
}
/// Get an object's contents and ETag for conditional updates.
pub async fn get_object_with_etag(&self, key: &str) -> Result<Option<(Vec<u8>, String)>> {
let resp = match self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
{
Ok(resp) => resp,
Err(sdk_err) => {
let service_err = sdk_err.into_service_error();
if service_err.is_no_such_key() || is_s3_compatible_not_found_code(&service_err) {
return Ok(None);
}
return Err(format_s3_service_error(
"GetObjectWithEtag",
&format!("bucket={} key={}", self.bucket, key),
&service_err,
));
}
};
let etag = resp.e_tag().ok_or_else(|| {
Error::internal(format!(
"S3 GetObjectWithEtag missing ETag: bucket={} key={key}",
self.bucket
))
})?.to_string();
let bytes = resp
.body
.collect()
.await
.map_err(|e| format_generic_s3_error("ReadBody", &self.bucket, key, e))?;
Ok(Some((bytes.to_vec(), etag)))
}
/// Get an object's contents with range request
pub async fn get_object_range(&self, key: &str, offset: u64, size: u64) -> Result<Vec<u8>> {
let range = if size == 0 {
format!("bytes={}-", offset)
} else {
format!("bytes={}-{}", offset, offset + size - 1)
};
let resp = match self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.range(range)
.send()
.await
{
Ok(resp) => resp,
Err(sdk_err) => {
let service_err = sdk_err.into_service_error();
if service_err.is_no_such_key() || is_s3_compatible_not_found_code(&service_err) {
return Err(Error::NotFound(key.to_string()));
}
return Err(format_s3_service_error(
"GetObjectRange",
&format!("bucket={} key={}", self.bucket, key),
&service_err,
));
}
};
let bytes = resp
.body
.collect()
.await
.map_err(|e| format_generic_s3_error("ReadBody", &self.bucket, key, e))?;
Ok(bytes.to_vec())
}
/// Upload an object
pub async fn put_object(&self, key: &str, data: Vec<u8>) -> Result<()> {
let mut request = self
.client
.put_object()
.bucket(&self.bucket)
.key(key)
.body(ByteStream::from(data));
if self.auto_detect_content_type {
if let Some(content_type) = detect_content_type_for_key(key) {
request = request.content_type(content_type);
}
}
request.send().await.map_err(|e| {
format_sdk_s3_error(
"PutObject",
&format!("bucket={} key={key}", self.bucket),
&e,
)
})?;
Ok(())
}
/// Upload an object only when the key does not already exist.
pub async fn put_object_create_new(&self, key: &str, data: Vec<u8>) -> Result<()> {
let mut request = self
.client
.put_object()
.bucket(&self.bucket)
.key(key)
.if_none_match("*")
.body(ByteStream::from(data));
if self.auto_detect_content_type {
if let Some(content_type) = detect_content_type_for_key(key) {
request = request.content_type(content_type);
}
}
request.send().await.map_err(|e| {
if is_s3_conditional_failure(&e) {
Error::already_exists(key)
} else {
format_sdk_s3_error(
"PutObjectCreateNew",
&format!("bucket={} key={key}", self.bucket),
&e,
)
}
})?;
Ok(())
}
/// Upload an object only when its ETag still matches `etag`.
pub async fn put_object_if_match(&self, key: &str, data: Vec<u8>, etag: &str) -> Result<bool> {
let mut request = self
.client
.put_object()
.bucket(&self.bucket)
.key(key)
.if_match(etag)
.body(ByteStream::from(data));
if self.auto_detect_content_type {
if let Some(content_type) = detect_content_type_for_key(key) {
request = request.content_type(content_type);
}
}
request.send().await.map_err(|e| {
if is_s3_conditional_failure(&e) {
Error::AlreadyExists(key.to_string())
} else {
format_sdk_s3_error(
"PutObjectIfMatch",
&format!("bucket={} key={key}", self.bucket),
&e,
)
}
}).map(|_| true).or_else(|e| {
if matches!(e, Error::AlreadyExists(_)) {
Ok(false)
} else {
Err(e)
}
})
}
/// Delete a single object
pub async fn delete_object(&self, key: &str) -> Result<()> {
self.client
.delete_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
.map_err(|e| {
format_sdk_s3_error(
"DeleteObject",
&format!("bucket={} key={key}", self.bucket),
&e,
)
})?;
Ok(())
}
/// Delete a single object only when its ETag still matches `etag`.
pub async fn delete_object_if_match(&self, key: &str, etag: &str) -> Result<bool> {
self.client
.delete_object()
.bucket(&self.bucket)
.key(key)
.if_match(etag)
.send()
.await
.map_err(|e| {
if is_s3_conditional_failure(&e) {
Error::AlreadyExists(key.to_string())
} else {
format_sdk_s3_error(
"DeleteObjectIfMatch",
&format!("bucket={} key={key}", self.bucket),
&e,
)
}
})
.map(|_| true)
.or_else(|e| {
if matches!(e, Error::AlreadyExists(_)) {
Ok(false)
} else {
Err(e)
}
})
}
/// Batch delete objects (up to 1000 per call)
/// If disable_batch_delete is true, use sequential single-object deletes
/// for S3-compatible services (e.g., Alibaba Cloud OSS) that require
/// Content-MD5 for DeleteObjects but AWS SDK v2 does not send it by default.
pub async fn delete_objects(&self, keys: &[String]) -> Result<()> {
if keys.is_empty() {
return Ok(());
}
if self.disable_batch_delete {
// Sequential single-object delete
for key in keys {
self.client
.delete_object()
.bucket(&self.bucket)
.key(key.as_str())
.send()
.await
.map_err(|e| {
format_sdk_s3_error(
"DeleteObject",
&format!("bucket={} key={}", self.bucket, key),
&e,
)
})?;
}
} else {
// S3 batch delete limit is 1000
for chunk in keys.chunks(1000) {
let objects: Vec<_> = chunk
.iter()
.map(|k| {
aws_sdk_s3::types::ObjectIdentifier::builder()
.key(k.as_str())
.build()
.unwrap()
})
.collect();
let delete = aws_sdk_s3::types::Delete::builder()
.set_objects(Some(objects))
.build()
.map_err(|e| format_bucket_s3_error("BuildDelete", &self.bucket, e))?;
let response = self
.client
.delete_objects()
.bucket(&self.bucket)
.delete(delete)
.send()
.await
.map_err(|e| {
format_sdk_s3_error("DeleteObjects", &format!("bucket={}", self.bucket), &e)
})?;
if let Some(error) = partial_delete_error(&self.bucket, response.errors()) {
return Err(error);
}
}
}
Ok(())
}
/// Get object metadata (HeadObject)
pub async fn head_object(&self, key: &str) -> Result<Option<ObjectMeta>> {
match self
.client
.head_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
{
Ok(resp) => {
let size = resp.content_length.unwrap_or(0);
let last_modified = resp
.last_modified()
.map(aws_datetime_to_systemtime)
.unwrap_or(UNIX_EPOCH);
let is_dir_marker = key.ends_with('/');
Ok(Some(ObjectMeta {
key: key.to_string(),
size,
last_modified,
is_dir_marker,
}))
}
Err(sdk_err) => {
// Check if it's a 404
if sdk_err
.as_service_error()
.map(|err| err.is_not_found() || is_s3_compatible_not_found_code(err))
.unwrap_or(false)
{
Ok(None)
} else {
Err(format_sdk_s3_error(
"HeadObject",
&format!("bucket={} key={}", self.bucket, key),
&sdk_err,
))
}
}
}
}
/// List objects with prefix and delimiter
pub async fn list_objects(&self, prefix: &str, delimiter: Option<&str>) -> Result<ListResult> {
let mut files = Vec::new();
let mut directories = Vec::new();
let mut continuation_token: Option<String> = None;
loop {
let mut req = self
.client
.list_objects_v2()
.bucket(&self.bucket)
.prefix(prefix);
if let Some(d) = delimiter {
req = req.delimiter(d);
}
if let Some(token) = &continuation_token {
req = req.continuation_token(token);
}
let resp = req.send().await.map_err(|e| {
format_sdk_s3_error(
"ListObjectsV2",
&format!("bucket={} prefix={prefix}", self.bucket),
&e,
)
})?;
// Process files (contents)
for obj in resp.contents() {
let key = obj.key().unwrap_or("");
// Skip the prefix itself and directory markers
if key == prefix || key.ends_with('/') {
continue;
}
let size = obj.size.unwrap_or(0);
let last_modified = obj
.last_modified()
.map(aws_datetime_to_systemtime)
.unwrap_or(UNIX_EPOCH);
files.push(ObjectMeta {
key: key.to_string(),
size,
last_modified,
is_dir_marker: false,
});
}
// Process directory prefixes (common prefixes)
for cp in resp.common_prefixes() {
if let Some(p) = cp.prefix() {
// Remove trailing slash for consistency
let dir = p.trim_end_matches('/').to_string();
if !dir.is_empty() {
directories.push(dir);
}
}
}
// Check if there are more results
if resp.is_truncated() == Some(true) {
continuation_token = resp.next_continuation_token().map(|s| s.to_string());
} else {
break;
}
}
Ok(ListResult { files, directories })
}
/// List all objects under a prefix (flat listing, no delimiter).
/// Preserves directory marker objects (keys ending with '/').
/// Used by tree_directory for efficient flat traversal.
pub async fn list_tree_objects(&self, prefix: &str) -> Result<Vec<ObjectMeta>> {
let mut objects = Vec::new();
let mut continuation_token: Option<String> = None;
loop {
let mut req = self
.client
.list_objects_v2()
.bucket(&self.bucket)
.prefix(prefix);
if let Some(token) = &continuation_token {
req = req.continuation_token(token);
}
let resp = req.send().await.map_err(|e| {
format_sdk_s3_error(
"ListObjectsV2",
&format!("bucket={} prefix={prefix}", self.bucket),
&e,
)
})?;
for obj in resp.contents() {
let key = obj.key().unwrap_or("");
if key == prefix {
continue;
}
let size = obj.size.unwrap_or(0);
let last_modified = obj
.last_modified()
.map(aws_datetime_to_systemtime)
.unwrap_or(UNIX_EPOCH);
let is_dir_marker = key.ends_with('/');
objects.push(ObjectMeta {
key: key.to_string(),
size,
last_modified,
is_dir_marker,
});
}
if resp.is_truncated() == Some(true) {
continuation_token = resp.next_continuation_token().map(|s| s.to_string());
} else {
break;
}
}
Ok(objects)
}
/// List one flat object page under a prefix (no delimiter).
pub async fn list_tree_objects_page(
&self,
prefix: &str,
continuation_token: Option<&str>,
max_keys: usize,
) -> Result<ListTreePage> {
let mut req = self
.client
.list_objects_v2()
.bucket(&self.bucket)
.prefix(prefix)
.max_keys(max_keys.min(1000) as i32);
if let Some(token) = continuation_token {
req = req.continuation_token(token);
}
let resp = req.send().await.map_err(|e| {
format_sdk_s3_error(
"ListObjectsV2",
&format!("bucket={} prefix={prefix}", self.bucket),
&e,
)
})?;
let mut objects = Vec::new();
for obj in resp.contents() {
let key = obj.key().unwrap_or("");
if key == prefix {
continue;
}
let size = obj.size.unwrap_or(0);
let last_modified = obj
.last_modified()
.map(aws_datetime_to_systemtime)
.unwrap_or(UNIX_EPOCH);
objects.push(ObjectMeta {
key: key.to_string(),
size,
last_modified,
is_dir_marker: key.ends_with('/'),
});
}
Ok(ListTreePage {
objects,
next_continuation_token: resp.next_continuation_token().map(|s| s.to_string()),
})
}
/// Copy an object
pub async fn copy_object(&self, src_key: &str, dst_key: &str) -> Result<()> {
let copy_source = format!("{}/{}", self.bucket, src_key);
self.client
.copy_object()
.bucket(&self.bucket)
.copy_source(©_source)
.key(dst_key)
.send()
.await
.map_err(|e| {
format_sdk_s3_error(
"CopyObject",
&format!(
"bucket={} src_key={} dst_key={}",
self.bucket, src_key, dst_key
),
&e,
)
})?;
Ok(())
}
/// Check if a directory exists (either marker or any children)
pub async fn directory_exists(&self, path: &str) -> Result<bool> {
let dir_key = self.build_key(path);
let dir_key_slash = if dir_key.ends_with('/') {
dir_key.clone()
} else {
format!("{}/", dir_key)
};
// Check if directory marker exists
if self.head_object(&dir_key_slash).await?.is_some() {
return Ok(true);
}
// Check if any objects exist with this prefix
let resp = self
.client
.list_objects_v2()
.bucket(&self.bucket)
.prefix(&dir_key_slash)
.max_keys(1)
.send()
.await
.map_err(|e| {
format_sdk_s3_error(
"ListObjectsV2",
&format!("bucket={} prefix={dir_key_slash}", self.bucket),
&e,
)
})?;
let has_contents = !resp.contents().is_empty();
let has_prefixes = !resp.common_prefixes().is_empty();
Ok(has_contents || has_prefixes)
}
/// Delete a directory and all its contents
pub async fn delete_directory(&self, path: &str) -> Result<()> {
let dir_key = self.build_key(path);
let prefix = if dir_key.ends_with('/') {
dir_key
} else {
format!("{}/", dir_key)
};
// List and delete all objects under prefix
loop {
let resp = self
.client
.list_objects_v2()
.bucket(&self.bucket)
.prefix(&prefix)
.max_keys(1000)
.send()
.await
.map_err(|e| {
format_sdk_s3_error(
"ListObjectsV2",
&format!("bucket={} prefix={prefix}", self.bucket),
&e,
)
})?;
let contents = resp.contents();
if contents.is_empty() {
break;
}
let keys: Vec<String> = contents
.iter()
.filter_map(|obj: &aws_sdk_s3::types::Object| obj.key().map(|k| k.to_string()))
.collect();
self.delete_objects(&keys).await?;
if contents.len() < 1000 {
break;
}
}
Ok(())
}
/// Create a directory marker object
pub async fn create_directory_marker(&self, path: &str) -> Result<()> {
if let Some(data) = self.marker_mode.marker_data() {
let dir_key = self.build_key(path);
let key = if dir_key.ends_with('/') {
dir_key
} else {
format!("{}/", dir_key)
};
self.put_object(&key, data).await?;
}
Ok(())
}
/// Get the marker mode
pub fn marker_mode(&self) -> &DirectoryMarkerMode {
&self.marker_mode
}
/// Get the bucket name
pub fn bucket(&self) -> &str {
&self.bucket
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::core::Error;
use aws_sdk_s3::error::{ErrorMetadata, ProvideErrorMetadata};
#[derive(Debug)]
struct FakeServiceError {
raw: &'static str,
meta: ErrorMetadata,
}
impl std::fmt::Display for FakeServiceError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.raw)
}
}
impl RequestId for FakeServiceError {
fn request_id(&self) -> Option<&str> {
self.meta.request_id()
}
}
impl RequestIdExt for FakeServiceError {
fn extended_request_id(&self) -> Option<&str> {
self.meta.extended_request_id()
}
}
impl ProvideErrorMetadata for FakeServiceError {
fn meta(&self) -> &ErrorMetadata {
&self.meta
}
}
fn test_client(prefix: &str, normalize_encoding_chars: &str) -> S3Client {
S3Client {
client: Client::from_conf(
aws_sdk_s3::Config::builder()
.behavior_version(BehaviorVersion::latest())
.region(Region::new("us-east-1"))
.build(),
),
bucket: "test-bucket".to_string(),
prefix: prefix.to_string(),
normalize_encoding_chars: normalize_encoding_chars.to_string(),
marker_mode: DirectoryMarkerMode::Empty,
disable_batch_delete: false,
auto_detect_content_type: false,
}
}
#[test]
fn test_key_codec_keeps_safe_segments() {
let path = "dir/ok-name_1.2!*'()";
assert_eq!(encode_path(path, "?#%+@"), path);
assert_eq!(decode_path(path, "?#%+@"), path);
}
#[test]
fn test_key_codec_encodes_only_target_characters() {
let encoded = encode_path("a b/c?d/#frag/%raw", "?#%+@");
assert_eq!(encoded, "a!20b/c!3Fd/!23frag/!25raw");
assert_eq!(decode_path(&encoded, "?#%+@"), "a b/c?d/#frag/%raw");
}
#[test]
fn test_key_codec_encodes_segments_with_reserved_prefix() {
let encoded = encode_path("!literal/normal", "?#%+@");
assert_eq!(encoded, "!literal/normal");
assert_eq!(decode_path(&encoded, "?#%+@"), "!literal/normal");
}
#[test]
fn test_key_codec_disabled_is_passthrough() {
let path = "a b/c?d";
assert_eq!(encode_path(path, ""), path);
assert_eq!(decode_path(path, ""), path);
}
#[test]
fn test_key_codec_preserves_non_target_characters_including_unicode() {
let path = "dir//safe-_.*/@scope+pkg/客户看板 file/";
let encoded = encode_path(path, "?#%+@");
assert_eq!(encoded, "dir//safe-_.*/!40scope!2Bpkg/客户看板!20file/");
assert_eq!(decode_path(&encoded, "?#%+@"), path);
}
#[test]
fn test_key_codec_leaves_other_characters_unchanged() {
let path = "目录/@scope&name=1";
assert_eq!(encode_path(path, "?#%+@"), "目录/!40scope&name=1");
assert_eq!(decode_path("目录/!40scope&name=1", "?#%+@"), path);
}
#[test]
fn test_build_key_applies_normalized_encoding_per_segment() {
let key = test_client("ns", "?#%+@").build_key("/dir/a b/c?d.txt");
assert_eq!(key, "ns/dir/a!20b/c!3Fd.txt");
}
#[test]
fn test_strip_prefix_decodes_normalized_segments() {
let client = test_client("ns", "?#%+@");
let key = client.build_key("/dir/a b/c?d.txt");
assert_eq!(client.strip_prefix(&key), "dir/a b/c?d.txt".to_string());
}
#[test]
fn test_build_key_preserves_segments_when_normalization_disabled() {
assert_eq!(test_client("", "").build_key("/a b"), "a b");
}
#[test]
fn test_partial_delete_error_reports_each_failed_key() {
let errors = [aws_sdk_s3::types::Error::builder()
.key("denied.txt")
.code("AccessDenied")
.message("denied")
.build()];
let error = partial_delete_error("bucket", &errors).unwrap();
assert!(error.to_string().contains("key=denied.txt"));
assert!(error.to_string().contains("code=AccessDenied"));
}
#[test]
fn test_detect_content_type_for_key_uses_filename_extension() {
assert_eq!(
detect_content_type_for_key("tenant/docs/readme.md").as_deref(),
Some("text/markdown")
);
assert_eq!(
detect_content_type_for_key("tenant/docs/data.json").as_deref(),
Some("application/json")
);
assert_eq!(
detect_content_type_for_key("tenant/images/logo.png").as_deref(),
Some("image/png")
);
}
#[test]
fn test_detect_content_type_for_key_handles_unknown_and_directory_marker() {
assert_eq!(
detect_content_type_for_key("tenant/blob/no-extension").as_deref(),
Some("application/octet-stream")
);
assert_eq!(detect_content_type_for_key("tenant/dir/"), None);
}
#[test]
fn test_format_generic_s3_error_includes_operation_bucket_key_and_raw_error() {
let err =
format_generic_s3_error("PutObject", "test-bucket", "tenant/a.txt", "service error");
match err {
Error::Internal(message) => {
assert!(message.contains("S3 PutObject error"));
assert!(message.contains("bucket=test-bucket"));
assert!(message.contains("key=tenant/a.txt"));
assert!(message.contains("raw=service error"));
}
other => panic!("expected internal error, got {other:?}"),
}
}
#[test]
fn test_format_s3_service_error_includes_metadata_fields() {
let service_err = FakeServiceError {
raw: "service error",
meta: ErrorMetadata::builder()
.code("AccessDenied")
.message("signature mismatch")
.custom("aws_request_id", "req-123")
.custom("s3_extended_request_id", "ext-456")
.build(),
};
let err = format_s3_service_error(
"PutObject",
"bucket=test-bucket key=tenant/a.txt",
&service_err,
);
match err {
Error::Internal(message) => {
assert!(message.contains("S3 PutObject error"));
assert!(message.contains("bucket=test-bucket key=tenant/a.txt"));
assert!(message.contains("code=AccessDenied"));
assert!(message.contains("message=signature mismatch"));
assert!(message.contains("request_id=req-123"));
assert!(message.contains("extended_request_id=ext-456"));
assert!(message.contains("raw=service error"));
}
other => panic!("expected internal error, got {other:?}"),
}
}
#[test]
fn test_is_s3_compatible_not_found_code_accepts_tos_code() {
let tos_not_found = FakeServiceError {
raw: "service error",
meta: ErrorMetadata::builder().code("NotFound").build(),
};
assert!(is_s3_compatible_not_found_code(&tos_not_found));
let no_such_key = FakeServiceError {
raw: "service error",
meta: ErrorMetadata::builder().code("NoSuchKey").build(),
};
assert!(!is_s3_compatible_not_found_code(&no_such_key));
let access_denied = FakeServiceError {
raw: "service error",
meta: ErrorMetadata::builder().code("AccessDenied").build(),
};
assert!(!is_s3_compatible_not_found_code(&access_denied));
let no_code = FakeServiceError {
raw: "service error",
meta: ErrorMetadata::builder().build(),
};
assert!(!is_s3_compatible_not_found_code(&no_code));
}
}