Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
//! Utility functions for Git module
use bytes::Bytes;
use std::io::{Read, Write};
use crate::git::error::RefStoreError;
/// Validate a ref name follows Git naming conventions.
///
/// Checks for:
/// - Empty name
/// - Contains ".."
/// - Starts or ends with "/"
/// - Contains invalid characters
pub fn validate_ref_name(ref_name: &str) -> Result<(), RefStoreError> {
if ref_name.is_empty() {
return Err(RefStoreError::InvalidName(
"ref name cannot be empty".to_string(),
));
}
if ref_name.contains("..") {
return Err(RefStoreError::InvalidName(
"ref name cannot contain '..'".to_string(),
));
}
if ref_name.starts_with('/') || ref_name.ends_with('/') {
return Err(RefStoreError::InvalidName(
"ref name cannot start or end with '/'".to_string(),
));
}
if ref_name.contains(' ')
|| ref_name.contains('\x00')
|| ref_name.contains('~')
|| ref_name.contains('^')
|| ref_name.contains(':')
|| ref_name.contains('?')
|| ref_name.contains('[')
|| ref_name.contains('*')
{
return Err(RefStoreError::InvalidName(
"ref name contains invalid characters".to_string(),
));
}
Ok(())
}
/// Compress data using zlib (for Git loose object storage).
pub fn zlib_compress(data: &[u8]) -> Result<Vec<u8>, std::io::Error> {
let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
encoder.write_all(data)?;
encoder.finish()
}
/// Decompress zlib-compressed data (for reading Git loose objects).
///
/// Pre-allocates a generous output buffer to amortize the `Vec` doubling cost
/// that hurts large blobs (a 100 MiB payload otherwise triggers ~27 reallocs).
/// The hint assumes a worst-case compression ratio of ~4×; capped at 64 MiB
/// so a pathologically small-but-compressible header doesn't reserve absurd
/// amounts of memory. Exceeding the hint still works — `Vec` will grow.
pub fn zlib_decompress(data: &[u8]) -> Result<Vec<u8>, std::io::Error> {
const HINT_CAP: usize = 64 * 1024 * 1024;
let hint = data.len().saturating_mul(4).min(HINT_CAP);
let mut decoded = Vec::with_capacity(hint);
let mut decoder = flate2::read::ZlibDecoder::new(data);
decoder.read_to_end(&mut decoded)?;
Ok(decoded)
}
/// Parse a Git loose object header, returning (kind, size, header_end_offset).
pub fn parse_object_header(data: &[u8]) -> Result<(gix_object::Kind, u64, usize), crate::git::error::ObjectStoreError> {
gix_object::decode::loose_header(data).map_err(|e| {
crate::git::error::ObjectStoreError::Backend(format!("invalid object header: {e}"))
})
}
/// Read and decompress a Git object from ObjectStore, returning the full
/// uncompressed bytes (including header).
pub async fn read_object(
store: &dyn crate::git::object_store::ObjectStore,
account: &str,
oid: &gix_hash::ObjectId,
) -> Result<Bytes, crate::git::error::ObjectStoreError> {
let compressed = store.get(account, oid).await?;
let decompressed = zlib_decompress(&compressed)
.map_err(|e| crate::git::error::ObjectStoreError::Zlib(e.to_string()))?;
Ok(Bytes::from(decompressed))
}
/// Read and decompress a loose object while bounding both compressed and
/// decompressed materialization. `max_payload_bytes` excludes the Git header.
pub async fn read_object_limited(
store: &dyn crate::git::object_store::ObjectStore,
account: &str,
oid: &gix_hash::ObjectId,
max_payload_bytes: u64,
) -> Result<Bytes, crate::git::error::ObjectStoreError> {
const MAX_LOOSE_HEADER_BYTES: u64 = 64;
let max_loose_bytes = max_payload_bytes.saturating_add(MAX_LOOSE_HEADER_BYTES);
let compressed_limit = zlib_compress_bound(max_loose_bytes);
let compressed = store.get_limited(account, oid, compressed_limit).await?;
let mut decoder = flate2::read::ZlibDecoder::new(compressed.as_ref());
let mut header = Vec::with_capacity(MAX_LOOSE_HEADER_BYTES as usize);
loop {
if header.len() >= MAX_LOOSE_HEADER_BYTES as usize {
return Err(crate::git::error::ObjectStoreError::Backend(
"invalid object header: exceeds 64 bytes".to_string(),
));
}
let mut byte = [0u8; 1];
let read = decoder.read(&mut byte)?;
if read == 0 {
return Err(crate::git::error::ObjectStoreError::Backend(
"invalid object header: missing terminator".to_string(),
));
}
header.push(byte[0]);
if byte[0] == 0 {
break;
}
}
let (_, payload_size, _) = parse_object_header(&header)?;
if payload_size > max_payload_bytes {
return Err(crate::git::error::ObjectStoreError::ReadLimitExceeded {
size: payload_size,
limit: max_payload_bytes,
});
}
let payload_limit = payload_size.saturating_add(1);
let mut payload = Vec::with_capacity(usize::try_from(payload_size).unwrap_or(usize::MAX));
decoder
.take(payload_limit)
.read_to_end(&mut payload)
.map_err(|e| crate::git::error::ObjectStoreError::Zlib(e.to_string()))?;
if payload.len() as u64 != payload_size {
return Err(crate::git::error::ObjectStoreError::Backend(format!(
"invalid object payload: header declares {payload_size} bytes, decoded {}",
payload.len()
)));
}
header.extend_from_slice(&payload);
Ok(Bytes::from(header))
}
fn zlib_compress_bound(source_len: u64) -> u64 {
source_len
.saturating_add(source_len >> 12)
.saturating_add(source_len >> 14)
.saturating_add(source_len >> 25)
.saturating_add(13)
}
/// Serialize, compress, and write a Git object to ObjectStore.
/// Returns the object's ObjectId.
pub async fn write_object(
store: &dyn crate::git::object_store::ObjectStore,
account: &str,
kind: gix_object::Kind,
data: &[u8],
) -> Result<gix_hash::ObjectId, crate::git::error::ObjectStoreError> {
let header = gix_object::encode::loose_header(kind, data.len() as u64);
let oid = gix_object::compute_hash(gix_hash::Kind::Sha1, kind, data);
let mut full = Vec::with_capacity(header.len() + data.len());
full.extend_from_slice(&header);
full.extend_from_slice(data);
let compressed = zlib_compress(&full)?;
store.put(account, &oid, Bytes::from(compressed)).await?;
Ok(oid)
}
/// Same as [`write_object`], but runs an `exists` precheck before compressing
/// and putting (Fast Path 3). If the object is already present, the zlib
/// compression and `put` are skipped and the oid is returned directly. `put`
/// is itself idempotent, so this precheck is purely a performance optimization
/// (saves S3 body upload / local zlib compression for duplicate blobs).
pub async fn write_object_if_absent(
store: &dyn crate::git::object_store::ObjectStore,
account: &str,
kind: gix_object::Kind,
data: &[u8],
) -> Result<gix_hash::ObjectId, crate::git::error::ObjectStoreError> {
let oid = gix_object::compute_hash(gix_hash::Kind::Sha1, kind, data);
if store.exists(account, &oid).await? {
return Ok(oid);
}
let header = gix_object::encode::loose_header(kind, data.len() as u64);
let mut full = Vec::with_capacity(header.len() + data.len());
full.extend_from_slice(&header);
full.extend_from_slice(data);
let compressed = zlib_compress(&full)?;
store.put(account, &oid, Bytes::from(compressed)).await?;
Ok(oid)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_validate_ref_name() {
assert!(validate_ref_name("refs/heads/main").is_ok());
assert!(validate_ref_name("refs/tags/v1.0").is_ok());
assert!(validate_ref_name("HEAD").is_ok());
assert!(validate_ref_name("").is_err());
assert!(validate_ref_name("..").is_err());
assert!(validate_ref_name("refs/../heads").is_err());
assert!(validate_ref_name("/refs/heads").is_err());
assert!(validate_ref_name("refs/heads/ ").is_err());
assert!(validate_ref_name("refs~head").is_err());
assert!(validate_ref_name("refs^head").is_err());
assert!(validate_ref_name("refs:head").is_err());
assert!(validate_ref_name("refs?head").is_err());
assert!(validate_ref_name("refs[head]").is_err());
assert!(validate_ref_name("refs*head").is_err());
}
#[test]
fn test_zlib_round_trip() {
let original = b"tree 15\0hello world!!!";
let compressed = zlib_compress(original).unwrap();
let decompressed = zlib_decompress(&compressed).unwrap();
assert_eq!(decompressed, original);
}
#[test]
fn test_parse_object_header_tree() {
let data = b"tree 15\0entries data";
let (kind, size, offset) = parse_object_header(data).unwrap();
assert_eq!(kind, gix_object::Kind::Tree);
assert_eq!(size, 15);
assert_eq!(offset, 8);
}
#[test]
fn test_parse_object_header_blob() {
let data = b"blob 5\0hello";
let (kind, size, offset) = parse_object_header(data).unwrap();
assert_eq!(kind, gix_object::Kind::Blob);
assert_eq!(size, 5);
assert_eq!(offset, 7);
}
#[tokio::test]
async fn test_write_read_object_round_trip() {
use tempfile::tempdir;
use crate::git::backends::local::LocalObjectStore;
let temp_dir = tempdir().unwrap();
let store = LocalObjectStore::new(temp_dir.path());
let data = b"hello tree bytes";
let kind = gix_object::Kind::Blob;
// Write the object
let oid = write_object(&store, "test-account", kind, data).await.unwrap();
// Read the object back
let raw = read_object(&store, "test-account", &oid).await.unwrap();
// Parse and validate header
let (parsed_kind, size, offset) = parse_object_header(&raw).unwrap();
assert_eq!(parsed_kind, kind);
assert_eq!(size, data.len() as u64);
// Validate body
assert_eq!(&raw[offset..], data);
// Validate OID matches expected
let expected_oid = gix_object::compute_hash(gix_hash::Kind::Sha1, kind, data);
assert_eq!(oid, expected_oid);
}
#[tokio::test]
async fn test_write_object_if_absent_skips_put_on_second_call() {
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use tempfile::tempdir;
use crate::git::backends::local::LocalObjectStore;
use crate::git::object_store::ObjectStore;
use crate::git::error::ObjectStoreError;
use gix_hash::ObjectId;
struct CountingStore {
inner: LocalObjectStore,
puts: AtomicUsize,
exists_calls: AtomicUsize,
}
#[async_trait::async_trait]
impl ObjectStore for CountingStore {
async fn put(
&self,
account: &str,
oid: &ObjectId,
zlib_body: Bytes,
) -> Result<(), ObjectStoreError> {
self.puts.fetch_add(1, Ordering::SeqCst);
self.inner.put(account, oid, zlib_body).await
}
async fn get(&self, account: &str, oid: &ObjectId) -> Result<Bytes, ObjectStoreError> {
self.inner.get(account, oid).await
}
async fn exists(&self, account: &str, oid: &ObjectId) -> Result<bool, ObjectStoreError> {
self.exists_calls.fetch_add(1, Ordering::SeqCst);
self.inner.exists(account, oid).await
}
}
let temp_dir = tempdir().unwrap();
let store = Arc::new(CountingStore {
inner: LocalObjectStore::new(temp_dir.path()),
puts: AtomicUsize::new(0),
exists_calls: AtomicUsize::new(0),
});
let data = b"duplicate blob content";
let kind = gix_object::Kind::Blob;
let oid1 = write_object_if_absent(store.as_ref(), "acct", kind, data)
.await
.unwrap();
assert_eq!(store.puts.load(Ordering::SeqCst), 1);
// Second call with identical data: should hit exists and skip put.
let oid2 = write_object_if_absent(store.as_ref(), "acct", kind, data)
.await
.unwrap();
assert_eq!(oid1, oid2);
assert_eq!(store.puts.load(Ordering::SeqCst), 1, "put must not be called again");
assert_eq!(store.exists_calls.load(Ordering::SeqCst), 2);
// Object is readable and oid matches compute_hash.
let raw = read_object(store.as_ref(), "acct", &oid1).await.unwrap();
let (parsed_kind, _size, offset) = parse_object_header(&raw).unwrap();
assert_eq!(parsed_kind, kind);
assert_eq!(&raw[offset..], data);
let expected = gix_object::compute_hash(gix_hash::Kind::Sha1, kind, data);
assert_eq!(oid1, expected);
}
}