Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
//! Persistence providers for V2 multi-write state.
use async_trait::async_trait;
use crate::core::errors::{Error, Result};
use crate::multibackend::codec::{decode_segment, sha256_hex};
use crate::multibackend::meta::MetadataStore;
use crate::multibackend::model::{
FlushResult, MarkerPosition, PartitionContext, PartitionState, PartitionsManifest,
PendingEvent, ScopeKey, SegmentDescriptor, SegmentManifest, SegmentRecord,
};
mod filesystem;
#[cfg(feature = "cache")]
mod cache;
pub use filesystem::FilesystemProvider;
#[cfg(feature = "cache")]
pub use cache::CacheProvider;
/// Persists and reads partitioned multi-write segment state.
#[async_trait]
pub trait MultiWriteProvider: Send + Sync {
/// Creates missing provider state without replacing an existing manifest.
async fn bootstrap_manifest(
&self,
scope: &ScopeKey,
manifest: SegmentManifest,
) -> Result<()>;
/// Flushes normal events after validating their current partition route.
async fn flush(
&self,
route_hint: Option<&PartitionContext>,
events: Vec<PendingEvent>,
) -> Result<FlushResult>;
/// Reads committed head records in the requested half-open sequence range.
async fn read_committed_head(
&self,
scope: &ScopeKey,
from_seq_inclusive: u64,
to_seq_exclusive: u64,
) -> Result<Vec<SegmentRecord>>;
/// Reads committed records across sealed segments and the current head.
async fn read_committed_range(
&self,
scope: &ScopeKey,
from_seq_inclusive: u64,
to_seq_exclusive: u64,
) -> Result<Vec<SegmentRecord>>;
/// Reads one immutable sealed segment named by a validated descriptor.
async fn read_sealed_segment(
&self,
scope: &ScopeKey,
descriptor: &SegmentDescriptor,
) -> Result<Vec<SegmentRecord>>;
/// Returns the current segment state for one scope.
async fn read_manifest(&self, scope: &ScopeKey) -> Result<SegmentManifest>;
/// Appends and seals one directory marker in the caller-selected scope.
async fn append_marker(
&self,
scope: &ScopeKey,
event: &PendingEvent,
) -> Result<MarkerPosition>;
/// Advances one backup's monotonic progress while its catch-up lease is held.
async fn advance_backend_state(
&self,
scope: &ScopeKey,
backend_id: &str,
synced_seq: u64,
) -> Result<()>;
/// Atomically removes selected sealed descriptors and returns the removed set.
async fn prune_sealed(
&self,
scope: &ScopeKey,
removable_paths: &[String],
) -> Result<Vec<SegmentDescriptor>>;
}
/// Bootstrap every stable provider partition from its raw-primary manifest.
pub(crate) async fn bootstrap_account(
provider: &dyn MultiWriteProvider,
store: &MetadataStore,
account_id: &str,
partitions: &PartitionsManifest,
) -> Result<()> {
for (&partition_id, partition) in &partitions.partitions {
if partition.state != PartitionState::Stable {
continue;
}
let scope = ScopeKey {
account_id: account_id.to_string(),
partition_id,
epoch: partitions.epoch,
};
let path = store.paths().segment_manifest(account_id, partition_id)?.0;
provider.bootstrap_manifest(&scope, store.read_json(&path).await?).await?;
}
Ok(())
}
/// Read and validate one immutable segment through the shared metadata store.
async fn read_sealed_segment(
store: &MetadataStore,
scope: &ScopeKey,
descriptor: &SegmentDescriptor,
) -> Result<Vec<SegmentRecord>> {
let directory = store
.paths()
.segments_dir(&scope.account_id, scope.partition_id)?
.0;
let bytes = store
.read_bytes(&format!("{directory}/{}", descriptor.path))
.await?;
if bytes.len() as u64 != descriptor.byte_len
|| sha256_hex(&bytes) != descriptor.checksum.as_deref().unwrap_or_default()
{
return Err(Error::Serialization(
"sealed segment bytes do not match descriptor".to_string(),
));
}
let decoded = decode_segment(&bytes)?;
if decoded.truncated_tail
|| decoded.valid_bytes != descriptor.byte_len
|| decoded.records.len() != descriptor.record_count as usize
|| decoded.records.iter().enumerate().any(|(index, record)| {
record.seq
!= descriptor
.segment_from_seq
.checked_add(index as u64)
.unwrap_or(u64::MAX)
})
{
return Err(Error::Serialization(
"sealed segment records do not match descriptor".to_string(),
));
}
Ok(decoded.records)
}
/// Select and validate one complete half-open sequence range.
fn select_record_range(
records: impl IntoIterator<Item = SegmentRecord>,
from_seq_inclusive: u64,
to_seq_exclusive: u64,
) -> Result<Vec<SegmentRecord>> {
let range_len = to_seq_exclusive
.checked_sub(from_seq_inclusive)
.ok_or_else(|| Error::invalid_operation("committed range is reversed"))?;
let expected_len = usize::try_from(range_len)
.map_err(|_| Error::invalid_operation("committed range is too large"))?;
let selected = records
.into_iter()
.filter(|record| {
record.seq >= from_seq_inclusive && record.seq < to_seq_exclusive
})
.collect::<Vec<_>>();
if selected.len() != expected_len
|| selected.iter().enumerate().any(|(index, record)| {
record.seq
!= from_seq_inclusive
.checked_add(index as u64)
.unwrap_or(u64::MAX)
})
{
return Err(Error::Serialization(
"committed record range is incomplete".to_string(),
));
}
Ok(selected)
}