:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! RocksDB wrapper: multi-CF atomic `WriteBatch`, async WAL.
// This module defines the disallowed `CohortStore` I/O methods, which call one another internally
// (e.g. `apply` replays through the `write_batch` path). The lint targets async callers, not the
// store's own implementation.
#![allow(clippy::disallowed_methods)]
use std::cell::Cell;
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::num::NonZeroUsize;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, RwLock};
use std::time::Instant;
use metrics::{counter, histogram};
use rocksdb::properties::{self, PropName};
use rocksdb::statistics::{StatsLevel, Ticker};
use rocksdb::{
BottommostLevelCompaction, Cache, ColumnFamily, CompactOptions, DBWithThreadMode, Direction,
FlushOptions, IteratorMode, Options, ReadOptions, SingleThreaded, WriteBatch, WriteOptions,
};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use tokio_util::sync::CancellationToken;
use tracing::{info, warn};
use cohort_core::partitioner::COHORT_PARTITION_COUNT;
use cohort_core::seed::CoverageStartMs;
use super::column_families::{self, Cf, OpaqueCf};
use super::coverage::{SliceCoverage, SliceTenure};
use super::keys::{
self, MergeAppliedKey, MergeDrainKey, PendingTransferKey, Stage2CohortPrefix, Stage2DirtyKey,
Stage2Key, Stage2TransferredRegisterKey, Stage2TransferredRegisterPersonPrefix, TombstoneKey,
STAGE2_DIRTY_KEY_LEN,
};
use super::keyspace::{
BehavioralKey, Keyspace, Meta, PersonPrefix, PersonRecordKey, SliceCoverageKey, SliceCoverages,
META_POD_PARTITIONS, META_SCHEMA_VERSION, META_SLICE_COVERAGE,
};
use super::staged::{StagedBatch, StagedOp};
use crate::observability::metrics::{
CHECKPOINT_DURATION_SECONDS, SLICES_ADOPTED_TOTAL, STAGE2_SCAN_UNDECODABLE_KEYS_TOTAL,
STORE_ERRORS_TOTAL, STORE_READS_TOTAL, STORE_READ_DURATION_SECONDS,
STORE_SCHEMA_MISMATCH_WIPES_TOTAL, STORE_WRITE_BATCH_TOTAL, STORE_WRITE_DURATION_SECONDS,
WAL_FSYNC_DURATION_SECONDS, WAL_FSYNC_ERRORS_TOTAL,
};
use crate::partitions::PartitionOwnership;
/// On-disk store schema version, stamped into `cf_meta` at first open and checked on every reopen.
/// A layout revision (key codec, value shape) that keeps the same CF set MUST bump this so an older
/// store fails fast instead of being misread; an on-disk CF *outside* `Cf::ALL` is caught independently
/// by `open_cf_descriptors` (see [`CohortStore::open_inner`]).
///
/// The same version is also stamped into checkpoint `metadata.json` and checked on restore
/// (`store/durability/recovery.rs`) with the live store's rule: a mismatched checkpoint blocks the
/// boot, unless `wipe_on_schema_mismatch` lets the restore pass over it as unusable. A future bump
/// must keep both guards in mind.
pub const STORE_SCHEMA_VERSION: u32 = 3;
const OP_OPEN: &str = "open";
const OP_DESTROY: &str = "destroy";
const OP_GET: &str = "get";
const OP_MULTI_GET: &str = "multi_get";
const OP_WRITE_BATCH: &str = "write_batch";
const OP_DELETE_PARTITION: &str = "delete_partition";
const OP_RESET_SLICES: &str = "reset_slices";
const OP_DB_IDENTITY: &str = "db_identity";
const OP_FLUSH: &str = "flush";
const OP_FLUSH_WAL: &str = "flush_wal";
const OP_SCAN: &str = "scan";
const OP_CHECKPOINT: &str = "checkpoint";
const DEFAULT_BLOCK_CACHE_BYTES: usize = 128 * 1024 * 1024;
const DEFAULT_WRITE_BUFFER_BYTES: usize = 64 * 1024 * 1024;
const DEFAULT_MAX_OPEN_FILES: i32 = 1024;
/// `get` is on the hot path, so sample the latency histogram 1-in-64 by default; the counter is exact.
const DEFAULT_READ_SAMPLE_RATIO: u32 = 64;
const DEFAULT_COMPACT_ON_DELETION_WINDOW: usize = 1000;
const DEFAULT_COMPACT_ON_DELETION_NUM_DELS_TRIGGER: usize = 500;
const DEFAULT_COMPACT_ON_DELETION_RATIO: f64 = 0.5;
/// RocksDB keeps the active info `LOG` and the newest rolled ones, `DEFAULT_KEEP_LOG_FILE_NUM` files
/// in total, so the info logs use at most about 10 × 256 MiB of the store volume.
const DEFAULT_MAX_LOG_FILE_SIZE_BYTES: NonZeroUsize = NonZeroUsize::new(256 * 1024 * 1024).unwrap();
const DEFAULT_KEEP_LOG_FILE_NUM: NonZeroUsize = NonZeroUsize::new(10).unwrap();
/// Resolved RocksDB settings.
#[derive(Debug, Clone)]
pub struct StoreConfig {
pub path: PathBuf,
/// Shared across all CFs; also holds index/filter blocks when `tuned_block_options` is set.
pub block_cache_bytes: usize,
pub write_buffer_bytes: usize,
pub max_open_files: i32,
pub create_if_missing: bool,
/// Destroy any existing database at `path` before opening.
pub wipe_on_start: bool,
/// On a schema-version mismatch at open, destroy and recreate the store instead of failing fast.
/// Off by default: a mismatch is a hard error so a stale store or checkpoint is never misread.
pub wipe_on_schema_mismatch: bool,
/// Enable RocksDB statistics so [`CohortStore::stats_snapshot`] reports live cache tickers; they
/// read 0 when off.
pub statistics_enabled: bool,
/// Sample 1-in-N reads into [`STORE_READ_DURATION_SECONDS`] (the read counter stays exact).
/// `1` records every read; clamped to `>= 1` at open.
pub read_sample_ratio: u32,
/// Cache and partition index/filter blocks so point lookups short-circuit on the bloom.
pub tuned_block_options: bool,
/// Mark tombstone-heavy SSTs for compaction.
pub compact_on_deletion: bool,
/// Window of recent entries the compact-on-deletion collector inspects.
pub compact_on_deletion_window: usize,
/// Tombstone count within the window that arms the collector.
pub compact_on_deletion_num_dels_trigger: usize,
/// Tombstone ratio that arms the collector; `<= 0` or `> 1` disables the ratio trigger.
pub compact_on_deletion_ratio: f64,
/// `0` disables it.
pub periodic_compaction_seconds: u64,
/// Non-positive leaves RocksDB's default untouched.
pub max_background_jobs: i32,
/// TTL in days for `cf_person_records`: a compaction filter drops a person record whose
/// `last_seen_ms` is older than this. `0` (the default) installs no filter. Attached to
/// `cf_person_records` **only** — never `cf_behavioral`, whose eviction deadlines are the sweep's
/// contract. See [`super::ttl_filter`].
pub person_record_ttl_days: u32,
/// The partitions this pod owns. The store records it, and refuses to open under a split its
/// record does not cover. A store that predates coverage records adopts all of its partitions
/// as complete.
pub ownership: PartitionOwnership,
/// Size at which RocksDB rolls the info `LOG` file. RocksDB reads `0` as "never roll", which
/// lets one info log grow without bound.
pub max_log_file_size_bytes: NonZeroUsize,
/// Number of info `LOG` files RocksDB keeps. RocksDB refuses to open with `0`.
pub keep_log_file_num: NonZeroUsize,
}
impl Default for StoreConfig {
fn default() -> Self {
Self {
path: PathBuf::from("cohort-store"),
block_cache_bytes: DEFAULT_BLOCK_CACHE_BYTES,
write_buffer_bytes: DEFAULT_WRITE_BUFFER_BYTES,
max_open_files: DEFAULT_MAX_OPEN_FILES,
create_if_missing: true,
wipe_on_start: false,
wipe_on_schema_mismatch: false,
statistics_enabled: true,
read_sample_ratio: DEFAULT_READ_SAMPLE_RATIO,
tuned_block_options: true,
compact_on_deletion: true,
compact_on_deletion_window: DEFAULT_COMPACT_ON_DELETION_WINDOW,
compact_on_deletion_num_dels_trigger: DEFAULT_COMPACT_ON_DELETION_NUM_DELS_TRIGGER,
compact_on_deletion_ratio: DEFAULT_COMPACT_ON_DELETION_RATIO,
periodic_compaction_seconds: 0,
max_background_jobs: 0,
person_record_ttl_days: 0,
ownership: PartitionOwnership::single(COHORT_PARTITION_COUNT as u16),
max_log_file_size_bytes: DEFAULT_MAX_LOG_FILE_SIZE_BYTES,
keep_log_file_num: DEFAULT_KEEP_LOG_FILE_NUM,
}
}
}
/// Errors from the state store.
#[derive(Debug, Error)]
pub enum StoreError {
#[error("opening RocksDB at {path:?}: {source}")]
Open {
path: PathBuf,
source: rocksdb::Error,
},
#[error("RocksDB {op}: {source}")]
Backend {
op: &'static str,
source: rocksdb::Error,
},
#[error("unknown column family: {0}")]
UnknownColumnFamily(&'static str),
#[error("decoding {kind} key: expected {expected} bytes, got {actual}")]
KeyDecode {
kind: &'static str,
expected: usize,
actual: usize,
},
#[error("decoding {kind} value: expected {expected} bytes, got {actual}")]
ValueDecode {
kind: &'static str,
expected: usize,
actual: usize,
},
/// A key of the right length that matches none of the keyspace's known literals (closed-set
/// keyspaces like `cf_meta`). Distinct from [`Self::KeyDecode`], which reports a length mismatch.
#[error("unknown {kind} key: matches no known literal")]
UnknownKey { kind: &'static str },
#[error(
"store schema mismatch: on-disk version {found:?} != expected {expected}; refusing to open \
(set COHORT_WIPE_ON_SCHEMA_MISMATCH=true to wipe and recreate)"
)]
SchemaMismatch { found: Option<u32>, expected: u32 },
/// The store's slices belong to a split that does not cover this pod's partitions, so its
/// slices for them may be missing or another pod's. `None` when the record is unreadable.
#[error(
"store ownership mismatch: the store was written as {}, which does not cover {expected}; \
refusing to open (wipe the store to start this pod's slices over)",
.found.map_or_else(|| "an unreadable ownership record".to_owned(), |found| found.to_string())
)]
OwnershipMismatch {
found: Option<PartitionOwnership>,
expected: PartitionOwnership,
},
/// A batched read answered fewer slots than it was asked keys. RocksDB answers one slot per
/// key, so this is a backend contract break rather than a data problem, and pairing a slot
/// with the wrong key would attribute one person's stored state to another.
#[error("{op}: batched read answered {answered} of {asked} keys")]
ShortRead {
op: &'static str,
asked: usize,
answered: usize,
},
#[error("store offload cancelled by runtime shutdown")]
OffloadCancelled,
}
/// The RocksDB DB id. A reopen and a checkpoint keep it; a created store gets a new one.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(transparent)]
pub struct DbIdentity(String);
#[cfg(test)]
impl DbIdentity {
pub(crate) fn for_test(id: &str) -> Self {
Self(id.to_owned())
}
}
/// One scanned key/value pair as raw bytes — the merge-CF GC decodes each per CF.
pub type RawKv = (Vec<u8>, Vec<u8>);
#[derive(Debug, Default)]
struct Stage2DirtyTracking {
/// Cohort prefixes with a reconcile drain currently capturing dirty rows, refcounted by lease.
///
/// No path holds two leases on one prefix today: the prefix carries `partition_id`, there is one
/// `ReconcileQueue` per partition worker, and only that queue's head is ever leased — kind-scoped
/// supersession puts a mixed cohort's two runs in the same queue, not in the same lease. Counting
/// rather than setting is what keeps this correct if that ever stops holding: the last release
/// wins instead of the first, so capture stays on for the surviving job rather than silently
/// dropping every concurrent mutation from its snapshot.
active: RwLock<HashMap<Stage2CohortPrefix, NonZeroUsize>>,
active_count: AtomicUsize,
}
impl Stage2DirtyTracking {
fn acquire(self: &Arc<Self>, prefix: Stage2CohortPrefix) -> Stage2DirtyTrackingGuard {
let mut active = self
.active
.write()
.expect("Stage 2 dirty-tracking lock is not held across fallible work");
active
.entry(prefix)
.and_modify(|leases| *leases = leases.saturating_add(1))
.or_insert(NonZeroUsize::MIN);
self.active_count.fetch_add(1, Ordering::Release);
Stage2DirtyTrackingGuard {
tracking: self.clone(),
prefix,
}
}
/// Cheap pre-check for the hot write path: whether any drain is capturing at all. A `false` lets
/// a reconcile-disabled pipeline skip decoding the Stage 2 key entirely.
fn any_active(&self) -> bool {
self.active_count.load(Ordering::Acquire) != 0
}
fn is_active(&self, prefix: Stage2CohortPrefix) -> bool {
if !self.any_active() {
return false;
}
self.active
.read()
.expect("Stage 2 dirty-tracking lock is not held across fallible work")
.contains_key(&prefix)
}
/// The dirty marker to stage for a `cf_stage2` write, or `None` when no drain tracks its cohort.
/// The single home of the "is this cohort captured / which marker do we write" decision, shared by
/// the live-event [`BatchBuilder`] path and the owned [`StagedBatch`] apply path so the rule for
/// emitting a marker cannot drift between them.
fn dirty_marker(&self, key: &Stage2Key) -> Option<[u8; STAGE2_DIRTY_KEY_LEN]> {
self.is_active(key.cohort_prefix())
.then(|| Stage2DirtyKey::new(*key).encode())
}
fn release(&self, prefix: Stage2CohortPrefix) {
let mut active = self
.active
.write()
.expect("Stage 2 dirty-tracking lock is not held across fallible work");
match active.entry(prefix) {
Entry::Occupied(mut entry) => match NonZeroUsize::new(entry.get().get() - 1) {
Some(remaining) => *entry.get_mut() = remaining,
None => {
entry.remove();
}
},
Entry::Vacant(_) => debug_assert!(false, "every guard owns one active lease"),
}
drop(active);
let previous = self.active_count.fetch_sub(1, Ordering::Release);
debug_assert!(previous > 0, "dirty-tracking reference count underflow");
}
}
/// In-memory lease enabling per-person dirty capture for one active reconcile cohort.
///
/// The lease is owned by the queued job. Dropping the job on completion, supersession, rebalance,
/// or shutdown disables future capture. Already-written markers are safe for GC to reclaim because
/// a replayed job starts from a full cohort scan.
#[must_use]
#[derive(Debug)]
pub struct Stage2DirtyTrackingGuard {
tracking: Arc<Stage2DirtyTracking>,
prefix: Stage2CohortPrefix,
}
impl Drop for Stage2DirtyTrackingGuard {
fn drop(&mut self) {
self.tracking.release(self.prefix);
}
}
/// One partitioned CF's range of a partition another pod owns, with its estimated SST size.
#[derive(Debug)]
struct UnownedRange {
cf: Cf,
partition: u16,
bytes: u64,
}
/// What one [`CohortStore::compact_unowned`] pass did. Sizes are RocksDB's estimates.
#[derive(Debug)]
pub struct UnownedCompaction {
/// The unowned ranges above the threshold when the pass began.
pub ranges: usize,
/// The bytes of those ranges.
pub bytes_before: u64,
/// The ranges compacted. Fewer than `ranges` when the stop fired.
pub compacted: usize,
/// The bytes of the ranges still above the threshold after the pass.
pub bytes_after: u64,
}
/// Handle to the per-process state store.
///
/// Writes have two layers. The closure-based [`Self::write_batch`] is for synchronous callers that
/// can borrow the store's CF handles for the duration of the call — it stages through a
/// [`BatchBuilder`] whose handles are tied to `&self`. [`StagedBatch`] plus [`Self::apply`] is the
/// owned staging path: keys and operands are encoded into owned bytes up front, so the batch is
/// `Send + 'static` and can be built on one thread and applied on another. Both funnel through the
/// same commit, so the two produce identical RocksDB writes.
#[derive(Clone)]
pub struct CohortStore {
db: Arc<DBWithThreadMode<SingleThreaded>>,
/// Retained so [`Self::stats_snapshot`] can read cache tickers: RocksDB shares one statistics
/// handle between these `Options` and the live DB, so reads here reflect ongoing activity.
db_opts: Arc<Options>,
/// See [`StoreConfig::read_sample_ratio`]; `>= 1`.
read_sample_ratio: u32,
dirty_tracking: Arc<Stage2DirtyTracking>,
}
impl CohortStore {
/// Open the column families at `config.path`, creating them if missing.
///
/// After open, the schema guard runs: a store that did not exist before this open is stamped with
/// [`STORE_SCHEMA_VERSION`]; an existing store's stamp is checked. A mismatch (or an absent stamp on
/// an existing store) is a hard [`StoreError::SchemaMismatch`] — unless `wipe_on_schema_mismatch`,
/// which destroys and recreates the store, then stamps the fresh one. A store carrying a CF outside
/// `Cf::ALL` never reaches this guard: it fails earlier in [`Self::open_inner`] (see there for the
/// extra-vs-subset asymmetry the version key exists to close).
///
/// An existing store without the coverage stamp adopts every partition as complete.
pub fn open(config: &StoreConfig) -> Result<Self, StoreError> {
let db_opts = db_options(config);
if config.wipe_on_start && config.path.exists() {
destroy_db(&db_opts, &config.path)?;
}
let mut path_existed = config.path.exists();
loop {
let store = Self::open_inner(config, &db_opts)?;
match store.check_schema(path_existed, config.wipe_on_schema_mismatch)? {
SchemaCheck::Ok => {
store.claim_ownership(config.ownership)?;
if path_existed {
store.settle_coverage(config.ownership.partition_count())?;
}
return Ok(store);
}
SchemaCheck::WipeAndRetry => {
// Drop the handle before destroy: RocksDB cannot destroy an open DB.
drop(store);
counter!(STORE_SCHEMA_MISMATCH_WIPES_TOTAL).increment(1);
warn!(
path = ?config.path,
expected = STORE_SCHEMA_VERSION,
"store schema mismatch with wipe-on-mismatch set: destroying and recreating",
);
destroy_db(&db_opts, &config.path)?;
// The recreated store is fresh, so the retry stamps it rather than re-checking.
path_existed = false;
}
}
}
}
/// Open the DB handle without the schema guard. Shared by the first open and the post-wipe reopen.
///
/// The descriptor list is the fixed `Cf::ALL` set, and this is where cross-era CF-set differences
/// are caught — but only in one direction, so the `cf_meta` version key is not redundant with it:
/// - A store with a CF *absent from* `Cf::ALL` (e.g. a pre-`cf_behavioral` store's `cf_stage1` /
/// `cf_person_index`) hard-fails here with a generic [`StoreError::Open`], before `check_schema`
/// and its `wipe_on_schema_mismatch` branch run. That is intentional: fail-fast and incapable of
/// misreading bytes. Such a store cannot exist once this schema is live (PoC posture wipes and
/// replays rather than migrates), so there is deliberately no legacy-CF upgrade path here.
/// - A store *missing* a CF that `Cf::ALL` adds (a subset) does NOT fail: `create_missing_column_
/// families(true)` creates it and the open succeeds. Same for an identical CF set with a changed
/// value layout. Those are exactly the cases the [`STORE_SCHEMA_VERSION`] stamp in `cf_meta`
/// guards against being silently misread.
fn open_inner(config: &StoreConfig, db_opts: &Options) -> Result<Self, StoreError> {
let cache = Cache::new_lru_cache(config.block_cache_bytes);
let descriptors = column_families::descriptors(config, &cache);
let db = DBWithThreadMode::<SingleThreaded>::open_cf_descriptors(
db_opts,
&config.path,
descriptors,
)
.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_OPEN).increment(1);
StoreError::Open {
path: config.path.clone(),
source,
}
})?;
Ok(Self::from_db(db, db_opts, config))
}
fn from_db(
db: DBWithThreadMode<SingleThreaded>,
db_opts: &Options,
config: &StoreConfig,
) -> Self {
Self {
db: Arc::new(db),
db_opts: Arc::new(db_opts.clone()),
// Floor at 1: `next % ratio` must not divide by zero.
read_sample_ratio: config.read_sample_ratio.max(1),
dirty_tracking: Arc::new(Stage2DirtyTracking::default()),
}
}
/// Check that the store at `path` is whole and of this build's schema, without writing to it: it
/// opens read-only with this build's column families and carries [`STORE_SCHEMA_VERSION`]. The
/// open's consistency check, on under RocksDB's default `paranoid_checks`, refuses a store
/// missing any SST its MANIFEST names or holding one at another size.
pub fn validate_read_only(path: &Path) -> Result<(), StoreError> {
let config = StoreConfig {
path: path.to_path_buf(),
..StoreConfig::default()
};
let db_opts = db_options(&config);
let cache = Cache::new_lru_cache(config.block_cache_bytes);
let db = DBWithThreadMode::<SingleThreaded>::open_cf_descriptors_read_only(
&db_opts,
path,
column_families::descriptors(&config, &cache),
false,
)
.map_err(|source| StoreError::Open {
path: path.to_path_buf(),
source,
})?;
let store = Self::from_db(db, &db_opts, &config);
store.check_schema(true, false)?;
Ok(())
}
pub fn db_identity(&self) -> Result<DbIdentity, StoreError> {
let id = self
.db
.get_db_identity()
.map_err(|source| StoreError::Backend {
op: OP_DB_IDENTITY,
source,
})?;
Ok(DbIdentity(String::from_utf8_lossy(&id).into_owned()))
}
/// Compare the `cf_meta` schema stamp against [`STORE_SCHEMA_VERSION`]. A fresh store (one that
/// did not exist before this open) is stamped and passes; an existing store must match. On
/// mismatch, either fail typed or signal a wipe-and-retry per `wipe_on_schema_mismatch`.
fn check_schema(
&self,
path_existed: bool,
wipe_on_schema_mismatch: bool,
) -> Result<SchemaCheck, StoreError> {
if !path_existed {
self.stamp_new_store()?;
return Ok(SchemaCheck::Ok);
}
let found = self
.get(Cf::Meta, META_SCHEMA_VERSION.0)?
.and_then(|bytes| {
bytes
.get(0..4)
.map(|b| u32::from_be_bytes([b[0], b[1], b[2], b[3]]))
});
if found == Some(STORE_SCHEMA_VERSION) {
Ok(SchemaCheck::Ok)
} else if wipe_on_schema_mismatch {
Ok(SchemaCheck::WipeAndRetry)
} else {
Err(StoreError::SchemaMismatch {
found,
expected: STORE_SCHEMA_VERSION,
})
}
}
/// Stamp a new store with the schema version (big-endian `u32`) and the coverage stamp.
fn stamp_new_store(&self) -> Result<(), StoreError> {
self.write_batch(|batch| {
batch.put::<Meta>(&META_SCHEMA_VERSION, &STORE_SCHEMA_VERSION.to_be_bytes());
batch.put::<Meta>(&META_SLICE_COVERAGE, &[]);
})
}
/// Records `ownership` in a store written under a split that covers it, and refuses any other
/// store. A covering split is this pod's own, or in a reshard the split that wrote the
/// parent's store or checkpoint. A store from before ownership records adopts the configured
/// one.
fn claim_ownership(&self, ownership: PartitionOwnership) -> Result<(), StoreError> {
if let Some(recorded) = self.get(Meta::CF, &Meta::encode(&META_POD_PARTITIONS))? {
match PartitionOwnership::decode(&recorded) {
Some(found) if found == ownership => return Ok(()),
Some(found) if found.covers(&ownership) => {}
found => {
return Err(StoreError::OwnershipMismatch {
found,
expected: ownership,
})
}
}
}
self.write_batch(|batch| {
batch.put::<Meta>(&META_POD_PARTITIONS, &ownership.encode());
})?;
info!(%ownership, "recorded the store's partition ownership");
Ok(())
}
/// Adopt every partition as complete in a store without the coverage stamp. The stamp rides the
/// same batch, so adoption happens once.
fn settle_coverage(&self, partition_count: u16) -> Result<(), StoreError> {
if self
.get(Meta::CF, &Meta::encode(&META_SLICE_COVERAGE))?
.is_some()
{
return Ok(());
}
let complete = SliceCoverage::Complete.encode();
self.write_batch(|batch| {
for partition_id in 0..partition_count {
batch.put::<SliceCoverages>(&SliceCoverageKey(partition_id), &complete);
}
batch.put::<Meta>(&META_SLICE_COVERAGE, &[]);
})?;
counter!(SLICES_ADOPTED_TOTAL).increment(u64::from(partition_count));
info!(
partitions = partition_count,
"store predates slice coverage records; adopted every partition as complete",
);
Ok(())
}
/// `None` when the slice has no readable record.
pub fn slice_coverage(&self, partition_id: u16) -> Result<Option<SliceCoverage>, StoreError> {
let key = SliceCoverages::encode(&SliceCoverageKey(partition_id));
let Some(value) = self.get(SliceCoverages::CF, &key)? else {
return Ok(None);
};
let coverage = SliceCoverage::decode(&value);
if coverage.is_none() {
warn!(
partition_id,
"undecodable slice coverage record; the slice begins again"
);
}
Ok(coverage)
}
/// Begins the slice at `start` when it has no record. The commit loop's fsync makes the write
/// durable before any offset commits.
pub fn resume_or_begin_slice(
&self,
partition_id: u16,
start: CoverageStartMs,
) -> Result<SliceTenure, StoreError> {
if let Some(coverage) = self.slice_coverage(partition_id)? {
return Ok(SliceTenure::Resumed(coverage));
}
self.write_batch(|batch| {
batch.put::<SliceCoverages>(
&SliceCoverageKey(partition_id),
&SliceCoverage::Since(start).encode(),
);
})?;
Ok(SliceTenure::Begun(start))
}
/// Read a raw value from any CF.
pub fn get(&self, cf: Cf, key: &[u8]) -> Result<Option<Vec<u8>>, StoreError> {
let handle = self.cf(cf)?;
// Sample 1-in-N; unsampled reads skip even `Instant::now()`. The counter below stays exact.
let started = should_sample_read(self.read_sample_ratio).then(Instant::now);
let result = self.db.get_cf(handle, key);
if let Some(started) = started {
histogram!(STORE_READ_DURATION_SECONDS, "op" => OP_GET)
.record(started.elapsed().as_secs_f64());
}
counter!(STORE_READS_TOTAL, "op" => OP_GET).increment(1);
result.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_GET).increment(1);
StoreError::Backend { op: OP_GET, source }
})
}
pub fn get_behavioral(&self, key: &BehavioralKey) -> Result<Option<Vec<u8>>, StoreError> {
self.get(Cf::Behavioral, &key.encode())
}
/// One batched read of `cf`, preserving input order. Every typed `multi_get_*` routes through
/// it, so none can skip the empty-batch or answer-length checks or label its errors differently.
fn multi_get_ordered<K, B: AsRef<[u8]>>(
&self,
cf: Cf,
keys: &[K],
encode: impl Fn(&K) -> B,
) -> Result<Vec<Option<Vec<u8>>>, StoreError> {
// An empty batch is not a read: skip it so it records no phantom read-latency sample.
if keys.is_empty() {
return Ok(Vec::new());
}
let handle = self.cf(cf)?;
let encoded: Vec<B> = keys.iter().map(encode).collect();
let started = Instant::now();
let results = self
.db
.multi_get_cf(encoded.iter().map(|key| (handle, key.as_ref())));
record_multi_get(started, keys.len());
if results.len() != keys.len() {
counter!(STORE_ERRORS_TOTAL, "op" => OP_MULTI_GET).increment(1);
return Err(StoreError::ShortRead {
op: OP_MULTI_GET,
asked: keys.len(),
answered: results.len(),
});
}
results
.into_iter()
.map(|result| {
result.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_MULTI_GET).increment(1);
StoreError::Backend {
op: OP_MULTI_GET,
source,
}
})
})
.collect()
}
/// Batch-read several `cf_behavioral` values in one call, preserving input order.
pub fn multi_get_behavioral(
&self,
keys: &[BehavioralKey],
) -> Result<Vec<Option<Vec<u8>>>, StoreError> {
self.multi_get_ordered(Cf::Behavioral, keys, BehavioralKey::encode)
}
/// Batch-read several `cf_person_records` values in one call, preserving input order.
pub fn multi_get_person_records(
&self,
keys: &[PersonRecordKey],
) -> Result<Vec<Option<Vec<u8>>>, StoreError> {
self.multi_get_ordered(Cf::PersonRecords, keys, PersonRecordKey::encode)
}
/// Batch-read several `cf_merge_tombstones` values in one call, preserving input order.
pub fn multi_get_tombstones(
&self,
keys: &[TombstoneKey],
) -> Result<Vec<Option<Vec<u8>>>, StoreError> {
self.multi_get_ordered(Cf::MergeTombstones, keys, TombstoneKey::encode)
}
/// Point-read one person's `cf_person_records` value as raw bytes. Decoding into a
/// [`PersonRecord`](crate::stage1::PersonRecord) lives with the caller.
pub fn get_person_record(&self, key: &PersonRecordKey) -> Result<Option<Vec<u8>>, StoreError> {
self.get(Cf::PersonRecords, &key.encode())
}
/// Read one event's full state snapshot in a single mixed-CF `multi_get`: the `behavioral` keys
/// (in order) plus, when `record` is given, the person's `cf_person_records` key as the final
/// lookup.
///
/// The result preserves order: `behavioral[i]` corresponds to the i-th requested behavioral key,
/// and `record` is `Some(_)` iff a record key was requested (`Some(None)` = requested but absent).
/// An empty behavioral set with no record key reads nothing.
pub fn read_event_snapshot(
&self,
behavioral: &[BehavioralKey],
record: Option<&PersonRecordKey>,
) -> Result<EventSnapshotRaw, StoreError> {
// Nothing to read: skip so no phantom read-latency sample is recorded.
if behavioral.is_empty() && record.is_none() {
return Ok(EventSnapshotRaw {
behavioral: Vec::new(),
record: None,
});
}
let behavioral_handle = self.cf(Cf::Behavioral)?;
let behavioral_encoded: Vec<_> = behavioral.iter().map(BehavioralKey::encode).collect();
// Behavioral handles first, the record handle last; `multi_get_cf` preserves this order, so the
// record's result is the trailing slot.
let mut pairs: Vec<(&ColumnFamily, &[u8])> = behavioral_encoded
.iter()
.map(|key| (behavioral_handle, key.as_slice()))
.collect();
let record_encoded = record.map(PersonRecordKey::encode);
if let Some(encoded) = record_encoded.as_ref() {
let record_handle = self.cf(Cf::PersonRecords)?;
pairs.push((record_handle, encoded.as_slice()));
}
let started = Instant::now();
let results = self.db.multi_get_cf(pairs);
record_multi_get(started, results.len());
let mut decoded: Vec<Option<Vec<u8>>> = Vec::with_capacity(results.len());
for result in results {
decoded.push(result.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_MULTI_GET).increment(1);
StoreError::Backend {
op: OP_MULTI_GET,
source,
}
})?);
}
let record_slot = if record.is_some() {
Some(
decoded
.pop()
.expect("a requested record key yields a result slot"),
)
} else {
None
};
Ok(EventSnapshotRaw {
behavioral: decoded,
record: record_slot,
})
}
pub fn get_stage2(&self, key: &Stage2Key) -> Result<Option<Vec<u8>>, StoreError> {
self.get(Cf::Stage2, &key.encode())
}
/// Enable coalescing dirty-person capture for one active reconcile cohort until the returned
/// lease is dropped. This is process-local by design: a crash replays the held reconcile tile
/// and starts a fresh full scan, so correctness never depends on preserving an inactive marker.
pub fn track_stage2_dirty(&self, prefix: Stage2CohortPrefix) -> Stage2DirtyTrackingGuard {
self.dirty_tracking.acquire(prefix)
}
/// Whether this process currently owns a reconcile scan for `prefix`.
///
/// Stage 2 GC uses this to distinguish live dirty work from metadata left by a completed,
/// discarded, or rebalanced job. Partition-worker serialization prevents a lease transition
/// from racing the corresponding GC tick.
pub(crate) fn is_stage2_dirty_tracking_active(&self, prefix: Stage2CohortPrefix) -> bool {
self.dirty_tracking.is_active(prefix)
}
pub fn get_stage2_transferred_register(
&self,
key: &Stage2TransferredRegisterKey,
) -> Result<Option<Vec<u8>>, StoreError> {
self.get(Cf::Stage2, &key.encode())
}
/// Batch-read transferred-register inventory values, preserving input order.
pub fn multi_get_stage2_transferred_registers(
&self,
keys: &[Stage2TransferredRegisterKey],
) -> Result<Vec<Option<Vec<u8>>>, StoreError> {
if keys.is_empty() {
return Ok(Vec::new());
}
let handle = self.cf(Cf::Stage2)?;
let encoded: Vec<_> = keys.iter().map(|key| key.encode()).collect();
let started = Instant::now();
let results = self
.db
.multi_get_cf(encoded.iter().map(|key| (handle, key.as_slice())));
record_multi_get(started, keys.len());
results
.into_iter()
.map(|result| {
result.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_MULTI_GET).increment(1);
StoreError::Backend {
op: OP_MULTI_GET,
source,
}
})
})
.collect()
}
/// Batch-read several `cf_stage2` values in one call, preserving input order.
pub fn multi_get_stage2(&self, keys: &[Stage2Key]) -> Result<Vec<Option<Vec<u8>>>, StoreError> {
self.multi_get_ordered(Cf::Stage2, keys, Stage2Key::encode)
}
/// Scan up to `limit` keys in one partition/team/cohort slice of `cf_stage2`, in person order,
/// resuming strictly after `start_after` when it decodes as a key in that slice.
///
/// Corrupt keys are counted and skipped. The scan continues until it collects `limit` valid
/// keys or exhausts the prefix, so a short result proves there are no later decodable rows.
pub fn scan_stage2_cohort(
&self,
prefix: Stage2CohortPrefix,
start_after: Option<&[u8]>,
limit: usize,
) -> Result<Vec<Stage2Key>, StoreError> {
if limit == 0 {
return Ok(Vec::new());
}
let (prefix_start, prefix_end) = prefix.range();
let begin = match start_after {
Some(cursor)
if cursor >= prefix_start.as_slice()
&& cursor < prefix_end.as_slice()
&& Stage2Key::decode(cursor).is_ok() =>
{
successor(cursor)
}
_ => prefix_start,
};
let handle = self.cf(Cf::Stage2)?;
let mut read_opts = ReadOptions::default();
read_opts.set_iterate_upper_bound(prefix_end);
let iter = self.db.iterator_cf_opt(
handle,
read_opts,
IteratorMode::From(&begin, Direction::Forward),
);
let mut out = Vec::with_capacity(limit.min(1024));
for item in iter {
if out.len() == limit {
break;
}
let (key_bytes, _value) = item.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_SCAN).increment(1);
StoreError::Backend {
op: OP_SCAN,
source,
}
})?;
if Stage2DirtyKey::decode(&key_bytes).is_ok()
|| Stage2TransferredRegisterKey::decode(&key_bytes).is_ok()
{
// Metadata is the partition tail, so no Stage 2 row can follow it. This is reachable
// only for the all-ones row prefix; ordinary cohort ranges end earlier.
break;
}
match Stage2Key::decode(&key_bytes) {
Ok(key) => out.push(key),
Err(_) => counter!(STAGE2_SCAN_UNDECODABLE_KEYS_TOTAL).increment(1),
}
}
Ok(out)
}
/// Scan up to `limit` dirty-person markers for one Stage 2 cohort, resuming strictly after a
/// marker in that same prefix. A malformed key inside the reserved namespace is an error: the
/// reconciler must stop rather than mistake corruption for an empty dirty set.
pub fn scan_stage2_dirty(
&self,
prefix: Stage2CohortPrefix,
start_after: Option<&[u8]>,
limit: usize,
) -> Result<Vec<Stage2DirtyKey>, StoreError> {
if limit == 0 {
return Ok(Vec::new());
}
let dirty_prefix = prefix.dirty_prefix();
let (prefix_start, prefix_end) = dirty_prefix.range();
let begin = match start_after {
Some(cursor)
if cursor >= prefix_start.as_slice()
&& cursor < prefix_end.as_slice()
&& Stage2DirtyKey::decode(cursor).is_ok() =>
{
successor(cursor)
}
_ => prefix_start,
};
let handle = self.cf(Cf::Stage2)?;
let mut read_opts = ReadOptions::default();
read_opts.set_iterate_upper_bound(prefix_end);
let iter = self.db.iterator_cf_opt(
handle,
read_opts,
IteratorMode::From(&begin, Direction::Forward),
);
let mut out = Vec::with_capacity(limit.min(1024));
for item in iter.take(limit) {
let (key_bytes, _value) = item.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_SCAN).increment(1);
StoreError::Backend {
op: OP_SCAN,
source,
}
})?;
out.push(Stage2DirtyKey::decode(&key_bytes)?);
}
Ok(out)
}
/// Scan the catalog-independent transferred-register inventory for one person.
pub fn scan_stage2_transferred_registers(
&self,
prefix: Stage2TransferredRegisterPersonPrefix,
) -> Result<Vec<(Stage2TransferredRegisterKey, Vec<u8>)>, StoreError> {
let (prefix_start, prefix_end) = prefix.range();
let handle = self.cf(Cf::Stage2)?;
let mut read_opts = ReadOptions::default();
read_opts.set_iterate_upper_bound(prefix_end);
let iter = self.db.iterator_cf_opt(
handle,
read_opts,
IteratorMode::From(&prefix_start, Direction::Forward),
);
let mut out = Vec::new();
for item in iter {
let (key_bytes, value) = item.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_SCAN).increment(1);
StoreError::Backend {
op: OP_SCAN,
source,
}
})?;
out.push((
Stage2TransferredRegisterKey::decode(&key_bytes)?,
value.to_vec(),
));
}
Ok(out)
}
/// Apply writes across CFs in one atomic `WriteBatch`.
pub fn write_batch<F>(&self, build: F) -> Result<(), StoreError>
where
F: FnOnce(&mut BatchBuilder<'_>),
{
let mut builder = BatchBuilder {
batch: WriteBatch::default(),
behavioral: self.cf(Cf::Behavioral)?,
person_records: self.cf(Cf::PersonRecords)?,
stage2: self.cf(Cf::Stage2)?,
merge_drains_applied: self.cf(Cf::MergeDrainsApplied)?,
pending_transfers: self.cf(Cf::PendingTransfers)?,
merge_applied: self.cf(Cf::MergeApplied)?,
merge_tombstones: self.cf(Cf::MergeTombstones)?,
meta: self.cf(Cf::Meta)?,
dirty_tracking: &self.dirty_tracking,
};
build(&mut builder);
self.commit(builder.batch, OP_WRITE_BATCH)
}
/// Replay an owned [`StagedBatch`] into one atomic `WriteBatch`, in staging order, and commit it.
///
/// This is the owned counterpart to [`Self::write_batch`]: the same operations staged either way
/// produce identical writes. It goes through the same commit funnel with the same op label, so
/// metrics do not distinguish the two paths.
pub fn apply(&self, staged: &StagedBatch) -> Result<(), StoreError> {
let mut batch = WriteBatch::default();
for op in staged.ops() {
match op {
StagedOp::Put { cf, key, value } => {
batch.put_cf(self.cf(*cf)?, key, value);
self.mark_staged_stage2_dirty(&mut batch, *cf, key);
}
StagedOp::Delete { cf, key } => {
batch.delete_cf(self.cf(*cf)?, key);
self.mark_staged_stage2_dirty(&mut batch, *cf, key);
}
}
}
self.commit(batch, OP_WRITE_BATCH)
}
fn mark_staged_stage2_dirty(&self, batch: &mut WriteBatch, cf: Cf, encoded_key: &[u8]) {
if !matches!(cf, Cf::Stage2) {
return;
}
// Skip decoding the key on the hot write path when no reconcile drain is capturing dirties.
if !self.dirty_tracking.any_active() {
return;
}
let Ok(key) = Stage2Key::decode(encoded_key) else {
return;
};
if let Some(marker) = self.dirty_tracking.dirty_marker(&key) {
batch.put_cf(
self.cf(Cf::Stage2)
.expect("the Stage 2 CF was opened before applying a batch"),
marker,
[],
);
}
}
pub fn get_merge_drain_applied(
&self,
key: &MergeDrainKey,
) -> Result<Option<Vec<u8>>, StoreError> {
self.get(Cf::MergeDrainsApplied, &key.encode())
}
pub fn get_pending_transfer(
&self,
key: &PendingTransferKey,
) -> Result<Option<Vec<u8>>, StoreError> {
self.get(Cf::PendingTransfers, &key.encode())
}
pub fn get_merge_applied(&self, key: &MergeAppliedKey) -> Result<Option<Vec<u8>>, StoreError> {
self.get(Cf::MergeApplied, &key.encode())
}
pub fn get_tombstone(&self, key: &TombstoneKey) -> Result<Option<Vec<u8>>, StoreError> {
self.get(Cf::MergeTombstones, &key.encode())
}
/// Clear one outbox slot once its transfer is acked.
pub fn clear_pending_transfer(&self, key: &PendingTransferKey) -> Result<(), StoreError> {
self.write_batch(|batch| batch.delete_pending_transfer(key))
}
/// Scan up to `limit` of one partition's `cf_pending_transfers` slice, returning `(key, value)`
/// in key order, resuming strictly *after* `start_after` (exclusive) when given. The per-tick
/// redrive passes a bounded `limit` and no cursor; the eager boot redrive paginates with a cursor
/// to drain the whole outbox (mirrors [`Self::scan_behavioral`] / [`Self::scan_merge_cf`]).
pub fn scan_pending_transfers(
&self,
partition_id: u16,
start_after: Option<&[u8]>,
limit: usize,
) -> Result<Vec<(PendingTransferKey, Vec<u8>)>, StoreError> {
let (prefix_start, prefix_end) = keys::partition_range(partition_id);
let handle = self.cf(Cf::PendingTransfers)?;
// Resume after the cursor when it falls inside this partition, else at the prefix start.
let begin: Vec<u8> = match start_after {
Some(cursor) if cursor >= prefix_start.as_slice() && cursor < prefix_end.as_slice() => {
successor(cursor)
}
_ => prefix_start.clone(),
};
let mut read_opts = ReadOptions::default();
read_opts.set_iterate_upper_bound(prefix_end);
let iter = self.db.iterator_cf_opt(
handle,
read_opts,
IteratorMode::From(&begin, Direction::Forward),
);
let mut out = Vec::with_capacity(limit.min(1024));
for item in iter {
if out.len() == limit {
break;
}
let (key_bytes, value) = item.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_SCAN).increment(1);
StoreError::Backend {
op: OP_SCAN,
source,
}
})?;
out.push((PendingTransferKey::decode(&key_bytes)?, value.to_vec()));
}
Ok(out)
}
/// Scan up to `limit` raw `(key, value)` pairs from one partition's slice of a CF, in key order,
/// resuming strictly *after* `start_after` (exclusive) when given.
///
/// Returns raw bytes (not typed keys/values) so each GC handler can decode the CF's own value
/// shape and keep the last key as its resume cursor. The mechanics are CF-generic: the merge-CF GC
/// passes the three time-stamped merge CFs (`DrainStamp` / `ApplyStamp` / `Tombstone`, see
/// [`crate::merge::gc`]), and the `cf_stage2` orphan GC passes `Cf::Stage2` (see
/// [`crate::workers::stage2_gc`]). `cf_pending_transfers` is the redrive's outbox and no GC path
/// passes it.
pub fn scan_merge_cf(
&self,
cf: Cf,
partition_id: u16,
start_after: Option<&[u8]>,
limit: usize,
) -> Result<Vec<RawKv>, StoreError> {
let (prefix_start, prefix_end) = keys::partition_range(partition_id);
let handle = self.cf(cf)?;
// Resume after the cursor when it falls inside this partition, else at the prefix start.
// `successor` is the smallest key strictly greater than the cursor, turning the inclusive
// `IteratorMode::From(_, Forward)` seek into an exclusive resume.
let begin: Vec<u8> = match start_after {
Some(cursor) if cursor >= prefix_start.as_slice() && cursor < prefix_end.as_slice() => {
successor(cursor)
}
_ => prefix_start.clone(),
};
let mut read_opts = ReadOptions::default();
read_opts.set_iterate_upper_bound(prefix_end);
let iter = self.db.iterator_cf_opt(
handle,
read_opts,
IteratorMode::From(&begin, Direction::Forward),
);
let mut out = Vec::with_capacity(limit.min(1024));
for item in iter {
if out.len() == limit {
break;
}
let (key_bytes, value) = item.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_SCAN).increment(1);
StoreError::Backend {
op: OP_SCAN,
source,
}
})?;
out.push((key_bytes.to_vec(), value.to_vec()));
}
Ok(out)
}
/// Reclaim all state for one partition on rebalance, and return whether it held any key.
/// Non-partitioned CFs (`cf_meta`) are skipped: their short literal keys collide with an
/// arbitrary partition's byte range, so a range delete would wipe store-wide guards like the
/// schema stamp.
///
/// The SST files that lie wholly inside the partition are dropped only after the range deletes
/// are durable. A file that straddles the partition boundary can hold older versions of keys
/// that a dropped file overwrote, and only the range deletes keep those versions hidden.
/// RocksDB syncs a file drop to its MANIFEST at once, so the WAL fsync comes first: a host crash
/// must not keep the drop and lose the range deletes, because the partition can be owned again
/// later.
pub fn delete_partition(&self, partition_id: u16) -> Result<bool, StoreError> {
let held_keys = self.slice_holds_keys(partition_id)?;
let mut batch = WriteBatch::default();
self.delete_partition_ranges(&mut batch, partition_id)?;
self.commit(batch, OP_DELETE_PARTITION)?;
self.flush_wal_sync()?;
let (start, end) = keys::partition_range(partition_id);
for cf in Cf::ALL {
if !cf.partitioned() {
continue;
}
self.db
.delete_file_in_range_cf(self.cf(cf)?, &start, &end)
.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_DELETE_PARTITION).increment(1);
StoreError::Backend {
op: OP_DELETE_PARTITION,
source,
}
})?;
}
Ok(held_keys)
}
/// Total-order seek, because `cf_behavioral`'s prefix extractor would otherwise end the seek at
/// the first person boundary.
fn slice_holds_keys(&self, partition_id: u16) -> Result<bool, StoreError> {
let (start, end) = keys::partition_range(partition_id);
for cf in Cf::ALL {
if !cf.partitioned() {
continue;
}
let mut read_opts = ReadOptions::default();
read_opts.set_iterate_upper_bound(end.clone());
read_opts.set_total_order_seek(true);
let first = self
.db
.iterator_cf_opt(
self.cf(cf)?,
read_opts,
IteratorMode::From(&start, Direction::Forward),
)
.next()
.transpose()
.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_SCAN).increment(1);
StoreError::Backend {
op: OP_SCAN,
source,
}
})?;
if first.is_some() {
return Ok(true);
}
}
Ok(false)
}
/// Delete the slices a checkpoint restore cannot replay, with their coverage records, so each
/// begins again behind the coverage fence. The fsync makes the deletes outlive a crash: a
/// restore ends by deleting its marker, and a reset still in the async WAL would come back with
/// a `Complete` coverage record and positions past the events it lost.
pub fn reset_slices(
&self,
partitions: impl IntoIterator<Item = u16>,
) -> Result<(), StoreError> {
let mut batch = WriteBatch::default();
for partition_id in partitions {
self.delete_partition_ranges(&mut batch, partition_id)?;
}
self.commit(batch, OP_RESET_SLICES)?;
self.flush_wal_sync()
}
fn delete_partition_ranges(
&self,
batch: &mut WriteBatch,
partition_id: u16,
) -> Result<(), StoreError> {
let (start, end) = keys::partition_range(partition_id);
for cf in Cf::ALL {
if !cf.partitioned() {
continue;
}
batch.delete_range_cf(self.cf(cf)?, start.as_slice(), end.as_slice());
}
Ok(())
}
/// Compact through the bottom level every range of a partition outside `ownership` that holds
/// more than `threshold` bytes of SST data in a partitioned CF, which drops the data its range
/// deletes cover. A manual compaction cannot be interrupted, so the ranges compact one at a
/// time, and none starts after `stop` fires.
pub fn compact_unowned(
&self,
ownership: &PartitionOwnership,
threshold: u64,
stop: &CancellationToken,
) -> Result<UnownedCompaction, StoreError> {
let ranges = self.unowned_ranges_above(ownership, threshold)?;
let mut opts = CompactOptions::default();
opts.set_bottommost_level_compaction(BottommostLevelCompaction::ForceOptimized);
let mut compacted = 0;
for range in &ranges {
if stop.is_cancelled() {
break;
}
let (start, end) = keys::partition_range(range.partition);
self.db
.compact_range_cf_opt(self.cf(range.cf)?, Some(start), Some(end), &opts);
compacted += 1;
}
// rust-rocksdb discards the compaction's error, so only a second measure shows what the
// pass dropped.
let bytes_after = self
.unowned_ranges_above(ownership, threshold)?
.iter()
.map(|range| range.bytes)
.sum();
Ok(UnownedCompaction {
ranges: ranges.len(),
bytes_before: ranges.iter().map(|range| range.bytes).sum(),
compacted,
bytes_after,
})
}
/// The ranges of the partitions outside `ownership` that hold more than `threshold` bytes of
/// SST data in a partitioned CF. RocksDB estimates the size from index blocks, and range deletes
/// do not shrink the estimate, so it counts deleted data that no compaction has dropped yet.
fn unowned_ranges_above(
&self,
ownership: &PartitionOwnership,
threshold: u64,
) -> Result<Vec<UnownedRange>, StoreError> {
let partitions: Vec<u16> = (0..ownership.partition_count())
.filter(|&partition| !ownership.owns(partition))
.collect();
let bounds: Vec<(Vec<u8>, Vec<u8>)> = partitions
.iter()
.map(|&partition| keys::partition_range(partition))
.collect();
let ranges: Vec<rocksdb::Range<'_>> = bounds
.iter()
.map(|(start, end)| rocksdb::Range::new(start, end))
.collect();
let mut above = Vec::new();
for cf in Cf::ALL {
if !cf.partitioned() {
continue;
}
let sizes = self.db.get_approximate_sizes_cf(self.cf(cf)?, &ranges);
above.extend(
partitions
.iter()
.zip(sizes)
.filter(|&(_, bytes)| bytes > threshold)
.map(|(&partition, bytes)| UnownedRange {
cf,
partition,
bytes,
}),
);
}
Ok(above)
}
/// Flush all CF memtables to SST. Checkpoint path and tests only.
pub fn flush(&self) -> Result<(), StoreError> {
let handles: Vec<&ColumnFamily> = Cf::ALL
.iter()
.map(|&cf| self.cf(cf))
.collect::<Result<_, _>>()?;
let mut flush_opts = FlushOptions::default();
flush_opts.set_wait(true);
self.db
.flush_cfs_opt(&handles, &flush_opts)
.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_FLUSH).increment(1);
StoreError::Backend {
op: OP_FLUSH,
source,
}
})
}
/// Synchronously fsync the WAL, making every write so far durable. The hot path writes with an
/// async WAL (`set_sync(false)`), so without this a committed Kafka offset could outrun the durable
/// state on a hard crash; callers fsync before every offset commit to keep `committed <= durable`.
/// On error the caller skips the commit (fail-stop).
pub fn flush_wal_sync(&self) -> Result<(), StoreError> {
let started = Instant::now();
let result = self.db.flush_wal(true);
histogram!(WAL_FSYNC_DURATION_SECONDS).record(started.elapsed().as_secs_f64());
result.map_err(|source| {
counter!(WAL_FSYNC_ERRORS_TOTAL).increment(1);
StoreError::Backend {
op: OP_FLUSH_WAL,
source,
}
})
}
/// Take a frozen, point-in-time RocksDB checkpoint of the whole DB into `path`. RocksDB
/// hard-links the immutable SSTs into `path`, so `path` must be on the **same filesystem** as
/// the store path and **must not be a child** of it (RocksDB refuses a nested checkpoint path).
pub fn create_checkpoint(&self, path: &Path) -> Result<(), StoreError> {
let started = Instant::now();
let result = rocksdb::checkpoint::Checkpoint::new(&self.db)
.and_then(|cp| cp.create_checkpoint(path));
histogram!(CHECKPOINT_DURATION_SECONDS).record(started.elapsed().as_secs_f64());
result.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_CHECKPOINT).increment(1);
StoreError::Backend {
op: OP_CHECKPOINT,
source,
}
})
}
/// Scan up to `limit` of one partition's `cf_behavioral` slice as `(BehavioralKey, raw_value)` in
/// key order, resuming strictly after `start_after` when given. The value stays raw so the caller
/// decodes the [`StatefulRecord`](crate::stage1::StatefulRecord) (and owns the decode-error metric)
/// and the resume cursor is the last scanned key.
///
/// This is a partition-wide scan that crosses many 26-byte person prefixes. `cf_behavioral` has a
/// fixed-prefix extractor, so the iterator defaults to prefix-seek mode and would silently stop at
/// the first person boundary; `set_total_order_seek(true)` forces a full-order iteration across all
/// prefixes.
pub fn scan_behavioral(
&self,
partition_id: u16,
start_after: Option<&[u8]>,
limit: usize,
) -> Result<Vec<(BehavioralKey, Vec<u8>)>, StoreError> {
let (prefix_start, prefix_end) = keys::partition_range(partition_id);
let handle = self.cf(Cf::Behavioral)?;
// Resume after the cursor when it falls inside this partition, else at the prefix start.
let begin: Vec<u8> = match start_after {
Some(cursor) if cursor >= prefix_start.as_slice() && cursor < prefix_end.as_slice() => {
successor(cursor)
}
_ => prefix_start.clone(),
};
let mut read_opts = ReadOptions::default();
read_opts.set_iterate_upper_bound(prefix_end);
read_opts.set_total_order_seek(true);
let iter = self.db.iterator_cf_opt(
handle,
read_opts,
IteratorMode::From(&begin, Direction::Forward),
);
let mut out = Vec::with_capacity(limit.min(1024));
for item in iter {
if out.len() == limit {
break;
}
let (key_bytes, value) = item.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_SCAN).increment(1);
StoreError::Backend {
op: OP_SCAN,
source,
}
})?;
out.push((BehavioralKey::decode(&key_bytes)?, value.to_vec()));
}
Ok(out)
}
/// Scan one person's whole `cf_behavioral` slice as `(BehavioralKey, raw_value)` in lsk order.
///
/// Bounded to the person's 26-byte prefix by both the iterate-upper-bound (the prefix successor)
/// and `set_prefix_same_as_start(true)`, which pins the iterator to the seek key's prefix so it can
/// never leak into an adjacent person's rows. Used by the merge drain to enumerate P_old's leaves.
pub fn scan_behavioral_prefix(
&self,
prefix: PersonPrefix,
) -> Result<Vec<(BehavioralKey, Vec<u8>)>, StoreError> {
let (start, end) = prefix.scan_range();
let handle = self.cf(Cf::Behavioral)?;
let mut read_opts = ReadOptions::default();
read_opts.set_iterate_upper_bound(end);
read_opts.set_prefix_same_as_start(true);
let iter = self.db.iterator_cf_opt(
handle,
read_opts,
IteratorMode::From(&start, Direction::Forward),
);
let mut out = Vec::new();
for item in iter {
let (key_bytes, value) = item.map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_SCAN).increment(1);
StoreError::Backend {
op: OP_SCAN,
source,
}
})?;
out.push((BehavioralKey::decode(&key_bytes)?, value.to_vec()));
}
Ok(out)
}
/// Snapshot the block-cache tickers and per-CF size properties. Tickers are cumulative since store
/// open and read 0 when `statistics_enabled` is off; per-CF properties read 0 when RocksDB has no
/// estimate yet (a fresh, never-written CF).
pub fn stats_snapshot(&self) -> StoreStats {
let ticker = |t: Ticker| self.db_opts.get_ticker_count(t);
StoreStats {
block_cache_hits: ticker(Ticker::BlockCacheHit),
block_cache_misses: ticker(Ticker::BlockCacheMiss),
block_cache_data_hits: ticker(Ticker::BlockCacheDataHit),
block_cache_data_misses: ticker(Ticker::BlockCacheDataMiss),
block_cache_index_hits: ticker(Ticker::BlockCacheIndexHit),
block_cache_index_misses: ticker(Ticker::BlockCacheIndexMiss),
block_cache_filter_hits: ticker(Ticker::BlockCacheFilterHit),
block_cache_filter_misses: ticker(Ticker::BlockCacheFilterMiss),
bloom_filter_useful: ticker(Ticker::BloomFilterUseful),
// The block cache is shared across every CF, so any CF handle reports the same usage.
block_cache_usage_bytes: self
.cf_property_u64(Cf::Behavioral, properties::BLOCK_CACHE_USAGE),
per_cf: Cf::ALL
.iter()
.map(|&cf| CfStats {
cf,
sst_bytes: self.cf_property_u64(cf, properties::TOTAL_SST_FILES_SIZE),
live_data_bytes: self.cf_property_u64(cf, properties::ESTIMATE_LIVE_DATA_SIZE),
num_keys: self.cf_property_u64(cf, properties::ESTIMATE_NUM_KEYS),
})
.collect(),
}
}
/// Read an integer RocksDB property for one CF; an absent handle, error, or missing estimate all
/// fold to 0.
fn cf_property_u64(&self, cf: Cf, name: &PropName) -> u64 {
let Ok(handle) = self.cf(cf) else {
return 0;
};
self.db
.property_int_value_cf(handle, name)
.ok()
.flatten()
.unwrap_or(0)
}
fn cf(&self, cf: Cf) -> Result<&ColumnFamily, StoreError> {
self.db
.cf_handle(cf.as_str())
.ok_or(StoreError::UnknownColumnFamily(cf.as_str()))
}
fn commit(&self, batch: WriteBatch, op: &'static str) -> Result<(), StoreError> {
let mut write_opts = WriteOptions::default();
write_opts.set_sync(false);
let started = Instant::now();
let result = self.db.write_opt(batch, &write_opts);
histogram!(STORE_WRITE_DURATION_SECONDS, "op" => op)
.record(started.elapsed().as_secs_f64());
match result {
Ok(()) => {
counter!(STORE_WRITE_BATCH_TOTAL, "op" => op).increment(1);
Ok(())
}
Err(source) => {
counter!(STORE_ERRORS_TOTAL, "op" => op).increment(1);
Err(StoreError::Backend { op, source })
}
}
}
}
/// Snapshot of the store's cache tickers and per-CF sizes, produced by
/// [`CohortStore::stats_snapshot`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StoreStats {
pub block_cache_hits: u64,
pub block_cache_misses: u64,
pub block_cache_data_hits: u64,
pub block_cache_data_misses: u64,
pub block_cache_index_hits: u64,
pub block_cache_index_misses: u64,
pub block_cache_filter_hits: u64,
pub block_cache_filter_misses: u64,
/// Point lookups the bloom filter let skip a data-block read.
pub bloom_filter_useful: u64,
pub block_cache_usage_bytes: u64,
/// One entry per [`Cf::ALL`].
pub per_cf: Vec<CfStats>,
}
/// Per-column-family size properties. See [`StoreStats`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CfStats {
pub cf: Cf,
pub sst_bytes: u64,
/// Estimated live (non-tombstone) data bytes.
pub live_data_bytes: u64,
/// Estimated key count (memtable-inclusive, so non-zero before a flush).
pub num_keys: u64,
}
/// The raw bytes of one event's state snapshot from a single mixed-CF `multi_get`, produced by
/// [`CohortStore::read_event_snapshot`]. Decoding lives with the caller.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct EventSnapshotRaw {
/// One slot per requested behavioral key, in request order; `None` = absent row.
pub behavioral: Vec<Option<Vec<u8>>>,
/// The person-record slot: outer `None` = no record key was requested; `Some(None)` = requested
/// but absent; `Some(Some(bytes))` = present.
pub record: Option<Option<Vec<u8>>>,
}
/// Typed builder for a multi-CF [`WriteBatch`].
pub struct BatchBuilder<'db> {
batch: WriteBatch,
behavioral: &'db ColumnFamily,
person_records: &'db ColumnFamily,
stage2: &'db ColumnFamily,
merge_drains_applied: &'db ColumnFamily,
pending_transfers: &'db ColumnFamily,
merge_applied: &'db ColumnFamily,
merge_tombstones: &'db ColumnFamily,
meta: &'db ColumnFamily,
dirty_tracking: &'db Stage2DirtyTracking,
}
impl<'db> BatchBuilder<'db> {
/// The held handle for one CF, returned at the store's `'db` lifetime rather than `&self` so it
/// does not alias the `&mut self.batch` callers immediately take.
fn handle(&self, cf: Cf) -> &'db ColumnFamily {
match cf {
Cf::Behavioral => self.behavioral,
Cf::PersonRecords => self.person_records,
Cf::Stage2 => self.stage2,
Cf::MergeDrainsApplied => self.merge_drains_applied,
Cf::PendingTransfers => self.pending_transfers,
Cf::MergeApplied => self.merge_applied,
Cf::MergeTombstones => self.merge_tombstones,
Cf::Meta => self.meta,
}
}
/// Put a typed key/value into its keyspace's CF. The [`Keyspace`] binding routes to the right CF,
/// so a key cannot be written to the wrong column family.
pub fn put<K: Keyspace>(&mut self, key: &K::Key, value: &[u8]) {
let handle = self.handle(K::CF);
self.batch.put_cf(handle, K::encode(key), value);
}
/// Delete a typed key from its keyspace's CF.
pub fn delete<K: Keyspace>(&mut self, key: &K::Key) {
let handle = self.handle(K::CF);
self.batch.delete_cf(handle, K::encode(key));
}
/// Range-delete one person's whole `cf_behavioral` slice in a single tombstone, reclaiming every
/// leaf under the person prefix at once (used when a merge drains P_old's state).
pub fn delete_behavioral_prefix(&mut self, prefix: &PersonPrefix) {
let (start, end) = prefix.scan_range();
self.batch
.delete_range_cf(self.behavioral, start.as_slice(), end.as_slice());
}
pub fn put_stage2(&mut self, key: &Stage2Key, value: &[u8]) {
self.batch.put_cf(self.stage2, key.encode(), value);
self.mark_stage2_dirty(key);
}
pub fn delete_stage2(&mut self, key: &Stage2Key) {
self.batch.delete_cf(self.stage2, key.encode());
self.mark_stage2_dirty(key);
}
/// Clear a reconcile dirty marker without creating another marker.
pub fn delete_stage2_dirty(&mut self, key: &Stage2DirtyKey) {
self.batch.delete_cf(self.stage2, key.encode());
}
/// Preserve a merge-carried register independently of the receiver's current catalog.
pub fn put_stage2_transferred_register(
&mut self,
key: &Stage2TransferredRegisterKey,
value: &[u8],
) {
self.batch.put_cf(self.stage2, key.encode(), value);
}
pub fn delete_stage2_transferred_register(&mut self, key: &Stage2TransferredRegisterKey) {
self.batch.delete_cf(self.stage2, key.encode());
}
/// Stage the Phase 1 idempotence marker for a drained merge message.
pub fn put_merge_drain_applied(&mut self, key: &MergeDrainKey, value: &[u8]) {
self.batch
.put_cf(self.merge_drains_applied, key.encode(), value);
}
/// GC-delete one expired Phase 1 idempotence marker.
pub fn delete_merge_drain_applied(&mut self, key: &MergeDrainKey) {
self.batch
.delete_cf(self.merge_drains_applied, key.encode());
}
/// Stage a packaged merge into the outbox.
pub fn put_pending_transfer(&mut self, key: &PendingTransferKey, value: &[u8]) {
self.batch
.put_cf(self.pending_transfers, key.encode(), value);
}
/// Clear an outbox slot once its transfer is acked.
pub fn delete_pending_transfer(&mut self, key: &PendingTransferKey) {
self.batch.delete_cf(self.pending_transfers, key.encode());
}
/// Stage the Phase 2 idempotence marker for an applied transfer message.
pub fn put_merge_applied(&mut self, key: &MergeAppliedKey, value: &[u8]) {
self.batch.put_cf(self.merge_applied, key.encode(), value);
}
/// GC-delete one expired Phase 2 idempotence marker.
pub fn delete_merge_applied(&mut self, key: &MergeAppliedKey) {
self.batch.delete_cf(self.merge_applied, key.encode());
}
/// Stage the redirect tombstone for a merged-away person.
pub fn put_tombstone(&mut self, key: &TombstoneKey, value: &[u8]) {
self.batch
.put_cf(self.merge_tombstones, key.encode(), value);
}
/// GC-delete one expired redirect tombstone.
pub fn delete_tombstone(&mut self, key: &TombstoneKey) {
self.batch.delete_cf(self.merge_tombstones, key.encode());
}
/// Raw put by pre-encoded key bytes. Restricted to [`OpaqueCf`].
pub fn put_raw(&mut self, cf: OpaqueCf, key: &[u8], value: &[u8]) {
self.batch.put_cf(self.handle(cf.cf()), key, value);
if matches!(cf, OpaqueCf::Stage2) {
if let Ok(key) = Stage2Key::decode(key) {
self.mark_stage2_dirty(&key);
}
}
}
fn mark_stage2_dirty(&mut self, key: &Stage2Key) {
if let Some(marker) = self.dirty_tracking.dirty_marker(key) {
self.batch.put_cf(self.stage2, marker, []);
}
}
}
thread_local! {
/// Thread-local to avoid cross-worker contention; each worker samples independently (~1-in-N).
static READ_SAMPLE_COUNTER: Cell<u32> = const { Cell::new(0) };
}
/// Fires once per `ratio` calls. Count-based, so sampled quantiles stay unbiased; `ratio >= 1`.
fn should_sample_read(ratio: u32) -> bool {
READ_SAMPLE_COUNTER.with(|counter| {
let next = counter.get().wrapping_add(1);
counter.set(next);
next % ratio == 0
})
}
/// Records one duration sample and `key_count` logical reads for a `multi_get` (a batch touches
/// `key_count` keys).
fn record_multi_get(started: Instant, key_count: usize) {
histogram!(STORE_READ_DURATION_SECONDS, "op" => OP_MULTI_GET)
.record(started.elapsed().as_secs_f64());
counter!(STORE_READS_TOTAL, "op" => OP_MULTI_GET).increment(key_count as u64);
}
/// The smallest byte string strictly greater than `key`: `key` with a trailing `0x00` appended.
/// Used to turn an inclusive `IteratorMode::From` seek into an exclusive resume past a cursor.
fn successor(key: &[u8]) -> Vec<u8> {
let mut next = Vec::with_capacity(key.len() + 1);
next.extend_from_slice(key);
next.push(0x00);
next
}
/// Outcome of the open-time schema-version check.
enum SchemaCheck {
/// The stamp matched (or a fresh store was just stamped).
Ok,
/// A mismatch under `wipe_on_schema_mismatch`: the caller destroys and reopens fresh.
WipeAndRetry,
}
/// Destroy the store at `path`, counting a metric on failure. Requires no open handle.
fn destroy_db(db_opts: &Options, path: &Path) -> Result<(), StoreError> {
DBWithThreadMode::<SingleThreaded>::destroy(db_opts, path).map_err(|source| {
counter!(STORE_ERRORS_TOTAL, "op" => OP_DESTROY).increment(1);
StoreError::Open {
path: path.to_path_buf(),
source,
}
})
}
fn db_options(config: &StoreConfig) -> Options {
let mut opts = Options::default();
opts.create_if_missing(config.create_if_missing);
opts.create_missing_column_families(true);
opts.set_atomic_flush(true);
opts.set_max_open_files(config.max_open_files);
opts.set_allow_mmap_reads(false);
opts.set_allow_mmap_writes(false);
if config.statistics_enabled {
// `ExceptHistogramOrTimers` keeps the cheap cache tickers and drops the expensive per-op
// histograms and timers.
opts.enable_statistics();
opts.set_statistics_level(StatsLevel::ExceptHistogramOrTimers);
}
if config.max_background_jobs > 0 {
opts.set_max_background_jobs(config.max_background_jobs);
}
opts.set_max_log_file_size(config.max_log_file_size_bytes.get());
opts.set_keep_log_file_num(config.keep_log_file_num.get());
opts
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
use uuid::Uuid;
use super::super::keyspace::{Behavioral, PersonRecords};
use crate::stage1::key::LeafStateKey;
use crate::store::durability::PodOrdinal;
fn behavioral_key() -> BehavioralKey {
BehavioralKey::new(3, 7, Uuid::from_u128(1), LeafStateKey([0xAB; 16]))
}
fn record_key(partition: u16, person: u128) -> PersonRecordKey {
PersonRecordKey::new(partition, 7, Uuid::from_u128(person))
}
fn tombstone_key(person: u128) -> TombstoneKey {
TombstoneKey {
partition_id: 3,
team_id: 7,
person: Uuid::from_u128(person),
}
}
#[test]
fn wipe_on_start_clears_existing_state_and_is_a_noop_when_off() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let key = behavioral_key();
// Seed a value, then close the database (drop the only handle).
{
let store = CohortStore::open(&StoreConfig {
path: path.clone(),
..StoreConfig::default()
})
.unwrap();
store
.write_batch(|b| b.put::<Behavioral>(&key, b"state"))
.unwrap();
assert_eq!(
store.get_behavioral(&key).unwrap().as_deref(),
Some(b"state".as_slice()),
);
}
// Reopen without wiping: the value survives a restart.
{
let store = CohortStore::open(&StoreConfig {
path: path.clone(),
wipe_on_start: false,
..StoreConfig::default()
})
.unwrap();
assert_eq!(
store.get_behavioral(&key).unwrap().as_deref(),
Some(b"state".as_slice()),
);
}
// Reopen with wiping: the previous owner's state is gone.
{
let store = CohortStore::open(&StoreConfig {
path: path.clone(),
wipe_on_start: true,
..StoreConfig::default()
})
.unwrap();
assert_eq!(store.get_behavioral(&key).unwrap(), None);
}
}
#[test]
fn wipe_on_start_opens_cleanly_when_no_store_exists_yet() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("fresh"),
wipe_on_start: true,
..StoreConfig::default()
})
.unwrap();
assert_eq!(store.get_behavioral(&behavioral_key()).unwrap(), None);
}
#[test]
fn fresh_store_stamps_the_schema_version_and_a_reopen_matches() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
{
let store = CohortStore::open(&StoreConfig {
path: path.clone(),
..StoreConfig::default()
})
.unwrap();
assert_eq!(
store
.get(Cf::Meta, META_SCHEMA_VERSION.0)
.unwrap()
.as_deref(),
Some(STORE_SCHEMA_VERSION.to_be_bytes().as_slice()),
"a fresh store stamps the current schema version",
);
}
// A reopen matches the stamp and opens cleanly.
CohortStore::open(&StoreConfig {
path,
wipe_on_start: false,
..StoreConfig::default()
})
.unwrap();
}
#[test]
fn a_version_mismatch_fails_typed_and_the_wipe_flag_recreates_empty() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let key = behavioral_key();
// Seed a store and corrupt its stamp to a bogus version.
{
let store = CohortStore::open(&StoreConfig {
path: path.clone(),
..StoreConfig::default()
})
.unwrap();
store
.write_batch(|b| b.put::<Behavioral>(&key, b"state"))
.unwrap();
store
.write_batch(|b| b.put::<Meta>(&META_SCHEMA_VERSION, &999u32.to_be_bytes()))
.unwrap();
}
// Without the wipe flag, a mismatch is a typed hard error and the store is untouched.
// (`CohortStore` is not `Debug`, so match the `Result` rather than `unwrap_err`.)
let result = CohortStore::open(&StoreConfig {
path: path.clone(),
wipe_on_start: false,
..StoreConfig::default()
});
assert!(
matches!(
result,
Err(StoreError::SchemaMismatch {
found: Some(999),
expected,
}) if expected == STORE_SCHEMA_VERSION
),
"expected a typed schema mismatch",
);
// With the wipe flag, the store is destroyed, recreated, and re-stamped — the old row is gone.
let store = CohortStore::open(&StoreConfig {
path,
wipe_on_start: false,
wipe_on_schema_mismatch: true,
..StoreConfig::default()
})
.unwrap();
assert_eq!(
store.get_behavioral(&key).unwrap(),
None,
"wipe-on-mismatch recreates an empty store",
);
assert_eq!(
store
.get(Cf::Meta, META_SCHEMA_VERSION.0)
.unwrap()
.as_deref(),
Some(STORE_SCHEMA_VERSION.to_be_bytes().as_slice()),
"the recreated store is re-stamped with the current version",
);
}
#[test]
fn multi_get_behavioral_preserves_order_and_reports_absent_keys() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let present = |person: u128, lsk: u8| {
BehavioralKey::new(3, 7, Uuid::from_u128(person), LeafStateKey([lsk; 16]))
};
let a = present(1, 0xA0);
let b = present(2, 0xB0);
let absent = present(9, 0xFF);
store
.write_batch(|batch| {
batch.put::<Behavioral>(&a, b"alpha");
batch.put::<Behavioral>(&b, b"bravo");
})
.unwrap();
// Order: present, absent, present — the absent key must surface as a `None` hole, not shift
// the others.
let results = store.multi_get_behavioral(&[a, absent, b]).unwrap();
assert_eq!(results.len(), 3);
assert_eq!(results[0].as_deref(), Some(b"alpha".as_slice()));
assert_eq!(results[1], None);
assert_eq!(results[2].as_deref(), Some(b"bravo".as_slice()));
assert!(
store.multi_get_behavioral(&[]).unwrap().is_empty(),
"an empty key set reads no values",
);
}
/// A hole that shifted the answers would pair one person's record with another person's key,
/// folding a seed onto the wrong state.
#[test]
fn multi_get_person_records_preserves_order_and_reports_absent_keys() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let a = record_key(3, 1);
let b = record_key(3, 2);
let absent = record_key(3, 9);
store
.write_batch(|batch| {
batch.put::<PersonRecords>(&a, b"alpha");
batch.put::<PersonRecords>(&b, b"bravo");
})
.unwrap();
let results = store.multi_get_person_records(&[a, absent, b]).unwrap();
assert_eq!(results.len(), 3);
assert_eq!(results[0].as_deref(), Some(b"alpha".as_slice()));
assert_eq!(results[1], None);
assert_eq!(results[2].as_deref(), Some(b"bravo".as_slice()));
assert!(store.multi_get_person_records(&[]).unwrap().is_empty());
}
/// A shifted answer here would route a seed through another person's merge chain.
#[test]
fn multi_get_tombstones_preserves_order_and_reports_absent_keys() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let a = tombstone_key(1);
let b = tombstone_key(2);
let absent = tombstone_key(9);
store
.write_batch(|batch| {
batch.put_tombstone(&a, b"alpha");
batch.put_tombstone(&b, b"bravo");
})
.unwrap();
let results = store.multi_get_tombstones(&[a, absent, b]).unwrap();
assert_eq!(results.len(), 3);
assert_eq!(results[0].as_deref(), Some(b"alpha".as_slice()));
assert_eq!(results[1], None);
assert_eq!(results[2].as_deref(), Some(b"bravo".as_slice()));
assert!(store.multi_get_tombstones(&[]).unwrap().is_empty());
}
#[test]
fn person_record_get_round_trips_through_its_cf() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let key = record_key(3, 1);
assert_eq!(
store.get_person_record(&key).unwrap(),
None,
"absent before write"
);
store
.write_batch(|b| b.put::<PersonRecords>(&key, b"record-bytes"))
.unwrap();
assert_eq!(
store.get_person_record(&key).unwrap().as_deref(),
Some(b"record-bytes".as_slice()),
);
}
#[test]
fn read_event_snapshot_aligns_behavioral_hits_misses_and_the_record_slot() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let b = |person: u128, lsk: u8| {
BehavioralKey::new(3, 7, Uuid::from_u128(person), LeafStateKey([lsk; 16]))
};
let present_a = b(1, 0xA0);
let present_b = b(1, 0xB0);
let absent = b(9, 0xFF);
let record = record_key(3, 1);
store
.write_batch(|batch| {
batch.put::<Behavioral>(&present_a, b"alpha");
batch.put::<Behavioral>(&present_b, b"bravo");
batch.put::<PersonRecords>(&record, b"rec");
})
.unwrap();
// Behavioral order present/absent/present, plus the record present.
let snap = store
.read_event_snapshot(&[present_a, absent, present_b], Some(&record))
.unwrap();
assert_eq!(snap.behavioral.len(), 3);
assert_eq!(snap.behavioral[0].as_deref(), Some(b"alpha".as_slice()));
assert_eq!(
snap.behavioral[1], None,
"the absent behavioral key is a hole"
);
assert_eq!(snap.behavioral[2].as_deref(), Some(b"bravo".as_slice()));
assert_eq!(snap.record, Some(Some(b"rec".to_vec())), "record present");
// Record requested but absent ⇒ Some(None); behavioral all present.
let absent_record = record_key(3, 999);
let snap = store
.read_event_snapshot(&[present_a], Some(&absent_record))
.unwrap();
assert_eq!(snap.behavioral.len(), 1);
assert_eq!(snap.record, Some(None), "requested but absent record");
// Record not requested ⇒ outer None; behavioral only.
let snap = store.read_event_snapshot(&[present_a], None).unwrap();
assert_eq!(snap.behavioral.len(), 1);
assert_eq!(snap.record, None, "no record key requested");
// Record-only (empty behavioral).
let snap = store.read_event_snapshot(&[], Some(&record)).unwrap();
assert!(snap.behavioral.is_empty());
assert_eq!(snap.record, Some(Some(b"rec".to_vec())));
// Empty both ⇒ no read, empty result.
let snap = store.read_event_snapshot(&[], None).unwrap();
assert!(snap.behavioral.is_empty());
assert_eq!(snap.record, None);
}
#[test]
fn delete_partition_wipes_person_records_for_that_partition_only() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let victim = record_key(5, 1);
let survivor = record_key(6, 1);
store
.write_batch(|batch| {
batch.put::<PersonRecords>(&victim, b"v");
batch.put::<PersonRecords>(&survivor, b"s");
})
.unwrap();
store.delete_partition(5).unwrap();
assert_eq!(
store.get_person_record(&victim).unwrap(),
None,
"partition 5's record is reclaimed",
);
assert_eq!(
store.get_person_record(&survivor).unwrap().as_deref(),
Some(b"s".as_slice()),
"partition 6's record is untouched",
);
}
#[test]
fn multi_get_stage2_preserves_order_and_reports_absent_keys() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let present = |person: u128, cohort: u64| Stage2Key {
partition_id: 3,
team_id: 7,
cohort_id: cohort,
person_id: Uuid::from_u128(person),
};
let a = present(1, 100);
let b = present(2, 200);
let absent = present(9, 999);
store
.write_batch(|batch| {
batch.put_stage2(&a, b"alpha");
batch.put_stage2(&b, b"bravo");
})
.unwrap();
// Order: present, absent, present — the absent key must surface as a `None` hole, not shift
// the others.
let results = store.multi_get_stage2(&[a, absent, b]).unwrap();
assert_eq!(results.len(), 3);
assert_eq!(results[0].as_deref(), Some(b"alpha".as_slice()));
assert_eq!(results[1], None);
assert_eq!(results[2].as_deref(), Some(b"bravo".as_slice()));
assert!(
store.multi_get_stage2(&[]).unwrap().is_empty(),
"an empty key set reads no values",
);
}
#[test]
fn stage2_mutations_atomically_coalesce_one_dirty_marker_per_person() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let key = |person_id: u128, cohort_id: u64| Stage2Key {
partition_id: 3,
team_id: 7,
cohort_id,
person_id: Uuid::from_u128(person_id),
};
let watched = key(1, 100);
let prefix = watched.cohort_prefix();
let _tracking = store.track_stage2_dirty(prefix);
assert!(store
.scan_stage2_dirty(prefix, None, 10)
.unwrap()
.is_empty());
store
.write_batch(|batch| {
batch.put_stage2(&watched, b"member");
batch.put_stage2(&watched, b"member-again");
batch.put_stage2(&key(2, 100), b"second person");
batch.put_stage2(&key(1, 101), b"other cohort");
})
.unwrap();
assert_eq!(
store
.scan_stage2_dirty(prefix, None, 10)
.unwrap()
.into_iter()
.map(Stage2DirtyKey::stage2_key)
.collect::<Vec<_>>(),
vec![watched, key(2, 100)],
"repeated writes coalesce and another cohort stays outside the prefix",
);
let mut staged = StagedBatch::default();
staged.delete_stage2(&watched);
store.apply(&staged).unwrap();
assert!(store.get_stage2(&watched).unwrap().is_none());
assert_eq!(
store.scan_stage2_dirty(prefix, None, 10).unwrap().len(),
2,
"a delete retains the coalesced marker even though the row is gone",
);
store
.write_batch(|batch| {
batch.delete_stage2_dirty(&Stage2DirtyKey::new(watched));
batch.delete_stage2_dirty(&Stage2DirtyKey::new(key(2, 100)));
})
.unwrap();
assert!(store
.scan_stage2_dirty(prefix, None, 10)
.unwrap()
.is_empty());
}
#[test]
fn stage2_dirty_capture_is_scoped_to_the_tracking_lease() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let key = Stage2Key {
partition_id: 3,
team_id: 7,
cohort_id: 100,
person_id: Uuid::from_u128(1),
};
let prefix = key.cohort_prefix();
store
.write_batch(|batch| batch.put_stage2(&key, b"before"))
.unwrap();
assert!(store
.scan_stage2_dirty(prefix, None, 10)
.unwrap()
.is_empty());
let tracking = store.track_stage2_dirty(prefix);
let mut staged = StagedBatch::default();
staged.put_stage2(&key, b"during");
store.apply(&staged).unwrap();
assert_eq!(store.scan_stage2_dirty(prefix, None, 10).unwrap().len(), 1);
store
.write_batch(|batch| batch.delete_stage2_dirty(&Stage2DirtyKey::new(key)))
.unwrap();
drop(tracking);
store
.write_batch(|batch| batch.put_stage2(&key, b"after"))
.unwrap();
assert!(
store
.scan_stage2_dirty(prefix, None, 10)
.unwrap()
.is_empty(),
"idle Stage 2 writes do not accumulate permanent dirty metadata",
);
}
#[test]
fn stage2_dirty_capture_survives_until_the_last_lease_releases() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let key = Stage2Key {
partition_id: 3,
team_id: 7,
cohort_id: 100,
person_id: Uuid::from_u128(1),
};
let prefix = key.cohort_prefix();
// No production path holds two leases on one prefix today, but the refcount is what keeps
// capture on for a surviving lease if that ever changes — the last release must win, not
// the first.
let first = store.track_stage2_dirty(prefix);
let second = store.track_stage2_dirty(prefix);
drop(first);
store
.write_batch(|batch| batch.put_stage2(&key, b"during"))
.unwrap();
assert_eq!(
store.scan_stage2_dirty(prefix, None, 10).unwrap().len(),
1,
"capture stays on while any lease is held",
);
drop(second);
store
.write_batch(|batch| {
batch.delete_stage2_dirty(&Stage2DirtyKey::new(key));
batch.put_stage2(&key, b"after");
})
.unwrap();
assert!(
store
.scan_stage2_dirty(prefix, None, 10)
.unwrap()
.is_empty(),
"the last release disables capture",
);
}
#[test]
fn scan_stage2_dirty_is_bounded_and_resumable() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let key = |person_id: u128| Stage2Key {
partition_id: 3,
team_id: 7,
cohort_id: 100,
person_id: Uuid::from_u128(person_id),
};
let prefix = key(1).cohort_prefix();
let _tracking = store.track_stage2_dirty(prefix);
store
.write_batch(|batch| {
for person_id in 1..=4 {
batch.put_stage2(&key(person_id), b"member");
}
})
.unwrap();
let first = store.scan_stage2_dirty(prefix, None, 2).unwrap();
assert_eq!(
first
.iter()
.map(|dirty| dirty.stage2_key().person_id)
.collect::<Vec<_>>(),
vec![Uuid::from_u128(1), Uuid::from_u128(2)],
);
let cursor = first.last().unwrap().encode();
let second = store.scan_stage2_dirty(prefix, Some(&cursor), 2).unwrap();
assert_eq!(
second
.iter()
.map(|dirty| dirty.stage2_key().person_id)
.collect::<Vec<_>>(),
vec![Uuid::from_u128(3), Uuid::from_u128(4)],
);
let cursor = second.last().unwrap().encode();
assert!(store
.scan_stage2_dirty(prefix, Some(&cursor), 2)
.unwrap()
.is_empty());
}
#[test]
fn transferred_register_inventory_scans_one_person_in_cohort_order() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("transferred-register-scan"),
..StoreConfig::default()
})
.unwrap();
let person_id = Uuid::from_u128(7);
let prefix = Stage2TransferredRegisterPersonPrefix::new(3, 7, person_id);
store
.write_batch(|batch| {
for cohort_id in [9, 2, 5] {
batch.put_stage2_transferred_register(
&Stage2TransferredRegisterKey::new(Stage2Key {
partition_id: 3,
team_id: 7,
cohort_id,
person_id,
}),
&[cohort_id as u8],
);
}
batch.put_stage2_transferred_register(
&Stage2TransferredRegisterKey::new(Stage2Key {
partition_id: 3,
team_id: 7,
cohort_id: 1,
person_id: Uuid::from_u128(8),
}),
b"foreign",
);
})
.unwrap();
let rows = store.scan_stage2_transferred_registers(prefix).unwrap();
assert_eq!(
rows.iter()
.map(|(key, value)| (key.stage2_key().cohort_id, value.clone()))
.collect::<Vec<_>>(),
vec![(2, vec![2]), (5, vec![5]), (9, vec![9])],
);
}
#[test]
fn scan_stage2_cohort_is_bounded_resumable_and_skips_corrupt_keys() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let prefix = Stage2CohortPrefix {
partition_id: 3,
team_id: 7,
cohort_id: 100,
};
let key = |partition_id: u16, team_id: u64, cohort_id: u64, person: u128| Stage2Key {
partition_id,
team_id,
cohort_id,
person_id: Uuid::from_u128(person),
};
store
.write_batch(|batch| {
for person in 1..=4 {
batch.put_stage2(&key(3, 7, 100, person), b"member");
}
batch.put_stage2(&key(3, 7, 101, 1), b"other-cohort");
batch.put_stage2(&key(3, 8, 100, 1), b"other-team");
batch.put_stage2(&key(4, 7, 100, 1), b"other-partition");
})
.unwrap();
// This malformed key is inside the prefix range and sorts before person 1. It must not
// consume the valid-key page limit or make the scan fail.
let mut malformed = prefix.encode().to_vec();
malformed.extend_from_slice(&[0; 15]);
store
.db
.put_cf(store.cf(Cf::Stage2).unwrap(), malformed, b"corrupt-prefix")
.unwrap();
let mut malformed_tail = prefix.encode().to_vec();
malformed_tail.extend_from_slice(Uuid::from_u128(5).as_bytes());
malformed_tail.push(0);
store
.db
.put_cf(
store.cf(Cf::Stage2).unwrap(),
&malformed_tail,
b"corrupt-tail",
)
.unwrap();
let first = store.scan_stage2_cohort(prefix, None, 2).unwrap();
assert_eq!(
first.iter().map(|key| key.person_id).collect::<Vec<_>>(),
vec![Uuid::from_u128(1), Uuid::from_u128(2)],
);
let cursor = first.last().unwrap().encode().to_vec();
let second = store.scan_stage2_cohort(prefix, Some(&cursor), 2).unwrap();
assert_eq!(
second.iter().map(|key| key.person_id).collect::<Vec<_>>(),
vec![Uuid::from_u128(3), Uuid::from_u128(4)],
);
let last = second.last().unwrap().encode().to_vec();
assert!(
store
.scan_stage2_cohort(prefix, Some(&last), 10)
.unwrap()
.is_empty(),
"the cohort prefix is exhausted after person 4",
);
assert_eq!(
store
.scan_stage2_cohort(prefix, Some(&malformed_tail), 10)
.unwrap()
.len(),
4,
"an in-prefix cursor must decode before it can skip rows",
);
let foreign_cursor = key(3, 7, 101, 1).encode();
assert_eq!(
store
.scan_stage2_cohort(prefix, Some(&foreign_cursor), 10)
.unwrap()
.len(),
4,
"an out-of-prefix cursor starts a fresh scan",
);
assert!(store
.scan_stage2_cohort(prefix, None, 0)
.unwrap()
.is_empty());
}
#[test]
fn scan_stage2_cohort_includes_the_all_ones_key() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let prefix = Stage2CohortPrefix {
partition_id: u16::MAX,
team_id: u64::MAX,
cohort_id: u64::MAX,
};
let key = Stage2Key {
partition_id: u16::MAX,
team_id: u64::MAX,
cohort_id: u64::MAX,
person_id: Uuid::from_u128(u128::MAX),
};
store
.write_batch(|batch| batch.put_stage2(&key, b"max"))
.unwrap();
assert_eq!(
store.scan_stage2_cohort(prefix, None, 1).unwrap(),
vec![key]
);
}
#[test]
fn scan_merge_cf_resumes_after_the_cursor_and_honors_the_limit_and_partition_bounds() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let tombstone_key = |partition_id: u16, person: u128| TombstoneKey {
partition_id,
team_id: 7,
person: Uuid::from_u128(person),
};
// Four tombstones in partition 5, one in partition 6 (must never surface for a p=5 scan).
store
.write_batch(|batch| {
for person in 1..=4u128 {
batch.put_tombstone(&tombstone_key(5, person), b"v");
}
batch.put_tombstone(&tombstone_key(6, 9), b"other-partition");
})
.unwrap();
let page1 = store
.scan_merge_cf(Cf::MergeTombstones, 5, None, 2)
.unwrap();
assert_eq!(page1.len(), 2);
let decoded: Vec<_> = page1
.iter()
.map(|(k, _)| TombstoneKey::decode(k).unwrap().person)
.collect();
assert_eq!(
decoded,
vec![Uuid::from_u128(1), Uuid::from_u128(2)],
"key-ordered ascending",
);
let cursor = page1.last().unwrap().0.clone();
let page2 = store
.scan_merge_cf(Cf::MergeTombstones, 5, Some(&cursor), 10)
.unwrap();
let decoded2: Vec<_> = page2
.iter()
.map(|(k, _)| TombstoneKey::decode(k).unwrap().person)
.collect();
assert_eq!(
decoded2,
vec![Uuid::from_u128(3), Uuid::from_u128(4)],
"resumes past the cursor, never re-emits it, and stays inside partition 5",
);
// A cursor at the partition's last key → empty (exhausted), wrapping is the caller's job.
let last_cursor = page2.last().unwrap().0.clone();
let exhausted = store
.scan_merge_cf(Cf::MergeTombstones, 5, Some(&last_cursor), 10)
.unwrap();
assert!(exhausted.is_empty(), "no keys past the partition's last");
// A cursor that is not in this partition falls back to a fresh prefix scan.
let foreign_cursor = tombstone_key(6, 9).encode().to_vec();
let fresh = store
.scan_merge_cf(Cf::MergeTombstones, 5, Some(&foreign_cursor), 10)
.unwrap();
assert_eq!(
fresh.len(),
4,
"out-of-partition cursor rescans from the start"
);
}
#[test]
fn flush_wal_sync_succeeds_and_persists_writes_across_a_reopen() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let key = behavioral_key();
{
let store = CohortStore::open(&StoreConfig {
path: path.clone(),
..StoreConfig::default()
})
.unwrap();
store
.write_batch(|b| b.put::<Behavioral>(&key, b"durable"))
.unwrap();
store.flush_wal_sync().unwrap();
// A second fsync with nothing new pending is a no-op, not an error.
store.flush_wal_sync().unwrap();
}
let reopened = CohortStore::open(&StoreConfig {
path,
wipe_on_start: false,
..StoreConfig::default()
})
.unwrap();
assert_eq!(
reopened.get_behavioral(&key).unwrap().as_deref(),
Some(b"durable".as_slice()),
);
}
#[test]
fn create_checkpoint_produces_an_openable_db_with_the_same_state() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
// The checkpoint must be a sibling of the store path, never a child (RocksDB hard-links SSTs).
let checkpoint = dir.path().join("checkpoint");
let key = behavioral_key();
let store = CohortStore::open(&StoreConfig {
path,
..StoreConfig::default()
})
.unwrap();
store
.write_batch(|b| b.put::<Behavioral>(&key, b"snapshot"))
.unwrap();
store.create_checkpoint(&checkpoint).unwrap();
let restored = CohortStore::open(&StoreConfig {
path: checkpoint,
wipe_on_start: false,
..StoreConfig::default()
})
.unwrap();
assert_eq!(
restored.get_behavioral(&key).unwrap().as_deref(),
Some(b"snapshot".as_slice()),
);
assert_eq!(
restored.db_identity().unwrap(),
store.db_identity().unwrap(),
"a restored checkpoint is the same DB, so an incremental upload may reuse its files",
);
let created = CohortStore::open(&StoreConfig {
path: dir.path().join("other"),
..StoreConfig::default()
})
.unwrap();
assert_ne!(created.db_identity().unwrap(), store.db_identity().unwrap());
}
// RocksDB's Checkpoint::create_checkpoint creates the destination dir itself and requires
// the leaf to NOT already exist: the parent must exist, the leaf must not.
#[test]
fn create_checkpoint_succeeds_when_only_the_parent_exists_and_the_leaf_does_not() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let key = behavioral_key();
store
.write_batch(|b| b.put::<Behavioral>(&key, b"snapshot"))
.unwrap();
// Parent exists, leaf does not — the correct usage.
let parent = dir.path().join("attempts");
std::fs::create_dir_all(&parent).unwrap();
let checkpoint = parent.join("attempt-0");
store.create_checkpoint(&checkpoint).unwrap();
let restored = CohortStore::open(&StoreConfig {
path: checkpoint,
wipe_on_start: false,
..StoreConfig::default()
})
.unwrap();
assert_eq!(
restored.get_behavioral(&key).unwrap().as_deref(),
Some(b"snapshot".as_slice()),
);
}
#[test]
fn create_checkpoint_errors_when_the_destination_dir_already_exists() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
// Pre-create the leaf: RocksDB requires it to not exist.
let checkpoint = dir.path().join("attempt-0");
std::fs::create_dir_all(&checkpoint).unwrap();
let err = store.create_checkpoint(&checkpoint).unwrap_err();
assert!(
matches!(err, StoreError::Backend { op, .. } if op == OP_CHECKPOINT),
"pre-existing checkpoint dir must surface a backend checkpoint error, got: {err}",
);
}
#[test]
fn scan_behavioral_resumes_after_the_cursor_and_honors_the_limit_and_partition_bounds() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let key = |partition_id: u16, person: u128| {
BehavioralKey::new(
partition_id,
7,
Uuid::from_u128(person),
LeafStateKey([0xAB; 16]),
)
};
// Three keys in partition 5, one in partition 6 (must never surface for a p=5 scan).
store
.write_batch(|batch| {
for person in 1..=3u128 {
batch.put::<Behavioral>(&key(5, person), format!("v{person}").as_bytes());
}
batch.put::<Behavioral>(&key(6, 9), b"other-partition");
})
.unwrap();
let page1 = store.scan_behavioral(5, None, 2).unwrap();
assert_eq!(page1.len(), 2);
assert_eq!(page1[0].0, key(5, 1));
assert_eq!(page1[0].1, b"v1");
assert_eq!(page1[1].0, key(5, 2));
let cursor = page1.last().unwrap().0.encode();
let page2 = store.scan_behavioral(5, Some(&cursor), 10).unwrap();
assert_eq!(page2.len(), 1, "only the remaining partition-5 key");
assert_eq!(page2[0].0, key(5, 3));
// A cursor at the partition's last key is exhausted.
let last = page2.last().unwrap().0.encode();
assert!(store
.scan_behavioral(5, Some(&last), 10)
.unwrap()
.is_empty());
assert!(
store.scan_behavioral(9, None, 10).unwrap().is_empty(),
"empty partition"
);
}
fn behavioral_key_for(partition_id: u16, person: u128) -> BehavioralKey {
BehavioralKey::new(
partition_id,
7,
Uuid::from_u128(person),
LeafStateKey([0xAB; 16]),
)
}
/// Five partition-5 keys plus one partition-6 key a partition-5 scan must never surface.
fn seed_p5_and_one_p6(batch: &mut BatchBuilder<'_>) {
for person in 1..=5u128 {
batch.put::<Behavioral>(
&behavioral_key_for(5, person),
format!("v{person}").as_bytes(),
);
}
batch.put::<Behavioral>(&behavioral_key_for(6, 9), b"other-partition");
}
/// Flushes to SST so the scan hits on-disk index/filter blocks, not the memtable.
fn flushed_p5_scan(
config: &StoreConfig,
seed: impl FnOnce(&mut BatchBuilder<'_>),
) -> Vec<(BehavioralKey, Vec<u8>)> {
let store = CohortStore::open(config).unwrap();
store.write_batch(seed).unwrap();
store.flush().unwrap();
let mut out = Vec::new();
let mut cursor: Option<Vec<u8>> = None;
loop {
let page = store.scan_behavioral(5, cursor.as_deref(), 2).unwrap();
let Some((last_key, _)) = page.last() else {
break;
};
cursor = Some(last_key.encode().to_vec());
out.extend(page);
}
out
}
/// Seeds several persons × several leaves in one partition plus neighbours in adjacent partitions,
/// flushes to SST (so iteration hits on-disk prefix blocks, not the memtable where truncation is
/// masked), and returns the store.
fn seed_multi_person_flushed() -> (TempDir, CohortStore) {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
tuned_block_options: true,
..StoreConfig::default()
})
.unwrap();
store
.write_batch(|batch| {
for person in 1..=4u128 {
for leaf in 0..3u8 {
let key = BehavioralKey::new(
5,
7,
Uuid::from_u128(person),
LeafStateKey([leaf; 16]),
);
batch.put::<Behavioral>(&key, b"v");
}
}
// Neighbours in adjacent partitions the partition-5 scan must never surface.
batch.put::<Behavioral>(&behavioral_key_for(4, 99), b"prev-partition");
batch.put::<Behavioral>(&behavioral_key_for(6, 99), b"next-partition");
})
.unwrap();
store.flush().unwrap();
(dir, store)
}
#[test]
fn scan_behavioral_crosses_every_person_prefix_in_the_partition() {
// The regression this pins: with the fixed-prefix extractor in place, a partition-wide scan
// must set total-order seek or it truncates at the first person's prefix boundary.
let (_dir, store) = seed_multi_person_flushed();
let mut all = Vec::new();
let mut cursor: Option<Vec<u8>> = None;
loop {
let page = store.scan_behavioral(5, cursor.as_deref(), 2).unwrap();
let Some((last, _)) = page.last() else {
break;
};
cursor = Some(last.encode().to_vec());
all.extend(page);
}
assert_eq!(
all.len(),
12,
"4 persons × 3 leaves, crossing every person prefix"
);
let persons: std::collections::BTreeSet<u128> =
all.iter().map(|(k, _)| k.person_id().as_u128()).collect();
assert_eq!(
persons,
[1, 2, 3, 4].into_iter().collect(),
"the scan reaches all four persons, not just the first prefix",
);
assert!(
all.iter().all(|(k, _)| k.partition_id() == 5),
"the adjacent-partition neighbours are excluded by the upper bound",
);
}
#[test]
fn scan_behavioral_prefix_returns_exactly_one_persons_leaves_in_lsk_order() {
let (_dir, store) = seed_multi_person_flushed();
let prefix = PersonPrefix::new(5, 7, Uuid::from_u128(2));
let rows = store.scan_behavioral_prefix(prefix).unwrap();
assert_eq!(rows.len(), 3, "only person 2's three leaves");
assert!(
rows.iter()
.all(|(k, _)| k.person_id() == Uuid::from_u128(2)),
"the scan never leaks past the 26-byte person prefix",
);
let lsks: Vec<[u8; 16]> = rows.iter().map(|(k, _)| k.lsk().0).collect();
let mut sorted = lsks.clone();
sorted.sort();
assert_eq!(lsks, sorted, "leaves come back in lsk-byte order");
}
#[test]
fn slice_coverage_is_settled_once_per_store_and_dies_with_its_slice() {
let t = CoverageStartMs(1_791_547_200_000);
let later = CoverageStartMs(t.0 + 60_000);
let created = TempDir::new().unwrap();
let config = StoreConfig {
path: created.path().join("db"),
ownership: PartitionOwnership::single(4),
..StoreConfig::default()
};
{
let store = CohortStore::open(&config).unwrap();
assert_eq!(
store.slice_coverage(0).unwrap(),
None,
"a created store adopts nothing"
);
for partition_id in [1, 2] {
assert_eq!(
store.resume_or_begin_slice(partition_id, t).unwrap(),
SliceTenure::Begun(t),
);
}
}
// Reopened: records resume, and nothing is adopted.
let store = CohortStore::open(&config).unwrap();
assert_eq!(store.slice_coverage(0).unwrap(), None);
assert_eq!(
store.resume_or_begin_slice(1, later).unwrap(),
SliceTenure::Resumed(SliceCoverage::Since(t)),
);
// The record dies with its slice.
store.delete_partition(1).unwrap();
assert_eq!(
store.resume_or_begin_slice(1, later).unwrap(),
SliceTenure::Begun(later),
);
assert_eq!(
store.slice_coverage(2).unwrap(),
Some(SliceCoverage::Since(t))
);
// A store without the stamp adopts every partition.
let legacy = TempDir::new().unwrap();
let config = StoreConfig {
path: legacy.path().join("db"),
..config
};
CohortStore::open(&config)
.unwrap()
.write_batch(|batch| batch.delete::<Meta>(&META_SLICE_COVERAGE))
.unwrap();
let store = CohortStore::open(&config).unwrap();
assert_eq!(
(0..5)
.map(|partition_id| store.slice_coverage(partition_id).unwrap())
.collect::<Vec<_>>(),
[[Some(SliceCoverage::Complete); 4].as_slice(), &[None]].concat(),
);
// Adoption happens once.
store.delete_partition(0).unwrap();
drop(store);
assert_eq!(
CohortStore::open(&config)
.unwrap()
.slice_coverage(0)
.unwrap(),
None
);
}
#[test]
fn a_store_opens_only_under_a_split_its_record_covers_and_then_records_that_split() {
let pods = std::num::NonZeroU16::new(2).unwrap();
let pod_zero = PartitionOwnership::new(pods, PodOrdinal::STANDALONE, 4).unwrap();
let pod_one = PartitionOwnership::new(pods, PodOrdinal::from(1), 4).unwrap();
let dir = TempDir::new().unwrap();
let under = |ownership| StoreConfig {
path: dir.path().join("db"),
ownership,
..StoreConfig::default()
};
drop(CohortStore::open(&under(pod_one)).unwrap());
CohortStore::open(&under(pod_one)).expect("the recorded ownership reopens");
for other in [pod_zero, PartitionOwnership::single(4)] {
assert!(
matches!(
CohortStore::open(&under(other)),
Err(StoreError::OwnershipMismatch { found: Some(found), expected })
if found == pod_one && expected == other
),
"{other}",
);
}
// A store from before ownership records adopts the ownership it opens under.
CohortStore::open(&under(pod_one))
.unwrap()
.write_batch(|batch| batch.delete::<Meta>(&META_POD_PARTITIONS))
.unwrap();
drop(CohortStore::open(&under(pod_zero)).unwrap());
assert!(matches!(
CohortStore::open(&under(pod_one)),
Err(StoreError::OwnershipMismatch { .. })
));
CohortStore::open(&under(pod_zero))
.unwrap()
.write_batch(|batch| batch.put::<Meta>(&META_POD_PARTITIONS, &[0xFF]))
.unwrap();
assert!(matches!(
CohortStore::open(&under(pod_zero)),
Err(StoreError::OwnershipMismatch { found: None, .. })
));
// A parent's store, or a checkpoint of it, opens under a split it covers and records it.
let parent = TempDir::new().unwrap();
let resharded = |ownership| StoreConfig {
path: parent.path().join("db"),
ownership,
..StoreConfig::default()
};
drop(CohortStore::open(&resharded(PartitionOwnership::single(4))).unwrap());
drop(CohortStore::open(&resharded(pod_one)).expect("a covering record opens"));
assert!(matches!(
CohortStore::open(&resharded(PartitionOwnership::single(4))),
Err(StoreError::OwnershipMismatch { found: Some(found), .. }) if found == pod_one
));
}
#[test]
fn delete_partition_wipes_partitioned_cfs_but_preserves_the_schema_stamp() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
let key = behavioral_key_for(5, 1);
store
.write_batch(|b| b.put::<Behavioral>(&key, b"state"))
.unwrap();
assert!(store.get_behavioral(&key).unwrap().is_some());
store.delete_partition(5).unwrap();
assert_eq!(
store.get_behavioral(&key).unwrap(),
None,
"the partition's behavioral rows are reclaimed",
);
assert_eq!(
store
.get(Cf::Meta, META_SCHEMA_VERSION.0)
.unwrap()
.as_deref(),
Some(STORE_SCHEMA_VERSION.to_be_bytes().as_slice()),
"cf_meta is exempt from the partition wipe, so the schema guard survives a rebalance",
);
}
#[test]
fn reset_slices_deletes_each_slice_and_its_coverage_record_durably() {
let dir = TempDir::new().unwrap();
let config = StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
};
let start = CoverageStartMs(1_791_547_200_000);
{
let store = CohortStore::open(&config).unwrap();
for partition in [4, 5, 6] {
store
.write_batch(|b| {
b.put::<Behavioral>(&behavioral_key_for(partition, 1), b"state")
})
.unwrap();
store.resume_or_begin_slice(partition, start).unwrap();
}
store.reset_slices([4, 6]).unwrap();
}
let reopened = CohortStore::open(&config).unwrap();
for partition in [4, 6] {
assert_eq!(
reopened
.get_behavioral(&behavioral_key_for(partition, 1))
.unwrap(),
None,
"partition {partition}",
);
assert_eq!(
reopened.slice_coverage(partition).unwrap(),
None,
"partition {partition}",
);
}
assert!(reopened
.get_behavioral(&behavioral_key_for(5, 1))
.unwrap()
.is_some());
assert_eq!(
reopened.slice_coverage(5).unwrap(),
Some(SliceCoverage::Since(start)),
);
}
#[test]
fn tuned_block_options_preserve_scan_semantics_across_flushed_ssts() {
// Partitioned filters + two-level index change SST metadata layout only, not scan order.
let tuned = TempDir::new().unwrap();
let plain = TempDir::new().unwrap();
let tuned_scan = flushed_p5_scan(
&StoreConfig {
path: tuned.path().join("db"),
tuned_block_options: true,
compact_on_deletion: true,
..StoreConfig::default()
},
seed_p5_and_one_p6,
);
let plain_scan = flushed_p5_scan(
&StoreConfig {
path: plain.path().join("db"),
tuned_block_options: false,
compact_on_deletion: false,
periodic_compaction_seconds: 0,
..StoreConfig::default()
},
seed_p5_and_one_p6,
);
assert_eq!(
tuned_scan.len(),
5,
"all five partition-5 keys, the partition-6 key excluded by the upper bound",
);
assert_eq!(
tuned_scan, plain_scan,
"tuned partitioned-filter / two-level-index layout must not change prefix iteration",
);
}
#[test]
fn tuned_options_keep_tombstone_visibility_correct_across_compaction() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
tuned_block_options: true,
compact_on_deletion: true,
..StoreConfig::default()
})
.unwrap();
// Ten partition-5 keys → SST.
store
.write_batch(|batch| {
for person in 1..=10u128 {
batch.put::<Behavioral>(&behavioral_key_for(5, person), b"v");
}
})
.unwrap();
store.flush().unwrap();
// Delete the even persons → a second SST carrying the tombstones over the first.
store
.write_batch(|batch| {
for person in (2..=10u128).step_by(2) {
batch.delete::<Behavioral>(&behavioral_key_for(5, person));
}
})
.unwrap();
store.flush().unwrap();
// Force a physical compaction so the tombstones actually rewrite the SSTs, exercising the
// compaction path instead of only the read-time merge iterator.
store.db.compact_range_cf(
store.cf(Cf::Behavioral).unwrap(),
None::<&[u8]>,
None::<&[u8]>,
);
let mut survivors = Vec::new();
let mut cursor: Option<Vec<u8>> = None;
loop {
let page = store.scan_behavioral(5, cursor.as_deref(), 3).unwrap();
let Some((last_key, _)) = page.last() else {
break;
};
cursor = Some(last_key.encode().to_vec());
survivors.extend(page.iter().map(|(key, _)| key.person_id().as_u128()));
}
assert_eq!(
survivors,
vec![1, 3, 5, 7, 9],
"compaction drops the deleted evens and keeps the odds ordered and bounded",
);
}
#[test]
fn deleting_unowned_partitions_drops_their_files_and_compaction_removes_the_rest() {
const KEYS: u128 = 1_000;
const THRESHOLD: u64 = 64 * 1024;
// Distinct in every 16-byte chunk, so LZ4 keeps each value near its 256 bytes and the size
// estimates stay far above `THRESHOLD`.
let value = |partition: u16, person: u128| -> Vec<u8> {
(0..16u128)
.flat_map(|chunk| {
((u128::from(partition) << 96) | (person << 8) | chunk)
.wrapping_mul(0x2545_F491_4F6C_DD1D_9E37_79B9_7F4A_7C15)
.to_le_bytes()
})
.collect()
};
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
// Partitions 0 and 1 each get a file of their own, and partitions 2 and 3 share one.
for partitions in [0..1u16, 1..2, 2..4] {
store
.write_batch(|batch| {
for partition in partitions.clone() {
for person in 0..KEYS {
batch.put::<Behavioral>(
&behavioral_key_for(partition, person),
&value(partition, person),
);
}
}
})
.unwrap();
store.flush().unwrap();
store.db.compact_range_cf(
store.cf(Cf::Behavioral).unwrap(),
Some(keys::partition_range(partitions.start).0),
Some(keys::partition_range(partitions.end - 1).1),
);
}
let behavioral_files = || -> Vec<rocksdb::LiveFile> {
store
.db
.live_files()
.unwrap()
.into_iter()
.filter(|file| file.column_family_name == Cf::Behavioral.as_str())
.collect()
};
assert_eq!(behavioral_files().len(), 3);
let pod_zero_of_two = PartitionOwnership::new(
std::num::NonZeroU16::new(2).unwrap(),
PodOrdinal::STANDALONE,
4,
)
.unwrap();
assert!(store.delete_partition(1).unwrap());
assert!(store.delete_partition(3).unwrap());
assert_eq!(behavioral_files().len(), 2);
assert_eq!(
store
.unowned_ranges_above(&pod_zero_of_two, THRESHOLD)
.unwrap()
.iter()
.map(|range| (range.cf, range.partition))
.collect::<Vec<_>>(),
[(Cf::Behavioral, 3)],
);
let stopped = CancellationToken::new();
stopped.cancel();
let skipped = store
.compact_unowned(&pod_zero_of_two, THRESHOLD, &stopped)
.unwrap();
assert_eq!(
(skipped.ranges, skipped.compacted, skipped.bytes_after),
(1, 0, skipped.bytes_before),
);
let pass = store
.compact_unowned(&pod_zero_of_two, THRESHOLD, &CancellationToken::new())
.unwrap();
assert_eq!((pass.ranges, pass.compacted, pass.bytes_after), (1, 1, 0));
let held: Vec<usize> = (0..4)
.map(|partition| {
store
.scan_behavioral(partition, None, usize::MAX)
.unwrap()
.len()
})
.collect();
assert_eq!(held, [KEYS as usize, 0, KEYS as usize, 0]);
assert_eq!(
behavioral_files()
.iter()
.map(|file| file.num_entries)
.sum::<u64>(),
2 * KEYS as u64,
);
}
#[test]
fn person_record_ttl_drops_ancient_keeps_fresh_and_malformed_and_never_touches_behavioral() {
use crate::stage1::person_record::{PersonRecord, Stamp};
use chrono::Utc;
// A 30-day TTL against a fixed enough gap: `last_seen_ms = 0` is ~epoch, far older than any
// real `now - 30d`, so it is ancient; a `last_seen_ms` at `now` is fresh.
let now_ms = Utc::now().timestamp_millis();
let fresh_ms = now_ms;
let ancient_ms = 0;
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
person_record_ttl_days: 30,
compact_on_deletion: false,
..StoreConfig::default()
})
.unwrap();
let record_bytes = |last_seen_ms: i64| {
let mut record = PersonRecord::absent();
record.last_seen_ms = last_seen_ms;
record.stamp = Stamp::new(last_seen_ms, 0);
record.encode()
};
let ancient_key = record_key(5, 1);
let fresh_key = record_key(5, 2);
// A malformed person-record value: the keyspace treats the value as opaque, so a raw put of
// garbage bytes seeds a row the codec cannot decode.
let malformed_key = record_key(5, 3);
let malformed_value = b"not a person record";
// An ancient behavioral row: the sweep owns its eviction, so the TTL filter must never touch it.
let behavioral_key = BehavioralKey::new(5, 7, Uuid::from_u128(4), LeafStateKey([0xCD; 16]));
store
.write_batch(|b| {
b.put::<PersonRecords>(&ancient_key, &record_bytes(ancient_ms));
b.put::<PersonRecords>(&fresh_key, &record_bytes(fresh_ms));
b.put::<PersonRecords>(&malformed_key, malformed_value);
b.put::<Behavioral>(&behavioral_key, &record_bytes(ancient_ms));
})
.unwrap();
store.flush().unwrap();
// Force a physical compaction of both CFs so the filter actually runs over the SSTs.
store.db.compact_range_cf(
store.cf(Cf::PersonRecords).unwrap(),
None::<&[u8]>,
None::<&[u8]>,
);
store.db.compact_range_cf(
store.cf(Cf::Behavioral).unwrap(),
None::<&[u8]>,
None::<&[u8]>,
);
assert_eq!(
store.get_person_record(&ancient_key).unwrap(),
None,
"a record older than the TTL is dropped by compaction",
);
assert_eq!(
store.get_person_record(&fresh_key).unwrap().as_deref(),
Some(record_bytes(fresh_ms).as_slice()),
"a record newer than the TTL survives",
);
assert_eq!(
store.get_person_record(&malformed_key).unwrap().as_deref(),
Some(malformed_value.as_slice()),
"a malformed value is never TTL-dropped — it surfaces as a read-time decode error instead",
);
assert_eq!(
store.get_behavioral(&behavioral_key).unwrap().as_deref(),
Some(record_bytes(ancient_ms).as_slice()),
"an ancient behavioral row is untouched: the TTL filter attaches to cf_person_records only",
);
}
#[test]
fn person_record_ttl_zero_installs_no_filter_so_ancient_records_survive() {
use crate::stage1::person_record::{PersonRecord, Stamp};
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
person_record_ttl_days: 0, // default: no compaction filter
compact_on_deletion: false,
..StoreConfig::default()
})
.unwrap();
let key = record_key(5, 1);
let mut record = PersonRecord::absent();
record.last_seen_ms = 0; // epoch-ancient
record.stamp = Stamp::new(0, 0);
let encoded = record.encode();
store
.write_batch(|b| b.put::<PersonRecords>(&key, &encoded))
.unwrap();
store.flush().unwrap();
store.db.compact_range_cf(
store.cf(Cf::PersonRecords).unwrap(),
None::<&[u8]>,
None::<&[u8]>,
);
assert_eq!(
store.get_person_record(&key).unwrap().as_deref(),
Some(encoded.as_slice()),
"with TTL=0 no filter is installed, so even an ancient record survives compaction",
);
}
#[test]
fn stats_snapshot_reports_keys_and_cache_activity_after_reads() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
statistics_enabled: true,
..StoreConfig::default()
})
.unwrap();
let key = |person: u128| {
BehavioralKey::new(3, 7, Uuid::from_u128(person), LeafStateKey([0xAB; 16]))
};
store
.write_batch(|batch| {
for person in 1..=8u128 {
batch.put::<Behavioral>(&key(person), b"state");
}
})
.unwrap();
// Only SST reads exercise the block cache (memtable hits short-circuit it), so flush first.
store.flush().unwrap();
for person in 1..=8u128 {
assert!(store.get_behavioral(&key(person)).unwrap().is_some());
}
let stats = store.stats_snapshot();
assert_eq!(
stats.per_cf.len(),
Cf::ALL.len(),
"one CfStats entry per column family",
);
let behavioral = stats
.per_cf
.iter()
.find(|cf| cf.cf == Cf::Behavioral)
.expect("Behavioral CF present in the snapshot");
assert!(
behavioral.num_keys > 0,
"estimate-num-keys counts the written keys"
);
assert!(
behavioral.sst_bytes > 0,
"flushed behavioral keys occupy SST bytes"
);
assert!(
stats.block_cache_hits + stats.block_cache_misses > 0,
"reads against the flushed SSTs drove block-cache lookups: hits={}, misses={}",
stats.block_cache_hits,
stats.block_cache_misses,
);
}
#[test]
fn info_log_caps_reach_the_persisted_rocksdb_options() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let _store = CohortStore::open(&StoreConfig {
path: path.clone(),
max_log_file_size_bytes: NonZeroUsize::new(1_048_576).unwrap(),
keep_log_file_num: NonZeroUsize::new(3).unwrap(),
..StoreConfig::default()
})
.unwrap();
let options_file = std::fs::read_dir(&path)
.unwrap()
.map(|entry| entry.unwrap().path())
.find(|p| {
p.file_name()
.unwrap()
.to_string_lossy()
.starts_with("OPTIONS-")
})
.expect("RocksDB writes an OPTIONS file at open");
let options = std::fs::read_to_string(options_file).unwrap();
for line in ["max_log_file_size=1048576", "keep_log_file_num=3"] {
assert!(
options.lines().any(|l| l.trim() == line),
"missing `{line}`"
);
}
}
#[test]
fn stats_snapshot_tickers_are_zero_when_statistics_disabled() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
statistics_enabled: false,
..StoreConfig::default()
})
.unwrap();
let key = behavioral_key();
store
.write_batch(|b| b.put::<Behavioral>(&key, b"state"))
.unwrap();
store.flush().unwrap();
assert!(store.get_behavioral(&key).unwrap().is_some());
let stats = store.stats_snapshot();
// Tickers read 0 with statistics off; size properties are not gated on statistics.
assert_eq!(stats.block_cache_hits, 0);
assert_eq!(stats.block_cache_misses, 0);
let behavioral = stats
.per_cf
.iter()
.find(|cf| cf.cf == Cf::Behavioral)
.unwrap();
assert!(
behavioral.num_keys > 0,
"size properties work without statistics"
);
}
#[test]
fn default_config_is_sane() {
let config = StoreConfig::default();
assert!(config.create_if_missing);
assert!(config.block_cache_bytes > 0);
assert!(config.write_buffer_bytes > 0);
assert!(config.max_open_files > 0);
assert!(config.statistics_enabled, "statistics default on");
assert_eq!(
config.read_sample_ratio, 64,
"read latency sampling defaults to 1-in-64",
);
assert!(config.tuned_block_options);
assert!(config.compact_on_deletion);
assert!(config.compact_on_deletion_window > 0);
assert!(config.compact_on_deletion_num_dels_trigger > 0);
assert!(config.compact_on_deletion_ratio > 0.0 && config.compact_on_deletion_ratio <= 1.0);
// Periodic compaction and the background-jobs cap are opt-in; `0` leaves RocksDB's own behavior.
assert_eq!(config.periodic_compaction_seconds, 0);
assert_eq!(config.max_background_jobs, 0);
}
#[test]
fn read_sampler_fires_once_per_ratio() {
// `ratio == 1` samples every read.
assert!((0..4).all(|_| should_sample_read(1)));
// Any 64 consecutive calls hold exactly one multiple of 64, so carry-over doesn't matter.
let hits = (0..64).filter(|_| should_sample_read(64)).count();
assert_eq!(hits, 1, "1-in-64 fires exactly once across 64 calls");
}
#[test]
fn read_sample_ratio_floors_zero_at_one() {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
read_sample_ratio: 0,
..StoreConfig::default()
})
.unwrap();
// `0` would panic `next % ratio` in `should_sample_read`; `open()` clamps it to 1.
assert!(store.get(Cf::Behavioral, b"missing").unwrap().is_none());
}
#[test]
fn cohort_store_is_send_and_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<CohortStore>();
}
}