Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cohorts): cap rocksdb info logs and gauge volume usage by file type (#114236)

posthog[bot]3de953df98e8
diff --git a/docs/internal/realtime-cohorts/state-store-and-durability.md b/docs/internal/realtime-cohorts/state-store-and-durability.md
index 52ef9fa82f2a..17cd74490b5b 100644
--- a/docs/internal/realtime-cohorts/state-store-and-durability.md
+++ b/docs/internal/realtime-cohorts/state-store-and-durability.md
@@ -327,6 +327,16 @@ Three metrics follow coverage:
- **Merge markers and tombstones** are removed after their retention period.
- **Orphaned behavioral rows**, left behind when a leaf's definition changes, a cohort is deleted, or a team leaves the catalog, are not collected.
They are removed only by a merge drain or a partition wipe.
+- **RocksDB's info logs** have a size cap.
+ The info `LOG` rolls at `COHORT_MAX_LOG_FILE_SIZE_BYTES` (256 MiB), and RocksDB keeps `COHORT_KEEP_LOG_FILE_NUM` (10) files, the active one included, so the info logs use at most about 2.5 GiB.
+ Each open and each roll delete the oldest rolled files beyond that count.
+ The first boot under the cap keeps the newest 9 rolled files that an uncapped build left, and they can be much larger than 256 MiB.
+ The next 9 rolls delete them.
+ Both values must be greater than 0, or the processor does not start.
+- **Usage by file type** of the store directory and the local checkpoints is the `store_file_bytes` gauge.
+ Its `kind` label is `sst`, `wal`, `info_log`, `manifest`, `other` or `checkpoint`.
+ The `checkpoint` value counts only local checkpoint files that the live store does not share, such as SSTs that compaction replaced.
+ It is 0 when checkpoints are off.
## Tuning that matters
diff --git a/rust/cohort-stream-processor/src/config.rs b/rust/cohort-stream-processor/src/config.rs
index bed6e0db83a7..57e325039c00 100644
--- a/rust/cohort-stream-processor/src/config.rs
+++ b/rust/cohort-stream-processor/src/config.rs
@@ -461,6 +461,14 @@ pub struct Config {
#[envconfig(from = "COHORT_MAX_BACKGROUND_JOBS", default = "0")]
pub cohort_max_background_jobs: i32,
+ /// Size at which RocksDB rolls its info `LOG` file.
+ #[envconfig(from = "COHORT_MAX_LOG_FILE_SIZE_BYTES", default = "268435456")]
+ pub cohort_max_log_file_size_bytes: NonZeroUsize,
+
+ /// Number of info `LOG` files RocksDB keeps.
+ #[envconfig(from = "COHORT_KEEP_LOG_FILE_NUM", default = "10")]
+ pub cohort_keep_log_file_num: NonZeroUsize,
+
/// TTL in days for `cf_person_records`: a compaction filter drops a person record whose
/// `last_seen_ms` is older than this. `0` (the default) disables it — no filter is installed, so a
/// persisted record never ages out. Attached to `cf_person_records` only; `cf_behavioral` eviction
@@ -1130,6 +1138,8 @@ impl Config {
compact_on_deletion: self.cohort_compact_on_deletion_enabled,
periodic_compaction_seconds: self.cohort_periodic_compaction_seconds,
max_background_jobs: self.cohort_max_background_jobs,
+ max_log_file_size_bytes: self.cohort_max_log_file_size_bytes,
+ keep_log_file_num: self.cohort_keep_log_file_num,
person_record_ttl_days: self.cohort_person_record_ttl_days,
ownership,
..StoreConfig::default()
@@ -1336,6 +1346,8 @@ mod tests {
cohort_compact_on_deletion_enabled: true,
cohort_periodic_compaction_seconds: 0,
cohort_max_background_jobs: 0,
+ cohort_max_log_file_size_bytes: NonZeroUsize::new(268_435_456).unwrap(),
+ cohort_keep_log_file_num: NonZeroUsize::new(10).unwrap(),
cohort_person_record_ttl_days: 0,
cohort_store_offload_mode: OffloadMode::All,
cohort_store_event_read_permits: 16,
@@ -2611,6 +2623,8 @@ mod tests {
assert!(defaults.cohort_compact_on_deletion_enabled);
assert_eq!(defaults.cohort_periodic_compaction_seconds, 0);
assert_eq!(defaults.cohort_max_background_jobs, 0);
+ assert_eq!(defaults.cohort_max_log_file_size_bytes.get(), 268_435_456);
+ assert_eq!(defaults.cohort_keep_log_file_num.get(), 10);
assert_eq!(defaults.partition_channel_buffer, 128);
let env: std::collections::HashMap<String, String> = [
@@ -2619,6 +2633,8 @@ mod tests {
("COHORT_COMPACT_ON_DELETION_ENABLED", "false"),
("COHORT_PERIODIC_COMPACTION_SECONDS", "3600"),
("COHORT_MAX_BACKGROUND_JOBS", "2"),
+ ("COHORT_MAX_LOG_FILE_SIZE_BYTES", "1048576"),
+ ("COHORT_KEEP_LOG_FILE_NUM", "3"),
("PARTITION_CHANNEL_BUFFER", "256"),
]
.into_iter()
@@ -2630,6 +2646,8 @@ mod tests {
assert!(!config.cohort_compact_on_deletion_enabled);
assert_eq!(config.cohort_periodic_compaction_seconds, 3600);
assert_eq!(config.cohort_max_background_jobs, 2);
+ assert_eq!(config.cohort_max_log_file_size_bytes.get(), 1_048_576);
+ assert_eq!(config.cohort_keep_log_file_num.get(), 3);
assert_eq!(config.partition_channel_buffer, 256);
}
@@ -2641,6 +2659,8 @@ mod tests {
config.cohort_compact_on_deletion_enabled = false;
config.cohort_periodic_compaction_seconds = 3600;
config.cohort_max_background_jobs = 2;
+ config.cohort_max_log_file_size_bytes = NonZeroUsize::new(1_048_576).unwrap();
+ config.cohort_keep_log_file_num = NonZeroUsize::new(3).unwrap();
let store = config.store_config(PartitionOwnership::single(64));
assert_eq!(store.block_cache_bytes, 3_221_225_472);
@@ -2648,6 +2668,8 @@ mod tests {
assert!(!store.compact_on_deletion);
assert_eq!(store.periodic_compaction_seconds, 3600);
assert_eq!(store.max_background_jobs, 2);
+ assert_eq!(store.max_log_file_size_bytes.get(), 1_048_576);
+ assert_eq!(store.keep_log_file_num.get(), 3);
}
#[test]
diff --git a/rust/cohort-stream-processor/src/main.rs b/rust/cohort-stream-processor/src/main.rs
index 58a76969fc4c..03c2427f1b92 100644
--- a/rust/cohort-stream-processor/src/main.rs
+++ b/rust/cohort-stream-processor/src/main.rs
@@ -23,6 +23,7 @@ use cohort_stream_processor::merge::gc::MergeGcSweeper;
use cohort_stream_processor::merge::redrive::RedriveSweeper;
use cohort_stream_processor::observability;
use cohort_stream_processor::observability::disk::SharedDiskUtilization;
+use cohort_stream_processor::observability::store_files::StoreFilesSweeper;
use cohort_stream_processor::observability::store_stats::{DiskProbe, StoreStatsSweeper};
use cohort_stream_processor::observability::tokio_monitor::TokioRuntimeMonitor;
use cohort_stream_processor::partitions::{
@@ -560,6 +561,17 @@ async fn async_main(config: Config) -> Result<()> {
"store_stats",
consumer_handle.shutdown_token(),
));
+ tokio::spawn(run_sweep_loop(
+ StoreFilesSweeper::new(
+ PathBuf::from(&config.store_path),
+ config
+ .checkpoint_enabled
+ .then(|| PathBuf::from(&config.checkpoint_local_dir)),
+ ),
+ config.stats_publish_interval(),
+ "store_files",
+ consumer_handle.shutdown_token(),
+ ));
tokio::spawn(
TokioRuntimeMonitor::new(
&tokio::runtime::Handle::current(),
diff --git a/rust/cohort-stream-processor/src/observability/metrics.rs b/rust/cohort-stream-processor/src/observability/metrics.rs
index 6da376469a19..8ec201630f3c 100644
--- a/rust/cohort-stream-processor/src/observability/metrics.rs
+++ b/rust/cohort-stream-processor/src/observability/metrics.rs
@@ -222,6 +222,10 @@ pub const STORE_SST_BYTES: &str = "store_sst_bytes";
pub const STORE_LIVE_DATA_BYTES: &str = "store_live_data_bytes";
/// Estimated key count, labelled by `cf` (gauge).
pub const STORE_ESTIMATE_NUM_KEYS: &str = "store_estimate_num_keys";
+/// Bytes in the store directory and the local checkpoints, labelled by file `kind`
+/// (`sst`|`wal`|`info_log`|`manifest`|`other`|`checkpoint`) (gauge). `checkpoint` counts only the
+/// local checkpoint files that the live store does not share.
+pub const STORE_FILE_BYTES: &str = "store_file_bytes";
/// Size of the filesystem holding the store (gauge, bytes).
pub const STORE_DISK_TOTAL_BYTES: &str = "store_disk_total_bytes";
@@ -939,6 +943,7 @@ mod tests {
assert_eq!(STORE_SST_BYTES, "store_sst_bytes");
assert_eq!(STORE_LIVE_DATA_BYTES, "store_live_data_bytes");
assert_eq!(STORE_ESTIMATE_NUM_KEYS, "store_estimate_num_keys");
+ assert_eq!(STORE_FILE_BYTES, "store_file_bytes");
assert_eq!(STORE_DISK_TOTAL_BYTES, "store_disk_total_bytes");
assert_eq!(STORE_DISK_AVAILABLE_BYTES, "store_disk_available_bytes");
assert_eq!(STORE_DISK_UTILIZATION_PCT, "store_disk_utilization_pct");
diff --git a/rust/cohort-stream-processor/src/observability/mod.rs b/rust/cohort-stream-processor/src/observability/mod.rs
index 4007a0a9bdd8..aa526fc694cb 100644
--- a/rust/cohort-stream-processor/src/observability/mod.rs
+++ b/rust/cohort-stream-processor/src/observability/mod.rs
@@ -4,5 +4,6 @@ pub mod canonical_log;
pub mod disk;
pub mod health;
pub mod metrics;
+pub mod store_files;
pub mod store_stats;
pub mod tokio_monitor;
diff --git a/rust/cohort-stream-processor/src/observability/store_files.rs b/rust/cohort-stream-processor/src/observability/store_files.rs
new file mode 100644
index 000000000000..3c0ca39ec378
--- /dev/null
+++ b/rust/cohort-stream-processor/src/observability/store_files.rs
@@ -0,0 +1,215 @@
+//! Per-file-type disk usage of the store directory and the local checkpoint directory. RocksDB
+//! reports only SST sizes, so this walk is the only view of the WAL, info `LOG`, `MANIFEST`, and
+//! checkpoint bytes on the store volume.
+
+use std::collections::HashSet;
+use std::fs::{self, DirEntry, Metadata};
+use std::io;
+use std::os::unix::fs::MetadataExt;
+use std::path::{Path, PathBuf};
+
+use async_trait::async_trait;
+use metrics::gauge;
+use tracing::{error, warn};
+
+use crate::observability::metrics::STORE_FILE_BYTES;
+use crate::sweep::Sweeper;
+
+/// Publishes the `store_file_bytes` gauge each tick. It has its own sweep loop, apart from
+/// [`DiskProbe`](crate::observability::store_stats::DiskProbe), so a slow walk cannot skip a
+/// sample of the seed disk gate. The loop awaits each walk, so two walks never overlap.
+pub struct StoreFilesSweeper {
+ store_path: PathBuf,
+ /// Set only when checkpoints are on, because only then does startup check that the directory
+ /// is outside the store and on the store's filesystem.
+ checkpoint_dir: Option<PathBuf>,
+}
+
+impl StoreFilesSweeper {
+ pub fn new(store_path: PathBuf, checkpoint_dir: Option<PathBuf>) -> Self {
+ Self {
+ store_path,
+ checkpoint_dir,
+ }
+ }
+}
+
+#[async_trait]
+impl Sweeper for StoreFilesSweeper {
+ async fn run_once(&self) {
+ let store_path = self.store_path.clone();
+ let checkpoint_dir = self.checkpoint_dir.clone();
+ let sample = tokio::task::spawn_blocking(move || {
+ sample_store_files(&store_path, checkpoint_dir.as_deref())
+ })
+ .await;
+ match sample {
+ Ok(Ok(usage)) => {
+ for (kind, bytes) in usage.by_kind() {
+ gauge!(STORE_FILE_BYTES, "kind" => kind).set(bytes as f64);
+ }
+ }
+ Ok(Err(err)) => {
+ warn!(error = %err, path = %self.store_path.display(), "store file usage sample failed");
+ }
+ Err(err) => error!(error = %err, "store file usage sample panicked"),
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+struct StoreFileUsage {
+ sst_bytes: u64,
+ wal_bytes: u64,
+ info_log_bytes: u64,
+ manifest_bytes: u64,
+ other_bytes: u64,
+ /// Checkpoint bytes that the store directory does not share: SSTs that the live store
+ /// compacted away, and the files the checkpoint copies instead of hard-links. Each file counts
+ /// once, however many checkpoint attempts hard-link it.
+ checkpoint_bytes: u64,
+}
+
+impl StoreFileUsage {
+ fn by_kind(&self) -> [(&'static str, u64); 6] {
+ [
+ ("sst", self.sst_bytes),
+ ("wal", self.wal_bytes),
+ ("info_log", self.info_log_bytes),
+ ("manifest", self.manifest_bytes),
+ ("other", self.other_bytes),
+ ("checkpoint", self.checkpoint_bytes),
+ ]
+ }
+}
+
+fn sample_store_files(
+ store_path: &Path,
+ checkpoint_dir: Option<&Path>,
+) -> io::Result<StoreFileUsage> {
+ let mut usage = StoreFileUsage::default();
+ // Inode numbers do not collide between the two trees, because startup checks that the
+ // checkpoint directory is on the store's filesystem.
+ let mut counted = HashSet::new();
+ for entry in fs::read_dir(store_path)? {
+ let entry = entry?;
+ let Some(meta) = file_metadata(&entry)? else {
+ continue;
+ };
+ let name = entry.file_name();
+ let name = name.to_string_lossy();
+ let bucket = if name.ends_with(".sst") {
+ &mut usage.sst_bytes
+ } else if name.ends_with(".log") {
+ // RocksDB names WAL files `<number>.log`; the info log is `LOG`, with no extension.
+ &mut usage.wal_bytes
+ } else if name == "LOG" || name.starts_with("LOG.old") {
+ &mut usage.info_log_bytes
+ } else if name.starts_with("MANIFEST-") {
+ &mut usage.manifest_bytes
+ } else {
+ &mut usage.other_bytes
+ };
+ *bucket += meta.len();
+ counted.insert(meta.ino());
+ }
+ if let Some(dir) = checkpoint_dir {
+ usage.checkpoint_bytes = unshared_bytes(dir, &mut counted)?;
+ }
+ Ok(usage)
+}
+
+fn unshared_bytes(root: &Path, counted: &mut HashSet<u64>) -> io::Result<u64> {
+ let mut total = 0;
+ let mut pending = vec![root.to_path_buf()];
+ while let Some(dir) = pending.pop() {
+ let entries = match fs::read_dir(&dir) {
+ Ok(entries) => entries,
+ // The checkpoint sweeper prunes old attempt directories while this walk runs.
+ Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
+ Err(err) => return Err(err),
+ };
+ for entry in entries {
+ let entry = entry?;
+ if entry.file_type()?.is_dir() {
+ pending.push(entry.path());
+ continue;
+ }
+ if let Some(meta) = file_metadata(&entry)? {
+ if counted.insert(meta.ino()) {
+ total += meta.len();
+ }
+ }
+ }
+ }
+ Ok(total)
+}
+
+/// `None` for a non-file entry or a file that compaction or pruning deleted after the listing.
+fn file_metadata(entry: &DirEntry) -> io::Result<Option<Metadata>> {
+ match entry.metadata() {
+ Ok(meta) if meta.is_file() => Ok(Some(meta)),
+ Ok(_) => Ok(None),
+ Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None),
+ Err(err) => Err(err),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn classifies_store_files_by_rocksdb_name() {
+ let dir = tempfile::tempdir().unwrap();
+ for (name, len) in [
+ ("000012.sst", 100),
+ ("000013.sst", 50),
+ ("000014.log", 7),
+ ("LOG", 3),
+ ("LOG.old.1700000000000000", 4),
+ ("MANIFEST-000005", 9),
+ ("OPTIONS-000007", 2),
+ ("CURRENT", 1),
+ ] {
+ fs::write(dir.path().join(name), vec![0u8; len]).unwrap();
+ }
+
+ let usage = sample_store_files(dir.path(), None).unwrap();
+
+ assert_eq!(
+ usage,
+ StoreFileUsage {
+ sst_bytes: 150,
+ wal_bytes: 7,
+ info_log_bytes: 7,
+ manifest_bytes: 9,
+ other_bytes: 3,
+ checkpoint_bytes: 0,
+ },
+ );
+ }
+
+ #[test]
+ fn checkpoint_counts_only_files_the_live_store_no_longer_shares() {
+ let root = tempfile::tempdir().unwrap();
+ let store = root.path().join("store");
+ let lineage = root.path().join("ckpt").join("topic").join("0");
+ let (older, newer) = (lineage.join("a1"), lineage.join("a2"));
+ fs::create_dir_all(&store).unwrap();
+ fs::create_dir_all(&older).unwrap();
+ fs::create_dir_all(&newer).unwrap();
+
+ fs::write(store.join("000020.sst"), vec![0u8; 40]).unwrap();
+ fs::hard_link(store.join("000020.sst"), newer.join("000020.sst")).unwrap();
+ // The live store compacted this SST away, so only the two checkpoint attempts hold it.
+ fs::write(older.join("000010.sst"), vec![0u8; 30]).unwrap();
+ fs::hard_link(older.join("000010.sst"), newer.join("000010.sst")).unwrap();
+ fs::write(newer.join("MANIFEST-000003"), vec![0u8; 5]).unwrap();
+
+ let usage = sample_store_files(&store, Some(&root.path().join("ckpt"))).unwrap();
+
+ assert_eq!(usage.sst_bytes, 40);
+ assert_eq!(usage.checkpoint_bytes, 35);
+ }
+}
diff --git a/rust/cohort-stream-processor/src/store/rocks.rs b/rust/cohort-stream-processor/src/store/rocks.rs
index cbbc00119371..50ffc595bf2f 100644
--- a/rust/cohort-stream-processor/src/store/rocks.rs
+++ b/rust/cohort-stream-processor/src/store/rocks.rs
@@ -82,6 +82,11 @@ const DEFAULT_COMPACT_ON_DELETION_WINDOW: usize = 1000;
const DEFAULT_COMPACT_ON_DELETION_NUM_DELS_TRIGGER: usize = 500;
const DEFAULT_COMPACT_ON_DELETION_RATIO: f64 = 0.5;
+/// RocksDB keeps the active info `LOG` and the newest rolled ones, `DEFAULT_KEEP_LOG_FILE_NUM` files
+/// in total, so the info logs use at most about 10 × 256 MiB of the store volume.
+const DEFAULT_MAX_LOG_FILE_SIZE_BYTES: NonZeroUsize = NonZeroUsize::new(256 * 1024 * 1024).unwrap();
+const DEFAULT_KEEP_LOG_FILE_NUM: NonZeroUsize = NonZeroUsize::new(10).unwrap();
+
/// Resolved RocksDB settings.
#[derive(Debug, Clone)]
pub struct StoreConfig {
@@ -125,6 +130,11 @@ pub struct StoreConfig {
/// record does not cover. A store that predates coverage records adopts all of its partitions
/// as complete.
pub ownership: PartitionOwnership,
+ /// Size at which RocksDB rolls the info `LOG` file. RocksDB reads `0` as "never roll", which
+ /// lets one info log grow without bound.
+ pub max_log_file_size_bytes: NonZeroUsize,
+ /// Number of info `LOG` files RocksDB keeps. RocksDB refuses to open with `0`.
+ pub keep_log_file_num: NonZeroUsize,
}
impl Default for StoreConfig {
@@ -148,6 +158,8 @@ impl Default for StoreConfig {
max_background_jobs: 0,
person_record_ttl_days: 0,
ownership: PartitionOwnership::single(COHORT_PARTITION_COUNT as u16),
+ max_log_file_size_bytes: DEFAULT_MAX_LOG_FILE_SIZE_BYTES,
+ keep_log_file_num: DEFAULT_KEEP_LOG_FILE_NUM,
}
}
}
@@ -1652,6 +1664,8 @@ fn db_options(config: &StoreConfig) -> Options {
if config.max_background_jobs > 0 {
opts.set_max_background_jobs(config.max_background_jobs);
}
+ opts.set_max_log_file_size(config.max_log_file_size_bytes.get());
+ opts.set_keep_log_file_num(config.keep_log_file_num.get());
opts
}
@@ -3303,6 +3317,37 @@ mod tests {
);
}
+ #[test]
+ fn info_log_caps_reach_the_persisted_rocksdb_options() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let _store = CohortStore::open(&StoreConfig {
+ path: path.clone(),
+ max_log_file_size_bytes: NonZeroUsize::new(1_048_576).unwrap(),
+ keep_log_file_num: NonZeroUsize::new(3).unwrap(),
+ ..StoreConfig::default()
+ })
+ .unwrap();
+
+ let options_file = std::fs::read_dir(&path)
+ .unwrap()
+ .map(|entry| entry.unwrap().path())
+ .find(|p| {
+ p.file_name()
+ .unwrap()
+ .to_string_lossy()
+ .starts_with("OPTIONS-")
+ })
+ .expect("RocksDB writes an OPTIONS file at open");
+ let options = std::fs::read_to_string(options_file).unwrap();
+ for line in ["max_log_file_size=1048576", "keep_log_file_num=3"] {
+ assert!(
+ options.lines().any(|l| l.trim() == line),
+ "missing `{line}`"
+ );
+ }
+ }
+
#[test]
fn stats_snapshot_tickers_are_zero_when_statistics_disabled() {
let dir = TempDir::new().unwrap();