Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Tag rollout compression metrics by trigger (#46047)

jif821ad43f9d6f
diff --git a/codex-rs/app-server/src/request_processors/rollout.rs b/codex-rs/app-server/src/request_processors/rollout.rs
index 8bf2eaef89..873ebdcd77 100644
--- a/codex-rs/app-server/src/request_processors/rollout.rs
+++ b/codex-rs/app-server/src/request_processors/rollout.rs
@@ -15,7 +15,10 @@ impl ThreadRequestProcessor {
return Err(unsupported_thread_store_operation("rollout/compress"));
}
- codex_rollout::spawn_rollout_compression_worker(self.config.codex_home.to_path_buf());
+ codex_rollout::spawn_rollout_compression_worker(
+ self.config.codex_home.to_path_buf(),
+ codex_rollout::RolloutCompressionTrigger::Rpc,
+ );
Ok(Some(RolloutCompressResponse {}.into()))
}
}
diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs
index ebfb570766..50807e628c 100644
--- a/codex-rs/core/src/thread_manager.rs
+++ b/codex-rs/core/src/thread_manager.rs
@@ -458,11 +458,17 @@ pub fn thread_store_from_config(
warn!("failed to migrate legacy rollouts on startup: {err}");
}
if compression_enabled {
- codex_rollout::spawn_rollout_compression_worker(codex_home);
+ codex_rollout::spawn_rollout_compression_worker(
+ codex_home,
+ codex_rollout::RolloutCompressionTrigger::Startup,
+ );
}
});
} else if compression_enabled {
- codex_rollout::spawn_rollout_compression_worker(config.codex_home.to_path_buf());
+ codex_rollout::spawn_rollout_compression_worker(
+ config.codex_home.to_path_buf(),
+ codex_rollout::RolloutCompressionTrigger::Startup,
+ );
}
store
}
diff --git a/codex-rs/rollout/src/compression.rs b/codex-rs/rollout/src/compression.rs
index de9fb65b6e..327a1eb420 100644
--- a/codex-rs/rollout/src/compression.rs
+++ b/codex-rs/rollout/src/compression.rs
@@ -28,13 +28,29 @@ const OPEN_ROLLOUT_LINE_READER_RETRY_DELAY: Duration = Duration::from_millis(50)
const TEMP_SUFFIX: &str = ".tmp";
static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
+/// The entry point that requested a compression pass, not a Statsig cohort.
+#[derive(Clone, Copy, Debug)]
+pub enum RolloutCompressionTrigger {
+ Startup,
+ Rpc,
+}
+
+impl RolloutCompressionTrigger {
+ fn tag(self) -> &'static str {
+ match self {
+ Self::Startup => "startup",
+ Self::Rpc => "rpc",
+ }
+ }
+}
+
/// Starts a best-effort background job that compresses cold local rollout files.
///
/// The worker is fire-and-forget: failures are logged, startup is not blocked,
/// and a run marker under `codex_home` prevents overlapping or too-frequent
/// compression runs from the same local store.
-pub fn spawn_rollout_compression_worker(codex_home: PathBuf) {
- worker::spawn(codex_home)
+pub fn spawn_rollout_compression_worker(codex_home: PathBuf, trigger: RolloutCompressionTrigger) {
+ worker::spawn(codex_home, trigger)
}
/// Returns the modified time for the existing plain or compressed rollout file.
@@ -300,6 +316,7 @@ mod worker {
use crate::ARCHIVED_SESSIONS_SUBDIR;
use crate::SESSIONS_SUBDIR;
+ use super::RolloutCompressionTrigger;
use super::RolloutFile;
use super::error_metrics::FailureMetric;
use super::metrics;
@@ -381,9 +398,9 @@ mod worker {
}
}
- pub(super) fn spawn(codex_home: PathBuf) {
+ pub(super) fn spawn(codex_home: PathBuf, trigger: RolloutCompressionTrigger) {
let Ok(handle) = tokio::runtime::Handle::try_current() else {
- metrics::run("skipped_no_runtime");
+ metrics::run(trigger, "skipped_no_runtime");
warn!(
"failed to start rollout compression worker for {}: no Tokio runtime",
codex_home.display()
@@ -391,7 +408,7 @@ mod worker {
return;
};
handle.spawn(async move {
- if let Err(err) = run(codex_home.clone()).await {
+ if let Err(err) = run(codex_home.clone(), trigger).await {
warn!(
"rollout compression worker failed for {}: {err}",
codex_home.display()
@@ -400,12 +417,15 @@ mod worker {
});
}
- pub(super) async fn run(codex_home: PathBuf) -> io::Result<()> {
+ pub(super) async fn run(
+ codex_home: PathBuf,
+ trigger: RolloutCompressionTrigger,
+ ) -> io::Result<()> {
let Some(_maintenance_guard) =
crate::try_acquire_rollout_maintenance_lock(codex_home.as_path())
- .inspect_err(|err| FailureMetric::Run.record("maintenance_lock", err))?
+ .inspect_err(|err| FailureMetric::Run(trigger).record("maintenance_lock", err))?
else {
- metrics::run("skipped_maintenance");
+ metrics::run(trigger, "skipped_maintenance");
debug!(
"rollout maintenance is already running for {}",
codex_home.display()
@@ -415,7 +435,7 @@ mod worker {
let marker = match CompressionRunMarker::try_claim(codex_home.as_path()) {
Ok(Some(marker)) => marker,
Ok(None) => {
- metrics::run("skipped_already_running");
+ metrics::run(trigger, "skipped_already_running");
debug!(
"rollout compression worker recently ran or is already running for {}",
codex_home.display()
@@ -423,18 +443,18 @@ mod worker {
return Ok(());
}
Err(err) => {
- FailureMetric::Run.record("run_marker", &err);
+ FailureMetric::Run(trigger).record("run_marker", &err);
return Err(err);
}
};
- metrics::run("started");
+ metrics::run(trigger, "started");
let started_at = Instant::now();
let writer_locks = Arc::new(crate::WriterLockCoordinator::new(&codex_home));
let mut stage = "temp_cleanup";
let result = async {
let mut stats = CompressionStats {
- cleanup_errors: cleanup_stale_temps(codex_home.as_path()).await?,
+ cleanup_errors: cleanup_stale_temps(codex_home.as_path(), trigger).await?,
..Default::default()
};
stage = "scan";
@@ -446,8 +466,14 @@ mod worker {
stats.time_budget_exhausted = true;
break;
}
- compress_rollouts_in_root(root.as_path(), started_at, &mut stats, &writer_locks)
- .await?;
+ compress_rollouts_in_root(
+ root.as_path(),
+ started_at,
+ &mut stats,
+ &writer_locks,
+ trigger,
+ )
+ .await?;
}
Ok::<_, io::Error>(stats)
}
@@ -455,8 +481,8 @@ mod worker {
let stats = match result {
Ok(stats) => stats,
Err(err) => {
- FailureMetric::Run.record(stage, &err);
- metrics::run_duration("failed", started_at.elapsed());
+ FailureMetric::Run(trigger).record(stage, &err);
+ metrics::run_duration(trigger, "failed", started_at.elapsed());
return Err(err);
}
};
@@ -473,6 +499,7 @@ mod worker {
};
let tags = [
("status", "completed"),
+ ("trigger", trigger.tag()),
("completion_reason", completion),
(
"file_errors",
@@ -516,12 +543,13 @@ mod worker {
started_at: Instant,
stats: &mut CompressionStats,
writer_locks: &Arc<crate::WriterLockCoordinator>,
+ trigger: RolloutCompressionTrigger,
) -> io::Result<()> {
if !tokio::fs::try_exists(root)
.await
.inspect_err(|err| {
stats.scan_errors = true;
- FailureMetric::Scan.record("check_root", err);
+ FailureMetric::Scan(trigger).record("check_root", err);
})
.unwrap_or(false)
{
@@ -538,7 +566,7 @@ mod worker {
Ok(read_dir) => read_dir,
Err(err) => {
stats.scan_errors = true;
- FailureMetric::Scan.record("read_directory", &err);
+ FailureMetric::Scan(trigger).record("read_directory", &err);
warn!(
"failed to read rollout compression directory {}: {err}",
dir.display()
@@ -551,7 +579,7 @@ mod worker {
Ok(Some(entry)) => entry,
Ok(None) => break,
Err(err) => {
- drain_compression_jobs(&mut jobs, stats).await;
+ drain_compression_jobs(&mut jobs, stats, trigger).await;
return Err(err);
}
};
@@ -564,7 +592,7 @@ mod worker {
Ok(file_type) => file_type,
Err(err) => {
stats.scan_errors = true;
- FailureMetric::Scan.record("read_file_type", &err);
+ FailureMetric::Scan(trigger).record("read_file_type", &err);
warn!(
"failed to read rollout compression file type {}: {err}",
path.display()
@@ -589,35 +617,39 @@ mod worker {
if crate::rollout_id_from_path(path.as_path()).is_none() {
stats.scan_errors = true;
stats.skipped = stats.skipped.saturating_add(1);
- metrics::file("skipped_unreadable_meta");
+ metrics::file(trigger, "skipped_unreadable_meta");
continue;
}
let thread_id = match crate::read_session_meta_line(path.as_path()).await {
Ok(metadata) => metadata.meta.id,
Err(err) => {
stats.scan_errors = true;
- FailureMetric::Scan.record("read_metadata", &err);
+ FailureMetric::Scan(trigger).record("read_metadata", &err);
stats.skipped = stats.skipped.saturating_add(1);
- metrics::file("skipped_unreadable_meta");
+ metrics::file(trigger, "skipped_unreadable_meta");
continue;
}
};
stats.scanned = stats.scanned.saturating_add(1);
- metrics::file("scanned");
+ metrics::file(trigger, "scanned");
while jobs.len() >= MAX_CONCURRENT_COMPRESSION_JOBS {
- collect_next_compression_job(&mut jobs, stats).await;
+ collect_next_compression_job(&mut jobs, stats, trigger).await;
}
let writer_locks = Arc::clone(writer_locks);
jobs.spawn_blocking(move || {
let started_at = Instant::now();
- let result =
- compress_rollout_if_cold_blocking(path.as_path(), &writer_locks, thread_id);
+ let result = compress_rollout_if_cold_blocking(
+ path.as_path(),
+ &writer_locks,
+ thread_id,
+ trigger,
+ );
let duration = started_at.elapsed();
(path, duration, result)
});
}
}
- drain_compression_jobs(&mut jobs, stats).await;
+ drain_compression_jobs(&mut jobs, stats, trigger).await;
Ok(())
}
@@ -672,15 +704,17 @@ mod worker {
async fn drain_compression_jobs(
jobs: &mut JoinSet<CompressionJobResult>,
stats: &mut CompressionStats,
+ trigger: RolloutCompressionTrigger,
) {
while !jobs.is_empty() {
- collect_next_compression_job(jobs, stats).await;
+ collect_next_compression_job(jobs, stats, trigger).await;
}
}
async fn collect_next_compression_job(
jobs: &mut JoinSet<CompressionJobResult>,
stats: &mut CompressionStats,
+ trigger: RolloutCompressionTrigger,
) {
let Some(result) = jobs.join_next().await else {
return;
@@ -699,28 +733,33 @@ mod worker {
stats.skipped = stats.skipped.saturating_add(1);
}
}
- metrics::file(outcome.tag());
- metrics::file_duration(outcome.tag(), duration);
+ metrics::file(trigger, outcome.tag());
+ metrics::file_duration(trigger, outcome.tag(), duration);
if let Some(source_bytes) = measurement.source_bytes {
- metrics::source_bytes(outcome.tag(), source_bytes);
+ metrics::source_bytes(trigger, outcome.tag(), source_bytes);
}
if let Some(compressed_bytes) = measurement.compressed_bytes {
- metrics::compressed_bytes(outcome.tag(), compressed_bytes);
+ metrics::compressed_bytes(trigger, outcome.tag(), compressed_bytes);
if let Some(source_bytes) = measurement.source_bytes {
- metrics::compression_ratio(outcome.tag(), source_bytes, compressed_bytes);
+ metrics::compression_ratio(
+ trigger,
+ outcome.tag(),
+ source_bytes,
+ compressed_bytes,
+ );
}
}
}
Ok((path, duration, Err(err))) => {
stats.failed = stats.failed.saturating_add(1);
// The failing operation records its stage before returning the error.
- metrics::file_duration("failed", duration);
+ metrics::file_duration(trigger, "failed", duration);
warn!("failed to compress rollout {}: {err}", path.display());
}
Err(err) => {
stats.failed = stats.failed.saturating_add(1);
warn!("rollout compression task failed: {err}");
- FailureMetric::File.record("task_join", &io::Error::other(err));
+ FailureMetric::File(trigger).record("task_join", &io::Error::other(err));
}
}
}
@@ -729,9 +768,10 @@ mod worker {
path: &Path,
writer_locks: &Arc<crate::WriterLockCoordinator>,
thread_id: codex_protocol::ThreadId,
+ trigger: RolloutCompressionTrigger,
) -> io::Result<CompressionMeasurement> {
let before = match cold_file_state(path)
- .inspect_err(|err| FailureMetric::File.record("read_metadata", err))?
+ .inspect_err(|err| FailureMetric::File(trigger).record("read_metadata", err))?
{
ColdFileState::Cold(state) => state,
ColdFileState::NotCold(state) => {
@@ -757,22 +797,22 @@ mod worker {
.filter(|parent| !parent.as_os_str().is_empty())
.unwrap_or_else(|| Path::new("."));
std::fs::create_dir_all(temp_dir)
- .inspect_err(|err| FailureMetric::File.record("prepare_directory", err))?;
+ .inspect_err(|err| FailureMetric::File(trigger).record("prepare_directory", err))?;
let mut temp_file = tempfile::Builder::new()
.prefix("rollout-compress-")
.suffix(TEMP_SUFFIX)
.tempfile_in(temp_dir)
- .inspect_err(|err| FailureMetric::File.record("create_temp", err))?;
+ .inspect_err(|err| FailureMetric::File(trigger).record("create_temp", err))?;
encode_zstd_to_writer(path, temp_file.as_file_mut())
- .inspect_err(|err| FailureMetric::File.record("encode_and_write", err))?;
+ .inspect_err(|err| FailureMetric::File(trigger).record("encode_and_write", err))?;
temp_file
.as_file_mut()
.flush()
- .inspect_err(|err| FailureMetric::File.record("flush", err))?;
+ .inspect_err(|err| FailureMetric::File(trigger).record("flush", err))?;
verify_zstd(temp_file.path())
- .inspect_err(|err| FailureMetric::File.record("verify", err))?;
+ .inspect_err(|err| FailureMetric::File(trigger).record("verify", err))?;
if !same_file_state(path, &before)
- .inspect_err(|err| FailureMetric::File.record("recheck_source", err))?
+ .inspect_err(|err| FailureMetric::File(trigger).record("recheck_source", err))?
{
return Ok(CompressionMeasurement::new(
CompressionOutcome::SkippedChanged,
@@ -781,22 +821,22 @@ mod worker {
));
}
set_file_metadata(temp_file.as_file(), before.modified, &before.permissions)
- .inspect_err(|err| FailureMetric::File.record("set_metadata", err))?;
+ .inspect_err(|err| FailureMetric::File(trigger).record("set_metadata", err))?;
temp_file
.as_file()
.sync_all()
- .inspect_err(|err| FailureMetric::File.record("sync", err))?;
+ .inspect_err(|err| FailureMetric::File(trigger).record("sync", err))?;
let compressed_bytes = temp_file
.as_file()
.metadata()
- .inspect_err(|err| FailureMetric::File.record("read_metadata", err))?
+ .inspect_err(|err| FailureMetric::File(trigger).record("read_metadata", err))?
.len();
// Encoding and verification do not block writers. Coordination prevents writer
// acquisition while we recheck, publish, and remove the original file.
let Some(_publication_guard) = writer_locks
.try_acquire_for_publication(thread_id)
- .inspect_err(|err| FailureMetric::File.record("writer_lock", err))?
+ .inspect_err(|err| FailureMetric::File(trigger).record("writer_lock", err))?
else {
return Ok(CompressionMeasurement::new(
CompressionOutcome::SkippedBusy,
@@ -805,7 +845,7 @@ mod worker {
));
};
if !same_file_state(path, &before)
- .inspect_err(|err| FailureMetric::File.record("recheck_source", err))?
+ .inspect_err(|err| FailureMetric::File(trigger).record("recheck_source", err))?
{
return Ok(CompressionMeasurement::new(
CompressionOutcome::SkippedChanged,
@@ -824,12 +864,12 @@ mod worker {
));
}
Err(err) => {
- FailureMetric::File.record("publish", &err.error);
+ FailureMetric::File(trigger).record("publish", &err.error);
return Err(err.error);
}
}
if !same_file_state(path, &before)
- .inspect_err(|err| FailureMetric::File.record("recheck_source", err))?
+ .inspect_err(|err| FailureMetric::File(trigger).record("recheck_source", err))?
{
let _ = std::fs::remove_file(compressed_path.as_path());
return Ok(CompressionMeasurement::new(
@@ -839,7 +879,7 @@ mod worker {
));
}
std::fs::remove_file(path)
- .inspect_err(|err| FailureMetric::File.record("remove_source", err))?;
+ .inspect_err(|err| FailureMetric::File(trigger).record("remove_source", err))?;
Ok(CompressionMeasurement::new(
CompressionOutcome::Compressed,
source_bytes,
@@ -916,24 +956,30 @@ mod worker {
file.set_permissions(permissions.clone())
}
- async fn cleanup_stale_temps(codex_home: &Path) -> io::Result<bool> {
+ async fn cleanup_stale_temps(
+ codex_home: &Path,
+ trigger: RolloutCompressionTrigger,
+ ) -> io::Result<bool> {
let mut errors = false;
for root in [
codex_home.join(SESSIONS_SUBDIR),
codex_home.join(ARCHIVED_SESSIONS_SUBDIR),
] {
- errors |= cleanup_stale_temps_in_root(root.as_path()).await?;
+ errors |= cleanup_stale_temps_in_root(root.as_path(), trigger).await?;
}
Ok(errors)
}
- async fn cleanup_stale_temps_in_root(root: &Path) -> io::Result<bool> {
+ async fn cleanup_stale_temps_in_root(
+ root: &Path,
+ trigger: RolloutCompressionTrigger,
+ ) -> io::Result<bool> {
let mut errors = false;
if !tokio::fs::try_exists(root)
.await
.inspect_err(|err| {
errors = true;
- FailureMetric::TempCleanup.record("check_root", err);
+ FailureMetric::TempCleanup(trigger).record("check_root", err);
})
.unwrap_or(false)
{
@@ -945,7 +991,7 @@ mod worker {
Ok(read_dir) => read_dir,
Err(err) => {
errors = true;
- FailureMetric::TempCleanup.record("read_directory", &err);
+ FailureMetric::TempCleanup(trigger).record("read_directory", &err);
warn!(
"failed to read rollout temp cleanup directory {}: {err}",
dir.display()
@@ -959,7 +1005,7 @@ mod worker {
Ok(file_type) => file_type,
Err(err) => {
errors = true;
- FailureMetric::TempCleanup.record("read_file_type", &err);
+ FailureMetric::TempCleanup(trigger).record("read_file_type", &err);
warn!(
"failed to read rollout temp cleanup file type {}: {err}",
path.display()
@@ -983,7 +1029,7 @@ mod worker {
.and_then(|metadata| metadata.modified())
.inspect_err(|err| {
errors = true;
- FailureMetric::TempCleanup.record("read_metadata", err);
+ FailureMetric::TempCleanup(trigger).record("read_metadata", err);
})
.ok()
.and_then(|modified| SystemTime::now().duration_since(modified).ok())
@@ -992,11 +1038,11 @@ mod worker {
continue;
}
match tokio::fs::remove_file(path.as_path()).await {
- Ok(()) => metrics::temp_cleanup("removed"),
+ Ok(()) => metrics::temp_cleanup(trigger, "removed"),
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
errors = true;
- FailureMetric::TempCleanup.record("remove_temp", &err);
+ FailureMetric::TempCleanup(trigger).record("remove_temp", &err);
warn!(
"failed to remove stale rollout temp {}: {err}",
path.display()
@@ -1011,6 +1057,7 @@ mod worker {
}
mod metrics {
+ use super::RolloutCompressionTrigger;
use std::time::Duration;
const FILE_COMPRESSED_BYTES_HISTOGRAM: &str = "codex.rollout_compression.file.compressed_bytes";
@@ -1025,31 +1072,51 @@ mod metrics {
const RATIO_BASIS_POINTS: u128 = 10_000;
pub(super) const TEMP_CLEANUP_COUNTER: &str = "codex.rollout_compression.temp_cleanup";
- pub(super) fn file(outcome: &'static str) {
- counter(FILE_COUNTER, &[("outcome", outcome)]);
+ pub(super) fn file(trigger: RolloutCompressionTrigger, outcome: &'static str) {
+ counter(
+ FILE_COUNTER,
+ &[("outcome", outcome), ("trigger", trigger.tag())],
+ );
}
- pub(super) fn file_duration(outcome: &'static str, duration: Duration) {
- duration_histogram(FILE_DURATION_HISTOGRAM, duration, &[("outcome", outcome)]);
+ pub(super) fn file_duration(
+ trigger: RolloutCompressionTrigger,
+ outcome: &'static str,
+ duration: Duration,
+ ) {
+ duration_histogram(
+ FILE_DURATION_HISTOGRAM,
+ duration,
+ &[("outcome", outcome), ("trigger", trigger.tag())],
+ );
}
- pub(super) fn source_bytes(outcome: &'static str, bytes: u64) {
+ pub(super) fn source_bytes(
+ trigger: RolloutCompressionTrigger,
+ outcome: &'static str,
+ bytes: u64,
+ ) {
histogram(
FILE_SOURCE_BYTES_HISTOGRAM,
saturating_i64(bytes),
- &[("outcome", outcome)],
+ &[("outcome", outcome), ("trigger", trigger.tag())],
);
}
- pub(super) fn compressed_bytes(outcome: &'static str, bytes: u64) {
+ pub(super) fn compressed_bytes(
+ trigger: RolloutCompressionTrigger,
+ outcome: &'static str,
+ bytes: u64,
+ ) {
histogram(
FILE_COMPRESSED_BYTES_HISTOGRAM,
saturating_i64(bytes),
- &[("outcome", outcome)],
+ &[("outcome", outcome), ("trigger", trigger.tag())],
);
}
pub(super) fn compression_ratio(
+ trigger: RolloutCompressionTrigger,
outcome: &'static str,
source_bytes: u64,
compressed_bytes: u64,
@@ -1062,7 +1129,7 @@ mod metrics {
histogram(
FILE_COMPRESSION_RATIO_HISTOGRAM,
saturating_i64(ratio),
- &[("outcome", outcome)],
+ &[("outcome", outcome), ("trigger", trigger.tag())],
);
}
@@ -1078,16 +1145,30 @@ mod metrics {
);
}
- pub(super) fn run(status: &'static str) {
- counter(RUN_COUNTER, &[("status", status)]);
+ pub(super) fn run(trigger: RolloutCompressionTrigger, status: &'static str) {
+ counter(
+ RUN_COUNTER,
+ &[("status", status), ("trigger", trigger.tag())],
+ );
}
- pub(super) fn run_duration(status: &'static str, duration: Duration) {
- duration_histogram(RUN_DURATION_HISTOGRAM, duration, &[("status", status)]);
+ pub(super) fn run_duration(
+ trigger: RolloutCompressionTrigger,
+ status: &'static str,
+ duration: Duration,
+ ) {
+ duration_histogram(
+ RUN_DURATION_HISTOGRAM,
+ duration,
+ &[("status", status), ("trigger", trigger.tag())],
+ );
}
- pub(super) fn temp_cleanup(outcome: &'static str) {
- counter(TEMP_CLEANUP_COUNTER, &[("outcome", outcome)]);
+ pub(super) fn temp_cleanup(trigger: RolloutCompressionTrigger, outcome: &'static str) {
+ counter(
+ TEMP_CLEANUP_COUNTER,
+ &[("outcome", outcome), ("trigger", trigger.tag())],
+ );
}
pub(super) fn counter(name: &str, tags: &[(&str, &str)]) {
diff --git a/codex-rs/rollout/src/compression/error_metrics.rs b/codex-rs/rollout/src/compression/error_metrics.rs
index 614e85ca9d..6a0729d46f 100644
--- a/codex-rs/rollout/src/compression/error_metrics.rs
+++ b/codex-rs/rollout/src/compression/error_metrics.rs
@@ -3,37 +3,44 @@
use std::io;
+use super::RolloutCompressionTrigger;
use super::metrics;
pub(super) enum FailureMetric {
- File,
+ File(RolloutCompressionTrigger),
Materialize,
- Run,
- Scan,
- TempCleanup,
+ Run(RolloutCompressionTrigger),
+ Scan(RolloutCompressionTrigger),
+ TempCleanup(RolloutCompressionTrigger),
}
impl FailureMetric {
pub(super) fn record(self, stage: &'static str, error: &io::Error) {
- let (name, outcome_key) = match self {
- Self::File => (metrics::FILE_COUNTER, "outcome"),
- Self::Materialize => (metrics::MATERIALIZE_COUNTER, "outcome"),
- Self::Run => (metrics::RUN_COUNTER, "status"),
- Self::Scan => ("codex.rollout_compression.scan", "outcome"),
- Self::TempCleanup => (metrics::TEMP_CLEANUP_COUNTER, "outcome"),
+ let (name, outcome_key, trigger) = match self {
+ Self::File(trigger) => (metrics::FILE_COUNTER, "outcome", Some(trigger)),
+ Self::Materialize => (metrics::MATERIALIZE_COUNTER, "outcome", None),
+ Self::Run(trigger) => (metrics::RUN_COUNTER, "status", Some(trigger)),
+ Self::Scan(trigger) => ("codex.rollout_compression.scan", "outcome", Some(trigger)),
+ Self::TempCleanup(trigger) => (metrics::TEMP_CLEANUP_COUNTER, "outcome", Some(trigger)),
};
let Some(metrics) = codex_otel::global() else {
return;
};
- let _ = metrics.counter(
- name,
- /*inc*/ 1,
- &[
- (outcome_key, "failed"),
- ("stage", stage),
- ("error_kind", error_kind(error)),
- ],
- );
+ let tags = [
+ (outcome_key, "failed"),
+ ("stage", stage),
+ ("error_kind", error_kind(error)),
+ (
+ "trigger",
+ trigger.map_or("", RolloutCompressionTrigger::tag),
+ ),
+ ];
+ let tags = if trigger.is_some() {
+ &tags[..]
+ } else {
+ &tags[..3]
+ };
+ let _ = metrics.counter(name, /*inc*/ 1, tags);
}
}
diff --git a/codex-rs/rollout/src/compression_tests.rs b/codex-rs/rollout/src/compression_tests.rs
index 4639100ad4..e77dbee344 100644
--- a/codex-rs/rollout/src/compression_tests.rs
+++ b/codex-rs/rollout/src/compression_tests.rs
@@ -17,6 +17,7 @@ use pretty_assertions::assert_eq;
use tempfile::TempDir;
use uuid::Uuid;
+use super::RolloutCompressionTrigger::Startup;
use super::*;
use crate::InitialHistory;
use crate::RolloutConfig;
@@ -289,7 +290,7 @@ async fn worker_compresses_old_active_and_archived_rollouts() -> anyhow::Result<
let fresh_temp = active_path.with_file_name("rollout-fresh.jsonl.zst.tmp");
fs::write(&fresh_temp, "fresh temp")?;
- worker::run(home.path().to_path_buf()).await?;
+ worker::run(home.path().to_path_buf(), Startup).await?;
assert!(!active_path.exists());
assert!(compressed_rollout_path(&active_path).exists());
@@ -319,12 +320,12 @@ async fn worker_waits_for_rollout_maintenance_before_compressing() -> anyhow::Re
let guard = crate::try_acquire_rollout_maintenance_lock(home.path())?
.expect("claim rollout maintenance lock");
- worker::run(home.path().to_path_buf()).await?;
+ worker::run(home.path().to_path_buf(), Startup).await?;
assert!(path.exists());
assert!(!compressed_rollout_path(&path).exists());
drop(guard);
- worker::run(home.path().to_path_buf()).await?;
+ worker::run(home.path().to_path_buf(), Startup).await?;
assert!(!path.exists());
assert!(compressed_rollout_path(&path).exists());
Ok(())
@@ -355,7 +356,7 @@ async fn worker_compresses_archived_fork_chain() -> anyhow::Result<()> {
let original_source = fs::read(&source_path)?;
let original_child = fs::read(&child_path)?;
- worker::run(home.path().to_path_buf()).await?;
+ worker::run(home.path().to_path_buf(), Startup).await?;
for (path, original) in [
(&source_path, original_source),
@@ -382,7 +383,7 @@ async fn worker_skips_unreadable_metadata_without_blocking_other_compression() -
let unreadable_path = rollout_path(home.path(), "2025-01-03T12-00-01", Uuid::from_u128(21));
fs::write(unreadable_path.as_path(), "{not json}\n")?;
- worker::run(home.path().to_path_buf()).await?;
+ worker::run(home.path().to_path_buf(), Startup).await?;
assert!(!source_path.exists());
assert!(compressed_rollout_path(&source_path).exists());
@@ -456,7 +457,7 @@ async fn compression_preserves_rollout_permissions() -> anyhow::Result<()> {
fs::set_permissions(&rollout_path, fs::Permissions::from_mode(0o600))?;
set_old_mtime(&rollout_path)?;
- worker::run(home.path().to_path_buf()).await?;
+ worker::run(home.path().to_path_buf(), Startup).await?;
let compressed_path = compressed_rollout_path(&rollout_path);
assert!(!rollout_path.exists());
@@ -538,7 +539,7 @@ async fn compression_preserves_read_only_rollout_permissions() -> anyhow::Result
fs::set_permissions(&rollout_path, fs::Permissions::from_mode(0o400))?;
let source_modified = fs::metadata(&rollout_path)?.modified()?;
- worker::run(home.path().to_path_buf()).await?;
+ worker::run(home.path().to_path_buf(), Startup).await?;
let compressed_path = compressed_rollout_path(&rollout_path);
let compressed_metadata = fs::metadata(&compressed_path)?;
@@ -559,7 +560,7 @@ async fn worker_skips_existing_compressed_archived_rollouts() -> anyhow::Result<
let compressed_path = compressed_rollout_path(&rollout_path);
set_old_mtime(&compressed_path)?;
- worker::run(home.path().to_path_buf()).await?;
+ worker::run(home.path().to_path_buf(), Startup).await?;
assert!(!rollout_path.exists());
assert!(compressed_path.exists());
@@ -583,7 +584,7 @@ async fn worker_skips_when_fresh_run_marker_exists() -> anyhow::Result<()> {
fs::create_dir_all(marker_dir.as_path())?;
fs::write(marker_dir.join("rollout-compression.lock"), "recent run")?;
- worker::run(home.path().to_path_buf()).await?;
+ worker::run(home.path().to_path_buf(), Startup).await?;
assert!(rollout_path.exists());
assert!(!compressed_rollout_path(&rollout_path).exists());
@@ -781,7 +782,7 @@ async fn worker_compresses_parallel_cold_candidates_without_skipping_other_publi
set_old_mtime(&path)?;
paths.push(path);
}
- worker::run(home.path().to_path_buf()).await?;
+ worker::run(home.path().to_path_buf(), Startup).await?;
let representations = paths
.iter()
.map(|path| (path.exists(), compressed_rollout_path(path).exists()))
diff --git a/codex-rs/rollout/src/lib.rs b/codex-rs/rollout/src/lib.rs
index 7f4f3ed6e4..c74d53ee98 100644
--- a/codex-rs/rollout/src/lib.rs
+++ b/codex-rs/rollout/src/lib.rs
@@ -93,6 +93,7 @@ pub static INTERACTIVE_SESSION_SOURCES: LazyLock<Vec<SessionSource>> = LazyLock:
});
pub use codex_protocol::protocol::SessionMeta;
+pub use compression::RolloutCompressionTrigger;
pub use compression::RolloutLineReader;
pub use compression::existing_rollout_path;
pub use compression::open_rollout_line_reader;
diff --git a/codex-rs/thread-store/src/local/compression_writer_tests.rs b/codex-rs/thread-store/src/local/compression_writer_tests.rs
index 56ed82867a..448b0338a4 100644
--- a/codex-rs/thread-store/src/local/compression_writer_tests.rs
+++ b/codex-rs/thread-store/src/local/compression_writer_tests.rs
@@ -80,7 +80,10 @@ async fn compress(home: &Path) -> TestResult<()> {
if marker.exists() {
fs::remove_file(&marker)?;
}
- codex_rollout::spawn_rollout_compression_worker(home.to_path_buf());
+ codex_rollout::spawn_rollout_compression_worker(
+ home.to_path_buf(),
+ codex_rollout::RolloutCompressionTrigger::Startup,
+ );
// The marker proves startup; the maintenance lock proves every blocking job finished.
wait_until(|| {
marker.exists()
diff --git a/codex-rs/tui/src/app/tests/session_lifecycle_requests.rs b/codex-rs/tui/src/app/tests/session_lifecycle_requests.rs
index 6b34df40a2..b7ddeb167d 100644
--- a/codex-rs/tui/src/app/tests/session_lifecycle_requests.rs
+++ b/codex-rs/tui/src/app/tests/session_lifecycle_requests.rs
@@ -3674,7 +3674,10 @@ terminal_visualization_instructions = true
.set_times(std::fs::FileTimes::new().set_modified(
std::time::SystemTime::now() - std::time::Duration::from_secs(8 * 24 * 60 * 60),
))?;
- codex_rollout::spawn_rollout_compression_worker(home.clone());
+ codex_rollout::spawn_rollout_compression_worker(
+ home.clone(),
+ codex_rollout::RolloutCompressionTrigger::Startup,
+ );
tokio::time::timeout(std::time::Duration::from_secs(10), async {
while saved_path.exists() || !saved_path.with_extension("jsonl.zst").is_file() {
tokio::time::sleep(std::time::Duration::from_millis(50)).await;