Lightweight coding agent that runs in your terminal
Rust
Warn users when SQLite diagnostic log writes fail (#47887)
Dmitriy Kovalenko3d6428510fee
diff --git a/codex-rs/app-server/src/in_process.rs b/codex-rs/app-server/src/in_process.rsindex 1fb7b51f17..b467a33ec2 100644--- a/codex-rs/app-server/src/in_process.rs+++ b/codex-rs/app-server/src/in_process.rs@@ -456,6 +456,18 @@ async fn start_uninitialized(mut args: InProcessStartArgs) -> IoResult<InProcessanalytics_events_client.clone(),));+ let log_write_warning = crate::log_write_warning::LogWriteWarningReporter::new(+ args.feedback.clone(),+ &outgoing_message_sender,+ &args.config,+ );+ if let Some(log_db) = &args.log_db {+ log_db.set_failure_reporter(log_write_warning.clone());+ if log_db.has_write_failure() {+ log_write_warning.notify_failure();+ }+ }+let (writer_tx, mut writer_rx) = mpsc::channel::<QueuedOutgoingMessage>(channel_capacity);let outbound_initialized = Arc::new(AtomicBool::new(false));let outbound_experimental_api_enabled = Arc::new(AtomicBool::new(false));diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rsindex 4b7e526085..ae748d401e 100644--- a/codex-rs/app-server/src/lib.rs+++ b/codex-rs/app-server/src/lib.rs@@ -120,6 +120,7 @@ mod fuzzy_file_search;mod gateway_oauth_notifications;mod image_url;pub mod in_process;+mod log_write_warning;mod mcp_refresh;mod message_processor;mod model_catalog;@@ -691,6 +692,12 @@ pub async fn run_main_with_transport_options(});}+ let analytics_events_client =+ analytics_events_client_from_config(Arc::clone(&auth_manager), &config);+ let outgoing_message_sender = Arc::new(OutgoingMessageSender::new(+ outgoing_tx,+ analytics_events_client.clone(),+ ));let feedback = CodexFeedback::new();// Install a simple subscriber so `tracing` output is visible. Users can@@ -710,11 +717,16 @@ pub async fn run_main_with_transport_options(.boxed(),};+ let log_write_warning = log_write_warning::LogWriteWarningReporter::new(+ feedback.clone(),+ &outgoing_message_sender,+ &config,+ );let feedback_layer = feedback.logger_layer();let feedback_metadata_layer = feedback.metadata_layer();let log_db = state_db.clone()- .map(|state_db| log_db::start(state_db, std::sync::Arc::new(feedback.clone())));+ .map(|state_db| log_db::start(state_db, log_write_warning.clone()));let log_db_layer = log_db.clone().map(|layer| layer.with_filter(log_db::default_filter()));@@ -947,12 +959,6 @@ pub async fn run_main_with_transport_options(let recovery_file = daemon_recovery_file_path(&config.codex_home);let processor_handle = tokio::spawn({let auth_manager = Arc::clone(&auth_manager);- let analytics_events_client =- analytics_events_client_from_config(Arc::clone(&auth_manager), &config);- let outgoing_message_sender = Arc::new(OutgoingMessageSender::new(- outgoing_tx,- analytics_events_client.clone(),- ));let initialize_notification_sender = outgoing_message_sender.clone();let outbound_control_tx = outbound_control_tx;let processor = Arc::new(MessageProcessor::new(MessageProcessorArgs {diff --git a/codex-rs/app-server/src/log_write_warning.rs b/codex-rs/app-server/src/log_write_warning.rsnew file mode 100644index 0000000000..f282b1a1e0--- /dev/null+++ b/codex-rs/app-server/src/log_write_warning.rs@@ -0,0 +1,67 @@+use std::sync::Arc;+use std::sync::Weak;+use std::sync::atomic::AtomicBool;+use std::sync::atomic::Ordering;++use codex_app_server_protocol::ServerNotification;+use codex_app_server_protocol::WarningNotification;+use codex_core::config::Config;+use codex_feedback::CodexFeedback;+use codex_state::LogWriteFailureReporter;++use crate::outgoing_message::OutgoingMessageSender;++const LOG_WRITE_WARNING_WITH_FEEDBACK: &str = "Codex couldn't save diagnostic logs to its local database. Use /feedback with logs included before closing Codex, or run `codex doctor` for diagnostics.";+const LOG_WRITE_WARNING_WITHOUT_FEEDBACK: &str = "Codex couldn't save diagnostic logs to its local database. Run `codex doctor` for diagnostics.";++pub(crate) struct LogWriteWarningReporter {+ feedback: CodexFeedback,+ outgoing: Weak<OutgoingMessageSender>,+ /// Permanently set by the first failure; limits warning delivery to one attempt+ /// for this reporter, even if later SQLite writes succeed.+ has_failure: AtomicBool,+ message: &'static str,+}++impl LogWriteWarningReporter {+ pub(crate) fn new(+ feedback: CodexFeedback,+ outgoing: &Arc<OutgoingMessageSender>,+ config: &Config,+ ) -> Arc<Self> {+ Arc::new(Self {+ feedback,+ outgoing: Arc::downgrade(outgoing),+ has_failure: AtomicBool::new(false),+ message: if config.feedback_enabled {+ LOG_WRITE_WARNING_WITH_FEEDBACK+ } else {+ LOG_WRITE_WARNING_WITHOUT_FEEDBACK+ },+ })+ }++ pub(crate) fn notify_failure(&self) {+ if !self.has_failure.swap(true, Ordering::Relaxed)+ && let Some(outgoing) = self.outgoing.upgrade()+ {+ outgoing.try_send_server_notification(ServerNotification::Warning(+ WarningNotification {+ thread_id: None,+ message: self.message.to_string(),+ },+ ));+ }+ }+}++impl LogWriteFailureReporter for LogWriteWarningReporter {+ fn report_failure(&self, diagnostic: &str) {+ self.feedback.report_failure(diagnostic);+ self.notify_failure();+ }+}++#[cfg(test)]+#[path = "log_write_warning_tests.rs"]+mod tests;diff --git a/codex-rs/app-server/src/log_write_warning_tests.rs b/codex-rs/app-server/src/log_write_warning_tests.rsnew file mode 100644index 0000000000..cbccc8d6da--- /dev/null+++ b/codex-rs/app-server/src/log_write_warning_tests.rs@@ -0,0 +1,171 @@+use super::*;+use crate::outgoing_message::OutgoingEnvelope;+use crate::outgoing_message::OutgoingMessage;+use codex_analytics::AnalyticsEventsClient;+use codex_core::config::ConfigBuilder;+use codex_feedback::CodexFeedback;+use codex_state::LogWriteFailureReporter;+use pretty_assertions::assert_eq;+use std::sync::Arc;+use tokio::sync::mpsc;++#[tokio::test]+async fn log_write_warning_is_broadcast_once() -> anyhow::Result<()> {+ let home = tempfile::tempdir()?;+ let mut config = ConfigBuilder::default()+ .codex_home(home.path().to_path_buf())+ .build()+ .await?;+ for feedback_enabled in [true, false] {+ config.feedback_enabled = feedback_enabled;+ let feedback = CodexFeedback::new();+ let (tx, mut rx) = mpsc::channel(/*buffer*/ 4);+ let outgoing = Arc::new(OutgoingMessageSender::new(+ tx,+ AnalyticsEventsClient::disabled(),+ ));+ let reporter = LogWriteWarningReporter::new(feedback.clone(), &outgoing, &config);+ reporter.report_failure("SQLite flush failed");+ let OutgoingEnvelope::Broadcast {+ message: OutgoingMessage::AppServerNotification(envelope),+ } = rx.try_recv()?+ else {+ panic!("expected a broadcast notification");+ };+ let expected = if feedback_enabled {+ "Codex couldn't save diagnostic logs to its local database. Use /feedback with logs included before closing Codex, or run `codex doctor` for diagnostics."+ } else {+ "Codex couldn't save diagnostic logs to its local database. Run `codex doctor` for diagnostics."+ };+ let ServerNotification::Warning(notification) = envelope.notification else {+ panic!("expected a warning");+ };+ assert_eq!(+ notification,+ WarningNotification {+ thread_id: None,+ message: expected.to_string(),+ }+ );+ reporter.report_failure("another SQLite flush failed");+ assert!(rx.try_recv().is_err());+ assert_eq!(+ feedback+ .snapshot(/*session_id*/ None)+ .log_attachment(/*logs_override*/ None)+ .buffer,+ b"SQLite flush failedanother SQLite flush failed"+ );+ }+ Ok(())+}++#[tokio::test]+async fn log_write_warning_keeps_feedback_when_sender_is_gone() -> anyhow::Result<()> {+ let home = tempfile::tempdir()?;+ let config = ConfigBuilder::default()+ .codex_home(home.path().to_path_buf())+ .build()+ .await?;+ let feedback = CodexFeedback::new();+ let (tx, mut rx) = mpsc::channel(/*buffer*/ 4);+ let outgoing = Arc::new(OutgoingMessageSender::new(+ tx,+ AnalyticsEventsClient::disabled(),+ ));+ let reporter = LogWriteWarningReporter::new(feedback.clone(), &outgoing, &config);+ drop(outgoing);+ reporter.report_failure("SQLite flush failed");+ assert_eq!(+ feedback+ .snapshot(/*session_id*/ None)+ .log_attachment(/*logs_override*/ None)+ .buffer,+ b"SQLite flush failed"+ );+ assert!(reporter.outgoing.upgrade().is_none());+ reporter.report_failure("second failure");+ assert_eq!(+ feedback+ .snapshot(/*session_id*/ None)+ .log_attachment(/*logs_override*/ None)+ .buffer,+ b"SQLite flush failedsecond failure"+ );+ assert!(rx.try_recv().is_err());+ Ok(())+}++#[tokio::test]+async fn log_write_warning_reports_failed_sqlite_flush() -> anyhow::Result<()> {+ use tracing_subscriber::layer::SubscriberExt;++ for attach_after_failure in [false, true] {+ let home = tempfile::tempdir()?;+ let config = ConfigBuilder::default()+ .codex_home(home.path().to_path_buf())+ .build()+ .await?;+ let feedback = CodexFeedback::new();+ let runtime = codex_state::StateRuntime::init(+ config.sqlite_config().clone(),+ "test-provider".to_string(),+ )+ .await?;+ let (tx, mut rx) = mpsc::channel(/*buffer*/ 4);+ let outgoing = Arc::new(OutgoingMessageSender::new(+ tx,+ AnalyticsEventsClient::disabled(),+ ));+ let reporter = LogWriteWarningReporter::new(feedback.clone(), &outgoing, &config);+ let initial_reporter: Arc<dyn LogWriteFailureReporter> = if attach_after_failure {+ Arc::new(feedback.clone())+ } else {+ reporter.clone()+ };+ let layer = codex_state::log_db::start(runtime.clone(), initial_reporter);+ runtime.close().await;+ tracing::subscriber::with_default(+ tracing_subscriber::registry().with(layer.clone()),+ || {+ tracing::info!("first failed log write");+ },+ );+ layer.flush().await;+ assert!(layer.has_write_failure());+ if attach_after_failure {+ layer.set_failure_reporter(reporter.clone());+ if layer.has_write_failure() {+ reporter.notify_failure();+ }+ }++ assert!(matches!(+ rx.try_recv(),+ Ok(OutgoingEnvelope::Broadcast {+ message: OutgoingMessage::AppServerNotification(_),+ ..+ })+ ));+ tracing::subscriber::with_default(+ tracing_subscriber::registry().with(layer.clone()),+ || {+ tracing::info!("second failed log write");+ },+ );+ layer.flush().await;+ assert!(rx.try_recv().is_err());+ assert_eq!(+ String::from_utf8(+ feedback+ .snapshot(/*session_id*/ None)+ .log_attachment(/*logs_override*/ None)+ .buffer+ )?+ .matches("failed to flush logs to SQLite")+ .count(),+ 2+ );+ }+ Ok(())+}diff --git a/codex-rs/app-server/src/outgoing_message.rs b/codex-rs/app-server/src/outgoing_message.rsindex e6a109ceb3..961eccc2aa 100644--- a/codex-rs/app-server/src/outgoing_message.rs+++ b/codex-rs/app-server/src/outgoing_message.rs@@ -750,6 +750,14 @@ impl OutgoingMessageSender {.await;}+ pub(crate) fn try_send_server_notification(&self, notification: ServerNotification) {+ if let Err(error) = self.sender.try_send(OutgoingEnvelope::Broadcast {+ message: timestamped_server_notification(notification),+ }) {+ warn!("failed to queue server notification to clients: {error}");+ }+ }+pub(crate) async fn send_server_notification_to_connections(&self,connection_ids: &[ConnectionId],diff --git a/codex-rs/app-server/src/request_processors/initialize_processor.rs b/codex-rs/app-server/src/request_processors/initialize_processor.rsindex 9ee058aef7..d68a29df9f 100644--- a/codex-rs/app-server/src/request_processors/initialize_processor.rs+++ b/codex-rs/app-server/src/request_processors/initialize_processor.rs@@ -31,6 +31,7 @@ pub(crate) struct InitializeRequestProcessor {}impl InitializeRequestProcessor {+ #[allow(clippy::too_many_arguments)]pub(crate) fn new(gateway_login_control: Arc<codex_login::GatewayLoginControl>,outgoing: Arc<OutgoingMessageSender>,diff --git a/codex-rs/app-server/tests/suite/v2/feedback.rs b/codex-rs/app-server/tests/suite/v2/feedback.rsindex c594d6237f..fbe609df32 100644--- a/codex-rs/app-server/tests/suite/v2/feedback.rs+++ b/codex-rs/app-server/tests/suite/v2/feedback.rs@@ -218,6 +218,20 @@ async fn feedback_upload_includes_sqlite_flush_and_query_failures() -> Result<()app_server.send_notification(ClientNotification::Initialized).await?;+ // The periodic writer must notify the user before they choose to submit feedback.+ if phase == Phase::WriteFailure {+ let notification = timeout(+ Duration::from_secs(/*secs*/ 20),+ app_server.read_stream_until_notification_message("warning"),+ )+ .await??;+ let warning: codex_app_server_protocol::WarningNotification =+ serde_json::from_value(notification.params.unwrap())?;+ assert_eq!(warning, codex_app_server_protocol::WarningNotification {+ thread_id: None,+ message: "Codex couldn't save diagnostic logs to its local database. Use /feedback with logs included before closing Codex, or run `codex doctor` for diagnostics.".to_string(),+ });+ }let request = app_server.send_raw_request("feedback/upload",@@ -256,6 +270,12 @@ async fn feedback_upload_includes_sqlite_flush_and_query_failures() -> Result<()}let logs = &attachments["codex-logs.log"];if phase == Phase::Corrupt {+ assert!(+ !app_server+ .pending_notification_methods()+ .iter()+ .any(|method| method == "warning")+ );assert!(logs.contains("failed to flush logs to SQLite error=\"corrupt\""));assert!(logs.contains("failed to query feedback logs from sqlite"));} else {diff --git a/codex-rs/state/src/log_db.rs b/codex-rs/state/src/log_db.rsindex 4b2d8a6c90..0361312cab 100644--- a/codex-rs/state/src/log_db.rs+++ b/codex-rs/state/src/log_db.rs@@ -24,6 +24,7 @@use std::future::Future;use std::sync::Arc;use std::sync::OnceLock;+use std::sync::RwLock;use std::sync::atomic::AtomicBool;use std::sync::atomic::Ordering;use std::time::Duration;@@ -136,6 +137,7 @@ pub trait LogWriteFailureReporter: Send + Sync {pub struct LogDbLayer {sender: mpsc::Sender<LogDbCommand>,has_write_failure: Arc<AtomicBool>,+ failure_reporter: Arc<RwLock<Arc<dyn LogWriteFailureReporter>>>,process_uuid: String,}@@ -152,6 +154,7 @@ impl Clone for LogDbLayer {Self {sender: self.sender.clone(),has_write_failure: self.has_write_failure.clone(),+ failure_reporter: self.failure_reporter.clone(),process_uuid: self.process_uuid.clone(),}}@@ -173,20 +176,30 @@ impl LogDbLayer {let config = config.normalized();let (sender, receiver) = mpsc::channel(config.queue_capacity);let has_write_failure = Arc::new(AtomicBool::new(false));+ let failure_reporter = Arc::new(RwLock::new(failure_reporter));tokio::spawn(run_inserter(state_db,receiver,config,- failure_reporter,+ failure_reporter.clone(),has_write_failure.clone(),));Self {sender,has_write_failure,+ failure_reporter,process_uuid: current_process_log_uuid().to_string(),}}+ /// Replaces the startup reporter once the client notification destination is available.+ pub fn set_failure_reporter(&self, reporter: Arc<dyn LogWriteFailureReporter>) {+ *self+ .failure_reporter+ .write()+ .unwrap_or_else(std::sync::PoisonError::into_inner) = reporter;+ }+/// Whether this writer has lost logs, making SQLite incomplete for feedback.pub fn has_write_failure(&self) -> bool {self.has_write_failure.load(Ordering::Relaxed)@@ -479,7 +492,7 @@ async fn run_inserter(state_db: std::sync::Arc<StateRuntime>,mut receiver: mpsc::Receiver<LogDbCommand>,config: LogSinkQueueConfig,- failure_reporter: Arc<dyn LogWriteFailureReporter>,+ failure_reporter: Arc<RwLock<Arc<dyn LogWriteFailureReporter>>>,has_write_failure: Arc<AtomicBool>,) {let mut buffer = Vec::with_capacity(config.batch_size);@@ -516,7 +529,7 @@ async fn run_inserter(async fn flush(state_db: &StateRuntime,buffer: &mut Vec<LogEntry>,- failure_reporter: &dyn LogWriteFailureReporter,+ failure_reporter: &RwLock<Arc<dyn LogWriteFailureReporter>>,has_write_failure: &AtomicBool,) {if buffer.is_empty() {@@ -534,10 +547,10 @@ async fn flush(let timestamp =chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, /*use_z*/ true);let entry_count = entries.len();- failure_reporter.report_failure(&format!(+ has_write_failure.store(true, Ordering::Relaxed);+ failure_reporter.read().unwrap_or_else(std::sync::PoisonError::into_inner).report_failure(&format!("{timestamp} ERROR failed to flush logs to SQLite error={error:?} entries={entry_count}\n"));- has_write_failure.store(true, Ordering::Relaxed);}crate::telemetry::record_log_write(@@ -896,6 +909,7 @@ mod tests {let layer = LogDbLayer {sender,has_write_failure: Arc::new(AtomicBool::new(false)),+ failure_reporter: Arc::new(RwLock::new(Arc::new(SharedWriter::default()))),process_uuid: "process-1".to_string(),};@@ -917,6 +931,7 @@ mod tests {let layer = LogDbLayer {sender,has_write_failure: Arc::new(AtomicBool::new(false)),+ failure_reporter: Arc::new(RwLock::new(Arc::new(SharedWriter::default()))),process_uuid: "process-1".to_string(),};diff --git a/codex-rs/tui/src/chatwidget/tests/app_server.rs b/codex-rs/tui/src/chatwidget/tests/app_server.rsindex 9779268c92..7eeed4e626 100644--- a/codex-rs/tui/src/chatwidget/tests/app_server.rs+++ b/codex-rs/tui/src/chatwidget/tests/app_server.rs@@ -2013,3 +2013,28 @@ async fn permission_discovery_invalidates_on_thread_settings_and_uses_updated_cw};assert_eq!(thread_cwd, Some(test_path_buf("/tmp/thread-settings")));}++#[tokio::test]+async fn sqlite_log_write_warning_is_visible_in_warnings() {+ let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;+ chat.warning_display_state.startup_complete = true;+ chat.handle_server_notification(+ ServerNotification::Warning(WarningNotification {+ thread_id: None,+ message: "Codex couldn't save diagnostic logs to its local database. Use /feedback with logs included before closing Codex, or run `codex doctor` for diagnostics.".to_string(),+ }),+ /*replay_kind*/ None,+ );+ let cells: Vec<Arc<dyn HistoryCell>> = std::iter::from_fn(|| rx.try_recv().ok())+ .filter_map(|event| match event {+ AppEvent::InsertHistoryCell(cell) => Some(Arc::from(cell)),+ _ => None,+ })+ .collect();+ assert_eq!(cells.len(), 1);+ chat.open_warnings(&cells);+ insta::assert_snapshot!(+ "sqlite_log_write_warning",+ super::helpers::render_bottom_popup(&chat, /*width*/ 80)+ );+}diff --git a/codex-rs/tui/src/chatwidget/tests/snapshots/codex_tui__chatwidget__tests__app_server__sqlite_log_write_warning.snap b/codex-rs/tui/src/chatwidget/tests/snapshots/codex_tui__chatwidget__tests__app_server__sqlite_log_write_warning.snapnew file mode 100644index 0000000000..14f59f8ef9--- /dev/null+++ b/codex-rs/tui/src/chatwidget/tests/snapshots/codex_tui__chatwidget__tests__app_server__sqlite_log_write_warning.snap@@ -0,0 +1,16 @@+---+source: tui/src/chatwidget/tests/app_server.rs+expression: "super::helpers::render_bottom_popup(&chat, 80)"+---+ Warnings · 1 of 1 · Warning++ Codex couldn't save diagnostic logs to its local database. Use /feedback+ with logs included before closing Codex, or run `codex doctor` for+ diagnostics.+++++++ esc back · ctrl+o copy · ←/→ warning · ↓ scroll