Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

refactor(db): delete dead migration_worker (#4099)

Louis Beaumont7c8d404b69bf
diff --git a/crates/screenpipe-db/src/lib.rs b/crates/screenpipe-db/src/lib.rs
index 07386639f..42ab4a382 100644
--- a/crates/screenpipe-db/src/lib.rs
+++ b/crates/screenpipe-db/src/lib.rs
@@ -4,7 +4,6 @@
mod db;
#[cfg(test)]
mod failpoint_vfs;
-mod migration_worker;
mod sqlite_error;
pub mod text_normalizer;
pub mod text_similarity;
@@ -17,10 +16,6 @@ pub use db::{
ImmediateTx, NewMeetingTranscriptSegment, MEETING_END_REASON_AUTO_END,
MEETING_END_REASON_EXPLICIT_STOP, MEETING_END_REASON_SHUTDOWN,
};
-pub use migration_worker::{
- create_migration_worker, MigrationCommand, MigrationConfig, MigrationResponse, MigrationStatus,
- MigrationWorker,
-};
pub use text_normalizer::{expand_search_query, sanitize_fts5_query};
pub use types::*;
pub use write_queue::{request_write_pause, request_write_resume, SyncTable};
diff --git a/crates/screenpipe-db/src/migration_worker.rs b/crates/screenpipe-db/src/migration_worker.rs
deleted file mode 100644
index 1f3a613ce..000000000
--- a/crates/screenpipe-db/src/migration_worker.rs
+++ /dev/null
@@ -1,602 +0,0 @@
-use anyhow::{anyhow, Result};
-use chrono::Utc;
-use serde::Serialize;
-use sqlx::{Row, SqlitePool};
-use std::sync::{
- atomic::{AtomicBool, Ordering},
- Arc,
-};
-use std::time::Duration;
-use tokio::{sync::mpsc, task::JoinHandle, time};
-use tracing::{debug, error, info, warn};
-
-use crate::DatabaseManager;
-
-/// Status of a migration job
-#[derive(Debug, Clone, PartialEq, Serialize)]
-pub enum MigrationStatus {
- /// Migration is currently running
- Running {
- total_records: i64,
- processed_records: i64,
- },
- /// Migration completed successfully
- Completed {
- total_records: i64,
- duration_secs: u64,
- },
- /// Migration is paused and can be resumed
- Paused {
- total_records: i64,
- processed_records: i64,
- },
- /// Migration failed with an error
- Failed {
- total_records: i64,
- processed_records: i64,
- error: String,
- },
- /// Migration has not been started yet
- NotStarted,
-}
-
-/// Commands that can be sent to control the migration worker
-#[derive(Debug, Clone)]
-pub enum MigrationCommand {
- /// Start or resume the migration
- Start,
- /// Pause the migration (can be resumed later)
- Pause,
- /// Stop the migration (cannot be resumed)
- Stop,
- /// Request current migration status
- Status,
-}
-
-/// Response from the migration worker, including current status
-#[derive(Serialize)]
-pub struct MigrationResponse {
- pub status: MigrationStatus,
-}
-
-/// Configuration for the migration worker
-#[derive(Debug, Clone)]
-pub struct MigrationConfig {
- /// Number of records to process in a single batch
- pub batch_size: i64,
- /// Delay between batches to reduce database load
- pub batch_delay_ms: u64,
- /// Whether to continue processing if an error occurs
- pub continue_on_error: bool,
-}
-
-impl Default for MigrationConfig {
- fn default() -> Self {
- Self {
- batch_size: 200,
- batch_delay_ms: 100,
- continue_on_error: true,
- }
- }
-}
-
-/// Worker that handles background migration of data
-pub struct MigrationWorker {
- db: Arc<DatabaseManager>,
- status: MigrationStatus,
- is_running: Arc<AtomicBool>,
- is_paused: Arc<AtomicBool>,
- config: MigrationConfig,
- cmd_rx: mpsc::Receiver<MigrationCommand>,
- status_tx: mpsc::Sender<MigrationResponse>,
- worker_handle: Option<JoinHandle<()>>,
-}
-
-impl MigrationWorker {
- /// Create a new migration worker
- pub fn new(
- db: Arc<DatabaseManager>,
- cmd_rx: mpsc::Receiver<MigrationCommand>,
- status_tx: mpsc::Sender<MigrationResponse>,
- config: MigrationConfig,
- ) -> Self {
- Self {
- db,
- status: MigrationStatus::NotStarted,
- is_running: Arc::new(AtomicBool::new(false)),
- is_paused: Arc::new(AtomicBool::new(false)),
- config,
- cmd_rx,
- status_tx,
- worker_handle: None,
- }
- }
-
- /// Start the migration worker to process commands
- pub fn start(mut self) -> JoinHandle<()> {
- tokio::spawn(async move {
- info!("Migration worker started");
- while let Some(cmd) = self.cmd_rx.recv().await {
- match cmd {
- MigrationCommand::Start => {
- self.start_migration().await;
- }
- MigrationCommand::Pause => {
- self.pause_migration();
- }
- MigrationCommand::Stop => {
- self.stop_migration();
- break;
- }
- MigrationCommand::Status => {
- let _ = self
- .status_tx
- .send(MigrationResponse {
- status: self.status.clone(),
- })
- .await;
- }
- }
- }
- info!("Migration worker stopped");
- })
- }
-
- async fn start_migration(&mut self) {
- if self.is_running.load(Ordering::SeqCst) {
- if self.is_paused.load(Ordering::SeqCst) {
- // Resume migration
- info!("Resuming migration");
- self.is_paused.store(false, Ordering::SeqCst);
- let _ = self
- .status_tx
- .send(MigrationResponse {
- status: self.status.clone(),
- })
- .await;
- return;
- }
- warn!("Migration is already running");
- return;
- }
-
- // Start the migration worker
- self.is_running.store(true, Ordering::SeqCst);
-
- // Create clones of shared resources for the worker task
- let db = self.db.clone();
- let config = self.config.clone();
- let is_running = self.is_running.clone();
- let is_paused = self.is_paused.clone();
- let status_tx = self.status_tx.clone();
-
- // Spawn a task to handle the actual migration work
- let handle = tokio::spawn(async move {
- // Start the migration process
- let result = migrate_ocr_data_to_frames(
- &db,
- config,
- is_running.clone(),
- is_paused.clone(),
- status_tx.clone(),
- )
- .await;
-
- // Handle completion or failure
- match result {
- Ok((total, duration)) => {
- info!("Migration completed successfully: {total} records processed in {duration} seconds");
- let _ = status_tx
- .send(MigrationResponse {
- status: MigrationStatus::Completed {
- total_records: total,
- duration_secs: duration,
- },
- })
- .await;
- }
- Err(e) => {
- let error_msg = e.to_string();
- error!("Migration failed: {error_msg}");
-
- // Try to get the current progress
- let processed = get_migration_progress(&db.pool).await.unwrap_or(0);
- let total = get_total_records(&db.pool).await.unwrap_or(0);
-
- let _ = status_tx
- .send(MigrationResponse {
- status: MigrationStatus::Failed {
- total_records: total,
- processed_records: processed,
- error: error_msg,
- },
- })
- .await;
- }
- }
-
- is_running.store(false, Ordering::SeqCst);
- });
-
- self.worker_handle = Some(handle);
- }
-
- fn pause_migration(&self) {
- if self.is_running.load(Ordering::SeqCst) {
- info!("Pausing migration");
- self.is_paused.store(true, Ordering::SeqCst);
- } else {
- warn!("Cannot pause migration: not running");
- }
- }
-
- fn stop_migration(&self) {
- if self.is_running.load(Ordering::SeqCst) {
- info!("Stopping migration");
- self.is_running.store(false, Ordering::SeqCst);
-
- // Cancel the worker task if it's running
- if let Some(handle) = &self.worker_handle {
- handle.abort();
- }
- } else {
- warn!("Cannot stop migration: not running");
- }
- }
-}
-
-/// Create a migration progress table if it doesn't exist
-async fn ensure_migration_table(pool: &SqlitePool) -> Result<()> {
- sqlx::query(
- r#"
- CREATE TABLE IF NOT EXISTS migration_progress (
- id INTEGER PRIMARY KEY,
- migration_name TEXT NOT NULL,
- last_processed_id INTEGER NOT NULL,
- total_records INTEGER NOT NULL,
- processed_records INTEGER NOT NULL,
- started_at TIMESTAMP NOT NULL,
- updated_at TIMESTAMP NOT NULL,
- completed_at TIMESTAMP
- )
- "#,
- )
- .execute(pool)
- .await?;
-
- Ok(())
-}
-
-/// Get the current migration progress
-async fn get_migration_progress(pool: &SqlitePool) -> Result<i64> {
- ensure_migration_table(pool).await?;
-
- let progress = sqlx::query_scalar::<_, i64>(
- r#"
- SELECT processed_records FROM migration_progress
- WHERE migration_name = 'ocr_text_to_frames'
- ORDER BY id DESC LIMIT 1
- "#,
- )
- .fetch_optional(pool)
- .await?
- .unwrap_or(0);
-
- Ok(progress)
-}
-
-/// Get the total number of records to migrate
-async fn get_total_records(pool: &SqlitePool) -> Result<i64> {
- let total = sqlx::query_scalar::<_, i64>(
- r#"
- SELECT COUNT(DISTINCT frame_id) FROM ocr_text
- WHERE (app_name IS NOT NULL AND app_name != '')
- OR (window_name IS NOT NULL AND window_name != '')
- OR (focused IS NOT NULL)
- "#,
- )
- .fetch_one(pool)
- .await?;
-
- Ok(total)
-}
-
-/// Update the migration progress
-async fn update_migration_progress(
- pool: &SqlitePool,
- last_id: i64,
- total: i64,
- processed: i64,
- completed: bool,
-) -> Result<()> {
- ensure_migration_table(pool).await?;
-
- let now = Utc::now();
- let completed_at = if completed { Some(now) } else { None };
-
- // Check if we have an existing record
- let existing = sqlx::query_scalar::<_, i64>(
- r#"
- SELECT id FROM migration_progress
- WHERE migration_name = 'ocr_text_to_frames'
- "#,
- )
- .fetch_optional(pool)
- .await?;
-
- if let Some(id) = existing {
- // Update existing record
- sqlx::query(
- r#"
- UPDATE migration_progress
- SET last_processed_id = ?,
- total_records = ?,
- processed_records = ?,
- updated_at = ?,
- completed_at = ?
- WHERE id = ?
- "#,
- )
- .bind(last_id)
- .bind(total)
- .bind(processed)
- .bind(now)
- .bind(completed_at)
- .bind(id)
- .execute(pool)
- .await?;
- } else {
- // Insert new record
- sqlx::query(
- r#"
- INSERT INTO migration_progress
- (migration_name, last_processed_id, total_records, processed_records, started_at, updated_at, completed_at)
- VALUES (?, ?, ?, ?, ?, ?, ?)
- "#,
- )
- .bind("ocr_text_to_frames")
- .bind(last_id)
- .bind(total)
- .bind(processed)
- .bind(now)
- .bind(now)
- .bind(completed_at)
- .execute(pool)
- .await?;
- }
-
- Ok(())
-}
-
-/// Perform the actual migration of data from ocr_text to frames
-async fn migrate_ocr_data_to_frames(
- db: &DatabaseManager,
- config: MigrationConfig,
- is_running: Arc<AtomicBool>,
- is_paused: Arc<AtomicBool>,
- status_tx: mpsc::Sender<MigrationResponse>,
-) -> Result<(i64, u64)> {
- let pool = &db.pool;
- let start_time = std::time::Instant::now();
-
- // Ensure migration table exists
- ensure_migration_table(pool).await?;
-
- // Get the total number of records to migrate
- let total_records = get_total_records(pool).await?;
-
- if total_records == 0 {
- info!("No records to migrate");
- return Ok((0, 0));
- }
-
- // Get the last processed ID
- let last_processed_id = sqlx::query_scalar::<_, i64>(
- r#"
- SELECT last_processed_id FROM migration_progress
- WHERE migration_name = 'ocr_text_to_frames'
- ORDER BY id DESC LIMIT 1
- "#,
- )
- .fetch_optional(pool)
- .await?
- .unwrap_or(0);
-
- let mut processed_records = get_migration_progress(pool).await?;
-
- info!(
- "Starting migration: total_records={}, already_processed={}, last_processed_id={}",
- total_records, processed_records, last_processed_id
- );
-
- // Update the initial status
- let _ = status_tx
- .send(MigrationResponse {
- status: MigrationStatus::Running {
- total_records,
- processed_records,
- },
- })
- .await;
-
- // Process in batches
- let batch_size = config.batch_size;
- let mut last_id = last_processed_id;
-
- while is_running.load(Ordering::SeqCst) && processed_records < total_records {
- // Check if we should pause
- while is_paused.load(Ordering::SeqCst) && is_running.load(Ordering::SeqCst) {
- let _ = status_tx
- .send(MigrationResponse {
- status: MigrationStatus::Paused {
- total_records,
- processed_records,
- },
- })
- .await;
- time::sleep(Duration::from_millis(500)).await;
- }
-
- if !is_running.load(Ordering::SeqCst) {
- break;
- }
-
- // Process a batch
- match process_batch(db, last_id, batch_size).await {
- Ok((batch_processed, new_last_id)) => {
- if batch_processed == 0 {
- // No more records to process
- break;
- }
-
- processed_records += batch_processed;
- last_id = new_last_id;
-
- // Update progress
- if let Err(e) = update_migration_progress(
- pool,
- last_id,
- total_records,
- processed_records,
- processed_records >= total_records,
- )
- .await
- {
- warn!("Failed to update migration progress: {}", e);
- }
-
- // Update status
- let _ = status_tx
- .send(MigrationResponse {
- status: MigrationStatus::Running {
- total_records,
- processed_records,
- },
- })
- .await;
-
- // Delay between batches to reduce database load
- time::sleep(Duration::from_millis(config.batch_delay_ms)).await;
- }
- Err(e) => {
- error!("Error processing batch: {}", e);
-
- if !config.continue_on_error {
- return Err(anyhow!("Migration failed: {}", e));
- }
-
- // Try to continue with the next batch
- last_id += batch_size;
-
- // Delay a bit longer after an error
- time::sleep(Duration::from_millis(config.batch_delay_ms * 5)).await;
- }
- }
- }
-
- let duration = start_time.elapsed().as_secs();
-
- // Final update to mark as completed
- if processed_records >= total_records {
- update_migration_progress(pool, last_id, total_records, processed_records, true).await?;
- }
-
- Ok((processed_records, duration))
-}
-
-/// Process a batch of records
-async fn process_batch(db: &DatabaseManager, last_id: i64, batch_size: i64) -> Result<(i64, i64)> {
- let mut tx = db.begin_immediate_with_retry().await?;
-
- // Query to get a batch of records with unique frame_ids that need migration
- let records = sqlx::query(
- r#"
- SELECT DISTINCT ocr_text.frame_id,
- ocr_text.app_name,
- ocr_text.window_name,
- ocr_text.focused
- FROM ocr_text
- JOIN frames ON ocr_text.frame_id = frames.id
- WHERE ocr_text.frame_id > ?
- AND (
- -- We need to migrate this data
- (ocr_text.app_name IS NOT NULL AND ocr_text.app_name != '' AND frames.app_name IS NULL) OR
- (ocr_text.window_name IS NOT NULL AND ocr_text.window_name != '' AND frames.window_name IS NULL) OR
- (ocr_text.focused IS NOT NULL AND frames.focused IS NULL)
- )
- ORDER BY ocr_text.frame_id
- LIMIT ?
- "#,
- )
- .bind(last_id)
- .bind(batch_size)
- .fetch_all(&mut **tx.conn())
- .await?;
-
- let mut count = 0;
- let mut max_id = last_id;
-
- // Process each record
- for record in records.iter() {
- let frame_id: i64 = record.try_get("frame_id")?;
- let app_name: Option<String> = record.try_get("app_name").unwrap_or(None);
- let window_name: Option<String> = record.try_get("window_name").unwrap_or(None);
- let focused: Option<bool> = record.try_get("focused").unwrap_or(None);
-
- // Update the frames table
- sqlx::query(
- r#"
- UPDATE frames
- SET app_name = COALESCE(?, app_name),
- window_name = COALESCE(?, window_name),
- focused = COALESCE(?, focused)
- WHERE id = ?
- "#,
- )
- .bind(app_name)
- .bind(window_name)
- .bind(focused)
- .bind(frame_id)
- .execute(&mut **tx.conn())
- .await?;
-
- count += 1;
- max_id = std::cmp::max(max_id, frame_id);
- }
-
- // Commit the transaction (auto-rollback on drop if this fails)
- tx.commit().await?;
-
- debug!("Processed batch: {} records, max_id={}", count, max_id);
-
- Ok((count, max_id))
-}
-
-impl MigrationConfig {
- pub fn new(batch_size: i64, batch_delay_ms: u64, continue_on_error: bool) -> Self {
- Self {
- batch_size,
- batch_delay_ms,
- continue_on_error,
- }
- }
-}
-
-/// Create a database migration worker and return channels to control it
-pub fn create_migration_worker(
- db: Arc<DatabaseManager>,
- config: Option<MigrationConfig>,
-) -> (
- mpsc::Sender<MigrationCommand>,
- mpsc::Receiver<MigrationResponse>,
- JoinHandle<()>,
-) {
- let (cmd_tx, cmd_rx) = mpsc::channel(100);
- let (status_tx, status_rx) = mpsc::channel(100);
-
- let worker = MigrationWorker::new(db, cmd_rx, status_tx, config.unwrap_or_default());
-
- let handle = worker.start();
-
- (cmd_tx, status_rx, handle)
-}