Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

4.2 KB
Raw Blame
use crate::process::{self, context::SearchProcessingContext};
use futures::StreamExt;

/// Spawns multiple search processing workers.
pub fn run_search_processing_workers(ctx: SearchProcessingContext, worker_count: u8) {
    for worker_id in 0..worker_count {
        let ctx_clone = ctx.clone();
        tokio::spawn(async move {
            tracing::info!(worker_id, "spawning worker");
            run_search_processing_worker_with_id(ctx_clone, worker_id).await;
        });
    }
}

#[tracing::instrument(skip(ctx))]
pub async fn run_search_processing_worker_with_id(ctx: SearchProcessingContext, worker_id: u8) {
    loop {
        let worker_result = tokio::spawn({
            let ctx = ctx.clone();
            async move {
                tracing::info!("worker started");
                let mut last_heartbeat = std::time::Instant::now();
                loop {
                    if last_heartbeat.elapsed() > std::time::Duration::from_secs(60) {
                        tracing::info!("worker heartbeat - still running");
                        last_heartbeat = std::time::Instant::now();
                    }

                    match ctx.worker.receive_messages().await {
                        Ok(messages) => {
                            if messages.is_empty() {
                                continue;
                            }
                            let result = futures::stream::iter(messages.into_iter())
                                .map(|message| {
                                    let ctx = ctx.clone();
                                    async move {
                                        let result = tokio::time::timeout(
                                            std::time::Duration::from_secs(300), // 5 minutes
                                            process::process_message(&ctx, &message)
                                        ).await;

                                        match result {
                                            Ok(Ok(_)) => Ok(()),
                                            Ok(Err(e)) => Err((message.message_id.clone().unwrap_or_default(), e)),
                                            Err(_) => {
                                                tracing::error!(
                                                    message_id = message.message_id.as_deref().unwrap_or("unknown"),
                                                    "Message processing timed out after 5 minutes"
                                                );
                                                Err((message.message_id.clone().unwrap_or_default(), anyhow::anyhow!("Processing timeout")))
                                            }
                                        }
                                    }
                                })
                                .buffer_unordered(10)
                                .collect::<Vec<Result<(), (String, anyhow::Error)>>>()
                                .await;

                            let errors = result
                                .into_iter()
                                .filter_map(|result| result.err())
                                .collect::<Vec<(String, anyhow::Error)>>();

                            if !errors.is_empty() {
                                for (message_id, error) in errors {
                                    tracing::error!(message_id, error=?error, "error processing message");
                                }
                            }
                        }
                        Err(e) => {
                            tracing::error!(error=?e, "error receiving messages");
                        }
                    }
                }
            }
        })
        .await;

        match worker_result {
            Ok(_) => {
                // This should never be hit
                tracing::error!("worker exited successfully?");
            }
            Err(e) => {
                tracing::error!(error=?e, "worker crashed with error");
            }
        }

        // Add a delay before restarting to avoid rapid restart loops
        tracing::info!("WORKER RESTARTING...");
        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
    }
}