Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

8.3 KB
Raw Blame
//! Bounded queue driver. Domain services own claims, durable outcomes and recovery.
//! No detached import futures survive shutdown or loss of a heartbeat.

use std::{future::Future, time::Duration};

use futures::future::join_all;
use tokio::time::{Instant, MissedTickBehavior, interval_at, sleep, timeout};
use tokio_util::sync::CancellationToken;

use crate::domain::{
    maintenance::Maintenance,
    models::*,
    ports::{ImportConsumer, ImportWorker, PortResult},
};

#[cfg(test)]
mod test;

/// Validated operational bounds. A restart changes enablement; disabled instances
/// still reconcile DLQ records, expired/cancelled leases and dirty search work.
#[derive(Clone, Copy)]
pub struct WorkerConfig {
    enabled: bool,
    concurrency: u8,
    max_receives: u32,
    heartbeat: Duration,
    operation_timeout: Duration,
    retry_delay: Duration,
    maintenance_interval: Duration,
    drain_timeout: Duration,
}

impl WorkerConfig {
    /// Default to one import at a time; permit at most two. Reserve ten seconds
    /// of the task's 120-second stop budget for runtime/telemetry teardown.
    pub fn new(enabled: bool, concurrency: u8, max_receives: u32) -> Result<Self, ImportError> {
        if !(1..=2).contains(&concurrency) || max_receives == 0 {
            return Err(ImportError::InvalidInput);
        }
        Ok(Self {
            enabled,
            concurrency,
            max_receives,
            heartbeat: Duration::from_secs(60),
            operation_timeout: Duration::from_secs(30),
            retry_delay: Duration::from_secs(5),
            maintenance_interval: Duration::from_secs(10),
            drain_timeout: Duration::from_secs(110),
        })
    }
}

/// Run independent claim, publication, recovery and DLQ loops. On cancellation,
/// stop receiving immediately and drain active imports with heartbeats intact.
/// At the deadline drop all futures: committed checkpoints and expired fenced
/// leases are recovered by another instance, never acknowledged speculatively.
pub async fn run(
    work: &impl ImportWorker,
    maintenance: &impl Maintenance,
    queue: &impl ImportConsumer,
    owner: WorkerId,
    config: WorkerConfig,
    stop: CancellationToken,
) {
    let lanes = async {
        if config.enabled {
            join_all(
                (0..config.concurrency)
                    .map(|_| consume(work, maintenance, queue, owner, config, &stop, false)),
            )
            .await;
        }
    };
    let loops = async {
        tokio::join!(
            lanes,
            consume(work, maintenance, queue, owner, config, &stop, true),
            maintain(maintenance, config, &stop, false),
            maintain(maintenance, config, &stop, true),
        );
    };
    tokio::pin!(loops);
    tokio::select! {
        biased;
        _ = stop.cancelled() => (),
        _ = &mut loops => return,
    }
    if timeout(config.drain_timeout, loops).await.is_err() {
        tracing::warn!(
            operation = "shutdown",
            outcome = "checkpoint_recovery",
            "slack import drain deadline"
        );
    }
}

async fn consume(
    work: &impl ImportWorker,
    maintenance: &impl Maintenance,
    queue: &impl ImportConsumer,
    owner: WorkerId,
    config: WorkerConfig,
    stop: &CancellationToken,
    dead_letter: bool,
) {
    loop {
        let received = tokio::select! {
            biased;
            _ = stop.cancelled() => return,
            result = bounded(config, queue.receive(dead_letter)) => result,
        };
        if stop.is_cancelled() {
            return;
        }
        match received {
            Ok(Some(delivery)) => {
                let result = handle(
                    work,
                    maintenance,
                    queue,
                    &delivery,
                    owner,
                    config,
                    dead_letter,
                )
                .await;
                match result {
                    Ok(WorkerOutcome::Acknowledge) => {
                        report("delete", bounded(config, queue.delete(&delivery)).await);
                    }
                    Ok(WorkerOutcome::Defer) => {
                        tracing::info!(
                            operation = "delivery",
                            outcome = "deferred",
                            dead_letter,
                            "slack import delivery"
                        );
                    }
                    Err(error) => report("delivery", Err(error)),
                }
            }
            Ok(None) => (),
            Err(error) => {
                report("receive", Err(error));
                tokio::select! {
                    _ = stop.cancelled() => return,
                    _ = sleep(config.retry_delay) => (),
                }
            }
        }
    }
}

async fn handle<Q: ImportConsumer>(
    work: &impl ImportWorker,
    maintenance: &impl Maintenance,
    queue: &Q,
    delivery: &Q::Delivery,
    owner: WorkerId,
    config: WorkerConfig,
    dead_letter: bool,
) -> PortResult<WorkerOutcome> {
    let (event, receive_count) = Q::envelope(delivery);
    let event = match event {
        Ok(event) => event,
        Err(code) => {
            tracing::warn!(
                ?code,
                dead_letter,
                operation = "malformed",
                "slack import invalid envelope"
            );
            // Main poison messages redrive to DLQ for a separate sanitized signal.
            // No trusted identity exists to update. Never log the raw payload.
            return Ok(if dead_letter {
                WorkerOutcome::Acknowledge
            } else {
                WorkerOutcome::Defer
            });
        }
    };
    let process = async {
        if dead_letter || receive_count >= config.max_receives {
            return bounded(config, maintenance.dead_letter(&event)).await;
        }
        match bounded(config, work.claim(&event, owner)).await? {
            ClaimOutcome::Obsolete => Ok(WorkerOutcome::Acknowledge),
            ClaimOutcome::ActiveLease => Ok(WorkerOutcome::Defer),
            ClaimOutcome::Claimed(context) => {
                // These futures are polled independently of storage reads, target
                // resolution and batch commits. A failed renewal drops the importer.
                tokio::select! {
                    biased;
                    result = heartbeat(config, || async { work.heartbeat(&context.lease).await.map(|_| ()) }) => result,
                    result = work.import(&context) => result,
                }
            }
        }
    };
    tokio::select! {
        biased;
        result = heartbeat(config, || queue.extend(delivery)) => result,
        result = process => result,
    }
}

async fn heartbeat<F, Fut>(config: WorkerConfig, mut renew: F) -> PortResult<WorkerOutcome>
where
    F: FnMut() -> Fut,
    Fut: Future<Output = PortResult<()>>,
{
    let mut ticks = interval_at(Instant::now() + config.heartbeat, config.heartbeat);
    ticks.set_missed_tick_behavior(MissedTickBehavior::Delay);
    loop {
        ticks.tick().await;
        bounded(config, renew()).await?;
    }
}

async fn maintain(
    service: &impl Maintenance,
    config: WorkerConfig,
    stop: &CancellationToken,
    publication: bool,
) {
    if publication && !config.enabled {
        return;
    }
    let operation = if publication { "outbox" } else { "reconcile" };
    loop {
        tokio::select! {
            biased;
            _ = stop.cancelled() => return,
            result = bounded(config, async {
                if publication { service.publish().await } else { service.reconcile().await }
            }) => report(operation, result),
        }
        tokio::select! {
            _ = stop.cancelled() => return,
            _ = sleep(config.maintenance_interval) => (),
        }
    }
}

async fn bounded<T>(
    config: WorkerConfig,
    operation: impl Future<Output = PortResult<T>>,
) -> PortResult<T> {
    timeout(config.operation_timeout, operation)
        .await
        .map_err(|_| ImportError::Retryable)?
}

fn report(operation: &'static str, result: PortResult<()>) {
    match result {
        Ok(()) => tracing::info!(operation, outcome = "success", "slack import operation"),
        Err(error) => {
            tracing::warn!(operation, code = ?error.current_context(), "slack import retry")
        }
    }
}