Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

28.3 KB
Raw Blame
//! Opt-in real PostgreSQL/S3/SQS contract test. No hosted endpoints or customer data.
//! SQLx creates a database per test; Docker creates a private LocalStack container.
//! Search acceptance is deliberately controlled, not an OpenSearch end-to-end test.

use std::{
    process::Command,
    sync::{
        Arc,
        atomic::{AtomicBool, Ordering},
    },
    time::Duration,
};

use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
use chrono::{DateTime, Utc};
use entity_access::{
    domain::{models::*, service::EntityAccessServiceImpl},
    outbound::PgAccessRepository,
};
use import::outbound::pg_import_repo::PgImportRepo;
use macro_queues::{SlackImportDlq, SlackImportQueue};
use macro_user_id::user_id::MacroUserIdStr;
use sha2::{Digest, Sha256};
use slack_import_worker::composition::{
    authorizer::WorkerAuthorizer, channel_sink::ChannelImportSink,
    reference_reconciliation::WorkerReferenceReconciler,
};
use slack_integration::{
    domain::{
        importer::{ConversationImporter, ImporterConfig},
        maintenance::{ImportMaintenance, Maintenance},
        models::*,
        ports::*,
        service::SlackImportService,
    },
    inbound::worker::{self, WorkerConfig},
    outbound::{
        import_ledger::CanonicalImportLedger, pg_slack_import_repo::PgSlackImportRepo,
        s3_storage::S3ImportStorage, sqs_queue::SqsImportQueue,
    },
};
use sqlx::PgPool;
use tokio::time::{sleep, timeout};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;

#[path = "archive_import/process_recovery.rs"]
mod process_recovery;

type Access = EntityAccessServiceImpl<PgAccessRepository>;

/// Only this test's container is removed, including on assertion failure. The image
/// must already exist: an ignored test must not silently pull arbitrary images.
struct LocalStack(String);

impl LocalStack {
    async fn start() -> (Self, String) {
        let container = Self(docker(&[
            "run",
            "--detach",
            "--rm",
            "--pull=never",
            "--publish",
            "127.0.0.1::4566",
            "--env",
            "SERVICES=s3,sqs",
            "--env",
            "SQS_ENDPOINT_STRATEGY=path",
            "localstack/localstack:4",
        ]));
        let address = docker(&["port", &container.0, "4566/tcp"]);
        let endpoint = format!("http://{address}");
        let client = reqwest::Client::new();
        timeout(Duration::from_secs(60), async {
            loop {
                if let Ok(response) = client
                    .get(format!("{endpoint}/_localstack/health"))
                    .send()
                    .await
                    && response.status().is_success()
                {
                    break;
                }
                sleep(Duration::from_millis(250)).await;
            }
        })
        .await
        .expect("isolated LocalStack did not start");
        (container, endpoint)
    }
}

impl Drop for LocalStack {
    fn drop(&mut self) {
        let _ = Command::new("docker")
            .args(["rm", "--force", &self.0])
            .output();
    }
}

fn docker(args: &[&str]) -> String {
    let output = Command::new("docker")
        .args(args)
        .output()
        .expect("Docker is required");
    assert!(
        output.status.success(),
        "docker failed: {}",
        String::from_utf8_lossy(&output.stderr)
    );
    String::from_utf8(output.stdout).unwrap().trim().to_owned()
}

struct Resources {
    storage: S3ImportStorage,
    queue: SqsImportQueue,
    process: process_recovery::ProcessConfig,
}

async fn resources(endpoint: &str) -> Resources {
    let suffix = Uuid::now_v7().simple().to_string();
    let bucket = format!("archive-test-{suffix}");
    let main = format!("archive-main-{suffix}");
    let dlq = format!("archive-dlq-{suffix}");
    let s3 = aws_sdk_s3::Client::from_conf(
        aws_sdk_s3::Config::builder()
            .behavior_version(BehaviorVersion::latest())
            .region(Region::new("us-east-1"))
            .credentials_provider(Credentials::new(
                "test",
                "test",
                None,
                None,
                "isolated-test",
            ))
            .endpoint_url(endpoint)
            .force_path_style(true)
            .build(),
    );
    s3.create_bucket().bucket(&bucket).send().await.unwrap();
    let sqs = aws_sdk_sqs::Client::from_conf(
        aws_sdk_sqs::Config::builder()
            .behavior_version(BehaviorVersion::latest())
            .region(Region::new("us-east-1"))
            .credentials_provider(Credentials::new(
                "test",
                "test",
                None,
                None,
                "isolated-test",
            ))
            .endpoint_url(endpoint)
            .build(),
    );
    for name in [&main, &dlq] {
        sqs.create_queue().queue_name(name).send().await.unwrap();
    }
    use aws_sdk_sqs::types::QueueAttributeName;
    let main_url = sqs
        .get_queue_url()
        .queue_name(&main)
        .send()
        .await
        .unwrap()
        .queue_url
        .unwrap();
    let dlq_url = sqs
        .get_queue_url()
        .queue_name(&dlq)
        .send()
        .await
        .unwrap()
        .queue_url
        .unwrap();
    let attributes = sqs
        .get_queue_attributes()
        .queue_url(dlq_url)
        .attribute_names(QueueAttributeName::QueueArn)
        .send()
        .await
        .unwrap();
    let arn = &attributes.attributes.unwrap()[&QueueAttributeName::QueueArn];
    sqs.set_queue_attributes()
        .queue_url(main_url)
        .attributes(
            QueueAttributeName::RedrivePolicy,
            serde_json::json!({"deadLetterTargetArn": arn, "maxReceiveCount": 2}).to_string(),
        )
        .send()
        .await
        .unwrap();
    let queue = SqsImportQueue::new(
        sqs.clone(),
        &SlackImportQueue::from_owned(main.clone()),
        &SlackImportDlq::from_owned(dlq.clone()),
    )
    .await
    .unwrap();
    let process = process_recovery::ProcessConfig {
        endpoint: endpoint.to_owned(),
        bucket: bucket.clone(),
        main,
        dlq,
    };
    let storage = S3ImportStorage::new(s3, bucket, ImportLimits::default()).unwrap();
    Resources {
        storage,
        queue,
        process,
    }
}

fn user() -> MacroUserIdStr<'static> {
    MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap()
}

async fn team(pool: &PgPool) -> TeamId {
    let account = Uuid::now_v7();
    let team = Uuid::now_v7();
    sqlx::query!("INSERT INTO macro_user (id, username, email, stripe_customer_id) VALUES ($1, 't17', '[email protected]', 't17-customer')", account).execute(pool).await.unwrap();
    sqlx::query!(r#"INSERT INTO "User" (id, email, macro_user_id) VALUES ('macro|[email protected]', '[email protected]', $1)"#, account).execute(pool).await.unwrap();
    sqlx::query!("INSERT INTO team (id, name, owner_id, auto_join_domain) VALUES ($1, 'Atomic history', 'macro|[email protected]', 'example.com')", team).execute(pool).await.unwrap();
    sqlx::query!("INSERT INTO team_user (team_id, user_id, team_role) VALUES ($1, 'macro|[email protected]', 'owner')", team).execute(pool).await.unwrap();
    team.try_into().unwrap()
}

fn access(team: TeamId) -> EntityAccessReceipt<AdminTeamRole> {
    EntityAccessReceipt::try_new(
        EntityAccessAuth::Authenticated(user()),
        Entity {
            entity_id: team.to_string(),
            entity_type: EntityType::Team,
        },
        EntityPermission::TeamRole {
            role: TeamRole::Admin,
        },
    )
    .unwrap()
}

// Production write authorization is retained. This harness intentionally hides
// all target IDs in public receipts; reference disclosure uses the real lookup.
struct Admin(WorkerAuthorizer<Access>);
impl ImportAuthorizer for Admin {
    async fn require_admin(&self, team: TeamId, user: &MacroUserIdStr<'_>) -> PortResult<()> {
        self.0.require_admin(team, user).await
    }
    async fn require_target(
        &self,
        team: TeamId,
        user: &MacroUserIdStr<'_>,
        metadata: &ConversationMetadata,
        target: Uuid,
    ) -> PortResult<()> {
        self.0.require_target(team, user, metadata, target).await
    }
}
impl ImportProgressAccess for Admin {
    async fn participating_targets(
        &self,
        _: &MacroUserIdStr<'_>,
        _: &[Uuid],
    ) -> PortResult<Vec<Uuid>> {
        Ok(vec![])
    }
}
struct NoGateway;
impl ImportNotifier for NoGateway {
    async fn invalidate(
        &self,
        _: &MacroUserIdStr<'_>,
        _: TeamId,
        _: JobId,
        _: u64,
        _: JobStatus,
    ) -> PortResult<()> {
        Ok(())
    }
}
struct SystemClock;
impl Clock for SystemClock {
    fn now(&self) -> DateTime<Utc> {
        Utc::now()
    }
}
#[derive(Clone, Default)]
struct SearchGate(Arc<AtomicBool>);
impl SearchBackfillClient for SearchGate {
    async fn submit(&self, _: &SearchBackfill) -> PortResult<Uuid> {
        Ok(Uuid::now_v7())
    }
    async fn progress(&self, receipt_id: Uuid) -> PortResult<SearchState> {
        if self.0.load(Ordering::SeqCst) {
            Ok(SearchState::Completed)
        } else {
            Ok(SearchState::Submitted { receipt_id })
        }
    }
}

fn command(history: bool) -> CreateImport {
    CreateImport {
        idempotency_token: Uuid::now_v7().try_into().unwrap(),
        source: SourceIdentity::ConfirmedUnknown,
        include_message_history: history,
        conversations: ["C1", "C2"]
            .map(|id| ConversationMetadata {
                slack_channel_id: id.parse().unwrap(),
                kind: ConversationKind::PublicChannel,
                name: "duplicate display name".into(),
                folder: id.parse().unwrap(),
                member_ids: vec!["U1".parse().unwrap(), "U3".parse().unwrap()],
                creator_id: None,
                created_at: Some("1.000001".parse().unwrap()),
                archived: false,
                message_count: None,
            })
            .to_vec(),
    }
}

fn descriptor(upload: UploadId, bytes: &[u8]) -> UploadDescriptor {
    let record_count = match upload {
        UploadId::Users => None,
        UploadId::ConversationPart { .. } => {
            Some(bytes.iter().filter(|byte| **byte == b'\n').count() as u32)
        }
    };
    UploadDescriptor {
        upload,
        sha256: format!("{:x}", Sha256::digest(bytes)).parse().unwrap(),
        byte_length: bytes.len() as u64,
        record_count,
    }
}

async fn put(grant: &UploadGrant, bytes: &[u8]) -> reqwest::StatusCode {
    let mut request = reqwest::Client::new().put(&grant.url).body(bytes.to_vec());
    for (name, value) in &grant.required_headers {
        request = request.header(name, value);
    }
    request.send().await.unwrap().status()
}

async fn upload_job(service: &impl ImportService, team: TeamId, history: bool) -> JobId {
    let request = command(history);
    let job = service.create(access(team), request.clone()).await.unwrap();
    assert_eq!(
        service
            .create(access(team), request.clone())
            .await
            .unwrap()
            .job_id,
        job.job_id
    );
    let mut changed = request;
    changed.conversations.pop();
    assert_eq!(
        service.create(access(team), changed).await.unwrap_err(),
        ImportError::Conflict
    );
    let users = serde_json::to_vec(&serde_json::json!([
        {"id":"U1", "name":"admin", "profile":{"email":"[email protected]"}},
        {"id":"U2", "name":"Unknown author"},
        {"id":"U3", "name":"External member", "profile":{"email":"[email protected]"}}
    ]))
    .unwrap();
    let users_descriptor = descriptor(UploadId::Users, &users);
    let grant = service
        .register_uploads(
            access(team),
            job.job_id,
            RegisterUploads {
                descriptors: vec![users_descriptor],
            },
        )
        .await
        .unwrap()
        .pop()
        .unwrap();
    assert!(put(&grant, &users).await.is_success());
    assert_eq!(
        put(&grant, &users).await,
        reqwest::StatusCode::PRECONDITION_FAILED
    );
    service
        .complete_uploads(
            access(team),
            job.job_id,
            CompleteUploads {
                uploads: vec![UploadId::Users],
                seal: None,
            },
        )
        .await
        .unwrap();
    // Registration and sealing cannot widen the persisted selection.
    let foreign = "CUNSELECTED".parse().unwrap();
    assert!(
        service
            .register_uploads(
                access(team),
                job.job_id,
                RegisterUploads {
                    descriptors: vec![descriptor(
                        UploadId::ConversationPart {
                            slack_channel_id: foreign,
                            part_index: 0
                        },
                        b"{}\n"
                    )]
                }
            )
            .await
            .is_err()
    );
    let seal = ConversationSeal::from_descriptors(
        "CUNSELECTED".parse().unwrap(),
        &[],
        &ImportLimits::default(),
    )
    .unwrap();
    assert!(
        service
            .complete_uploads(
                access(team),
                job.job_id,
                CompleteUploads {
                    uploads: vec![],
                    seal: Some(seal)
                }
            )
            .await
            .is_err()
    );
    assert!(
        service
            .progress(
                access(Uuid::now_v7().try_into().unwrap()),
                JobCommand { job_id: job.job_id }
            )
            .await
            .is_err()
    );
    // Seal in reverse order. No message can rely on queue order for its links.
    for id in ["C2", "C1"] {
        let mut descriptors = vec![];
        if history {
            let other = if id == "C1" { "C2" } else { "C1" };
            let records = [
                serde_json::json!({"type":"message", "user":"U2", "ts":"2.000001", "text": format!("<@U1> <!channel> See <#{other}|other> and <https://unproven.slack.com/archives/{other}/p0000000002000001|original>"), "reactions":[{"name":"thumbsup", "users":["U1"], "count":1}]}),
                serde_json::json!({"type":"message", "user":"U1", "ts":"3.000001", "thread_ts":"2.000001", "text":"historical reply"}),
            ];
            let bytes = records
                .iter()
                .map(|record| format!("{record}\n"))
                .collect::<String>()
                .into_bytes();
            let part = descriptor(
                UploadId::ConversationPart {
                    slack_channel_id: id.parse().unwrap(),
                    part_index: 0,
                },
                &bytes,
            );
            let grant = service
                .register_uploads(
                    access(team),
                    job.job_id,
                    RegisterUploads {
                        descriptors: vec![part.clone()],
                    },
                )
                .await
                .unwrap()
                .pop()
                .unwrap();
            assert!(put(&grant, &bytes).await.is_success());
            service
                .complete_uploads(
                    access(team),
                    job.job_id,
                    CompleteUploads {
                        uploads: vec![part.upload.clone()],
                        seal: None,
                    },
                )
                .await
                .unwrap();
            descriptors.push(part);
        }
        let receipt = service
            .progress(access(team), JobCommand { job_id: job.job_id })
            .await
            .unwrap();
        assert_eq!(
            receipt
                .conversations
                .iter()
                .find(|conversation| conversation.slack_channel_id.as_str() == id)
                .unwrap()
                .status,
            ConversationStatus::AwaitingUploads
        );
        let seal = ConversationSeal::from_descriptors(
            id.parse().unwrap(),
            &descriptors,
            &ImportLimits::default(),
        )
        .unwrap();
        service
            .complete_uploads(
                access(team),
                job.job_id,
                CompleteUploads {
                    uploads: vec![],
                    seal: Some(seal),
                },
            )
            .await
            .unwrap();
    }
    let receipt = service
        .finalize(access(team), JobCommand { job_id: job.job_id })
        .await
        .unwrap();
    assert_eq!(receipt.conversations.len(), 2);
    assert_eq!(receipt.include_message_history, history);
    job.job_id
}

async fn progress(repo: &PgSlackImportRepo, team: TeamId, job: JobId) -> ImportProgress {
    repo.progress(team, job).await.unwrap().unwrap()
}

async fn assert_no_notification_or_activity_rows(pool: &PgPool) {
    let counts = sqlx::query!(
        r#"
        SELECT
            (SELECT count(*) FROM notification) AS "notifications!",
            (SELECT count(*) FROM user_notification) AS "user_notifications!",
            (SELECT count(*) FROM notification_email_sent) AS "emails!",
            (SELECT count(*) FROM channel_notification_email_sent) AS "channel_emails!",
            (SELECT count(*) FROM comms_activity) AS "channel_activity!",
            (SELECT count(*) FROM activity_events) AS "activity_events!"
        "#
    )
    .fetch_one(pool)
    .await
    .unwrap();
    assert_eq!(
        counts.notifications, 0,
        "history must not create notifications"
    );
    assert_eq!(
        counts.user_notifications, 0,
        "history must not create unread notifications"
    );
    assert_eq!(counts.emails, 0, "history must not send emails");
    assert_eq!(
        counts.channel_emails, 0,
        "history must not send channel invitations"
    );
    assert_eq!(
        counts.channel_activity, 0,
        "history must not create per-user activity"
    );
    assert_eq!(
        counts.activity_events, 0,
        "history must not create live activity events"
    );
}

#[ignore = "requires Docker with localstack/localstack:4 and a local PostgreSQL test role"]
#[sqlx::test(migrations = "../../crates/macro_db_client/migrations")]
async fn sealed_uploads_queue_replay_and_search_completion_barrier(pool: PgPool) {
    let (_container, endpoint) = LocalStack::start().await;
    let Resources { storage, queue, .. } = resources(&endpoint).await;
    let team = team(&pool).await;
    let limits = ImportLimits::default();
    let repo = PgSlackImportRepo::new(pool.clone(), limits);
    let authorizer = WorkerAuthorizer::new(
        pool.clone(),
        Access::new(PgAccessRepository::new(pool.clone())),
    );
    let ledger = CanonicalImportLedger::new(PgImportRepo::new(pool.clone()));
    let service = SlackImportService::new(
        repo.clone(),
        storage.clone(),
        ledger.clone(),
        NoGateway,
        Admin(authorizer.clone()),
        |_| true,
        limits,
    )
    .unwrap();
    let sink = ChannelImportSink::new(pool.clone(), authorizer.clone(), limits).unwrap();
    let search = SearchGate::default();
    let maintenance = Arc::new(ImportMaintenance::new(
        repo.clone(),
        queue.clone(),
        search.clone(),
        SystemClock,
        WorkerReferenceReconciler::new(pool.clone(), limits),
    ));
    let importer = Arc::new(
        ConversationImporter::new(
            repo.clone(),
            storage,
            ledger,
            sink.clone(),
            sink.clone(),
            authorizer,
            ImporterConfig { limits },
        )
        .unwrap(),
    );
    assert_no_notification_or_activity_rows(&pool).await;
    // Shape creation followed by history and an exact repeat reuse the same IDs.
    let mut canonical = None;
    for history in [false, true, true] {
        search.0.store(false, Ordering::SeqCst);
        let job = upload_job(&service, team, history).await;
        let events = repo.pending_events(50).await.unwrap();
        assert_eq!(events.len(), 2);
        // Fault boundary: SQS accepted the send, but the process died before the
        // PostgreSQL publication acknowledgement. The real outbox republishes it.
        for event in &events {
            queue.publish(event).await.unwrap();
        }
        // Expire only this test job's publication reservation, as wall time would
        // after a publisher crash. pending_events is a reservation, not a read.
        sqlx::query!("UPDATE slack_import_outbox SET available_at = clock_timestamp() - interval '1 second' WHERE job_id = $1", Uuid::from(job)).execute(&pool).await.unwrap();
        maintenance.publish().await.unwrap();
        assert!(repo.pending_events(50).await.unwrap().is_empty());
        let stop = CancellationToken::new();
        let driver = {
            let (importer, maintenance, queue, stop) = (
                importer.clone(),
                maintenance.clone(),
                queue.clone(),
                stop.clone(),
            );
            tokio::spawn(async move {
                worker::run(
                    importer.as_ref(),
                    maintenance.as_ref(),
                    &queue,
                    Uuid::now_v7().try_into().unwrap(),
                    WorkerConfig::new(true, 2, 5).unwrap(),
                    stop,
                )
                .await;
            })
        };
        timeout(Duration::from_secs(60), async {
            loop {
                let receipt = progress(&repo, team, job).await;
                if receipt
                    .conversations
                    .iter()
                    .all(|c| c.status == ConversationStatus::Completed)
                {
                    break;
                }
                sleep(Duration::from_millis(100)).await;
            }
        })
        .await
        .expect("queue driver did not settle selected conversations");
        stop.cancel();
        timeout(Duration::from_secs(35), driver)
            .await
            .unwrap()
            .unwrap();
        // Reconcile links and publish search without a competing maintenance loop.
        maintenance.reconcile().await.unwrap();
        let receipt = progress(&repo, team, job).await;
        let targets: Vec<_> = receipt
            .conversations
            .iter()
            .map(|c| c.channel_id.unwrap())
            .collect();
        if let Some(ref expected) = canonical {
            assert_eq!(&targets, expected);
        } else {
            canonical = Some(targets.clone());
        }
        if history
            && receipt
                .conversations
                .iter()
                .any(|c| c.counters.imported > 0)
        {
            assert_eq!(
                receipt.status,
                JobStatus::Processing,
                "search acceptance is not completion"
            );
            for (index, id) in ["C1", "C2"].iter().enumerate() {
                let sources = ["2.000001", "3.000001"].map(|ts| SourceMessageId {
                    team_id: team,
                    slack_channel_id: id.parse().unwrap(),
                    ts: ts.parse().unwrap(),
                });
                let mapped = sink.lookup(&sources).await.unwrap();
                assert_eq!(mapped.len(), 2);
                let root = mapped
                    .iter()
                    .find(|(source, _)| source.ts == sources[0].ts)
                    .unwrap()
                    .1;
                let body =
                    sqlx::query_scalar!("SELECT content FROM comms_messages WHERE id = $1", root)
                        .fetch_one(&pool)
                        .await
                        .unwrap();
                assert!(
                    body.contains(&targets[1 - index].to_string()),
                    "reference must contain final canonical target UUID"
                );
                assert!(body.contains("<m-link>"));
                assert!(
                    body.contains("unproven.slack.com"),
                    "unproven domain must stay external"
                );
                let replies = sqlx::query_scalar!(
                    "SELECT thread_id FROM comms_messages WHERE parent_entity_type = 'channel' AND parent_entity_id = $1::uuid::text AND id <> $2",
                    targets[index],
                    root
                )
                .fetch_all(&pool)
                .await
                .unwrap();
                assert_eq!(replies, vec![Some(root)]);
            }
        }
        search.0.store(true, Ordering::SeqCst);
        // Make the 60-second search receipt poll due in this isolated fixture.
        sqlx::query!("UPDATE slack_import_outbox SET available_at = clock_timestamp() - interval '1 second' WHERE job_id = $1", Uuid::from(job)).execute(&pool).await.unwrap();
        maintenance.reconcile().await.unwrap();
        assert_eq!(
            progress(&repo, team, job).await.status,
            JobStatus::Completed
        );
        assert_no_notification_or_activity_rows(&pool).await;
    }
    assert_eq!(
        sqlx::query_scalar!("SELECT count(*) FROM comms_messages")
            .fetch_one(&pool)
            .await
            .unwrap(),
        Some(4)
    );

    // Exhaust real receives (without a worker deleting them) into the configured
    // DLQ. Disabling admission must still run the production DLQ driver.
    // Cancelled HTTP long polls can still reserve messages server-side until
    // their timeout. A fresh queue keeps this redrive scenario independent.
    let Resources { storage, queue, .. } = resources(&endpoint).await;
    let authorizer = WorkerAuthorizer::new(
        pool.clone(),
        Access::new(PgAccessRepository::new(pool.clone())),
    );
    let service = SlackImportService::new(
        repo.clone(),
        storage,
        CanonicalImportLedger::new(PgImportRepo::new(pool.clone())),
        NoGateway,
        Admin(authorizer),
        |_| true,
        limits,
    )
    .unwrap();
    let maintenance = Arc::new(ImportMaintenance::new(
        repo.clone(),
        queue.clone(),
        search,
        SystemClock,
        WorkerReferenceReconciler::new(pool.clone(), limits),
    ));
    let exhausted = upload_job(&service, team, false).await;
    maintenance.publish().await.unwrap();
    let mut twice = std::collections::HashSet::new();
    timeout(Duration::from_secs(60), async {
        while twice.len() != 2 {
            let Some(delivery) = ImportConsumer::receive(&queue, false).await.unwrap() else {
                continue;
            };
            let (event, count) = SqsImportQueue::envelope(&delivery);
            let event = event.unwrap();
            if event.job_id != exhausted {
                ImportConsumer::delete(&queue, &delivery).await.unwrap();
                continue;
            }
            if count >= 2 {
                twice.insert(event.slack_channel_id);
            }
            queue.extend_visibility(&delivery, 0).await.unwrap();
        }
        // The next main receive triggers redrive, rather than fabricating a DLQ send.
        assert!(
            ImportConsumer::receive(&queue, false)
                .await
                .unwrap()
                .is_none()
        );
    })
    .await
    .expect("main queue did not exhaust into the DLQ");
    let stop = CancellationToken::new();
    let driver = {
        let (importer, maintenance, queue, stop) = (
            importer.clone(),
            maintenance.clone(),
            queue.clone(),
            stop.clone(),
        );
        tokio::spawn(async move {
            worker::run(
                importer.as_ref(),
                maintenance.as_ref(),
                &queue,
                Uuid::now_v7().try_into().unwrap(),
                WorkerConfig::new(false, 1, 5).unwrap(),
                stop,
            )
            .await;
        })
    };
    timeout(Duration::from_secs(60), async {
        loop {
            let receipt = progress(&repo, team, exhausted).await;
            if receipt.status == JobStatus::Failed {
                assert!(
                    receipt
                        .conversations
                        .iter()
                        .all(|conversation| conversation.status == ConversationStatus::Failed)
                );
                break;
            }
            sleep(Duration::from_millis(100)).await;
        }
    })
    .await
    .expect("disabled worker did not persist exhausted work as Failed");
    assert_no_notification_or_activity_rows(&pool).await;
    stop.cancel();
    timeout(Duration::from_secs(35), driver)
        .await
        .unwrap()
        .unwrap();
}