Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

6.8 KB
Raw Blame
use crate::{
    SQS,
    search::{
        calendar_event::UpsertCalendarEvent,
        call::{CallRecordMessage, RemoveCallRecord},
        channel::ChannelMessageUpdate,
        chat::ChatMessage,
        document::SearchExtractorMessage,
        email::EmailThreadBatchMessage,
        project::UpsertProject,
    },
};
use anyhow::Context;
use aws_sdk_sqs::{self as sqs, types::SendMessageBatchRequestEntry};
use futures::StreamExt;
use strum::Display;

pub mod calendar_event;
pub mod call;
pub mod channel;
pub mod chat;
pub mod document;
pub mod email;
pub mod project;

use crate::{MAX_BATCH_SIZE, PrimaryId};

impl SQS {
    pub fn search_event_queue(mut self, search_event_queue: &str) -> Self {
        self.search_event_queue = Some(search_event_queue.to_string());
        self
    }

    pub async fn send_message_to_search_event_queue(
        &self,
        message: SearchQueueMessage,
    ) -> anyhow::Result<String> {
        if let Some(search_event_queue) = &self.search_event_queue {
            enqueue_search_text_extractor(&self.inner, search_event_queue, message).await
        } else {
            Err(anyhow::anyhow!("search_event_queue is not configured"))
        }
    }

    pub async fn bulk_send_message_to_search_event_queue(
        &self,
        items: Vec<SearchQueueMessage>,
    ) -> Result<(), anyhow::Error> {
        if let Some(search_event_queue) = &self.search_event_queue {
            bulk_enqueue_search_text_extractor(&self.inner, search_event_queue, items).await
        } else {
            Err(anyhow::anyhow!("search_event_queue is not configured"))
        }
    }
}

#[derive(serde::Serialize, Debug, Display, strum::EnumString)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum Operation {
    /// Extract text from a particular source to be processed into OpenSearch
    ExtractText,
    /// Updates the metadata for a given item
    UpdateMetadata,
    /// Remove an item from the search index
    Remove,
    /// Extract text from live collab documents to be processed into OpenSearch
    ExtractSync,
}

#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Eq)]
pub enum SearchQueueMessage {
    // Document
    ExtractDocumentText(SearchExtractorMessage),
    ExtractSync(SearchExtractorMessage),
    // Chat
    /// SQS backfill work-queue contract for reconciling a chat message.
    ChatMessage(ChatMessage),
    // Email backfills
    ExtractEmailThreadBatch(EmailThreadBatchMessage),
    // Channel
    ChannelMessageUpdate(ChannelMessageUpdate),
    // Call
    CallRecord(CallRecordMessage),
    RemoveCallRecord(RemoveCallRecord),
    // Project
    UpsertProject(UpsertProject),
    // Calendar
    UpsertCalendarEvent(UpsertCalendarEvent),

    // User
    RemoveUserProfile(String),
}

impl PrimaryId for SearchQueueMessage {
    fn id(&self) -> String {
        match self {
            SearchQueueMessage::ExtractDocumentText(message) => message.document_id.clone(),
            SearchQueueMessage::ExtractSync(message) => message.document_id.clone(),
            // The message id keeps entries unique within an SQS batch.
            SearchQueueMessage::ChatMessage(message) => message.message_id.clone(),
            SearchQueueMessage::ExtractEmailThreadBatch(message) => {
                message.thread_ids.first().cloned().unwrap_or_default()
            }
            SearchQueueMessage::ChannelMessageUpdate(message) => message.message_id.clone(),
            SearchQueueMessage::CallRecord(message) => message.call_id.clone(),
            SearchQueueMessage::RemoveCallRecord(message) => format!(
                "{}{}",
                message.channel_id,
                message.call_id.clone().unwrap_or_default()
            ),
            SearchQueueMessage::UpsertProject(message) => message.project_id.clone(),
            SearchQueueMessage::UpsertCalendarEvent(message) => message.event_id.clone(),

            SearchQueueMessage::RemoveUserProfile(message) => message.clone(),
        }
    }
}

impl SearchQueueMessage {
    pub fn operation(&self) -> Operation {
        match self {
            // Document
            SearchQueueMessage::ExtractDocumentText(_) => Operation::ExtractText,
            SearchQueueMessage::ExtractSync(_) => Operation::ExtractSync,
            // Chat
            SearchQueueMessage::ChatMessage(_) => Operation::ExtractText,
            // Email backfills
            SearchQueueMessage::ExtractEmailThreadBatch(_) => Operation::ExtractText,
            // Channels
            SearchQueueMessage::ChannelMessageUpdate(_) => Operation::ExtractText,
            // Calls
            SearchQueueMessage::CallRecord(_) => Operation::ExtractText,
            SearchQueueMessage::RemoveCallRecord(_) => Operation::Remove,
            // Projects
            SearchQueueMessage::UpsertProject(_) => Operation::UpdateMetadata,
            // Calendar
            SearchQueueMessage::UpsertCalendarEvent(_) => Operation::UpdateMetadata,
            // Users
            SearchQueueMessage::RemoveUserProfile(_) => Operation::Remove,
        }
    }
}

#[tracing::instrument(skip(sqs_client))]
pub async fn enqueue_search_text_extractor(
    sqs_client: &sqs::Client,
    queue_url: &str,
    message: SearchQueueMessage,
) -> anyhow::Result<String> {
    let message_str = serde_json::to_string(&message)?;

    let result = sqs_client
        .send_message()
        .queue_url(queue_url)
        .message_body(message_str)
        .send()
        .await?;

    Ok(result
        .message_id
        .context("expected a message id")?
        .to_string())
}

const MAX_CONCURRENT_BATCHES: usize = 20;

/// Bulk enqueues items to the search text extractor queue
#[tracing::instrument(skip(sqs_client, items))]
pub async fn bulk_enqueue_search_text_extractor(
    sqs_client: &aws_sdk_sqs::Client,
    queue_url: &str,
    items: Vec<SearchQueueMessage>,
) -> anyhow::Result<()> {
    let mut entries: Vec<SendMessageBatchRequestEntry> = vec![];
    for item in items {
        tracing::trace!(item=?item, "enqueueing search text extractor");
        let message_str = serde_json::to_string(&item)?;
        let batch_request = SendMessageBatchRequestEntry::builder()
            .id(item.id())
            .message_body(message_str)
            .build()?;
        entries.push(batch_request);
    }

    if entries.is_empty() {
        tracing::warn!("no entries to enqueue");
        return Ok(());
    }

    let results: Vec<_> = futures::stream::iter(
        entries
            .chunks(MAX_BATCH_SIZE)
            .map(|chunk| chunk.to_vec())
            .collect::<Vec<_>>(),
    )
    .map(|chunk| {
        sqs_client
            .send_message_batch()
            .set_entries(Some(chunk))
            .queue_url(queue_url)
            .send()
    })
    .buffer_unordered(MAX_CONCURRENT_BATCHES)
    .collect()
    .await;

    for result in results {
        result?;
    }

    Ok(())
}