Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
use crate::{
SQS,
search::{
calendar_event::UpsertCalendarEvent,
call::{CallRecordMessage, RemoveCallRecord},
channel::ChannelMessageUpdate,
chat::ChatMessage,
document::SearchExtractorMessage,
email::EmailThreadBatchMessage,
project::UpsertProject,
},
};
use anyhow::Context;
use aws_sdk_sqs::{self as sqs, types::SendMessageBatchRequestEntry};
use futures::StreamExt;
use strum::Display;
pub mod calendar_event;
pub mod call;
pub mod channel;
pub mod chat;
pub mod document;
pub mod email;
pub mod project;
use crate::{MAX_BATCH_SIZE, PrimaryId};
impl SQS {
pub fn search_event_queue(mut self, search_event_queue: &str) -> Self {
self.search_event_queue = Some(search_event_queue.to_string());
self
}
pub async fn send_message_to_search_event_queue(
&self,
message: SearchQueueMessage,
) -> anyhow::Result<String> {
if let Some(search_event_queue) = &self.search_event_queue {
enqueue_search_text_extractor(&self.inner, search_event_queue, message).await
} else {
Err(anyhow::anyhow!("search_event_queue is not configured"))
}
}
pub async fn bulk_send_message_to_search_event_queue(
&self,
items: Vec<SearchQueueMessage>,
) -> Result<(), anyhow::Error> {
if let Some(search_event_queue) = &self.search_event_queue {
bulk_enqueue_search_text_extractor(&self.inner, search_event_queue, items).await
} else {
Err(anyhow::anyhow!("search_event_queue is not configured"))
}
}
}
#[derive(serde::Serialize, Debug, Display, strum::EnumString)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum Operation {
/// Extract text from a particular source to be processed into OpenSearch
ExtractText,
/// Updates the metadata for a given item
UpdateMetadata,
/// Remove an item from the search index
Remove,
/// Extract text from live collab documents to be processed into OpenSearch
ExtractSync,
}
#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Eq)]
pub enum SearchQueueMessage {
// Document
ExtractDocumentText(SearchExtractorMessage),
ExtractSync(SearchExtractorMessage),
// Chat
/// SQS backfill work-queue contract for reconciling a chat message.
ChatMessage(ChatMessage),
// Email backfills
ExtractEmailThreadBatch(EmailThreadBatchMessage),
// Channel
ChannelMessageUpdate(ChannelMessageUpdate),
// Call
CallRecord(CallRecordMessage),
RemoveCallRecord(RemoveCallRecord),
// Project
UpsertProject(UpsertProject),
// Calendar
UpsertCalendarEvent(UpsertCalendarEvent),
// User
RemoveUserProfile(String),
}
impl PrimaryId for SearchQueueMessage {
fn id(&self) -> String {
match self {
SearchQueueMessage::ExtractDocumentText(message) => message.document_id.clone(),
SearchQueueMessage::ExtractSync(message) => message.document_id.clone(),
// The message id keeps entries unique within an SQS batch.
SearchQueueMessage::ChatMessage(message) => message.message_id.clone(),
SearchQueueMessage::ExtractEmailThreadBatch(message) => {
message.thread_ids.first().cloned().unwrap_or_default()
}
SearchQueueMessage::ChannelMessageUpdate(message) => message.message_id.clone(),
SearchQueueMessage::CallRecord(message) => message.call_id.clone(),
SearchQueueMessage::RemoveCallRecord(message) => format!(
"{}{}",
message.channel_id,
message.call_id.clone().unwrap_or_default()
),
SearchQueueMessage::UpsertProject(message) => message.project_id.clone(),
SearchQueueMessage::UpsertCalendarEvent(message) => message.event_id.clone(),
SearchQueueMessage::RemoveUserProfile(message) => message.clone(),
}
}
}
impl SearchQueueMessage {
pub fn operation(&self) -> Operation {
match self {
// Document
SearchQueueMessage::ExtractDocumentText(_) => Operation::ExtractText,
SearchQueueMessage::ExtractSync(_) => Operation::ExtractSync,
// Chat
SearchQueueMessage::ChatMessage(_) => Operation::ExtractText,
// Email backfills
SearchQueueMessage::ExtractEmailThreadBatch(_) => Operation::ExtractText,
// Channels
SearchQueueMessage::ChannelMessageUpdate(_) => Operation::ExtractText,
// Calls
SearchQueueMessage::CallRecord(_) => Operation::ExtractText,
SearchQueueMessage::RemoveCallRecord(_) => Operation::Remove,
// Projects
SearchQueueMessage::UpsertProject(_) => Operation::UpdateMetadata,
// Calendar
SearchQueueMessage::UpsertCalendarEvent(_) => Operation::UpdateMetadata,
// Users
SearchQueueMessage::RemoveUserProfile(_) => Operation::Remove,
}
}
}
#[tracing::instrument(skip(sqs_client))]
pub async fn enqueue_search_text_extractor(
sqs_client: &sqs::Client,
queue_url: &str,
message: SearchQueueMessage,
) -> anyhow::Result<String> {
let message_str = serde_json::to_string(&message)?;
let result = sqs_client
.send_message()
.queue_url(queue_url)
.message_body(message_str)
.send()
.await?;
Ok(result
.message_id
.context("expected a message id")?
.to_string())
}
const MAX_CONCURRENT_BATCHES: usize = 20;
/// Bulk enqueues items to the search text extractor queue
#[tracing::instrument(skip(sqs_client, items))]
pub async fn bulk_enqueue_search_text_extractor(
sqs_client: &aws_sdk_sqs::Client,
queue_url: &str,
items: Vec<SearchQueueMessage>,
) -> anyhow::Result<()> {
let mut entries: Vec<SendMessageBatchRequestEntry> = vec![];
for item in items {
tracing::trace!(item=?item, "enqueueing search text extractor");
let message_str = serde_json::to_string(&item)?;
let batch_request = SendMessageBatchRequestEntry::builder()
.id(item.id())
.message_body(message_str)
.build()?;
entries.push(batch_request);
}
if entries.is_empty() {
tracing::warn!("no entries to enqueue");
return Ok(());
}
let results: Vec<_> = futures::stream::iter(
entries
.chunks(MAX_BATCH_SIZE)
.map(|chunk| chunk.to_vec())
.collect::<Vec<_>>(),
)
.map(|chunk| {
sqs_client
.send_message_batch()
.set_entries(Some(chunk))
.queue_url(queue_url)
.send()
})
.buffer_unordered(MAX_CONCURRENT_BATCHES)
.collect()
.await;
for result in results {
result?;
}
Ok(())
}