Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

8.1 KB
Raw Blame
//! Maps document lifecycle events to search-index actions and processes them.

use documents::domain::events::{DocumentMacroEvent, DocumentTopicEvent};
use macro_event_broker::MacroEvent as _;
use model::document::FileType;
use sqs_client::search::document::SearchExtractorMessage;

use super::{
    EventOutcome, KafkaProcessingContext, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY,
    retry_processing,
};
use crate::process::document::{
    process_extract_sync_message, process_extract_text_message, process_remove_message,
    process_update_name_message,
};

#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) enum DocumentIndexAction {
    ExtractText {
        owner: String,
        file_type: FileType,
        document_version_id: Option<String>,
    },
    ExtractSync {
        file_type: FileType,
        document_version_id: Option<String>,
    },
    RefreshName,
    Remove,
    Ignore,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct DocumentEventDescription {
    pub(super) action: DocumentIndexAction,
    pub(super) document_id: String,
    pub(super) event_type: &'static str,
}

pub(super) fn describe_document_event(event: &DocumentTopicEvent) -> DocumentEventDescription {
    match event {
        DocumentTopicEvent::Created(metadata) => DocumentEventDescription {
            action: DocumentIndexAction::Ignore,
            document_id: metadata.document_id.clone(),
            event_type: "document.created",
        },
        DocumentTopicEvent::Updated(metadata) => DocumentEventDescription {
            action: if metadata.document_name.is_some() {
                DocumentIndexAction::RefreshName
            } else {
                DocumentIndexAction::Ignore
            },
            document_id: metadata.document_id.clone(),
            event_type: "document.updated",
        },
        DocumentTopicEvent::Deleted(metadata) => DocumentEventDescription {
            action: DocumentIndexAction::Ignore,
            document_id: metadata.document_id.clone(),
            event_type: "document.deleted",
        },
        DocumentTopicEvent::ContentUploaded(metadata) => DocumentEventDescription {
            action: DocumentIndexAction::ExtractText {
                owner: metadata.owner.to_string(),
                file_type: metadata.file_type,
                document_version_id: metadata.document_version_id.clone(),
            },
            document_id: metadata.document_id.clone(),
            event_type: "document.content_uploaded",
        },
        DocumentTopicEvent::SyncContentUpdated(metadata) => DocumentEventDescription {
            action: DocumentIndexAction::ExtractSync {
                file_type: metadata.file_type,
                document_version_id: metadata.document_version_id.clone(),
            },
            document_id: metadata.document_id.clone(),
            event_type: "document.sync_content_updated",
        },
        DocumentTopicEvent::Purged(metadata) => DocumentEventDescription {
            action: DocumentIndexAction::Remove,
            document_id: metadata.document_id.clone(),
            event_type: "document.purged",
        },
        DocumentTopicEvent::Copied(metadata) => DocumentEventDescription {
            action: DocumentIndexAction::Ignore,
            document_id: metadata.document_id.clone(),
            event_type: "document.copied",
        },
        DocumentTopicEvent::Interaction(metadata) => DocumentEventDescription {
            action: DocumentIndexAction::Ignore,
            document_id: metadata.document_id.clone(),
            event_type: "document.interaction",
        },
    }
}

pub(super) fn stored_extractor_message(
    document_id: &str,
    owner: String,
    file_type: FileType,
    document_version_id: Option<String>,
) -> SearchExtractorMessage {
    SearchExtractorMessage {
        user_id: owner,
        document_id: document_id.to_string(),
        file_type,
        document_version_id,
        index_override: None,
    }
}

pub(super) fn sync_extractor_message(
    document_id: &str,
    file_type: FileType,
    document_version_id: Option<String>,
) -> SearchExtractorMessage {
    SearchExtractorMessage {
        user_id: String::new(),
        document_id: document_id.to_string(),
        file_type,
        document_version_id,
        index_override: None,
    }
}

async fn process_document_index_action(
    context: &KafkaProcessingContext,
    document_id: &str,
    action: DocumentIndexAction,
) -> anyhow::Result<()> {
    let opensearch_client = context.opensearch_client.as_ref();

    match action {
        DocumentIndexAction::ExtractText {
            owner,
            file_type,
            document_version_id,
        } => {
            let message =
                stored_extractor_message(document_id, owner, file_type, document_version_id);
            process_extract_text_message(
                opensearch_client,
                &context.db,
                context.s3_client.as_ref(),
                &context.document_storage_bucket,
                &message,
            )
            .await
        }
        DocumentIndexAction::ExtractSync {
            file_type,
            document_version_id,
        } => {
            let message = sync_extractor_message(document_id, file_type, document_version_id);
            process_extract_sync_message(
                opensearch_client,
                &context.db,
                context.s3_client.as_ref(),
                &context.document_storage_bucket,
                context.lexical_client.as_ref(),
                &message,
            )
            .await
        }
        DocumentIndexAction::RefreshName => {
            process_update_name_message(opensearch_client, &context.db, document_id).await
        }
        DocumentIndexAction::Remove => process_remove_message(opensearch_client, document_id).await,
        DocumentIndexAction::Ignore => Ok(()),
    }
}

pub(super) async fn process_document_event(
    context: &KafkaProcessingContext,
    event: &DocumentMacroEvent,
    partition: i32,
    offset: i64,
) -> EventOutcome {
    let description = describe_document_event(&event.event().event);
    let document_id = description.document_id.as_str();
    let event_type = description.event_type;

    if description.action == DocumentIndexAction::Ignore {
        tracing::trace!(
            document_id,
            event_type,
            partition,
            offset,
            "ignoring document event without a search-index action"
        );
        return EventOutcome::Ignored;
    }

    let result = retry_processing(|attempt| {
        let action = description.action.clone();
        async move {
            tracing::trace!(
                document_id,
                event_type,
                partition,
                offset,
                attempt,
                "processing document search-index event"
            );
            process_document_index_action(context, document_id, action)
                .await
                .inspect_err(|error| {
                    if attempt < MAX_PROCESSING_ATTEMPTS {
                        let retry_delay =
                            PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));
                        tracing::warn!(
                            error = ?error,
                            document_id,
                            event_type,
                            partition,
                            offset,
                            attempt,
                            delay_secs = retry_delay.as_secs(),
                            "document search-index processing failed, retrying"
                        );
                    }
                })
        }
    })
    .await;

    match result {
        Ok(()) => EventOutcome::Indexed,
        Err(error) => {
            tracing::error!(
                error = ?error,
                document_id,
                event_type,
                partition,
                offset,
                attempts = MAX_PROCESSING_ATTEMPTS,
                "dropping document event after processing retries were exhausted"
            );
            EventOutcome::Dropped
        }
    }
}