Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

23.5 KB
Raw Blame
use std::str::FromStr;

use anyhow::Context;
use chrono::Utc;
use document_sub_type::DocumentSubType;
use lexical_client::types::MarkdownParseResult;
use macro_db_client::document::get_document_version_id;
use model::document::{DocumentMetadata, FileType};
use model_owner::Owner;
use models_properties::EntityType;
use models_search::unified::is_searchable_file_type;
use opensearch_client::{
    OpensearchClient, date_format::EpochMillis, upsert::document::UpsertDocumentArgs,
};
use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;
use s3_key::{
    CONVERTED_DOCUMENT_FILE_NAME, build_cloud_storage_bucket_document_key,
    build_docx_to_pdf_converted_document_key,
};

#[cfg(feature = "pdf")]
use crate::parsers::pdf::parse_pdf_pages;
use crate::{
    parsers::{
        ai::parse_ai_text, fig::parse_fig_pages, markdown::parse_markdown_legacy,
        psd::parse_psd_text,
    },
    process::document::document_info::{DocumentInfo, get_document_info},
    process::properties::to_indexed_properties,
};

use super::SearchExtractorMessage;

/// The node id of the one chunk a Photoshop or Illustrator document indexes
/// as, fixed so that a new version replaces it.
const DESIGN_TEXT_NODE_ID: &str = "0";

async fn upsert_document(
    opensearch_client: &OpensearchClient,
    search_extractor_message: &SearchExtractorMessage,
    upserts: Vec<UpsertDocumentArgs>,
) -> anyhow::Result<()> {
    let index_override = search_extractor_message.index_override.as_deref();
    // Delete existing documents for the document id
    // This ensures we replace any old nodes with new ones for editable files
    // (a design's removed pages, for example)
    match search_extractor_message.file_type {
        FileType::Md
        | FileType::Canvas
        | FileType::Fig
        | FileType::Psd
        | FileType::Psb
        | FileType::Ai => {
            tracing::debug!("deleting existing search results");
            opensearch_client
                .delete_document(&search_extractor_message.document_id, index_override)
                .await
                .context("unable to delete existing search results")?;
        }
        _ => {}
    }

    let results = opensearch_client
        .bulk_upsert_documents(&upserts, index_override)
        .await
        .context("unable to bulk upsert documents in opensearch")?;

    if !results.errors.is_empty() {
        tracing::error!(errors=?results.errors, "bulk upsert failed");

        // delete document that failed to upsert
        opensearch_client
            .delete_document(&search_extractor_message.document_id, index_override)
            .await
            .context("failed to delete document for failed bulk upsert")?;

        anyhow::bail!("failed to upsert documents");
    }

    tracing::trace!("upserted document");

    Ok(())
}

/// Properties are keyed under the task entity type for tasks, otherwise the
/// document entity type.
fn properties_entity_type(sub_type: Option<&str>) -> EntityType {
    sub_type
        .and_then(|s| s.parse::<DocumentSubType>().ok())
        .map(EntityType::from)
        .unwrap_or(EntityType::Document)
}

/// Fetch the entity's indexed properties and attach them to every chunk upsert
/// (parent metadata is denormalized identically across chunks).
///
/// A full index overwrites the parent doc, so an *empty* `properties` clears
/// previously-indexed values — the deliberate "omit == remove" behavior used
/// for `sub_type`. That is correct when the entity genuinely has no
/// properties, but a fetch *failure* must not be mistaken for "empty": we
/// propagate the error so the doc isn't overwritten and the message is retried
/// (mirroring the partial-update path).
async fn attach_indexed_properties(
    db: &sqlx::Pool<sqlx::Postgres>,
    document_id: &str,
    sub_type: Option<&str>,
    upserts: &mut [UpsertDocumentArgs],
) -> anyhow::Result<()> {
    if upserts.is_empty() {
        return Ok(());
    }
    let properties =
        get_entity_properties_for_index(db, document_id, properties_entity_type(sub_type))
            .await
            .context("failed to fetch properties for search index")?;
    if properties.is_empty() {
        return Ok(());
    }
    let indexed = to_indexed_properties(properties);
    for upsert in upserts.iter_mut() {
        upsert.properties = indexed.clone();
    }
    Ok(())
}

/// Builds the parent-only upsert for a file type with no indexable content.
/// Returns `None` when the document has no file type.
fn generate_parent_only_upsert(
    document_info: DocumentMetadata,
) -> anyhow::Result<Option<UpsertDocumentArgs>> {
    let Some(file_type) = document_info.file_type else {
        return Ok(None);
    };
    Ok(Some(UpsertDocumentArgs {
        document_id: document_info.document_id,
        node_id: String::new(),
        raw_content: None,
        document_name: document_info.document_name,
        content: String::new(),
        owner_id: document_info.owner.to_string(),
        file_type,
        updated_at_millis: EpochMillis::new(Utc::now().timestamp_millis())?,
        sub_type: document_info.sub_type.map(|st| st.to_string()),
        properties: vec![],
    }))
}

/// Indexes a name-only parent doc (no content chunks) for file types whose
/// content isn't searchable, so the document stays findable by name and
/// filterable by its indexed properties.
async fn update_search_with_parent_only_document(
    opensearch_client: &OpensearchClient,
    db: &sqlx::Pool<sqlx::Postgres>,
    search_extractor_message: &SearchExtractorMessage,
) -> anyhow::Result<()> {
    let document_info = match get_document_info(db, search_extractor_message)
        .await
        .context("failed to get document info")?
    {
        DocumentInfo::Active(info) => *info,
        DocumentInfo::Removable => {
            tracing::trace!("document is deleted or missing, removing from search index");
            opensearch_client
                .delete_document(
                    &search_extractor_message.document_id,
                    search_extractor_message.index_override.as_deref(),
                )
                .await
                .context("failed to delete document from search index")?;
            return Ok(());
        }
        DocumentInfo::Skip => {
            tracing::trace!("no document info returned");
            return Ok(());
        }
    };

    let Some(args) = generate_parent_only_upsert(document_info)? else {
        tracing::debug!("file type is none");
        return Ok(());
    };

    let sub_type = args.sub_type.clone();
    let mut upserts = vec![args];
    attach_indexed_properties(
        db,
        &search_extractor_message.document_id,
        sub_type.as_deref(),
        &mut upserts,
    )
    .await?;

    opensearch_client
        .upsert_parent_document(
            &upserts[0],
            search_extractor_message.index_override.as_deref(),
        )
        .await
        .context("unable to upsert parent-only document in opensearch")?;

    Ok(())
}

fn should_index_parent_only(file_type: &FileType) -> bool {
    matches!(file_type, FileType::Canvas) || !is_searchable_file_type(file_type)
}

/// The searchable text of a Photoshop or Illustrator document. Decoding is
/// CPU work, so it runs off the async workers; the engine failing on a
/// damaged file is a read error like any other.
async fn read_design_text(file_type: FileType, content: Vec<u8>) -> anyhow::Result<String> {
    let parse: fn(&[u8]) -> anyhow::Result<String> = match file_type {
        FileType::Ai => parse_ai_text,
        FileType::Psd | FileType::Psb => parse_psd_text,
        other => anyhow::bail!("{other} files are not read as designs"),
    };
    tokio::task::spawn_blocking(move || parse(&content))
        .await
        .unwrap_or_else(|e| Err(anyhow::anyhow!("the design engine failed: {e}")))
}

/// Indexes a file its parser cannot read (a password-protected or damaged
/// PDF, or a design that is old, damaged, or too large to decode here) by
/// name only, dropping the text an earlier version left, so it stays
/// findable. Reading it again would fail the same way, so this is not an
/// error to retry.
async fn update_search_with_unreadable_file(
    opensearch_client: &OpensearchClient,
    db: &sqlx::Pool<sqlx::Postgres>,
    search_extractor_message: &SearchExtractorMessage,
) -> anyhow::Result<()> {
    opensearch_client
        .delete_document(
            &search_extractor_message.document_id,
            search_extractor_message.index_override.as_deref(),
        )
        .await
        .context("unable to delete existing search results")?;
    update_search_with_parent_only_document(opensearch_client, db, search_extractor_message).await
}

/// The version whose stored content search indexes. A PDF is served from its
/// first version, and a save adds a version without rewriting that file, so
/// text read from any other version (a message delivered out of order, or the
/// bytes of an API save) is not what the document shows. A DOCX is indexed
/// from its converted PDF, which each conversion overwrites in place. Every
/// other file is indexed at its latest version.
async fn indexed_version_id(
    db: &sqlx::Pool<sqlx::Postgres>,
    search_extractor_message: &SearchExtractorMessage,
    latest_version_id: i64,
) -> anyhow::Result<String> {
    if !matches!(
        search_extractor_message.file_type,
        FileType::Pdf | FileType::Docx
    ) {
        return Ok(latest_version_id.to_string());
    }
    let message_version_id = search_extractor_message
        .document_version_id
        .as_deref()
        .context("expected document version id to be provided for pdf/docx")?;
    if message_version_id == CONVERTED_DOCUMENT_FILE_NAME {
        return Ok(CONVERTED_DOCUMENT_FILE_NAME.to_string());
    }
    let (served_version_id, _) = get_document_version_id(db, &search_extractor_message.document_id)
        .await
        .context("failed to get the served document version")?;
    Ok(served_version_id.to_string())
}

/// The S3 key of the content to index, or `None` when the message names a
/// version other than the indexed one. Content is stored under the
/// document's owner; sync messages carry no `user_id`.
fn raw_document_key(
    owner: &Owner,
    search_extractor_message: &SearchExtractorMessage,
    document_version_id: &str,
) -> Option<String> {
    let document_id = &search_extractor_message.document_id;
    if document_version_id == CONVERTED_DOCUMENT_FILE_NAME {
        return Some(build_docx_to_pdf_converted_document_key(owner, document_id));
    }
    match search_extractor_message.document_version_id.as_deref() {
        Some(msg_version_id) if msg_version_id != document_version_id => None,
        _ => Some(build_cloud_storage_bucket_document_key(
            owner,
            document_id,
            document_version_id,
        )),
    }
}

/// Processes a message for a standard document and reads the updated contents from s3 and updates
/// the document in opensearch.
#[tracing::instrument(skip(opensearch_client, db, s3_client, document_storage_bucket, search_extractor_message), fields(document_id=search_extractor_message.document_id, file_type=?search_extractor_message.file_type))]
pub async fn update_search_with_raw_document(
    opensearch_client: &OpensearchClient,
    db: &sqlx::Pool<sqlx::Postgres>,
    s3_client: &s3_client::S3,
    document_storage_bucket: &str,
    search_extractor_message: &SearchExtractorMessage,
) -> anyhow::Result<()> {
    if should_index_parent_only(&search_extractor_message.file_type) {
        return update_search_with_parent_only_document(
            opensearch_client,
            db,
            search_extractor_message,
        )
        .await;
    }

    // This ensures we only process the latest version
    let document_info = match get_document_info(db, search_extractor_message)
        .await
        .context("failed to get document info")?
    {
        DocumentInfo::Active(info) => *info,
        DocumentInfo::Removable => {
            tracing::trace!("document is deleted or missing, removing from search index");
            opensearch_client
                .delete_document(
                    &search_extractor_message.document_id,
                    search_extractor_message.index_override.as_deref(),
                )
                .await
                .context("failed to delete document from search index")?;
            return Ok(());
        }
        DocumentInfo::Skip => {
            tracing::trace!("no document info returned");
            return Ok(());
        }
    };

    let document_name = document_info.document_name;
    let owner_id = document_info.owner.to_string();
    let sub_type = document_info.sub_type.map(|st| st.to_string());

    // TODO: this is hacky, update the search event message to use the correctly serialized
    // document key parts from the s3_key crate
    let document_version_id = indexed_version_id(
        db,
        search_extractor_message,
        document_info.document_version_id,
    )
    .await?;

    if document_info.file_type.is_none() {
        tracing::debug!("file type is none");
        return Ok(());
    }

    let file_type = FileType::from_str(
        document_info
            .file_type
            .context("expected a file type")?
            .as_str(),
    )?;

    let Some(key) = raw_document_key(
        &document_info.owner,
        search_extractor_message,
        &document_version_id,
    ) else {
        tracing::debug!(
            msg_version_id = ?search_extractor_message.document_version_id,
            document_version_id,
            "message names a version search does not index, skipping"
        );
        return Ok(());
    };

    let content = s3_client
        .get(document_storage_bucket, &key)
        .await
        .context("unable to get file")?;

    // Handle empty content for things like new markdown/canvas files
    if content.is_empty() {
        tracing::debug!("empty content");
        return Ok(());
    }

    tracing::trace!("got raw file content");

    let updated_at_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
    let uuid = macro_uuid::generate_uuid_v7().to_string();

    let mut upserts: Vec<UpsertDocumentArgs> = match file_type {
        FileType::Pdf | FileType::Docx => {
            #[cfg(feature = "pdf")]
            {
                let Some(pages_content) =
                    parse_pdf_pages(content).context("unable to parse pdf")?
                else {
                    return update_search_with_unreadable_file(
                        opensearch_client,
                        db,
                        search_extractor_message,
                    )
                    .await;
                };
                pages_content
                    .iter()
                    .enumerate()
                    .map(|(i, page_content)| UpsertDocumentArgs {
                        document_id: search_extractor_message.document_id.clone(),
                        node_id: i.to_string(), // page number
                        raw_content: None,
                        document_name: document_name.clone(),
                        content: page_content.clone(),
                        owner_id: owner_id.clone(),
                        file_type: file_type.to_string(),
                        updated_at_millis,
                        sub_type: sub_type.clone(),
                        properties: vec![],
                    })
                    .collect()
            }
            #[cfg(not(feature = "pdf"))]
            {
                let _ = content;
                tracing::debug!("pdf/docx indexing skipped: pdf feature disabled");
                vec![]
            }
        }
        FileType::Fig => {
            // Decoding a design is CPU work.
            let pages = tokio::task::spawn_blocking(move || parse_fig_pages(&content))
                .await
                .context("decoding the design")?
                .context("unable to parse the design")?;
            pages
                .into_iter()
                .map(|page| UpsertDocumentArgs {
                    document_id: search_extractor_message.document_id.clone(),
                    // The page's node id, so a page keeps its chunk.
                    node_id: page.node_id,
                    raw_content: None,
                    document_name: document_name.clone(),
                    content: page.content,
                    owner_id: owner_id.clone(),
                    file_type: file_type.to_string(),
                    updated_at_millis,
                    sub_type: sub_type.clone(),
                    properties: vec![],
                })
                .collect()
        }
        FileType::Psd | FileType::Psb | FileType::Ai => {
            match read_design_text(file_type, content).await {
                Ok(text) if !text.trim().is_empty() => vec![UpsertDocumentArgs {
                    document_id: search_extractor_message.document_id.clone(),
                    node_id: DESIGN_TEXT_NODE_ID.to_string(),
                    raw_content: None,
                    document_name,
                    content: text,
                    owner_id,
                    file_type: file_type.to_string(),
                    updated_at_millis,
                    sub_type: sub_type.clone(),
                    properties: vec![],
                }],
                read => {
                    if let Err(error) = &read {
                        tracing::warn!(error = ?error, "indexing an unreadable design by name only");
                    }
                    return update_search_with_unreadable_file(
                        opensearch_client,
                        db,
                        search_extractor_message,
                    )
                    .await;
                }
            }
        }
        FileType::Md => {
            // NOTE: this is legacy now. MD parsing mainly happens through sync service via
            // LexicalClient
            tracing::trace!("markdown parsing from DSS is deprecated");
            let result = parse_markdown_legacy(&String::from_utf8(content)?)
                .context("unable to parse markdown")?;
            result
                .into_iter()
                .map(|result| UpsertDocumentArgs {
                    document_id: search_extractor_message.document_id.clone(),
                    node_id: result.node_id,
                    raw_content: Some(result.raw_content),
                    document_name: document_name.clone(),
                    content: result.content,
                    owner_id: owner_id.clone(),
                    file_type: file_type.to_string(),
                    updated_at_millis,
                    sub_type: sub_type.clone(),
                    properties: vec![],
                })
                .collect::<Vec<UpsertDocumentArgs>>()
        }
        file_type => {
            let content = String::from_utf8(content)?;
            if content.is_empty() {
                vec![]
            } else {
                vec![UpsertDocumentArgs {
                    document_id: search_extractor_message.document_id.clone(),
                    node_id: uuid,
                    raw_content: None,
                    document_name,
                    content: content.clone(),
                    owner_id,
                    file_type: file_type.to_string(),
                    updated_at_millis,
                    sub_type: sub_type.clone(),
                    properties: vec![],
                }]
            }
        }
    };

    attach_indexed_properties(
        db,
        &search_extractor_message.document_id,
        sub_type.as_deref(),
        &mut upserts,
    )
    .await?;

    upsert_document(opensearch_client, search_extractor_message, upserts).await?;

    Ok(())
}

fn generate_upserts(
    document_info: DocumentMetadata,
    markdown_result: Vec<MarkdownParseResult>,
) -> anyhow::Result<Vec<UpsertDocumentArgs>> {
    let result = markdown_result;
    let updated_at_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
    let file_type = FileType::from_str(
        document_info
            .file_type
            .context("expected a file type")?
            .as_str(),
    )?;
    let document_name = document_info.document_name;
    let sub_type = document_info.sub_type.map(|st| st.to_string());

    let upserts = result
        .into_iter()
        .map(|result| UpsertDocumentArgs {
            document_id: document_info.document_id.clone(),
            node_id: result.node_id,
            raw_content: Some(result.raw_content),
            document_name: document_name.clone(),
            content: result.content,
            owner_id: document_info.owner.to_string(),
            file_type: file_type.to_string(),
            updated_at_millis,
            sub_type: sub_type.clone(),
            properties: vec![],
        })
        .collect::<Vec<UpsertDocumentArgs>>();

    Ok(upserts)
}

/// Processes a message for a standard document and reads the updated contents from sync service and updates
/// the document in opensearch.
#[tracing::instrument(skip(opensearch_client, search_extractor_message, db, s3_client, document_storage_bucket, lexical_client), fields(document_id=search_extractor_message.document_id, file_type=?search_extractor_message.file_type))]
pub async fn update_search_with_sync_document(
    opensearch_client: &OpensearchClient,
    db: &sqlx::Pool<sqlx::Postgres>,
    s3_client: &s3_client::S3,
    document_storage_bucket: &str,
    lexical_client: &lexical_client::LexicalClient,
    search_extractor_message: &SearchExtractorMessage,
) -> anyhow::Result<()> {
    match search_extractor_message.file_type.macro_app_path() {
        model_file_type::FileAssociation::Md(_) => {}
        _ => {
            tracing::warn!("unsupported file type");
            return Ok(());
        }
    }

    let document_info = match get_document_info(db, search_extractor_message)
        .await
        .context("failed to get document info")?
    {
        DocumentInfo::Active(info) => *info,
        DocumentInfo::Removable => {
            tracing::trace!("document is deleted or missing, removing from search index");
            opensearch_client
                .delete_document(
                    &search_extractor_message.document_id,
                    search_extractor_message.index_override.as_deref(),
                )
                .await
                .context("failed to delete document from search index")?;
            return Ok(());
        }
        DocumentInfo::Skip => {
            tracing::trace!("no document info returned");
            return Ok(());
        }
    };

    let document_id = &search_extractor_message.document_id;
    let result = match lexical_client.parse_markdown(document_id).await {
        Ok(result) => result,
        Err(e) => {
            tracing::warn!(error=?e, "failed to parse markdown with lexical");
            // call DSS as fallback if lexical/sync service fails
            update_search_with_raw_document(
                opensearch_client,
                db,
                s3_client,
                document_storage_bucket,
                search_extractor_message,
            )
            .await?;
            return Ok(());
        }
    };

    let sub_type = document_info.sub_type.as_ref().map(|st| st.to_string());
    let mut upserts =
        generate_upserts(document_info, result).context("unable to generate upserts")?;

    attach_indexed_properties(db, document_id, sub_type.as_deref(), &mut upserts).await?;

    let chunk_count = upserts.len();
    upsert_document(opensearch_client, search_extractor_message, upserts).await?;
    tracing::info!(document_id = %document_id, chunk_count, "sync document indexed");

    Ok(())
}

#[cfg(test)]
mod test;