Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

17.3 KB
Raw Blame
use std::collections::HashSet;

use models_opensearch::SearchIndex;

use super::BulkUpsertResult;
use super::properties::IndexedProperty;
use crate::{
    Result,
    date_format::EpochMillis,
    error::{OpensearchClientError, OpensearchOperation},
};

/// Relation name for parent docs in the join field.
const PARENT_RELATION: &str = "document";

/// Relation name for child (chunk) docs in the join field.
const CHILD_RELATION: &str = "chunk";

/// Cap on indexed chunk `content`, in characters. Markdown chunks are
/// node-sized and pdf/docx chunks page-sized, but formats without block
/// structure (code files, plain text) arrive as one whole-file chunk
/// that can run to many megabytes. Oversized chunks bloat the index and can
/// never be fully highlighted (`index.highlight.max_analyzed_offset` is
/// 1,000,000), so the tail is dropped at write time.
const MAX_CHUNK_CONTENT_CHARS: usize = 100_000;

/// Truncate chunk content to [`MAX_CHUNK_CONTENT_CHARS`] on a char boundary.
fn cap_chunk_content(chunk: &UpsertDocumentArgs) -> &str {
    match chunk.content.char_indices().nth(MAX_CHUNK_CONTENT_CHARS) {
        Some((byte_idx, _)) => {
            tracing::warn!(
                document_id = %chunk.document_id,
                node_id = %chunk.node_id,
                content_len = chunk.content.len(),
                "chunk content exceeds indexing cap, truncating"
            );
            &chunk.content[..byte_idx]
        }
        None => &chunk.content,
    }
}

/// The arguments for upserting a document into the opensearch index
#[derive(Debug, serde::Serialize)]
pub struct UpsertDocumentArgs {
    /// The id of the document
    #[serde(rename = "entity_id")]
    pub document_id: String,
    /// The node id of the document
    ///
    /// The node id can represent various things dependent on the file type of the document.
    /// For markdown/canvas, the node id is the root node id for a given block in the document.
    /// For pdf/docx, the node id is the page number.
    /// For other file types, this is just randomly generated at the moment.
    pub node_id: String,
    /// The name of the document
    pub document_name: String,
    /// The file type
    pub file_type: String,
    /// The owner id of the document
    pub owner_id: String,
    /// The raw content of the document if present
    /// At the moment, this is only used in markdown to store the raw json node
    #[serde(skip_serializing_if = "Option::is_none")]
    pub raw_content: Option<String>,
    /// The content of the document
    pub content: String,
    /// The updated at time of the document, in milliseconds
    pub updated_at_millis: EpochMillis,
    /// The sub type of the document (e.g. task)
    #[serde(skip_serializing_if = "Option::is_none")]
    pub sub_type: Option<String>,
    /// Denormalized parent entity properties (status, priority, assignees,
    /// custom) used for search filtering. Empty for documents without
    /// properties.
    pub properties: Vec<IndexedProperty>,
}

/// Resolve `index_override` to the physical/alias name we'll write to.
fn resolve_destination(index_override: Option<&str>) -> &str {
    index_override.unwrap_or(SearchIndex::Documents.as_ref())
}

/// Builds the JSON document body for a parent doc, given any one
/// `UpsertDocumentArgs` belonging to that parent (parent metadata is
/// denormalized identically across all chunks of the same document, so any
/// chunk's metadata is authoritative).
fn parent_doc_body(any_chunk: &UpsertDocumentArgs) -> serde_json::Value {
    let mut doc = serde_json::json!({
        "entity_id": &any_chunk.document_id,
        "document_name": &any_chunk.document_name,
        "owner_id": &any_chunk.owner_id,
        "file_type": &any_chunk.file_type,
        "updated_at_millis": any_chunk.updated_at_millis,
        "document_relation": PARENT_RELATION,
    });
    if let Some(sub_type) = &any_chunk.sub_type {
        doc["sub_type"] = serde_json::Value::String(sub_type.clone());
    }
    if !any_chunk.properties.is_empty()
        && let Ok(properties) = serde_json::to_value(&any_chunk.properties)
    {
        doc["properties"] = properties;
    }
    doc
}

/// Builds the JSON document body for a child (chunk) doc.
fn child_doc_body(chunk: &UpsertDocumentArgs) -> serde_json::Value {
    let mut doc = serde_json::json!({
        "entity_id": &chunk.document_id,
        "node_id": &chunk.node_id,
        "content": cap_chunk_content(chunk),
        "document_relation": {
            "name": CHILD_RELATION,
            "parent": &chunk.document_id,
        },
    });
    if let Some(raw) = &chunk.raw_content {
        doc["raw_content"] = serde_json::Value::String(raw.clone());
    }
    doc
}

/// Process a single chunk of documents.
///
/// Emits, for each chunk:
///   - one parent `index` op with `routing = parent_id` (deduped within
///     this batch). We use full-overwrite `index` semantics rather than
///     `update + doc_as_upsert` so omitted optional fields like `sub_type`
///     get cleared when a document transitions from having them to not.
///   - one child `index` op with `routing = parent_id`.
///
/// Cross-batch duplicate parent writes are accepted as cheap last-writer-
/// wins overwrites; all writers should agree on parent metadata since it's
/// denormalized from the same source.
async fn bulk_upsert_single_chunk(
    client: &opensearch::OpenSearch,
    documents: &[UpsertDocumentArgs],
    index: &str,
) -> Result<BulkUpsertResult> {
    let mut bulk_body = Vec::new();
    let mut seen_parents: HashSet<&str> = HashSet::new();

    for doc in documents {
        let parent_id = doc.document_id.as_str();
        let routing = parent_id;

        if seen_parents.insert(parent_id) {
            let parent_action = serde_json::json!({
                "index": {
                    "_id": parent_id,
                    "routing": routing,
                }
            });
            bulk_body.push(parent_action.to_string());
            bulk_body.push(parent_doc_body(doc).to_string());
        }

        let child_id = format!("{}:{}", doc.document_id, doc.node_id);
        let child_action = serde_json::json!({
            "index": {
                "_id": child_id,
                "routing": routing,
            }
        });
        bulk_body.push(child_action.to_string());
        bulk_body.push(child_doc_body(doc).to_string());
    }

    let result = super::bulk_upsert_to_index(
        client,
        index,
        bulk_body,
        OpensearchOperation::BulkUpsertSingleChunk,
    )
    .await?;

    tracing::trace!(
        chunk_total = documents.len(),
        parent_count = seen_parents.len(),
        successful = result.successful,
        failed = result.failed,
        version_conflicts = result.version_conflicts,
        "bulk upsert chunk completed"
    );

    Ok(result)
}

// ---------------------------------------------------------------------------
// Public entry points
// ---------------------------------------------------------------------------

/// Bulk upsert documents to reduce version conflicts with automatic chunking
#[tracing::instrument(skip(client, documents))]
pub(crate) async fn bulk_upsert_documents(
    client: &opensearch::OpenSearch,
    documents: &[UpsertDocumentArgs],
    index_override: Option<&str>,
) -> Result<BulkUpsertResult> {
    if documents.is_empty() {
        return Ok(BulkUpsertResult::default());
    }

    const CHUNK_SIZE: usize = 100;
    let mut overall_result = BulkUpsertResult::default();

    let index = resolve_destination(index_override);

    let chunks: Vec<_> = documents.chunks(CHUNK_SIZE).collect();

    tracing::info!(
        total_documents = documents.len(),
        total_chunks = chunks.len(),
        chunk_size = CHUNK_SIZE,
        index = %index,
        "starting chunked bulk upsert"
    );

    for (chunk_idx, chunk) in chunks.into_iter().enumerate() {
        tracing::debug!(
            chunk_index = chunk_idx,
            chunk_size = chunk.len(),
            "processing chunk"
        );

        let chunk_result = bulk_upsert_single_chunk(client, chunk, index).await;

        match chunk_result {
            Ok(r) => {
                overall_result.successful += r.successful;
                overall_result.failed += r.failed;
                overall_result.version_conflicts += r.version_conflicts;
                overall_result.errors.extend(r.errors);
            }
            Err(e) => {
                tracing::error!(
                    chunk_index = chunk_idx,
                    chunk_size = chunk.len(),
                    error = ?e,
                    "chunk completely failed"
                );
                overall_result.failed += chunk.len();
                overall_result
                    .errors
                    .push(format!("Chunk {}: {}", chunk_idx, e));
            }
        }
    }

    tracing::info!(
        total = documents.len(),
        successful = overall_result.successful,
        failed = overall_result.failed,
        version_conflicts = overall_result.version_conflicts,
        "chunked bulk upsert completed"
    );

    Ok(overall_result)
}

/// Upsert a single chunk: parent index followed by child index. Uses a
/// 2-op bulk so both land in one request. Full-overwrite `index`
/// semantics on the parent ensure omitted optional fields (e.g.
/// `sub_type`) get cleared on Some→None transitions.
#[tracing::instrument(skip(client))]
pub(crate) async fn upsert_document(
    client: &opensearch::OpenSearch,
    args: &UpsertDocumentArgs,
    index_override: Option<&str>,
) -> Result<()> {
    let index = resolve_destination(index_override);
    let parent_id = args.document_id.as_str();
    let routing = parent_id;

    let parent_action = serde_json::json!({
        "index": {
            "_id": parent_id,
            "routing": routing,
        }
    });

    let child_id = format!("{}:{}", args.document_id, args.node_id);
    let child_action = serde_json::json!({
        "index": {
            "_id": child_id,
            "routing": routing,
        }
    });

    let bulk_body = vec![
        parent_action.to_string(),
        parent_doc_body(args).to_string(),
        child_action.to_string(),
        child_doc_body(args).to_string(),
    ];

    let result = super::bulk_upsert_to_index(
        client,
        index,
        bulk_body,
        OpensearchOperation::UpsertDocument,
    )
    .await?;

    if result.failed > 0 {
        return Err(OpensearchClientError::Unknown {
            details: format!(
                "upsert_document had {} failures: {:?}",
                result.failed, result.errors
            ),
            operation: OpensearchOperation::UpsertDocument,
        });
    }

    tracing::trace!(id=%child_id, parent=%parent_id, "document upserted successfully");
    Ok(())
}

/// Builds the 2-line bulk body for a parent-only write: one `index` op
/// (routing = parent _id = document_id) followed by the parent doc body.
fn parent_only_bulk_body(args: &UpsertDocumentArgs) -> Vec<String> {
    let parent_id = args.document_id.as_str();
    let parent_action = serde_json::json!({
        "index": {
            "_id": parent_id,
            "routing": parent_id,
        }
    });
    vec![parent_action.to_string(), parent_doc_body(args).to_string()]
}

/// Upsert only the parent doc for a document with no indexable content
/// (e.g. archives, media). Writes the same full-overwrite parent body as
/// the chunked paths — `node_id`/`content` on `args` are ignored — and
/// touches no children.
#[tracing::instrument(skip(client))]
pub(crate) async fn upsert_parent_document(
    client: &opensearch::OpenSearch,
    args: &UpsertDocumentArgs,
    index_override: Option<&str>,
) -> Result<()> {
    let index = resolve_destination(index_override);
    let bulk_body = parent_only_bulk_body(args);

    let result = super::bulk_upsert_to_index(
        client,
        index,
        bulk_body,
        OpensearchOperation::UpsertParentDocument,
    )
    .await?;

    if result.failed > 0 {
        return Err(OpensearchClientError::Unknown {
            details: format!(
                "upsert_parent_document had {} failures: {:?}",
                result.failed, result.errors
            ),
            operation: OpensearchOperation::UpsertParentDocument,
        });
    }

    tracing::trace!(parent=%args.document_id, "parent document upserted successfully");
    Ok(())
}

/// Update the denormalized `document_name` for an existing document.
///
/// A single partial-update on the parent doc; children don't carry
/// `document_name`. Routing must match the parent's routing (= parent
/// _id = document_id).
pub(crate) async fn update_document_metadata(
    client: &opensearch::OpenSearch,
    document_id: &str,
    document_name: &str,
) -> Result<()> {
    use serde_json::json;

    let body = json!({
        "doc": {
            "document_name": document_name,
        }
    });

    let response = client
        .update(opensearch::UpdateParts::IndexId(
            SearchIndex::Documents.as_ref(),
            document_id,
        ))
        .routing(document_id)
        .body(body)
        .send()
        .await
        .map_err(|err| {
            OpensearchClientError::request_failed(OpensearchOperation::UpdateDocumentMetadata, &err)
        })?;

    let status_code = response.status_code();
    if status_code.is_success() {
        tracing::info!(
            document_id=%document_id,
            document_name=%document_name,
            "document metadata updated successfully"
        );
        return Ok(());
    }

    let body = response.text().await.map_err(|err| {
        OpensearchClientError::deserialization_failed(
            OpensearchOperation::UpdateDocumentMetadata,
            &err,
        )
    })?;

    // A *missing document* 404 is a no-op: the doc isn't indexed yet, so the
    // next full index will carry the current name. A *missing index* 404
    // (`index_not_found_exception`) is a real outage and must propagate. This
    // mirrors `update_document_properties`.
    if status_code.as_u16() == 404 {
        let error_type = serde_json::from_str::<serde_json::Value>(&body)
            .ok()
            .and_then(|value| value["error"]["type"].as_str().map(str::to_owned));
        if error_type.as_deref() == Some("document_missing_exception") {
            tracing::debug!(
                document_id=%document_id,
                "document not indexed yet; skipping name update"
            );
            return Ok(());
        }
    }

    tracing::error!(
        status_code=?status_code,
        body=?body,
        document_id=%document_id,
        "error updating document metadata",
    );

    Err(OpensearchClientError::Unknown {
        details: body,
        operation: OpensearchOperation::UpdateDocumentMetadata,
    })
}

/// Update only the denormalized `properties` on an existing parent doc,
/// without touching content. Used when an entity's properties change
/// independently of its body. A missing doc (404) is treated as a no-op —
/// the next full index will include the properties.
pub(crate) async fn update_document_properties(
    client: &opensearch::OpenSearch,
    document_id: &str,
    properties: &[IndexedProperty],
    index_override: Option<&str>,
) -> Result<()> {
    use serde_json::json;

    let index = resolve_destination(index_override);
    let properties_value =
        serde_json::to_value(properties).map_err(|err| OpensearchClientError::Unknown {
            details: err.to_string(),
            operation: OpensearchOperation::UpdateDocumentProperties,
        })?;
    let body = json!({ "doc": { "properties": properties_value } });

    let response = client
        .update(opensearch::UpdateParts::IndexId(index, document_id))
        .routing(document_id)
        .body(body)
        .send()
        .await
        .map_err(|err| {
            OpensearchClientError::request_failed(
                OpensearchOperation::UpdateDocumentProperties,
                &err,
            )
        })?;

    let status_code = response.status_code();
    if status_code.is_success() {
        tracing::trace!(document_id=%document_id, "document properties updated");
        return Ok(());
    }
    let body = response.text().await.map_err(|err| {
        OpensearchClientError::deserialization_failed(
            OpensearchOperation::UpdateDocumentProperties,
            &err,
        )
    })?;

    // A *missing document* 404 is a no-op: the doc isn't indexed yet, so the
    // next full index will include its properties. A *missing index* 404
    // (`index_not_found_exception`) is a real outage and must propagate. The
    // type is read from the error body the same way `parse_bulk_response`
    // reads `error.reason`.
    if status_code.as_u16() == 404 {
        let error_type = serde_json::from_str::<serde_json::Value>(&body)
            .ok()
            .and_then(|value| value["error"]["type"].as_str().map(str::to_owned));
        if error_type.as_deref() == Some("document_missing_exception") {
            tracing::debug!(
                document_id=%document_id,
                "document not indexed yet; skipping property update"
            );
            return Ok(());
        }
    }

    tracing::error!(
        status_code=?status_code,
        body=?body,
        document_id=%document_id,
        "error updating document properties",
    );

    Err(OpensearchClientError::Unknown {
        details: body,
        operation: OpensearchOperation::UpdateDocumentProperties,
    })
}

#[cfg(test)]
mod test;