Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(search): index parent-only docs for non-searchable file types (#4620)
gbirman38abe34d3597
diff --git a/rust/cloud-storage/opensearch_client/src/document.rs b/rust/cloud-storage/opensearch_client/src/document.rsindex 608255f659..f05a82678d 100644--- a/rust/cloud-storage/opensearch_client/src/document.rs+++ b/rust/cloud-storage/opensearch_client/src/document.rs@@ -10,6 +10,17 @@ impl OpensearchClient {upsert::document::upsert_document(&self.inner, upsert_document_args, None).await}+ /// Upserts only the parent doc (no content chunks) into the opensearch index+ #[tracing::instrument(skip(self), err)]+ pub async fn upsert_parent_document(+ &self,+ upsert_document_args: &UpsertDocumentArgs,+ index_override: Option<&str>,+ ) -> Result<()> {+ upsert::document::upsert_parent_document(&self.inner, upsert_document_args, index_override)+ .await+ }+/// Bulk upserts documents into the opensearch index#[tracing::instrument(skip(self, documents))]pub async fn bulk_upsert_documents(diff --git a/rust/cloud-storage/opensearch_client/src/upsert/document.rs b/rust/cloud-storage/opensearch_client/src/upsert/document.rsindex 1fbd92c331..2c5530c404 100644--- a/rust/cloud-storage/opensearch_client/src/upsert/document.rs+++ b/rust/cloud-storage/opensearch_client/src/upsert/document.rs@@ -279,6 +279,49 @@ pub(crate) async fn upsert_document(Ok(())}+/// Builds the 2-line bulk body for a parent-only write: one `index` op+/// (routing = parent _id = document_id) followed by the parent doc body.+fn parent_only_bulk_body(args: &UpsertDocumentArgs) -> Vec<String> {+ let parent_id = args.document_id.as_str();+ let parent_action = serde_json::json!({+ "index": {+ "_id": parent_id,+ "routing": parent_id,+ }+ });+ vec![parent_action.to_string(), parent_doc_body(args).to_string()]+}++/// Upsert only the parent doc for a document with no indexable content+/// (e.g. archives, media). Writes the same full-overwrite parent body as+/// the chunked paths — `node_id`/`content` on `args` are ignored — and+/// touches no children.+#[tracing::instrument(skip(client), err)]+pub(crate) async fn upsert_parent_document(+ client: &opensearch::OpenSearch,+ args: &UpsertDocumentArgs,+ index_override: Option<&str>,+) -> Result<()> {+ let index = resolve_destination(index_override);+ let bulk_body = parent_only_bulk_body(args);++ let result =+ super::bulk_upsert_to_index(client, index, bulk_body, "upsert_parent_document").await?;++ if result.failed > 0 {+ return Err(OpensearchClientError::Unknown {+ details: format!(+ "upsert_parent_document had {} failures: {:?}",+ result.failed, result.errors+ ),+ method: Some("upsert_parent_document".to_string()),+ });+ }++ tracing::trace!(parent=%args.document_id, "parent document upserted successfully");+ Ok(())+}+/// Update the denormalized `document_name` for an existing document.////// A single partial-update on the parent doc; children don't carrydiff --git a/rust/cloud-storage/opensearch_client/src/upsert/document/test.rs b/rust/cloud-storage/opensearch_client/src/upsert/document/test.rsindex 9ccc3f5d79..d312e0d344 100644--- a/rust/cloud-storage/opensearch_client/src/upsert/document/test.rs+++ b/rust/cloud-storage/opensearch_client/src/upsert/document/test.rs@@ -102,6 +102,38 @@ fn child_doc_body_omits_raw_content_when_none() {assert!(doc.get("raw_content").is_none());}+#[test]+fn parent_only_bulk_body_writes_parent_and_no_children() {+ let mut a = args("doc1", "n1");+ a.sub_type = Some("task".to_string());+ a.properties = vec![IndexedProperty {+ definition_id: "tag-def".to_string(),+ values: vec!["opt-1".to_string()],+ ..Default::default()+ }];+ let body = parent_only_bulk_body(&a);++ assert_eq!(body.len(), 2);++ let action: serde_json::Value = serde_json::from_str(&body[0]).unwrap();+ assert_eq!(action["index"]["_id"], "doc1");+ assert_eq!(action["index"]["routing"], "doc1");++ let doc: serde_json::Value = serde_json::from_str(&body[1]).unwrap();+ assert_eq!(doc["entity_id"], "doc1");+ assert_eq!(doc["document_name"], "name-doc1");+ assert_eq!(doc["owner_id"], "owner-doc1");+ assert_eq!(doc["file_type"], "md");+ assert_eq!(doc["document_relation"], "document");+ assert_eq!(doc["sub_type"], "task");+ assert_eq!(doc["properties"][0]["definition_id"], "tag-def");+ assert_eq!(doc["properties"][0]["values"][0], "opt-1");+ // No chunk is written and no chunk fields leak onto the parent.+ assert!(doc.get("content").is_none());+ assert!(doc.get("node_id").is_none());+ assert!(doc.get("raw_content").is_none());+}+#[test]fn resolve_destination_defaults_to_documents_alias() {assert_eq!(resolve_destination(None), SearchIndex::Documents.as_ref());diff --git a/rust/cloud-storage/search_processing_service/src/process/document/mod.rs b/rust/cloud-storage/search_processing_service/src/process/document/mod.rsindex 7bd7e6da5a..62e43a6ed8 100644--- a/rust/cloud-storage/search_processing_service/src/process/document/mod.rs+++ b/rust/cloud-storage/search_processing_service/src/process/document/mod.rs@@ -1,5 +1,4 @@use anyhow::Context;-use model::document::FileTypeExt;use opensearch_client::OpensearchClient;use sqs_client::search::document::{DocumentId, SearchExtractorMessage};@@ -24,11 +23,6 @@ pub async fn process_extract_text_message(document_storage_bucket: &str,search_extractor_message: &SearchExtractorMessage,) -> anyhow::Result<()> {- if search_extractor_message.file_type.is_image() {- tracing::trace!("image file, ignoring");- return Ok(());- }-raw_document::update_search_with_raw_document(opensearch_client,db,diff --git a/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs b/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rsindex dff92b3b6f..f3aeb08b23 100644--- a/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs+++ b/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs@@ -108,6 +108,84 @@ async fn attach_indexed_properties(Ok(())}+/// Builds the parent-only upsert for a file type with no indexable content.+/// Returns `None` when the document has no file type.+fn generate_parent_only_upsert(+ document_info: DocumentMetadata,+) -> anyhow::Result<Option<UpsertDocumentArgs>> {+ let Some(file_type) = document_info.file_type else {+ return Ok(None);+ };+ Ok(Some(UpsertDocumentArgs {+ document_id: document_info.document_id,+ node_id: String::new(),+ raw_content: None,+ document_name: document_info.document_name,+ content: String::new(),+ owner_id: document_info.owner.to_string(),+ file_type,+ updated_at_seconds: EpochSeconds::new(Utc::now().timestamp())?,+ sub_type: document_info.sub_type.map(|st| st.to_string()),+ properties: vec![],+ }))+}++/// Indexes a name-only parent doc (no content chunks) for file types whose+/// content isn't searchable, so the document stays findable by name and+/// filterable by its indexed properties.+async fn update_search_with_parent_only_document(+ opensearch_client: &OpensearchClient,+ db: &sqlx::Pool<sqlx::Postgres>,+ search_extractor_message: &SearchExtractorMessage,+) -> anyhow::Result<()> {+ let document_info = match get_document_info(db, search_extractor_message)+ .await+ .context("failed to get document info")?+ {+ DocumentInfo::Active(info) => *info,+ DocumentInfo::Removable => {+ tracing::trace!("document is deleted or missing, removing from search index");+ opensearch_client+ .delete_document(+ &search_extractor_message.document_id,+ search_extractor_message.index_override.as_deref(),+ )+ .await+ .context("failed to delete document from search index")?;+ return Ok(());+ }+ DocumentInfo::Skip => {+ tracing::trace!("no document info returned");+ return Ok(());+ }+ };++ let Some(args) = generate_parent_only_upsert(document_info)? else {+ tracing::debug!("file type is none");+ return Ok(());+ };++ let sub_type = args.sub_type.clone();+ let mut upserts = vec![args];+ attach_indexed_properties(+ db,+ &search_extractor_message.document_id,+ sub_type.as_deref(),+ &mut upserts,+ )+ .await?;++ opensearch_client+ .upsert_parent_document(+ &upserts[0],+ search_extractor_message.index_override.as_deref(),+ )+ .await+ .context("unable to upsert parent-only document in opensearch")?;++ Ok(())+}+/// Processes a message for a standard document and reads the updated contents from s3 and updates/// the document in opensearch.#[tracing::instrument(skip(opensearch_client, db, s3_client, document_storage_bucket, search_extractor_message), fields(document_id=search_extractor_message.document_id, file_type=?search_extractor_message.file_type))]@@ -119,8 +197,12 @@ pub async fn update_search_with_raw_document(search_extractor_message: &SearchExtractorMessage,) -> anyhow::Result<()> {if !is_searchable_association(&search_extractor_message.file_type.macro_app_path()) {- tracing::warn!("unsupported file type");- return Ok(());+ return update_search_with_parent_only_document(+ opensearch_client,+ db,+ search_extractor_message,+ )+ .await;}// This ensures we only process the latest version@@ -505,4 +587,50 @@ mod tests {assert_eq!(upserts.len(), 1);assert_eq!(upserts[0].sub_type, Some("task".to_string()));}++ fn parent_only_document_info(file_type: Option<&str>) -> DocumentMetadata {+ DocumentMetadata {+ document_id: "CCC".to_string(),+ document_version_id: 0,+ owner: MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap(),+ document_name: "pdf copy".to_string(),+ file_type: file_type.map(|ft| ft.to_string()),+ sha: None,+ project_id: None,+ project_name: None,+ branched_from_id: None,+ branched_from_version_id: None,+ document_family_id: None,+ document_bom: None,+ modification_data: None,+ created_at: None,+ updated_at: None,+ sub_type: None,+ deleted_at: None,+ }+ }++ #[test]+ fn test_generate_parent_only_upsert() {+ let args = generate_parent_only_upsert(parent_only_document_info(Some("zip")))+ .expect("could not generate parent-only upsert")+ .expect("expected upsert args");++ assert_eq!(args.document_id, "CCC");+ assert_eq!(args.document_name, "pdf copy");+ assert_eq!(args.owner_id, "macro|[email protected]");+ assert_eq!(args.file_type, "zip");+ assert_eq!(args.sub_type, None);+ assert_eq!(args.content, "");+ assert_eq!(args.raw_content, None);+ assert!(args.properties.is_empty());+ }++ #[test]+ fn test_generate_parent_only_upsert_without_file_type() {+ let args = generate_parent_only_upsert(parent_only_document_info(None))+ .expect("could not generate parent-only upsert");++ assert!(args.is_none());+ }}