Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(search): index parent-only docs for non-searchable file types (#4620)

gbirman38abe34d3597
diff --git a/rust/cloud-storage/opensearch_client/src/document.rs b/rust/cloud-storage/opensearch_client/src/document.rs
index 608255f659..f05a82678d 100644
--- a/rust/cloud-storage/opensearch_client/src/document.rs
+++ b/rust/cloud-storage/opensearch_client/src/document.rs
@@ -10,6 +10,17 @@ impl OpensearchClient {
upsert::document::upsert_document(&self.inner, upsert_document_args, None).await
}
+ /// Upserts only the parent doc (no content chunks) into the opensearch index
+ #[tracing::instrument(skip(self), err)]
+ pub async fn upsert_parent_document(
+ &self,
+ upsert_document_args: &UpsertDocumentArgs,
+ index_override: Option<&str>,
+ ) -> Result<()> {
+ upsert::document::upsert_parent_document(&self.inner, upsert_document_args, index_override)
+ .await
+ }
+
/// Bulk upserts documents into the opensearch index
#[tracing::instrument(skip(self, documents))]
pub async fn bulk_upsert_documents(
diff --git a/rust/cloud-storage/opensearch_client/src/upsert/document.rs b/rust/cloud-storage/opensearch_client/src/upsert/document.rs
index 1fbd92c331..2c5530c404 100644
--- a/rust/cloud-storage/opensearch_client/src/upsert/document.rs
+++ b/rust/cloud-storage/opensearch_client/src/upsert/document.rs
@@ -279,6 +279,49 @@ pub(crate) async fn upsert_document(
Ok(())
}
+/// Builds the 2-line bulk body for a parent-only write: one `index` op
+/// (routing = parent _id = document_id) followed by the parent doc body.
+fn parent_only_bulk_body(args: &UpsertDocumentArgs) -> Vec<String> {
+ let parent_id = args.document_id.as_str();
+ let parent_action = serde_json::json!({
+ "index": {
+ "_id": parent_id,
+ "routing": parent_id,
+ }
+ });
+ vec![parent_action.to_string(), parent_doc_body(args).to_string()]
+}
+
+/// Upsert only the parent doc for a document with no indexable content
+/// (e.g. archives, media). Writes the same full-overwrite parent body as
+/// the chunked paths — `node_id`/`content` on `args` are ignored — and
+/// touches no children.
+#[tracing::instrument(skip(client), err)]
+pub(crate) async fn upsert_parent_document(
+ client: &opensearch::OpenSearch,
+ args: &UpsertDocumentArgs,
+ index_override: Option<&str>,
+) -> Result<()> {
+ let index = resolve_destination(index_override);
+ let bulk_body = parent_only_bulk_body(args);
+
+ let result =
+ super::bulk_upsert_to_index(client, index, bulk_body, "upsert_parent_document").await?;
+
+ if result.failed > 0 {
+ return Err(OpensearchClientError::Unknown {
+ details: format!(
+ "upsert_parent_document had {} failures: {:?}",
+ result.failed, result.errors
+ ),
+ method: Some("upsert_parent_document".to_string()),
+ });
+ }
+
+ tracing::trace!(parent=%args.document_id, "parent document upserted successfully");
+ Ok(())
+}
+
/// Update the denormalized `document_name` for an existing document.
///
/// A single partial-update on the parent doc; children don't carry
diff --git a/rust/cloud-storage/opensearch_client/src/upsert/document/test.rs b/rust/cloud-storage/opensearch_client/src/upsert/document/test.rs
index 9ccc3f5d79..d312e0d344 100644
--- a/rust/cloud-storage/opensearch_client/src/upsert/document/test.rs
+++ b/rust/cloud-storage/opensearch_client/src/upsert/document/test.rs
@@ -102,6 +102,38 @@ fn child_doc_body_omits_raw_content_when_none() {
assert!(doc.get("raw_content").is_none());
}
+#[test]
+fn parent_only_bulk_body_writes_parent_and_no_children() {
+ let mut a = args("doc1", "n1");
+ a.sub_type = Some("task".to_string());
+ a.properties = vec![IndexedProperty {
+ definition_id: "tag-def".to_string(),
+ values: vec!["opt-1".to_string()],
+ ..Default::default()
+ }];
+ let body = parent_only_bulk_body(&a);
+
+ assert_eq!(body.len(), 2);
+
+ let action: serde_json::Value = serde_json::from_str(&body[0]).unwrap();
+ assert_eq!(action["index"]["_id"], "doc1");
+ assert_eq!(action["index"]["routing"], "doc1");
+
+ let doc: serde_json::Value = serde_json::from_str(&body[1]).unwrap();
+ assert_eq!(doc["entity_id"], "doc1");
+ assert_eq!(doc["document_name"], "name-doc1");
+ assert_eq!(doc["owner_id"], "owner-doc1");
+ assert_eq!(doc["file_type"], "md");
+ assert_eq!(doc["document_relation"], "document");
+ assert_eq!(doc["sub_type"], "task");
+ assert_eq!(doc["properties"][0]["definition_id"], "tag-def");
+ assert_eq!(doc["properties"][0]["values"][0], "opt-1");
+ // No chunk is written and no chunk fields leak onto the parent.
+ assert!(doc.get("content").is_none());
+ assert!(doc.get("node_id").is_none());
+ assert!(doc.get("raw_content").is_none());
+}
+
#[test]
fn resolve_destination_defaults_to_documents_alias() {
assert_eq!(resolve_destination(None), SearchIndex::Documents.as_ref());
diff --git a/rust/cloud-storage/search_processing_service/src/process/document/mod.rs b/rust/cloud-storage/search_processing_service/src/process/document/mod.rs
index 7bd7e6da5a..62e43a6ed8 100644
--- a/rust/cloud-storage/search_processing_service/src/process/document/mod.rs
+++ b/rust/cloud-storage/search_processing_service/src/process/document/mod.rs
@@ -1,5 +1,4 @@
use anyhow::Context;
-use model::document::FileTypeExt;
use opensearch_client::OpensearchClient;
use sqs_client::search::document::{DocumentId, SearchExtractorMessage};
@@ -24,11 +23,6 @@ pub async fn process_extract_text_message(
document_storage_bucket: &str,
search_extractor_message: &SearchExtractorMessage,
) -> anyhow::Result<()> {
- if search_extractor_message.file_type.is_image() {
- tracing::trace!("image file, ignoring");
- return Ok(());
- }
-
raw_document::update_search_with_raw_document(
opensearch_client,
db,
diff --git a/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs b/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs
index dff92b3b6f..f3aeb08b23 100644
--- a/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs
+++ b/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs
@@ -108,6 +108,84 @@ async fn attach_indexed_properties(
Ok(())
}
+/// Builds the parent-only upsert for a file type with no indexable content.
+/// Returns `None` when the document has no file type.
+fn generate_parent_only_upsert(
+ document_info: DocumentMetadata,
+) -> anyhow::Result<Option<UpsertDocumentArgs>> {
+ let Some(file_type) = document_info.file_type else {
+ return Ok(None);
+ };
+ Ok(Some(UpsertDocumentArgs {
+ document_id: document_info.document_id,
+ node_id: String::new(),
+ raw_content: None,
+ document_name: document_info.document_name,
+ content: String::new(),
+ owner_id: document_info.owner.to_string(),
+ file_type,
+ updated_at_seconds: EpochSeconds::new(Utc::now().timestamp())?,
+ sub_type: document_info.sub_type.map(|st| st.to_string()),
+ properties: vec![],
+ }))
+}
+
+/// Indexes a name-only parent doc (no content chunks) for file types whose
+/// content isn't searchable, so the document stays findable by name and
+/// filterable by its indexed properties.
+async fn update_search_with_parent_only_document(
+ opensearch_client: &OpensearchClient,
+ db: &sqlx::Pool<sqlx::Postgres>,
+ search_extractor_message: &SearchExtractorMessage,
+) -> anyhow::Result<()> {
+ let document_info = match get_document_info(db, search_extractor_message)
+ .await
+ .context("failed to get document info")?
+ {
+ DocumentInfo::Active(info) => *info,
+ DocumentInfo::Removable => {
+ tracing::trace!("document is deleted or missing, removing from search index");
+ opensearch_client
+ .delete_document(
+ &search_extractor_message.document_id,
+ search_extractor_message.index_override.as_deref(),
+ )
+ .await
+ .context("failed to delete document from search index")?;
+ return Ok(());
+ }
+ DocumentInfo::Skip => {
+ tracing::trace!("no document info returned");
+ return Ok(());
+ }
+ };
+
+ let Some(args) = generate_parent_only_upsert(document_info)? else {
+ tracing::debug!("file type is none");
+ return Ok(());
+ };
+
+ let sub_type = args.sub_type.clone();
+ let mut upserts = vec![args];
+ attach_indexed_properties(
+ db,
+ &search_extractor_message.document_id,
+ sub_type.as_deref(),
+ &mut upserts,
+ )
+ .await?;
+
+ opensearch_client
+ .upsert_parent_document(
+ &upserts[0],
+ search_extractor_message.index_override.as_deref(),
+ )
+ .await
+ .context("unable to upsert parent-only document in opensearch")?;
+
+ Ok(())
+}
+
/// Processes a message for a standard document and reads the updated contents from s3 and updates
/// the document in opensearch.
#[tracing::instrument(skip(opensearch_client, db, s3_client, document_storage_bucket, search_extractor_message), fields(document_id=search_extractor_message.document_id, file_type=?search_extractor_message.file_type))]
@@ -119,8 +197,12 @@ pub async fn update_search_with_raw_document(
search_extractor_message: &SearchExtractorMessage,
) -> anyhow::Result<()> {
if !is_searchable_association(&search_extractor_message.file_type.macro_app_path()) {
- tracing::warn!("unsupported file type");
- return Ok(());
+ return update_search_with_parent_only_document(
+ opensearch_client,
+ db,
+ search_extractor_message,
+ )
+ .await;
}
// This ensures we only process the latest version
@@ -505,4 +587,50 @@ mod tests {
assert_eq!(upserts.len(), 1);
assert_eq!(upserts[0].sub_type, Some("task".to_string()));
}
+
+ fn parent_only_document_info(file_type: Option<&str>) -> DocumentMetadata {
+ DocumentMetadata {
+ document_id: "CCC".to_string(),
+ document_version_id: 0,
+ owner: MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap(),
+ document_name: "pdf copy".to_string(),
+ file_type: file_type.map(|ft| ft.to_string()),
+ sha: None,
+ project_id: None,
+ project_name: None,
+ branched_from_id: None,
+ branched_from_version_id: None,
+ document_family_id: None,
+ document_bom: None,
+ modification_data: None,
+ created_at: None,
+ updated_at: None,
+ sub_type: None,
+ deleted_at: None,
+ }
+ }
+
+ #[test]
+ fn test_generate_parent_only_upsert() {
+ let args = generate_parent_only_upsert(parent_only_document_info(Some("zip")))
+ .expect("could not generate parent-only upsert")
+ .expect("expected upsert args");
+
+ assert_eq!(args.document_id, "CCC");
+ assert_eq!(args.document_name, "pdf copy");
+ assert_eq!(args.owner_id, "macro|[email protected]");
+ assert_eq!(args.file_type, "zip");
+ assert_eq!(args.sub_type, None);
+ assert_eq!(args.content, "");
+ assert_eq!(args.raw_content, None);
+ assert!(args.properties.is_empty());
+ }
+
+ #[test]
+ fn test_generate_parent_only_upsert_without_file_type() {
+ let args = generate_parent_only_upsert(parent_only_document_info(None))
+ .expect("could not generate parent-only upsert");
+
+ assert!(args.is_none());
+ }
}