Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(search): make highlighting survive oversized chunks, surface shard failures (#4929)
gbirmand4423155d233
diff --git a/crates/opensearch_client/src/search/call_records.rs b/crates/opensearch_client/src/search/call_records.rsindex 68f11099db..52f7cb8701 100644--- a/crates/opensearch_client/src/search/call_records.rs+++ b/crates/opensearch_client/src/search/call_records.rs@@ -252,6 +252,7 @@ impl CallRecordQueryBuilder {fn inner_hits_content_highlight(highlight_query: &serde_json::Value) -> serde_json::Value {serde_json::json!({"require_field_match": true,+ "max_analyzer_offset": super::HIGHLIGHT_MAX_ANALYZER_OFFSET,"pre_tags": ["<macro_em>"],"post_tags": ["</macro_em>"],"fields": {diff --git a/crates/opensearch_client/src/search/channels.rs b/crates/opensearch_client/src/search/channels.rsindex b38ff06689..0f8bdf9eb1 100644--- a/crates/opensearch_client/src/search/channels.rs+++ b/crates/opensearch_client/src/search/channels.rs@@ -205,6 +205,7 @@ fn build_channel_search_request(};let highlight = Highlight::new().require_field_match(true)+ .max_analyzer_offset(super::HIGHLIGHT_MAX_ANALYZER_OFFSET).field("content", em_field().number_of_fragments(1));request_builder.highlight(highlight);@@ -247,6 +248,7 @@ pub(crate) async fn search_channel(details: e.to_string(),raw_body: String::from_utf8_lossy(&bytes).to_string(),})?;+ result.warn_on_shard_failures("search_channel");let total = result.hits.total.value;diff --git a/crates/opensearch_client/src/search/chats.rs b/crates/opensearch_client/src/search/chats.rsindex 40116c12bc..b3b2d4980f 100644--- a/crates/opensearch_client/src/search/chats.rs+++ b/crates/opensearch_client/src/search/chats.rs@@ -189,6 +189,7 @@ impl ChatQueryBuilder {fn inner_hits_content_highlight(highlight_query: &serde_json::Value) -> serde_json::Value {serde_json::json!({"require_field_match": true,+ "max_analyzer_offset": super::HIGHLIGHT_MAX_ANALYZER_OFFSET,"pre_tags": ["<macro_em>"],"post_tags": ["</macro_em>"],"fields": {diff --git a/crates/opensearch_client/src/search/documents.rs b/crates/opensearch_client/src/search/documents.rsindex 9d30a06c05..786b72980b 100644--- a/crates/opensearch_client/src/search/documents.rs+++ b/crates/opensearch_client/src/search/documents.rs@@ -304,6 +304,7 @@ fn build_property_filter<'a>(filter: &PropertyFilterArg) -> Option<QueryType<'a>fn inner_hits_content_highlight(highlight_query: &serde_json::Value) -> serde_json::Value {serde_json::json!({"require_field_match": true,+ "max_analyzer_offset": super::HIGHLIGHT_MAX_ANALYZER_OFFSET,"pre_tags": ["<macro_em>"],"post_tags": ["</macro_em>"],"fields": {diff --git a/crates/opensearch_client/src/search/documents/test.rs b/crates/opensearch_client/src/search/documents/test.rsindex 3d3706806c..69903c471e 100644--- a/crates/opensearch_client/src/search/documents/test.rs+++ b/crates/opensearch_client/src/search/documents/test.rs@@ -150,6 +150,12 @@ fn test_build_bool_query_join_shape_has_inner_hits() -> anyhow::Result<()> {format!("term_{idx}"),"inner_hits name should be term_<idx>");+ assert_eq!(+ inner["highlight"]["max_analyzer_offset"],+ serde_json::json!(super::super::HIGHLIGHT_MAX_ANALYZER_OFFSET),+ "inner_hits highlight must cap analyzed offset so an oversized \+ chunk can't fail the shard fetch: {inner:?}"+ );}Ok(())}diff --git a/crates/opensearch_client/src/search/mod.rs b/crates/opensearch_client/src/search/mod.rsindex b5015a3227..a35d563692 100644--- a/crates/opensearch_client/src/search/mod.rs+++ b/crates/opensearch_client/src/search/mod.rs@@ -1,3 +1,12 @@+/// Request-level cap on how many characters of a field OpenSearch analyzes+/// for highlighting. Must stay below the index-level+/// `index.highlight.max_analyzed_offset` (default 1,000,000): with the+/// request-level cap set, an oversized field gets a truncated highlight,+/// while without it the plain highlighter throws and OpenSearch fails the+/// entire shard's fetch phase — silently dropping every hit on that shard+/// from the (HTTP 200, partial) response.+pub(crate) const HIGHLIGHT_MAX_ANALYZER_OFFSET: u32 = 999_999;+mod builder;pub mod call_records;pub mod channels;diff --git a/crates/opensearch_client/src/search/model/mod.rs b/crates/opensearch_client/src/search/model/mod.rsindex 9ec21f0dc9..3fff826b0d 100644--- a/crates/opensearch_client/src/search/model/mod.rs+++ b/crates/opensearch_client/src/search/model/mod.rs@@ -335,6 +335,9 @@ pub(crate) struct Shards {pub successful: i32,pub skipped: i32,pub failed: i32,+ /// Per-shard failure details, present only when `failed > 0`.+ #[serde(default, skip_serializing_if = "Vec::is_empty")]+ pub failures: Vec<serde_json::Value>,}pub(crate) type DefaultSearchResponse<T> = SearchResponse<T>;@@ -347,6 +350,23 @@ pub(crate) struct SearchResponse<T> {pub _shards: Shards,}+impl<T> SearchResponse<T> {+ /// OpenSearch returns HTTP 200 with partial results when individual+ /// shards fail their query or fetch phase, so every hit on a failed+ /// shard is silently absent unless callers check `_shards`.+ pub(crate) fn warn_on_shard_failures(&self, operation: &str) {+ if self._shards.failed > 0 {+ tracing::warn!(+ operation,+ failed = self._shards.failed,+ total = self._shards.total,+ failures = ?self._shards.failures,+ "opensearch returned partial results: some shards failed"+ );+ }+ }+}+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct SearchGotoDocument {/// The node id of the document.diff --git a/crates/opensearch_client/src/search/unified.rs b/crates/opensearch_client/src/search/unified.rsindex f2f6bbaab4..bf190ba3ba 100644--- a/crates/opensearch_client/src/search/unified.rs+++ b/crates/opensearch_client/src/search/unified.rs@@ -724,6 +724,7 @@ fn build_unified_search_request(args: &UnifiedSearchArgs) -> Result<SearchReques};let highlight = Highlight::new().require_field_match(true)+ .max_analyzer_offset(super::HIGHLIGHT_MAX_ANALYZER_OFFSET).field("content", em_field().number_of_fragments(1)).field("document_name", em_field().number_of_fragments(0)).field("name", em_field().number_of_fragments(0))@@ -824,6 +825,8 @@ pub(crate) async fn search_unified(raw_body: String::from_utf8_lossy(&bytes).to_string(),})?;+ result.warn_on_shard_failures("search_unified");+tracing::info!(response_body_bytes = bytes.len(),opensearch_took_ms = result.took,diff --git a/crates/opensearch_client/src/search/unified/test.rs b/crates/opensearch_client/src/search/unified/test.rsindex 9d39b0f225..67bc2e865b 100644--- a/crates/opensearch_client/src/search/unified/test.rs+++ b/crates/opensearch_client/src/search/unified/test.rs@@ -535,7 +535,8 @@ fn test_build_unified_search_request_content() -> anyhow::Result<()> {"bcc": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },"bcc_names": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" }},- "require_field_match": true+ "require_field_match": true,+ "max_analyzer_offset": 999999},"query": {"bool": {@@ -856,7 +857,8 @@ fn test_build_unified_search_request_single_index() -> anyhow::Result<()> {"bcc": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },"bcc_names": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" }},- "require_field_match": true+ "require_field_match": true,+ "max_analyzer_offset": 999999},"query": {"bool": {diff --git a/crates/opensearch_client/src/upsert/document.rs b/crates/opensearch_client/src/upsert/document.rsindex 28ef44745b..6aaf12683b 100644--- a/crates/opensearch_client/src/upsert/document.rs+++ b/crates/opensearch_client/src/upsert/document.rs@@ -12,6 +12,30 @@ const PARENT_RELATION: &str = "document";/// Relation name for child (chunk) docs in the join field.const CHILD_RELATION: &str = "chunk";+/// Cap on indexed chunk `content`, in characters. Markdown chunks are+/// node-sized and pdf/docx chunks page-sized, but formats without block+/// structure (code files, plain text) arrive as one whole-file chunk+/// that can run to many megabytes. Oversized chunks bloat the index and can+/// never be fully highlighted (`index.highlight.max_analyzed_offset` is+/// 1,000,000), so the tail is dropped at write time.+const MAX_CHUNK_CONTENT_CHARS: usize = 100_000;++/// Truncate chunk content to [`MAX_CHUNK_CONTENT_CHARS`] on a char boundary.+fn cap_chunk_content(chunk: &UpsertDocumentArgs) -> &str {+ match chunk.content.char_indices().nth(MAX_CHUNK_CONTENT_CHARS) {+ Some((byte_idx, _)) => {+ tracing::warn!(+ document_id = %chunk.document_id,+ node_id = %chunk.node_id,+ content_len = chunk.content.len(),+ "chunk content exceeds indexing cap, truncating"+ );+ &chunk.content[..byte_idx]+ }+ None => &chunk.content,+ }+}+/// The arguments for upserting a document into the opensearch index#[derive(Debug, serde::Serialize)]pub struct UpsertDocumentArgs {@@ -82,7 +106,7 @@ fn child_doc_body(chunk: &UpsertDocumentArgs) -> serde_json::Value {let mut doc = serde_json::json!({"entity_id": &chunk.document_id,"node_id": &chunk.node_id,- "content": &chunk.content,+ "content": cap_chunk_content(chunk),"document_relation": {"name": CHILD_RELATION,"parent": &chunk.document_id,diff --git a/crates/opensearch_client/src/upsert/document/test.rs b/crates/opensearch_client/src/upsert/document/test.rsindex bded2aa8b6..d620efec9d 100644--- a/crates/opensearch_client/src/upsert/document/test.rs+++ b/crates/opensearch_client/src/upsert/document/test.rs@@ -201,3 +201,37 @@ fn parent_body_omits_sub_type_when_none_so_index_clears_it() {let doc = parent_doc_body(&args("doc1", "n1"));assert!(doc.get("sub_type").is_none());}++#[test]+fn child_body_keeps_content_under_cap_intact() {+ let mut a = args("doc1", "n1");+ a.content = "x".repeat(MAX_CHUNK_CONTENT_CHARS);+ let doc = child_doc_body(&a);+ assert_eq!(+ doc["content"].as_str().unwrap().chars().count(),+ MAX_CHUNK_CONTENT_CHARS+ );+}++#[test]+fn child_body_truncates_oversized_content() {+ let mut a = args("doc1", "n1");+ a.content = "x".repeat(MAX_CHUNK_CONTENT_CHARS + 10);+ let doc = child_doc_body(&a);+ assert_eq!(+ doc["content"].as_str().unwrap().chars().count(),+ MAX_CHUNK_CONTENT_CHARS+ );+}++#[test]+fn child_body_truncates_on_char_boundary() {+ let mut a = args("doc1", "n1");+ // é is 2 bytes per char, so a byte-index split would panic mid-char.+ a.content = "é".repeat(MAX_CHUNK_CONTENT_CHARS + 5);+ let doc = child_doc_body(&a);+ assert_eq!(+ doc["content"].as_str().unwrap().chars().count(),+ MAX_CHUNK_CONTENT_CHARS+ );+}diff --git a/crates/opensearch_query_builder/src/request/highlight.rs b/crates/opensearch_query_builder/src/request/highlight.rsindex 3a99fedc97..0a263b26bf 100644--- a/crates/opensearch_query_builder/src/request/highlight.rs+++ b/crates/opensearch_query_builder/src/request/highlight.rs@@ -16,6 +16,12 @@ pub struct Highlight<'a> {/// Require field match#[serde(skip_serializing_if = "Option::is_none")]pub require_field_match: Option<bool>,+ /// Cap on how many characters of a field are analyzed for highlighting.+ /// Must stay below the index-level `index.highlight.max_analyzer_offset`+ /// (default 1,000,000) — with this set, oversized fields get truncated+ /// highlights instead of failing the whole shard's fetch phase.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub max_analyzer_offset: Option<u32>,}impl<'a> Highlight<'a> {@@ -39,6 +45,12 @@ impl<'a> Highlight<'a> {self.require_field_match = Some(require_field_match);self}++ /// Set the max analyzed offset for highlighting+ pub fn max_analyzer_offset(mut self, max_analyzer_offset: u32) -> Self {+ self.max_analyzer_offset = Some(max_analyzer_offset);+ self+ }}impl<'a> ToOpenSearchJson for Highlight<'a> {@@ -60,6 +72,13 @@ impl<'a> ToOpenSearchJson for Highlight<'a> {);}+ if let Some(max_analyzer_offset) = self.max_analyzer_offset {+ result.insert(+ "max_analyzer_offset".to_string(),+ Value::Number(max_analyzer_offset.into()),+ );+ }+Value::Object(result)}}