Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(search): restore channel message snippets (#7829)
Eric Hayes24bd20fb801a
diff --git a/crates/opensearch_client/src/search/builder.rs b/crates/opensearch_client/src/search/builder.rsindex d20f42a148..82f978f623 100644--- a/crates/opensearch_client/src/search/builder.rs+++ b/crates/opensearch_client/src/search/builder.rs@@ -98,6 +98,8 @@ pub trait SearchQueryConfig {const TITLE_KEY: &'static str;/// Content fieldconst CONTENT_KEY: &'static str = "content";+ /// Fields searched for each content term; highlights still use `CONTENT_KEY`.+ const CONTENT_FIELDS: &'static [&'static str] = &[Self::CONTENT_KEY];/// The entity index for the search queryconst ENTITY_INDEX: OpenSearchEntityType;}@@ -282,9 +284,9 @@ impl<T: SearchQueryConfig> SearchQueryBuilder<T> {let terms: Cow<'_, [&str]> =Cow::Owned(self.terms.iter().map(|t| t.as_str()).collect::<Vec<&str>>());- let must_array = vec![generate_terms_must_query(+ let must_array = vec![super::query::generate_terms_must_query_for_fields(query_key,- T::CONTENT_KEY,+ T::CONTENT_FIELDS,terms,self.term_combine,)];diff --git a/crates/opensearch_client/src/search/channels.rs b/crates/opensearch_client/src/search/channels.rsindex 0f8bdf9eb1..9632a27b95 100644--- a/crates/opensearch_client/src/search/channels.rs+++ b/crates/opensearch_client/src/search/channels.rs@@ -41,6 +41,9 @@ pub(crate) struct ChannelMessageSearchConfig;impl SearchQueryConfig for ChannelMessageSearchConfig {const USER_ID_KEY: Option<&'static str> = Some("sender_id");const TITLE_KEY: &'static str = "name";+ // Search authors directly: copying them into content hides body highlights+ // in OpenSearch, including messages with no imported author.+ const CONTENT_FIELDS: &'static [&'static str] = &["content", "imported_author"];const ENTITY_INDEX: OpenSearchEntityType = OpenSearchEntityType::Channels;}diff --git a/crates/opensearch_client/src/search/channels/test.rs b/crates/opensearch_client/src/search/channels/test.rsindex d394f78ef6..04861b19d4 100644--- a/crates/opensearch_client/src/search/channels/test.rs+++ b/crates/opensearch_client/src/search/channels/test.rs@@ -84,7 +84,10 @@ fn test_build_bool_query() -> anyhow::Result<()> {"bool": {"minimum_should_match": 1,"should": [- {"match_phrase": {"content": "test"}}+ {"bool": {"minimum_should_match": 1, "should": [+ {"match_phrase": {"content": "test"}},+ {"match_phrase": {"imported_author": "test"}}+ ]}}]}}@@ -114,8 +117,7 @@ fn test_build_bool_query() -> anyhow::Result<()> {#[test]fn test_build_bool_query_multi_term_ands_inside_opensearch() -> anyhow::Result<()> {- // Two terms — each becomes its own `match_phrase` clause and they- // combine with `must` so both must appear in the same message.+ // Terms may match the body or author but must belong to the same message.// Quoted phrases like "foo bar" arrive here as a single token and use// the same `match_phrase` path.let builder = ChannelMessageQueryBuilder::new(vec!["foo".to_string(), "bar baz".to_string()])@@ -135,8 +137,14 @@ fn test_build_bool_query_multi_term_ands_inside_opensearch() -> anyhow::Result<({"bool": {"must": [- { "match_phrase": { "content": "foo" } },- { "match_phrase": { "content": "bar baz" } }+ {"bool": {"minimum_should_match": 1, "should": [+ {"match_phrase": {"content": "foo"}},+ {"match_phrase": {"imported_author": "foo"}}+ ]}},+ {"bool": {"minimum_should_match": 1, "should": [+ {"match_phrase": {"content": "bar baz"}},+ {"match_phrase": {"imported_author": "bar baz"}}+ ]}}]}}diff --git a/crates/opensearch_client/src/search/query.rs b/crates/opensearch_client/src/search/query.rsindex 2228963168..76f0e83ca2 100644--- a/crates/opensearch_client/src/search/query.rs+++ b/crates/opensearch_client/src/search/query.rs@@ -86,17 +86,36 @@ pub(crate) fn generate_terms_must_query<'a>(field: &'a str,terms: impl Into<Cow<'a, [&'a str]>>,combine: TermCombine,+) -> QueryType<'a> {+ generate_terms_must_query_for_fields(query_key, &[field], terms, combine)+}++/// Each term must match at least one field on the same document.+pub(crate) fn generate_terms_must_query_for_fields<'a>(+ query_key: QueryKey,+ fields: &[&'a str],+ terms: impl Into<Cow<'a, [&'a str]>>,+ combine: TermCombine,) -> QueryType<'a> {let terms = terms.into();let queries: Vec<_> = terms.iter().map(|term| {- create_query(CreateQueryParams {- query_key,- field,- term,- })+ let mut alternatives = BoolQueryBuilder::new();+ alternatives.minimum_should_match(1);+ for field in fields {+ let query = create_query(CreateQueryParams {+ query_key,+ field,+ term,+ });+ if fields.len() == 1 {+ return query;+ }+ alternatives.should(query);+ }+ alternatives.build().into()}).collect();diff --git a/crates/opensearch_client/src/search/unified/test.rs b/crates/opensearch_client/src/search/unified/test.rsindex 4f1d5b15a1..929b1dff7f 100644--- a/crates/opensearch_client/src/search/unified/test.rs+++ b/crates/opensearch_client/src/search/unified/test.rs@@ -810,9 +810,10 @@ fn test_build_unified_search_request_content() -> anyhow::Result<()> {"minimum_should_match": 1,"should": [{- "match_phrase": {- "content": "test"- }+ "bool": {"minimum_should_match": 1, "should": [+ {"match_phrase": {"content": "test"}},+ {"match_phrase": {"imported_author": "test"}}+ ]}}]}diff --git a/crates/opensearch_client/src/upsert/channel_message/test.rs b/crates/opensearch_client/src/upsert/channel_message/test.rsindex 6e2669d8aa..129ca53158 100644--- a/crates/opensearch_client/src/upsert/channel_message/test.rs+++ b/crates/opensearch_client/src/upsert/channel_message/test.rs@@ -1,7 +1,7 @@use super::*;use serde_json::json;-fn args() -> UpsertChannelMessageArgs {+fn message_args() -> UpsertChannelMessageArgs {let message_id = uuid::Uuid::now_v7().to_string();UpsertChannelMessageArgs {channel_id: uuid::Uuid::now_v7().to_string(),@@ -20,7 +20,7 @@ fn args() -> UpsertChannelMessageArgs {#[test]fn author_is_text_not_content_or_sender() {- let mut args = args();+ let mut args = message_args();for author in [Some("quartzarchivebot".to_owned()), None] {args.imported_author = author.clone();let value = serde_json::to_value(&args).unwrap();@@ -32,14 +32,14 @@ fn author_is_text_not_content_or_sender() {}}-/// Scratch-index integration of copy_to with the production access/sender query.+/// Repair a legacy copy_to mapping and search both existing and new messages.#[tokio::test]#[ignore = "requires local OpenSearch at http://localhost:9200"]-async fn imported_author_matches_only_with_channel_access() {+async fn imported_author_search_preserves_body_highlights_and_channel_access() {use crate::{OpensearchClient, search::channels::ChannelMessageQueryBuilder};use opensearch::{SearchParts,- indices::{IndicesDeleteParts, IndicesRefreshParts},+ indices::{IndicesDeleteParts, IndicesPutMappingParts, IndicesRefreshParts},};use opensearch_query_builder::ToOpenSearchJson;@@ -60,17 +60,39 @@ async fn imported_author_matches_only_with_channel_access() {})).await.unwrap();let result: anyhow::Result<()> = async {- let args = args();+ let args = message_args();client.upsert_channel_message(&args, Some(&index)).await?;+ // This native message was indexed before the repair and has no author.+ let mut native = message_args();+ native.channel_id = args.channel_id.clone();+ native.thread_id = args.message_id.clone();+ native.imported_author = None;+ native.content = "Nebula deployment checklist".into();+ client.upsert_channel_message(&native, Some(&index)).await?;+ client.inner.indices().put_mapping(IndicesPutMappingParts::Index(&[&index]))+ .body(json!({"properties": {"imported_author": {"type": "text", "analyzer": "content_text", "copy_to": []}}}))+ .send().await?.error_for_status_code()?;+ // A post-repair import must match its author without copied content tokens.+ let mut fresh = message_args();+ fresh.channel_id = args.channel_id.clone();+ fresh.imported_author = Some("newarchivebot".into());+ fresh.content = "Aurora launch checklist".into();+ client.upsert_channel_message(&fresh, Some(&index)).await?;client.inner.indices().refresh(IndicesRefreshParts::Index(&[&index])).send().await?.error_for_status_code()?;- for (channel_id, sender_ids, expected) in [- (args.channel_id.clone(), vec![], 1),- (uuid::Uuid::now_v7().to_string(), vec![], 0),- (args.channel_id.clone(), vec!["quartzarchivebot".into()], 0),- (args.channel_id.clone(), vec![args.sender_id.clone()], 1),+ for (terms, mode, channel_id, sender_ids, expected, highlighted) in [+ (vec!["quartzarchivebot"], "exact", args.channel_id.clone(), vec![], Some(&args), None),+ (vec!["quartzarchivebot"], "exact", uuid::Uuid::now_v7().to_string(), vec![], None, None),+ (vec!["quartzarchivebot"], "exact", args.channel_id.clone(), vec!["quartzarchivebot".into()], None, None),+ (vec!["quartzarchivebot"], "exact", args.channel_id.clone(), vec![args.sender_id.clone()], Some(&args), None),+ (vec!["Nebula"], "partial", args.channel_id.clone(), vec![], Some(&native), Some("Nebula")),+ (vec!["Historical message"], "exact", args.channel_id.clone(), vec![], Some(&args), Some("Historical")),+ (vec!["newarchivebot"], "exact", args.channel_id.clone(), vec![], Some(&fresh), None),+ (vec!["newarchive", "Aurora"], "partial", args.channel_id.clone(), vec![], Some(&fresh), Some("Aurora")),+ (vec!["newarchivebot", "Nebula"], "exact", args.channel_id.clone(), vec![], None, None),] {- let builder = ChannelMessageQueryBuilder::new(vec!["quartzarchivebot".into()])+ let builder = ChannelMessageQueryBuilder::new(terms.into_iter().map(str::to_owned).collect())+ .match_type(mode).user_id("macro|[email protected]").ids_only(true).ids(vec![channel_id]).sender_ids(sender_ids);let mut query = builder.build_bool_query()?.build().to_json();@@ -81,12 +103,16 @@ async fn imported_author_matches_only_with_channel_access() {let response: serde_json::Value = client.inner.search(SearchParts::Index(&[&index])).body(json!({"query": query, "highlight": {"fields": {"content": {"type": "plain"}}}})).send().await?.error_for_status_code()?.json().await?;- assert_eq!(response["hits"]["total"]["value"], expected);- if expected == 1 {+ assert_eq!(response["hits"]["total"]["value"], usize::from(expected.is_some()));+ if let Some(expected) = expected {let hit = &response["hits"]["hits"][0];- assert_eq!(hit["_source"]["content"], args.content);- assert_eq!(hit["_source"]["message_id"], args.message_id);- assert_eq!(hit["_source"]["thread_id"], args.thread_id);+ assert_eq!(hit["_source"]["content"], expected.content);+ assert_eq!(hit["_source"]["message_id"], expected.message_id);+ assert_eq!(hit["_source"]["thread_id"], expected.thread_id);+ if let Some(word) = highlighted {+ let snippet = hit["highlight"]["content"][0].as_str().expect("body snippet");+ assert!(snippet.contains(&format!("<em>{word}</em>")), "{snippet}");+ }}}Ok(())diff --git a/infra/stacks/opensearch/helpers/README.md b/infra/stacks/opensearch/helpers/README.mdindex ee5a7daa1a..bac88054ad 100644--- a/infra/stacks/opensearch/helpers/README.md+++ b/infra/stacks/opensearch/helpers/README.md@@ -42,6 +42,22 @@ that can be swapped without a code deploy.All migration scripts default to `DRY_RUN=true`; pass `DRY_RUN=false` to apply.+## Repair missing channel message snippets++Deploy the search code that queries `imported_author` directly, then run+`DRY_RUN=false bun scripts/add_imported_author.ts` with the intended cluster's+`OPENSEARCH_*` configuration. Preview with `DRY_RUN=true`; `INDEX` optionally+targets a physical index instead of the `channels` alias.++This explicitly clears `imported_author.copy_to`. On OpenSearch 3.5, copying+the author into `content` makes body highlighting read the author source instead+of the message body, even for native messages with no imported author. The+mapping repair restores existing body snippets immediately; no reindex or+message rewrite is needed. The direct author query preserves author searches+for future imports. Running `create_indices.ts` alone does not clear an existing+`copy_to` parameter. After applying, verify a body search returns+`highlight.content` and an imported-author search still finds its message.+## Manual provisioningProvision indices and aliases manually using the canonical declarations indiff --git a/infra/stacks/opensearch/helpers/scripts/add_imported_author.test.ts b/infra/stacks/opensearch/helpers/scripts/add_imported_author.test.tsindex 8fca5e925f..24ac45414d 100644--- a/infra/stacks/opensearch/helpers/scripts/add_imported_author.test.ts+++ b/infra/stacks/opensearch/helpers/scripts/add_imported_author.test.ts@@ -41,6 +41,10 @@ function mappingServer(options: MappingServerOptions = {}): {}describe('addImportedAuthor', () => {+ test('explicitly removes copy_to from existing author mappings', () => {+ expect(IMPORTED_AUTHOR_MAPPING.copy_to).toEqual([]);+ });+test('adds only author mapping to the live alias and can be rerun', async () => {const { client, server, requests } = mappingServer();try {diff --git a/infra/stacks/opensearch/helpers/scripts/add_imported_author.ts b/infra/stacks/opensearch/helpers/scripts/add_imported_author.tsindex 09fa455fc2..4df10a6360 100644--- a/infra/stacks/opensearch/helpers/scripts/add_imported_author.ts+++ b/infra/stacks/opensearch/helpers/scripts/add_imported_author.ts@@ -5,12 +5,16 @@ import { CHANNELS_ALIAS, IS_DRY_RUN } from '../constants';export const IMPORTED_AUTHOR_MAPPING = {type: 'text' as const,analyzer: 'content_text',- copy_to: 'content',+ // Explicitly clear existing copy_to mappings; omitting it preserves them.+ // OpenSearch otherwise loads the author instead of the body for highlights.+ copy_to: [] as string[],};/*** Add the field to the live alias (including legacy physical `channels` indices).- * Run before deploying the writer, then backfill affected channels: mapping+ * Deploy direct imported_author search before running this on an existing index.+ * Clearing copy_to restores body highlights immediately, without a reindex.+ * If adding the field for the first time, backfill affected channels: mapping* updates do not index fields already present in old documents' _source.* No index creation, alias swap, or message-content rewrite is performed.* From helpers/: DRY_RUN=false bun scripts/add_imported_author.ts@@ -38,7 +42,7 @@ export async function addImportedAuthor(if (!response.body.acknowledged) {throw new Error(`Failed to add imported_author mapping to ${index}`);}- console.log(`${index}: imported_author mapped; backfill affected channels`);+ console.log(`${index}: imported_author mapped without copy_to`);}if (import.meta.main) {diff --git a/infra/stacks/opensearch/helpers/scripts/create_indices.test.ts b/infra/stacks/opensearch/helpers/scripts/create_indices.test.tsindex 6ed335b6c6..a91a6eecfa 100644--- a/infra/stacks/opensearch/helpers/scripts/create_indices.test.ts+++ b/infra/stacks/opensearch/helpers/scripts/create_indices.test.ts@@ -6,7 +6,7 @@ import {selectIndexSpecs,} from './create_indices';-test('channel fallback authors are searchable content, not sender identities', () => {+test('channel fallback authors preserve body highlights and sender identities', () => {const spec = INDEX_SPECS.find((spec) => spec.aliasName === 'channels');expect(spec?.body).toMatchObject({mappings: {@@ -15,7 +15,7 @@ test('channel fallback authors are searchable content, not sender identities', (imported_author: {type: 'text',analyzer: 'content_text',- copy_to: 'content',+ copy_to: [],},content: { type: 'text', analyzer: 'content_text' },sender_id: { type: 'keyword', index: true },diff --git a/infra/stacks/opensearch/helpers/scripts/create_indices.ts b/infra/stacks/opensearch/helpers/scripts/create_indices.tsindex 9879873c03..c41ab638b5 100644--- a/infra/stacks/opensearch/helpers/scripts/create_indices.ts+++ b/infra/stacks/opensearch/helpers/scripts/create_indices.ts@@ -425,7 +425,7 @@ const CHANNEL_BODY = {type: 'keyword',index: true,},- // copy_to indexes the name without modifying the message's _source.content.+ // Queried directly alongside content so message-body highlights stay intact.imported_author: IMPORTED_AUTHOR_MAPPING,mentions: {type: 'keyword',