Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(search): restore channel message snippets (#7829)

Eric Hayes24bd20fb801a
diff --git a/crates/opensearch_client/src/search/builder.rs b/crates/opensearch_client/src/search/builder.rs
index d20f42a148..82f978f623 100644
--- a/crates/opensearch_client/src/search/builder.rs
+++ b/crates/opensearch_client/src/search/builder.rs
@@ -98,6 +98,8 @@ pub trait SearchQueryConfig {
const TITLE_KEY: &'static str;
/// Content field
const CONTENT_KEY: &'static str = "content";
+ /// Fields searched for each content term; highlights still use `CONTENT_KEY`.
+ const CONTENT_FIELDS: &'static [&'static str] = &[Self::CONTENT_KEY];
/// The entity index for the search query
const ENTITY_INDEX: OpenSearchEntityType;
}
@@ -282,9 +284,9 @@ impl<T: SearchQueryConfig> SearchQueryBuilder<T> {
let terms: Cow<'_, [&str]> =
Cow::Owned(self.terms.iter().map(|t| t.as_str()).collect::<Vec<&str>>());
- let must_array = vec![generate_terms_must_query(
+ let must_array = vec![super::query::generate_terms_must_query_for_fields(
query_key,
- T::CONTENT_KEY,
+ T::CONTENT_FIELDS,
terms,
self.term_combine,
)];
diff --git a/crates/opensearch_client/src/search/channels.rs b/crates/opensearch_client/src/search/channels.rs
index 0f8bdf9eb1..9632a27b95 100644
--- a/crates/opensearch_client/src/search/channels.rs
+++ b/crates/opensearch_client/src/search/channels.rs
@@ -41,6 +41,9 @@ pub(crate) struct ChannelMessageSearchConfig;
impl SearchQueryConfig for ChannelMessageSearchConfig {
const USER_ID_KEY: Option<&'static str> = Some("sender_id");
const TITLE_KEY: &'static str = "name";
+ // Search authors directly: copying them into content hides body highlights
+ // in OpenSearch, including messages with no imported author.
+ const CONTENT_FIELDS: &'static [&'static str] = &["content", "imported_author"];
const ENTITY_INDEX: OpenSearchEntityType = OpenSearchEntityType::Channels;
}
diff --git a/crates/opensearch_client/src/search/channels/test.rs b/crates/opensearch_client/src/search/channels/test.rs
index d394f78ef6..04861b19d4 100644
--- a/crates/opensearch_client/src/search/channels/test.rs
+++ b/crates/opensearch_client/src/search/channels/test.rs
@@ -84,7 +84,10 @@ fn test_build_bool_query() -> anyhow::Result<()> {
"bool": {
"minimum_should_match": 1,
"should": [
- {"match_phrase": {"content": "test"}}
+ {"bool": {"minimum_should_match": 1, "should": [
+ {"match_phrase": {"content": "test"}},
+ {"match_phrase": {"imported_author": "test"}}
+ ]}}
]
}
}
@@ -114,8 +117,7 @@ fn test_build_bool_query() -> anyhow::Result<()> {
#[test]
fn test_build_bool_query_multi_term_ands_inside_opensearch() -> anyhow::Result<()> {
- // Two terms — each becomes its own `match_phrase` clause and they
- // combine with `must` so both must appear in the same message.
+ // Terms may match the body or author but must belong to the same message.
// Quoted phrases like "foo bar" arrive here as a single token and use
// the same `match_phrase` path.
let builder = ChannelMessageQueryBuilder::new(vec!["foo".to_string(), "bar baz".to_string()])
@@ -135,8 +137,14 @@ fn test_build_bool_query_multi_term_ands_inside_opensearch() -> anyhow::Result<(
{
"bool": {
"must": [
- { "match_phrase": { "content": "foo" } },
- { "match_phrase": { "content": "bar baz" } }
+ {"bool": {"minimum_should_match": 1, "should": [
+ {"match_phrase": {"content": "foo"}},
+ {"match_phrase": {"imported_author": "foo"}}
+ ]}},
+ {"bool": {"minimum_should_match": 1, "should": [
+ {"match_phrase": {"content": "bar baz"}},
+ {"match_phrase": {"imported_author": "bar baz"}}
+ ]}}
]
}
}
diff --git a/crates/opensearch_client/src/search/query.rs b/crates/opensearch_client/src/search/query.rs
index 2228963168..76f0e83ca2 100644
--- a/crates/opensearch_client/src/search/query.rs
+++ b/crates/opensearch_client/src/search/query.rs
@@ -86,17 +86,36 @@ pub(crate) fn generate_terms_must_query<'a>(
field: &'a str,
terms: impl Into<Cow<'a, [&'a str]>>,
combine: TermCombine,
+) -> QueryType<'a> {
+ generate_terms_must_query_for_fields(query_key, &[field], terms, combine)
+}
+
+/// Each term must match at least one field on the same document.
+pub(crate) fn generate_terms_must_query_for_fields<'a>(
+ query_key: QueryKey,
+ fields: &[&'a str],
+ terms: impl Into<Cow<'a, [&'a str]>>,
+ combine: TermCombine,
) -> QueryType<'a> {
let terms = terms.into();
let queries: Vec<_> = terms
.iter()
.map(|term| {
- create_query(CreateQueryParams {
- query_key,
- field,
- term,
- })
+ let mut alternatives = BoolQueryBuilder::new();
+ alternatives.minimum_should_match(1);
+ for field in fields {
+ let query = create_query(CreateQueryParams {
+ query_key,
+ field,
+ term,
+ });
+ if fields.len() == 1 {
+ return query;
+ }
+ alternatives.should(query);
+ }
+ alternatives.build().into()
})
.collect();
diff --git a/crates/opensearch_client/src/search/unified/test.rs b/crates/opensearch_client/src/search/unified/test.rs
index 4f1d5b15a1..929b1dff7f 100644
--- a/crates/opensearch_client/src/search/unified/test.rs
+++ b/crates/opensearch_client/src/search/unified/test.rs
@@ -810,9 +810,10 @@ fn test_build_unified_search_request_content() -> anyhow::Result<()> {
"minimum_should_match": 1,
"should": [
{
- "match_phrase": {
- "content": "test"
- }
+ "bool": {"minimum_should_match": 1, "should": [
+ {"match_phrase": {"content": "test"}},
+ {"match_phrase": {"imported_author": "test"}}
+ ]}
}
]
}
diff --git a/crates/opensearch_client/src/upsert/channel_message/test.rs b/crates/opensearch_client/src/upsert/channel_message/test.rs
index 6e2669d8aa..129ca53158 100644
--- a/crates/opensearch_client/src/upsert/channel_message/test.rs
+++ b/crates/opensearch_client/src/upsert/channel_message/test.rs
@@ -1,7 +1,7 @@
use super::*;
use serde_json::json;
-fn args() -> UpsertChannelMessageArgs {
+fn message_args() -> UpsertChannelMessageArgs {
let message_id = uuid::Uuid::now_v7().to_string();
UpsertChannelMessageArgs {
channel_id: uuid::Uuid::now_v7().to_string(),
@@ -20,7 +20,7 @@ fn args() -> UpsertChannelMessageArgs {
#[test]
fn author_is_text_not_content_or_sender() {
- let mut args = args();
+ let mut args = message_args();
for author in [Some("quartzarchivebot".to_owned()), None] {
args.imported_author = author.clone();
let value = serde_json::to_value(&args).unwrap();
@@ -32,14 +32,14 @@ fn author_is_text_not_content_or_sender() {
}
}
-/// Scratch-index integration of copy_to with the production access/sender query.
+/// Repair a legacy copy_to mapping and search both existing and new messages.
#[tokio::test]
#[ignore = "requires local OpenSearch at http://localhost:9200"]
-async fn imported_author_matches_only_with_channel_access() {
+async fn imported_author_search_preserves_body_highlights_and_channel_access() {
use crate::{OpensearchClient, search::channels::ChannelMessageQueryBuilder};
use opensearch::{
SearchParts,
- indices::{IndicesDeleteParts, IndicesRefreshParts},
+ indices::{IndicesDeleteParts, IndicesPutMappingParts, IndicesRefreshParts},
};
use opensearch_query_builder::ToOpenSearchJson;
@@ -60,17 +60,39 @@ async fn imported_author_matches_only_with_channel_access() {
})).await.unwrap();
let result: anyhow::Result<()> = async {
- let args = args();
+ let args = message_args();
client.upsert_channel_message(&args, Some(&index)).await?;
+ // This native message was indexed before the repair and has no author.
+ let mut native = message_args();
+ native.channel_id = args.channel_id.clone();
+ native.thread_id = args.message_id.clone();
+ native.imported_author = None;
+ native.content = "Nebula deployment checklist".into();
+ client.upsert_channel_message(&native, Some(&index)).await?;
+ client.inner.indices().put_mapping(IndicesPutMappingParts::Index(&[&index]))
+ .body(json!({"properties": {"imported_author": {"type": "text", "analyzer": "content_text", "copy_to": []}}}))
+ .send().await?.error_for_status_code()?;
+ // A post-repair import must match its author without copied content tokens.
+ let mut fresh = message_args();
+ fresh.channel_id = args.channel_id.clone();
+ fresh.imported_author = Some("newarchivebot".into());
+ fresh.content = "Aurora launch checklist".into();
+ client.upsert_channel_message(&fresh, Some(&index)).await?;
client.inner.indices().refresh(IndicesRefreshParts::Index(&[&index]))
.send().await?.error_for_status_code()?;
- for (channel_id, sender_ids, expected) in [
- (args.channel_id.clone(), vec![], 1),
- (uuid::Uuid::now_v7().to_string(), vec![], 0),
- (args.channel_id.clone(), vec!["quartzarchivebot".into()], 0),
- (args.channel_id.clone(), vec![args.sender_id.clone()], 1),
+ for (terms, mode, channel_id, sender_ids, expected, highlighted) in [
+ (vec!["quartzarchivebot"], "exact", args.channel_id.clone(), vec![], Some(&args), None),
+ (vec!["quartzarchivebot"], "exact", uuid::Uuid::now_v7().to_string(), vec![], None, None),
+ (vec!["quartzarchivebot"], "exact", args.channel_id.clone(), vec!["quartzarchivebot".into()], None, None),
+ (vec!["quartzarchivebot"], "exact", args.channel_id.clone(), vec![args.sender_id.clone()], Some(&args), None),
+ (vec!["Nebula"], "partial", args.channel_id.clone(), vec![], Some(&native), Some("Nebula")),
+ (vec!["Historical message"], "exact", args.channel_id.clone(), vec![], Some(&args), Some("Historical")),
+ (vec!["newarchivebot"], "exact", args.channel_id.clone(), vec![], Some(&fresh), None),
+ (vec!["newarchive", "Aurora"], "partial", args.channel_id.clone(), vec![], Some(&fresh), Some("Aurora")),
+ (vec!["newarchivebot", "Nebula"], "exact", args.channel_id.clone(), vec![], None, None),
] {
- let builder = ChannelMessageQueryBuilder::new(vec!["quartzarchivebot".into()])
+ let builder = ChannelMessageQueryBuilder::new(terms.into_iter().map(str::to_owned).collect())
+ .match_type(mode)
.user_id("macro|[email protected]")
.ids_only(true).ids(vec![channel_id]).sender_ids(sender_ids);
let mut query = builder.build_bool_query()?.build().to_json();
@@ -81,12 +103,16 @@ async fn imported_author_matches_only_with_channel_access() {
let response: serde_json::Value = client.inner.search(SearchParts::Index(&[&index]))
.body(json!({"query": query, "highlight": {"fields": {"content": {"type": "plain"}}}}))
.send().await?.error_for_status_code()?.json().await?;
- assert_eq!(response["hits"]["total"]["value"], expected);
- if expected == 1 {
+ assert_eq!(response["hits"]["total"]["value"], usize::from(expected.is_some()));
+ if let Some(expected) = expected {
let hit = &response["hits"]["hits"][0];
- assert_eq!(hit["_source"]["content"], args.content);
- assert_eq!(hit["_source"]["message_id"], args.message_id);
- assert_eq!(hit["_source"]["thread_id"], args.thread_id);
+ assert_eq!(hit["_source"]["content"], expected.content);
+ assert_eq!(hit["_source"]["message_id"], expected.message_id);
+ assert_eq!(hit["_source"]["thread_id"], expected.thread_id);
+ if let Some(word) = highlighted {
+ let snippet = hit["highlight"]["content"][0].as_str().expect("body snippet");
+ assert!(snippet.contains(&format!("<em>{word}</em>")), "{snippet}");
+ }
}
}
Ok(())
diff --git a/infra/stacks/opensearch/helpers/README.md b/infra/stacks/opensearch/helpers/README.md
index ee5a7daa1a..bac88054ad 100644
--- a/infra/stacks/opensearch/helpers/README.md
+++ b/infra/stacks/opensearch/helpers/README.md
@@ -42,6 +42,22 @@ that can be swapped without a code deploy.
All migration scripts default to `DRY_RUN=true`; pass `DRY_RUN=false` to apply.
+## Repair missing channel message snippets
+
+Deploy the search code that queries `imported_author` directly, then run
+`DRY_RUN=false bun scripts/add_imported_author.ts` with the intended cluster's
+`OPENSEARCH_*` configuration. Preview with `DRY_RUN=true`; `INDEX` optionally
+targets a physical index instead of the `channels` alias.
+
+This explicitly clears `imported_author.copy_to`. On OpenSearch 3.5, copying
+the author into `content` makes body highlighting read the author source instead
+of the message body, even for native messages with no imported author. The
+mapping repair restores existing body snippets immediately; no reindex or
+message rewrite is needed. The direct author query preserves author searches
+for future imports. Running `create_indices.ts` alone does not clear an existing
+`copy_to` parameter. After applying, verify a body search returns
+`highlight.content` and an imported-author search still finds its message.
+
## Manual provisioning
Provision indices and aliases manually using the canonical declarations in
diff --git a/infra/stacks/opensearch/helpers/scripts/add_imported_author.test.ts b/infra/stacks/opensearch/helpers/scripts/add_imported_author.test.ts
index 8fca5e925f..24ac45414d 100644
--- a/infra/stacks/opensearch/helpers/scripts/add_imported_author.test.ts
+++ b/infra/stacks/opensearch/helpers/scripts/add_imported_author.test.ts
@@ -41,6 +41,10 @@ function mappingServer(options: MappingServerOptions = {}): {
}
describe('addImportedAuthor', () => {
+ test('explicitly removes copy_to from existing author mappings', () => {
+ expect(IMPORTED_AUTHOR_MAPPING.copy_to).toEqual([]);
+ });
+
test('adds only author mapping to the live alias and can be rerun', async () => {
const { client, server, requests } = mappingServer();
try {
diff --git a/infra/stacks/opensearch/helpers/scripts/add_imported_author.ts b/infra/stacks/opensearch/helpers/scripts/add_imported_author.ts
index 09fa455fc2..4df10a6360 100644
--- a/infra/stacks/opensearch/helpers/scripts/add_imported_author.ts
+++ b/infra/stacks/opensearch/helpers/scripts/add_imported_author.ts
@@ -5,12 +5,16 @@ import { CHANNELS_ALIAS, IS_DRY_RUN } from '../constants';
export const IMPORTED_AUTHOR_MAPPING = {
type: 'text' as const,
analyzer: 'content_text',
- copy_to: 'content',
+ // Explicitly clear existing copy_to mappings; omitting it preserves them.
+ // OpenSearch otherwise loads the author instead of the body for highlights.
+ copy_to: [] as string[],
};
/**
* Add the field to the live alias (including legacy physical `channels` indices).
- * Run before deploying the writer, then backfill affected channels: mapping
+ * Deploy direct imported_author search before running this on an existing index.
+ * Clearing copy_to restores body highlights immediately, without a reindex.
+ * If adding the field for the first time, backfill affected channels: mapping
* updates do not index fields already present in old documents' _source.
* No index creation, alias swap, or message-content rewrite is performed.
* From helpers/: DRY_RUN=false bun scripts/add_imported_author.ts
@@ -38,7 +42,7 @@ export async function addImportedAuthor(
if (!response.body.acknowledged) {
throw new Error(`Failed to add imported_author mapping to ${index}`);
}
- console.log(`${index}: imported_author mapped; backfill affected channels`);
+ console.log(`${index}: imported_author mapped without copy_to`);
}
if (import.meta.main) {
diff --git a/infra/stacks/opensearch/helpers/scripts/create_indices.test.ts b/infra/stacks/opensearch/helpers/scripts/create_indices.test.ts
index 6ed335b6c6..a91a6eecfa 100644
--- a/infra/stacks/opensearch/helpers/scripts/create_indices.test.ts
+++ b/infra/stacks/opensearch/helpers/scripts/create_indices.test.ts
@@ -6,7 +6,7 @@ import {
selectIndexSpecs,
} from './create_indices';
-test('channel fallback authors are searchable content, not sender identities', () => {
+test('channel fallback authors preserve body highlights and sender identities', () => {
const spec = INDEX_SPECS.find((spec) => spec.aliasName === 'channels');
expect(spec?.body).toMatchObject({
mappings: {
@@ -15,7 +15,7 @@ test('channel fallback authors are searchable content, not sender identities', (
imported_author: {
type: 'text',
analyzer: 'content_text',
- copy_to: 'content',
+ copy_to: [],
},
content: { type: 'text', analyzer: 'content_text' },
sender_id: { type: 'keyword', index: true },
diff --git a/infra/stacks/opensearch/helpers/scripts/create_indices.ts b/infra/stacks/opensearch/helpers/scripts/create_indices.ts
index 9879873c03..c41ab638b5 100644
--- a/infra/stacks/opensearch/helpers/scripts/create_indices.ts
+++ b/infra/stacks/opensearch/helpers/scripts/create_indices.ts
@@ -425,7 +425,7 @@ const CHANNEL_BODY = {
type: 'keyword',
index: true,
},
- // copy_to indexes the name without modifying the message's _source.content.
+ // Queried directly alongside content so message-body highlights stay intact.
imported_author: IMPORTED_AUTHOR_MAPPING,
mentions: {
type: 'keyword',