Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(search): ignore bare punctuation in exact and partial search terms (#8691)

gbirman1c17557fc33f
diff --git a/crates/search_service/AGENTS.md b/crates/search_service/AGENTS.md
index 492eaa3e30..4229dbe694 100644
--- a/crates/search_service/AGENTS.md
+++ b/crates/search_service/AGENTS.md
@@ -50,6 +50,13 @@ per-entity-type search plus a unified search across them — mounted into
- `cargo test -p search_service` (`SQLX_OFFLINE` unset).
- `SQLX_OFFLINE=true cargo check` (`just check` recipe in `justfile`).
+## Gotchas
+
+- Every whitespace-split term is ANDed, so a term the standard tokenizer
+ analyzes to nothing (`-`, `@`) matches no document and empties the search.
+ `terms::drop_wordless_terms` removes such terms for exact and partial
+ queries; run new term handling through it.
+
## Related
- Rules: `.agents/rules/rust/architecture.md`
diff --git a/crates/search_service/src/api/search/channel.rs b/crates/search_service/src/api/search/channel.rs
index 877f645300..9711f7c3f6 100644
--- a/crates/search_service/src/api/search/channel.rs
+++ b/crates/search_service/src/api/search/channel.rs
@@ -1,5 +1,5 @@
use crate::api::search::simple::{SearchError, simple_channel};
-use crate::api::search::terms::split_search_terms;
+use crate::api::search::terms::{drop_wordless_terms, record_term_counts, split_search_terms};
use chrono::{DateTime, Utc};
use indexmap::IndexMap;
use std::collections::HashMap;
@@ -383,6 +383,15 @@ pub async fn name_handler(
(status = 500, body=model::response::ErrorResponse),
)
)]
+#[tracing::instrument(
+ skip_all,
+ fields(
+ search.match_type = %req.match_type,
+ search.terms = tracing::field::Empty,
+ search.dropped_terms = tracing::field::Empty,
+ ),
+ err
+)]
pub async fn handler(
State(ctx): State<SearchHandlerState>,
authorization: MacroAuthorizationExtractor<SearchAuthorizationService, UserOrInternal>,
@@ -420,6 +429,13 @@ pub async fn handler(
} else {
split_search_terms(&raw_terms)
};
+ let split_count = terms.len();
+ let terms = drop_wordless_terms(terms, req.match_type);
+ record_term_counts(
+ &tracing::Span::current(),
+ terms.len(),
+ split_count - terms.len(),
+ );
if terms.is_empty() {
return Err(SearchError::NoQueryOrTermsProvided);
}
diff --git a/crates/search_service/src/api/search/simple/simple_unified.rs b/crates/search_service/src/api/search/simple/simple_unified.rs
index a5a26dfb4a..43a6b70511 100644
--- a/crates/search_service/src/api/search/simple/simple_unified.rs
+++ b/crates/search_service/src/api/search/simple/simple_unified.rs
@@ -129,7 +129,7 @@ fn compute_next_cursor(
}
}
-use crate::api::search::terms::split_search_terms;
+use crate::api::search::terms::{drop_wordless_terms, record_term_counts, split_search_terms};
/// Per-request cap on whitespace-delimited search terms. Each term expands
/// into its own `has_child` clause and is embedded in every clause's highlight
@@ -186,7 +186,16 @@ fn to_property_filter_args(filters: &[item_filters::PropertyFilter]) -> Vec<Prop
/// Creates a unified search request and performs the search
/// by calling individual simple search endpoints for each entity type
-#[tracing::instrument(skip(ctx, user_context, query_params), fields(user_id = %user_context.user_id), err)]
+#[tracing::instrument(
+ skip(ctx, user_context, query_params),
+ fields(
+ user_id = %user_context.user_id,
+ search.match_type = %req.match_type,
+ search.terms = tracing::field::Empty,
+ search.dropped_terms = tracing::field::Empty,
+ ),
+ err
+)]
pub(in crate::api::search) async fn perform_unified_search(
ctx: &SearchHandlerState,
user_context: &UserContext,
@@ -235,9 +244,15 @@ pub(in crate::api::search) async fn perform_unified_search(
return Err(SearchError::InvalidQuerySize);
}
let terms: Vec<String> = vec![query];
- let search_terms = enforce_term_limits(split_search_terms(&terms))?;
-
let match_type = req.match_type;
+ let split_terms = split_search_terms(&terms);
+ let split_count = split_terms.len();
+ let search_terms = enforce_term_limits(drop_wordless_terms(split_terms, match_type))?;
+ record_term_counts(
+ &tracing::Span::current(),
+ search_terms.len(),
+ split_count - search_terms.len(),
+ );
// Property filters live at the top level of the request and only apply to
// the OpenSearch documents index, so capture them before the conversion
diff --git a/crates/search_service/src/api/search/terms.rs b/crates/search_service/src/api/search/terms.rs
index 6277cda77d..dfcd7cf50a 100644
--- a/crates/search_service/src/api/search/terms.rs
+++ b/crates/search_service/src/api/search/terms.rs
@@ -3,7 +3,10 @@
//! [`split_search_terms`] tokenizes user input by whitespace while preserving
//! double-quoted phrases so callers can build AND-of-terms queries where each
//! quoted span counts as a single phrase term. Used by both the unified
-//! dispatch and the channel-only endpoint.
+//! dispatch and the channel-only endpoint. [`drop_wordless_terms`] then removes
+//! the terms no indexed text can match.
+
+use models_search::MatchType;
/// Splits search terms by whitespace, preserving double-quoted phrases.
/// e.g. `["hello world"]` → `["hello", "world"]`
@@ -34,5 +37,36 @@ pub(in crate::api::search) fn split_search_terms(terms: &[String]) -> Vec<String
result
}
+/// Drops the terms the index can never match. The indexed text fields use the
+/// standard tokenizer, which turns a term with no letter or digit (`-`, `@`,
+/// `&`) into no tokens at all, and one such clause among the ANDed terms makes
+/// the whole query match nothing: `OOO - therapy` missed `OOO - therapy`.
+/// Regexp and query-string input keep every term, because punctuation is
+/// syntax there, and a query made only of such terms is left unchanged.
+pub(in crate::api::search) fn drop_wordless_terms(
+ terms: Vec<String>,
+ match_type: MatchType,
+) -> Vec<String> {
+ if matches!(match_type, MatchType::Regexp | MatchType::Query) {
+ return terms;
+ }
+ let has_word = |term: &String| term.chars().any(char::is_alphanumeric);
+ if !terms.iter().any(has_word) {
+ return terms;
+ }
+ terms.into_iter().filter(has_word).collect()
+}
+
+/// Records on `span` how many terms a search kept and how many
+/// [`drop_wordless_terms`] removed. The span declares both fields.
+pub(in crate::api::search) fn record_term_counts(
+ span: &tracing::Span,
+ kept: usize,
+ dropped: usize,
+) {
+ span.record("search.terms", kept);
+ span.record("search.dropped_terms", dropped);
+}
+
#[cfg(test)]
mod test;
diff --git a/crates/search_service/src/api/search/terms/test.rs b/crates/search_service/src/api/search/terms/test.rs
index cb877a2609..dbae640bc8 100644
--- a/crates/search_service/src/api/search/terms/test.rs
+++ b/crates/search_service/src/api/search/terms/test.rs
@@ -58,3 +58,55 @@ fn test_split_search_terms_leading_trailing_whitespace() {
vec!["hello", "world"]
);
}
+
+fn terms(input: &[&str]) -> Vec<String> {
+ input.iter().map(|term| term.to_string()).collect()
+}
+
+#[test]
+fn drop_wordless_terms_removes_bare_punctuation_from_word_queries() {
+ for match_type in [MatchType::Partial, MatchType::Exact] {
+ assert_eq!(
+ drop_wordless_terms(terms(&["OOO", "-", "therapy"]), match_type),
+ vec!["OOO", "therapy"]
+ );
+ assert_eq!(
+ drop_wordless_terms(terms(&["ux", "invite", "@", "Sat", "&", "|"]), match_type),
+ vec!["ux", "invite", "Sat"]
+ );
+ }
+}
+
+#[test]
+fn drop_wordless_terms_keeps_punctuation_inside_words() {
+ let input = terms(&[
+ "[jackson]",
+ "(Orillia)",
+ "fold-in",
+ "@gab",
+ "11:30am",
+ "café",
+ "日本",
+ ]);
+ assert_eq!(
+ drop_wordless_terms(input.clone(), MatchType::Partial),
+ input
+ );
+}
+
+#[test]
+fn drop_wordless_terms_keeps_regexp_and_query_syntax() {
+ for match_type in [MatchType::Regexp, MatchType::Query] {
+ let input = terms(&["foo", ".*", "-"]);
+ assert_eq!(drop_wordless_terms(input.clone(), match_type), input);
+ }
+}
+
+#[test]
+fn drop_wordless_terms_leaves_a_query_of_only_punctuation_unchanged() {
+ let input = terms(&["---", "@"]);
+ assert_eq!(
+ drop_wordless_terms(input.clone(), MatchType::Partial),
+ input
+ );
+}