Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
#[cfg(test)]
mod test;
use std::{collections::HashMap, fmt::Display};
use chrono::{DateTime, Utc};
use models_opensearch::SearchEntityType;
use crate::search::query::Keys;
/// Injects `fragment_size` into every highlight field config in the query JSON.
/// The `opensearch_query_builder` crate doesn't support `fragment_size` natively,
/// so we patch the serialized JSON after building.
pub(crate) fn inject_fragment_size(query: &mut serde_json::Value, fragment_size: u32) {
if let Some(fields) = query
.pointer_mut("/highlight/fields")
.and_then(|v| v.as_object_mut())
{
for (_name, config) in fields.iter_mut() {
if let Some(obj) = config.as_object_mut() {
obj.insert(
"fragment_size".to_string(),
serde_json::json!(fragment_size),
);
}
}
}
}
/// Excludes `content` from `_source` in the query JSON. The content field is
/// only needed in highlights, not in the raw source — excluding it cuts ~40%
/// of the OpenSearch response payload.
pub(crate) fn exclude_source_content(query: &mut serde_json::Value) {
if let Some(obj) = query.as_object_mut() {
obj.insert(
"_source".to_string(),
serde_json::json!({"excludes": ["content"]}),
);
}
}
const MAX_VISIBLE_FRAGMENT_CHARS: usize = 1000;
const OPEN_TAG: &str = "<macro_em>";
const CLOSE_TAG: &str = "</macro_em>";
const CHARS_BEFORE_HIGHLIGHT: usize = 200;
/// Merges adjacent highlight spans separated only by `@` into a single span.
/// The `plain` highlighter wraps each matched token individually, so a hit on
/// `[email protected]` comes back as
/// `<macro_em>hutch</macro_em>@<macro_em>macro.com</macro_em>`. Merging across
/// `@` renders it as `<macro_em>[email protected]</macro_em>`, which reads as
/// one email address.
pub(crate) fn merge_highlights_across_at(fragment: &str) -> String {
fragment.replace(&format!("{CLOSE_TAG}@{OPEN_TAG}"), "@")
}
fn normalize_highlight_fragment(fragment: &str) -> String {
let stripped: String = fragment
.chars()
.filter(|&c| {
!matches!(c,
'\u{2800}'..='\u{28FF}' | '\u{200B}'..='\u{200F}' |
'\u{2028}'..='\u{202F}' | '\u{2060}'..='\u{206F}' |
'\u{FEFF}' | '\u{00AD}' | '\u{034F}'
)
})
.collect();
let mut prev_space = false;
let normalized: String = stripped
.chars()
.filter_map(|c| {
if c == '\n' || c == '\r' || c.is_whitespace() {
if prev_space {
return None;
}
prev_space = true;
Some(' ')
} else {
prev_space = false;
Some(c)
}
})
.collect();
let merged = merge_highlights_across_at(normalized.trim());
window_around_highlight(&merged, MAX_VISIBLE_FRAGMENT_CHARS)
}
/// Returns a window of `max_chars` visible characters around the first
/// `<macro_em>` highlight tag. If the highlight is near the start, the window
/// starts from the beginning. Otherwise, the front is trimmed (on a word
/// boundary) to keep the highlight visible. If no highlight tag is found,
/// truncates from the start.
fn window_around_highlight(s: &str, max_chars: usize) -> String {
let tag_byte_pos = match s.find(OPEN_TAG) {
Some(pos) => pos,
None => return truncate_preserving_tags(s, max_chars),
};
let visible_before_tag = s[..tag_byte_pos]
.replace(CLOSE_TAG, "")
.replace(OPEN_TAG, "")
.chars()
.count();
if visible_before_tag <= CHARS_BEFORE_HIGHLIGHT {
return truncate_preserving_tags(s, max_chars);
}
let target_visible_skip = visible_before_tag.saturating_sub(CHARS_BEFORE_HIGHLIGHT);
let prefix = &s[..tag_byte_pos];
let no_tags = prefix.replace(OPEN_TAG, "").replace(CLOSE_TAG, "");
let skip_byte_len: usize = no_tags
.chars()
.take(target_visible_skip)
.map(|c| c.len_utf8())
.sum();
let cut_point = find_tag_aware_byte_offset(prefix, skip_byte_len);
let after_cut = &s[cut_point..];
let word_break = after_cut
.find(|c: char| c.is_whitespace())
.map(|i| i + 1)
.unwrap_or(0);
let windowed = &after_cut[word_break..];
let truncated = truncate_preserving_tags(windowed, max_chars);
format!("...{truncated}")
}
/// Maps a visible-char byte length to the actual byte offset in text that may
/// contain `<macro_em>` / `</macro_em>` tags (skipping tag bytes).
fn find_tag_aware_byte_offset(s: &str, visible_byte_len: usize) -> usize {
let mut visible_bytes = 0;
let mut byte_offset = 0;
while byte_offset < s.len() {
if s[byte_offset..].starts_with(OPEN_TAG) {
byte_offset += OPEN_TAG.len();
continue;
}
if s[byte_offset..].starts_with(CLOSE_TAG) {
byte_offset += CLOSE_TAG.len();
continue;
}
if visible_bytes >= visible_byte_len {
break;
}
let c = s[byte_offset..].chars().next().unwrap();
visible_bytes += c.len_utf8();
byte_offset += c.len_utf8();
}
byte_offset
}
/// Truncates a highlight fragment to `max_chars` visible characters (excluding
/// `<macro_em>`/`</macro_em>` tags from the count). If truncation lands inside
/// an open tag, the closing tag is appended. Adds "..." when truncated.
fn truncate_preserving_tags(s: &str, max_chars: usize) -> String {
let mut result = String::new();
let mut visible_count = 0;
let mut inside_tag = false;
let mut chars = s.chars().peekable();
while let Some(&c) = chars.peek() {
if visible_count >= max_chars {
break;
}
if c == '<' {
let rest: String = chars.clone().collect();
if rest.starts_with(OPEN_TAG) {
for _ in 0..OPEN_TAG.len() {
result.push(chars.next().unwrap());
}
inside_tag = true;
continue;
} else if rest.starts_with(CLOSE_TAG) {
for _ in 0..CLOSE_TAG.len() {
result.push(chars.next().unwrap());
}
inside_tag = false;
continue;
}
}
result.push(c);
visible_count += 1;
chars.next();
}
if visible_count >= max_chars && chars.peek().is_some() {
if inside_tag {
result.push_str(CLOSE_TAG);
}
result.push_str("...");
}
result
}
/// macro open/close tags for highlight matches
#[derive(Debug, PartialEq)]
pub(crate) enum MacroEm {
/// Open tag <macro_em>
Open,
/// Close tag </macro_em>
Close,
}
impl Display for MacroEm {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Open => write!(f, "<macro_em>"),
Self::Close => write!(f, "</macro_em>"),
}
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
pub(crate) struct Hit<T> {
/// Physical index this hit came from, e.g. `documents_v2`. Kept for
/// diagnostics only — it is a deployment detail on a naming convention
/// nothing enforces, so it must not be used to identify the entity.
#[serde(rename = "_index")]
pub index: String,
/// Names of the query clauses this hit matched. The unified query names
/// each entity's clause after that entity, which is how a hit is resolved
/// to a type. Absent when the query named no clauses.
#[serde(default)]
pub matched_queries: Vec<String>,
#[serde(rename = "_score")]
pub score: Option<f64>,
#[serde(rename = "_source")]
pub source: T,
/// Highlights may or may not be present since we could match
/// purely on the title of the item
pub highlight: Option<HashMap<String, Vec<String>>>,
/// `inner_hits` come back attached to a parent hit when the query
/// used `has_child` (the documents join shape). Kept as raw JSON
/// because the structure is per-has_child-clause and only the
/// documents branch consumes it today.
#[serde(default)]
pub inner_hits: Option<serde_json::Value>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Default, Clone)]
pub struct Highlight {
/// The highlight name match if present
#[serde(skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
/// The highlight content matches
#[serde(skip_serializing_if = "Vec::is_empty")]
pub content: Vec<String>,
/// The highlight match on the user (owner) of the entity
#[serde(skip_serializing_if = "Option::is_none")]
pub user_id: Option<String>,
/// The highlight match on the sender (email only)
#[serde(skip_serializing_if = "Option::is_none")]
pub sender: Option<String>,
/// The highlight match on the recipients (email only)
#[serde(skip_serializing_if = "Vec::is_empty")]
pub recipients: Vec<String>,
/// The highlight match on the cc (email only)
#[serde(skip_serializing_if = "Vec::is_empty")]
pub cc: Vec<String>,
/// The highlight match on the bcc (email only)
#[serde(skip_serializing_if = "Vec::is_empty")]
pub bcc: Vec<String>,
}
pub(crate) fn parse_highlight_hit(
highlight: HashMap<String, Vec<String>>,
keys: Keys,
) -> Highlight {
// The highlight on the user id
let user_id = highlight
.get("user_id")
.and_then(|v| v.first())
.map(|v| v.to_string());
// If the user id is not present we should try the owner_id which is the
// user field in documents index
let user_id = if user_id.is_none() {
highlight
.get("owner_id")
.and_then(|v| v.first())
.map(|v| v.to_string())
} else {
user_id
};
Highlight {
user_id,
// A calendar event found only through another copy's title has its
// match under `source_names`, so that stands in for the name.
name: highlight
.get(keys.title_key)
.or_else(|| highlight.get("source_names"))
.and_then(|v| v.first())
.map(|v| v.to_string()),
content: highlight
.get(keys.content_key)
.map(|v| v.iter().map(|f| normalize_highlight_fragment(f)).collect())
.unwrap_or_default(),
sender: highlight
.get("sender_name")
.or_else(|| highlight.get("sender"))
.and_then(|v| v.first())
.map(|v| v.to_string()),
recipients: highlight
.get("recipient_names")
.or_else(|| highlight.get("recipients"))
.map(|v| v.to_vec())
.unwrap_or_default(),
cc: highlight
.get("cc_names")
.or_else(|| highlight.get("cc"))
.map(|v| v.to_vec())
.unwrap_or_default(),
bcc: highlight
.get("bcc_names")
.or_else(|| highlight.get("bcc"))
.map(|v| v.to_vec())
.unwrap_or_default(),
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
pub(crate) struct Total {
pub value: i64,
pub relation: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
pub(crate) struct Hits<T> {
pub total: Total,
pub max_score: Option<f64>,
pub hits: Vec<Hit<T>>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
pub(crate) struct Shards {
pub total: i32,
pub successful: i32,
pub skipped: i32,
pub failed: i32,
/// Per-shard failure details, present only when `failed > 0`.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub failures: Vec<serde_json::Value>,
}
pub(crate) type DefaultSearchResponse<T> = SearchResponse<T>;
#[derive(Debug, serde::Serialize, serde::Deserialize)]
pub(crate) struct SearchResponse<T> {
pub hits: Hits<T>,
pub took: i32,
pub timed_out: bool,
pub _shards: Shards,
}
impl<T> SearchResponse<T> {
/// OpenSearch returns HTTP 200 with partial results when individual
/// shards fail their query or fetch phase, so every hit on a failed
/// shard is silently absent unless callers check `_shards`.
pub(crate) fn warn_on_shard_failures(&self, operation: crate::error::OpensearchOperation) {
if self._shards.failed > 0 {
tracing::warn!(
%operation,
failed = self._shards.failed,
total = self._shards.total,
failures = ?self._shards.failures,
"opensearch returned partial results: some shards failed"
);
}
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct SearchGotoDocument {
/// The node id of the document.
/// This is either a 0-indexed page number for pdfs (and docx since they are pdfs)
/// or a uuid of a lexical node for md. This can be ignored for all other
/// file types.
pub node_id: String,
/// The raw content of the document
pub raw_content: Option<String>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct SearchGotoChat {
/// The chat message id
pub chat_message_id: uuid::Uuid,
/// The role of the chat message
pub role: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct SearchGotoEmail {
/// The email message id
pub email_message_id: uuid::Uuid,
/// The bcc of the email
pub bcc: Vec<String>,
/// The cc of the email
pub cc: Vec<String>,
/// The labels of the email
pub labels: Vec<String>,
/// The sent_at timestamp of the email
pub sent_at: Option<chrono::DateTime<chrono::Utc>>,
/// The sender of the email
pub sender: String,
/// The recipients of the email
pub recipients: Vec<String>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct SearchGotoChannel {
/// The channel message id
pub channel_message_id: uuid::Uuid,
pub thread_id: Option<uuid::Uuid>,
pub sender_id: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub updated_at: chrono::DateTime<chrono::Utc>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct SearchGotoCallRecord {
pub channel_id: Option<uuid::Uuid>,
pub transcript_id: uuid::Uuid,
pub speaker_id: String,
pub sequence_num: i32,
pub started_at: chrono::DateTime<chrono::Utc>,
pub ended_at: Option<chrono::DateTime<chrono::Utc>>,
pub participant_ids: Vec<String>,
}
/// Enum containing structs for all data needed to handle search "goto" in the frontend
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
#[serde(untagged)]
pub enum SearchGotoContent {
AgentSessions(SearchGotoAgentSession),
Documents(SearchGotoDocument),
Chats(SearchGotoChat),
Emails(SearchGotoEmail),
Channels(SearchGotoChannel),
CallRecords(SearchGotoCallRecord),
// there is no goto needed for projects
}
/// Identity of a user-visible folded message within a session.
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct SearchGotoAgentSession {
/// Fold-assigned turn number.
pub message_turn: u32,
/// Side of the conversation within that turn.
pub author: AgentSessionAuthor,
}
/// Author sides emitted by the ACP fold.
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum AgentSessionAuthor {
/// User-authored prompt.
User,
/// Agent-authored response.
Agent,
}
/// This struct represents a single search hit for a given entity
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct SearchHit {
/// The id of the entity
pub entity_id: uuid::Uuid,
/// The entity type
pub entity_type: SearchEntityType,
/// The score of the match
pub score: Option<f64>,
/// The highlight of the match
pub highlight: Highlight,
/// The goto content for the entity
pub goto: Option<SearchGotoContent>,
/// Timestamp for sorting across sources (None sorts to bottom)
#[serde(skip_serializing_if = "Option::is_none")]
pub updated_at: Option<DateTime<Utc>>,
}