Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

20.1 KB
Raw Blame
use std::collections::HashMap;

use crate::{
    Result, delegate_methods,
    search::{
        builder::{SearchQueryBuilder, SearchQueryConfig},
        model::{Highlight, SearchGotoContent, SearchGotoDocument, SearchHit, parse_highlight_hit},
        properties::{PROPERTIES_PATH, build_tag_filter},
        query::Keys,
    },
};

use chrono::{DateTime, Utc};
use models_opensearch::{OpenSearchEntityType, SearchEntityType};
use opensearch_query_builder::{
    BoolQueryBuilder, HasChildQuery, InnerHits, MatchPhrasePrefixQuery, MatchPhraseQuery,
    NestedQuery, QueryType, ToOpenSearchJson,
};

/// Relation names for the join field. Kept in sync with the upsert path
/// in `upsert::document`.
const PARENT_RELATION: &str = "document";
const CHILD_RELATION: &str = "chunk";

/// Minimum prefix length before we emit a `match_phrase_prefix`. Matches
/// the email keyword-field threshold — shorter prefixes explode the term
/// set on the analyzer and risk hitting `max_clause_count`.
const MIN_PREFIX_LEN: usize = 3;

/// Cap on chunks returned per `has_child` clause inside `inner_hits`.
/// Applied per term, so an N-term query can surface up to N times this many
/// chunks per document. Highlight cost and response size scale with it (and
/// with the term count), so it stays small. A grouped result row only previews
/// a handful of matching passages per document.
const INNER_HITS_PER_TERM: u32 = 10;

/// Cap on terms a `match_phrase_prefix` may expand the last word to.
/// OpenSearch's default of 50 is too aggressive — a prefix like `wo`
/// can expand to far more real tokens. Picking a fixed ceiling keeps
/// query cost bounded without truncating common cases.
const MATCH_PHRASE_PREFIX_MAX_EXPANSIONS: u32 = 256;

/// Which fields a document search matches against.
///
/// `Content` (the default) preserves the chunk-only `has_child` behavior.
/// `Name` matches the parent `document_name`. `NameContent` matches either.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum DocumentSearchMode {
    /// Match terms against the parent `document_name` only.
    Name,
    /// Match terms against child chunk `content` only.
    #[default]
    Content,
    /// Match a document by its name or its content.
    NameContent,
}

/// UTC bounds for an indexed DATE property.
#[derive(Debug, Clone, Copy)]
pub struct PropertyDateRange {
    /// Inclusive lower bound, when provided.
    pub start: Option<DateTime<Utc>>,
    /// Exclusive upper bound, when provided.
    pub end: Option<DateTime<Utc>>,
}

/// A property-equality filter applied against the indexed `properties` nested
/// field. Matches parents that have a nested entry whose `definition_id`
/// equals `definition_id` and whose `values` contains any of `values`.
/// Empty `values` imposes no membership constraint; date predicates still apply.
#[derive(Debug, Clone)]
pub struct PropertyFilterArg {
    /// The property definition id to match on.
    pub definition_id: String,
    /// Candidate values (OR'd). Select-option UUIDs and entity-ref ids both
    /// live in the indexed `values` keyword array.
    pub values: Vec<String>,
    /// UTC range with optional inclusive start and exclusive end.
    pub date_range: Option<PropertyDateRange>,
    /// True for a non-null DATE; false for absent, null, or cleared values.
    pub has_date: Option<bool>,
}

#[derive(Clone)]
pub(crate) struct DocumentSearchConfig;

impl SearchQueryConfig for DocumentSearchConfig {
    const USER_ID_KEY: Option<&'static str> = Some("owner_id");
    const TITLE_KEY: &'static str = "document_name";
    const ENTITY_INDEX: OpenSearchEntityType = OpenSearchEntityType::Documents;
}

pub(crate) struct DocumentQueryBuilder {
    inner: SearchQueryBuilder<DocumentSearchConfig>,
    sub_types: Vec<String>,
    mode: DocumentSearchMode,
    property_filters: Vec<PropertyFilterArg>,
    tag_option_ids: Vec<String>,
    match_all_tags: bool,
}

impl DocumentQueryBuilder {
    pub fn new(terms: Vec<String>) -> Self {
        Self {
            inner: SearchQueryBuilder::new(terms),
            sub_types: Vec::new(),
            mode: DocumentSearchMode::default(),
            property_filters: Vec::new(),
            tag_option_ids: Vec::new(),
            match_all_tags: false,
        }
    }

    pub fn mode(mut self, mode: DocumentSearchMode) -> Self {
        self.mode = mode;
        self
    }

    // Copy function signature from SearchQueryBuilder
    delegate_methods! {
        fn match_type(match_type: &str) -> Self;
        fn page(page: u32) -> Self;
        fn page_size(page_size: u32) -> Self;
        fn user_id(user_id: &str) -> Self;
        fn collapse(collapse: bool) -> Self;
        fn ids(ids: Vec<String>) -> Self;
        fn ids_only(ids_only: bool) -> Self;
    }

    pub fn sub_types(mut self, sub_types: Vec<String>) -> Self {
        self.sub_types = sub_types;
        self
    }

    pub fn property_filters(mut self, property_filters: Vec<PropertyFilterArg>) -> Self {
        self.property_filters = property_filters;
        self
    }

    pub fn tag_option_ids(mut self, tag_option_ids: Vec<String>) -> Self {
        self.tag_option_ids = tag_option_ids;
        self
    }

    pub fn match_all_tags(mut self, match_all_tags: bool) -> Self {
        self.match_all_tags = match_all_tags;
        self
    }

    /// Parent/child join query: one `has_child` clause per term, ANDed
    /// inside `bool.must`. Parent metadata filters (owner, ids,
    /// sub_type) sit on `bool.filter` directly because they live on
    /// the parent doc.
    pub fn build_bool_query<'a>(&'a self) -> Result<BoolQueryBuilder<'a>> {
        if self.inner.ids_only && self.inner.ids.is_empty() {
            return Err(crate::error::OpensearchClientError::EmptyIdsWithIdsOnly(
                DocumentSearchConfig::ENTITY_INDEX,
            ));
        }
        if self.inner.terms.is_empty() {
            return Err(crate::error::OpensearchClientError::NoTermsProvided);
        }

        let mut bool_query = BoolQueryBuilder::new();

        // Restrict to parent documents in the documents alias.
        bool_query.filter(QueryType::term(
            "_index",
            DocumentSearchConfig::ENTITY_INDEX.index_name().to_string(),
        ));
        bool_query.filter(QueryType::term(
            "document_relation",
            PARENT_RELATION.to_string(),
        ));

        // Access control: filter on parent fields (owner_id and/or entity_id).
        bool_query.filter(self.build_parent_filter()?);

        // Optional sub_type filter (parent field).
        if !self.sub_types.is_empty() {
            bool_query.filter(QueryType::terms("sub_type", self.sub_types.clone()));
        }

        // Property filters: one nested clause per filter, ANDed via bool.filter.
        // Each matches a `properties` entry whose definition_id and value line
        // up within the same nested object. On bool.filter, so they constrain
        // both name and content matches.
        for filter in &self.property_filters {
            if let Some(nested) = build_property_filter(filter) {
                bool_query.filter(nested);
            }
        }

        // Tag filter: nested clause(s) matching `properties.values` with no
        // definition_id constraint. Option ids are globally unique, so this
        // ORs (or ANDs, for match-all) tags across definitions.
        if let Some(nested) = build_tag_filter(&self.tag_option_ids, self.match_all_tags) {
            bool_query.filter(nested);
        }

        // Match clause(s) per mode: the parent `document_name` (Name), child
        // chunk `content` via has_child (Content), or either (NameContent).
        let include_name = matches!(
            self.mode,
            DocumentSearchMode::Name | DocumentSearchMode::NameContent
        );
        let include_content = matches!(
            self.mode,
            DocumentSearchMode::Content | DocumentSearchMode::NameContent
        );

        if include_name && include_content {
            // A document matches when its name matches every term, or every
            // term matches some chunk's content.
            let mut content_bool = BoolQueryBuilder::new();
            for clause in self.build_content_has_child_clauses() {
                content_bool.must(clause);
            }
            let mut matched = BoolQueryBuilder::new();
            matched.minimum_should_match(1);
            matched.should(content_bool.build().into());
            matched.should(self.inner.build_title_term_query()?);
            bool_query.must(matched.build().into());
        } else if include_name {
            bool_query.must(self.inner.build_title_term_query()?);
        } else {
            // Content-only: one has_child clause per term, ANDed via bool.must.
            for clause in self.build_content_has_child_clauses() {
                bool_query.must(clause);
            }
        }

        Ok(bool_query)
    }

    /// One `has_child` clause per term, each carrying its own `inner_hits` so
    /// highlights and chunk-nav data come back alongside the parent. `size` is
    /// bumped well above OpenSearch's default of 3 so a doc with many matching
    /// chunks returns all of them. The shared `highlight_query` ORs every term
    /// so a returned chunk gets tagged for every term it contains, not just the
    /// one its clause matched.
    fn build_content_has_child_clauses<'a>(&'a self) -> Vec<QueryType<'a>> {
        let highlight_query =
            build_all_terms_highlight_query(&self.inner.terms, &self.inner.match_type);
        self.inner
            .terms
            .iter()
            .enumerate()
            .map(|(idx, term)| {
                let inner_query = build_child_content_query(term, &self.inner.match_type);
                let inner_hits = InnerHits::new()
                    .name(format!("term_{idx}"))
                    .size(INNER_HITS_PER_TERM)
                    .highlight(inner_hits_content_highlight(&highlight_query));
                HasChildQuery::new(CHILD_RELATION, inner_query)
                    .inner_hits(inner_hits)
                    .into()
            })
            .collect()
    }

    /// Build the access-control filter using parent-side fields: either
    /// `entity_id ∈ ids` (ids_only), `owner_id` alone, or a should-bool of
    /// both when ids are provided alongside the owner.
    fn build_parent_filter<'a>(&'a self) -> Result<QueryType<'a>> {
        let owner_key =
            DocumentSearchConfig::USER_ID_KEY.expect("documents config has owner_id key");

        if self.inner.ids_only {
            return Ok(QueryType::terms("entity_id", self.inner.ids.clone()));
        }
        let owner_query = QueryType::term(owner_key.to_string(), self.inner.user_id.clone());
        if self.inner.ids.is_empty() {
            return Ok(owner_query);
        }
        let mut filter = BoolQueryBuilder::new();
        filter.minimum_should_match(1);
        filter.should(QueryType::terms("entity_id", self.inner.ids.clone()));
        filter.should(owner_query);
        Ok(filter.build().into())
    }
}

/// Build a `nested` query over `properties` for one property filter:
/// a parent matches when it has a nested entry with `definition_id` equal to
/// the filter's id AND `values` containing any of the filter's values.
/// Returns `None` when the filter carries no values.
fn build_property_filter<'a>(filter: &PropertyFilterArg) -> Option<QueryType<'a>> {
    if filter.values.is_empty() && filter.date_range.is_none() && filter.has_date.is_none() {
        return None;
    }
    let definition_field = format!("{PROPERTIES_PATH}.definition_id");
    let date_field = format!("{PROPERTIES_PATH}.date_value");
    let nested = |value_query: QueryType<'a>| {
        let mut inner = BoolQueryBuilder::new();
        inner.filter(QueryType::term(
            definition_field.clone(),
            filter.definition_id.clone(),
        ));
        inner.filter(value_query);
        NestedQuery::new(PROPERTIES_PATH, inner.build().into())
            .ignore_unmapped(true)
            .into()
    };
    if filter.date_range.is_none() && filter.has_date.is_none() {
        return Some(nested(QueryType::terms(
            format!("{PROPERTIES_PATH}.values"),
            filter.values.clone(),
        )));
    }
    let mut outer = BoolQueryBuilder::new();
    if !filter.values.is_empty() {
        outer.filter(nested(QueryType::terms(
            format!("{PROPERTIES_PATH}.values"),
            filter.values.clone(),
        )));
    }
    if let Some(bounds) = filter.date_range {
        let mut range = QueryType::range(date_field.clone());
        if let Some(start) = bounds.start {
            range.gte(start.timestamp_millis());
        }
        if let Some(end) = bounds.end {
            range.lt(end.timestamp_millis());
        }
        outer.filter(nested(range.build().into()));
    }
    match filter.has_date {
        Some(true) => outer.filter(nested(QueryType::Exists(date_field.into()))),
        Some(false) => outer.must_not(nested(QueryType::Exists(date_field.into()))),
        None => &mut outer,
    };
    Some(outer.build().into())
}

/// Highlight config attached to each `has_child` inner_hits block.
/// Matches the top-level documents highlight (plain highlighter,
/// `<macro_em>` tags, single fragment) so chunk hits come back with
/// the same shape downstream consumers already handle.
///
/// `highlight_query` lets each clause tag every search term in the
/// returned chunk content, not just the term its own has_child matched.
fn inner_hits_content_highlight(highlight_query: &serde_json::Value) -> serde_json::Value {
    serde_json::json!({
        "require_field_match": true,
        "max_analyzer_offset": super::HIGHLIGHT_MAX_ANALYZER_OFFSET,
        "pre_tags": ["<macro_em>"],
        "post_tags": ["</macro_em>"],
        "fields": {
            "content": {
                "type": "plain",
                "number_of_fragments": 1,
                "fragment_size": 1000,
                "highlight_query": highlight_query,
            }
        }
    })
}

/// Combined OR-of-all-terms query used as the inner_hits highlight_query.
/// Single-term queries return that one term's query directly; multi-term
/// builds a `bool.should` so the highlighter tags every search term it
/// finds in the chunk's content.
fn build_all_terms_highlight_query(terms: &[String], match_type: &str) -> serde_json::Value {
    let term_queries: Vec<serde_json::Value> = terms
        .iter()
        .map(|t| build_child_content_query(t, match_type).to_json())
        .collect();
    if term_queries.len() == 1 {
        return term_queries.into_iter().next().unwrap();
    }
    serde_json::json!({
        "bool": {
            "should": term_queries,
            "minimum_should_match": 1,
        }
    })
}

/// Build the per-term query that runs inside `has_child` against `content`.
///
/// - Quoted phrases (term contains whitespace) → `match_phrase` (exact).
/// - Short terms (< `MIN_PREFIX_LEN`) → `match_phrase` (no prefix expansion).
/// - `match_type` = "exact" → `match_phrase` always.
/// - Otherwise → `match_phrase_prefix`.
fn build_child_content_query<'a>(term: &str, match_type: &str) -> QueryType<'a> {
    let exact = match_type == "exact"
        || term.chars().any(|c| c.is_whitespace())
        || term.chars().count() < MIN_PREFIX_LEN;
    if exact {
        QueryType::MatchPhrase(MatchPhraseQuery::new(
            "content".to_string(),
            term.to_string(),
        ))
    } else {
        QueryType::MatchPhrasePrefix(
            MatchPhrasePrefixQuery::new("content".to_string(), term.to_string())
                .max_expansions(MATCH_PHRASE_PREFIX_MAX_EXPANSIONS),
        )
    }
}

#[derive(Debug, serde::Serialize, serde::Deserialize)]
pub(crate) struct DocumentIndex {
    pub entity_id: uuid::Uuid,
    pub document_name: String,
    pub owner_id: String,
    pub file_type: String,
    #[serde(default)]
    pub updated_at_millis: Option<i64>,
}

#[derive(Debug)]
pub struct DocumentSearchArgs {
    pub terms: Vec<String>,
    pub user_id: String,
    pub document_ids: Vec<String>,
    pub page: u32,
    pub page_size: u32,
    pub match_type: String,
    pub collapse: bool,
    pub ids_only: bool,
    pub sub_types: Vec<String>,
    pub mode: DocumentSearchMode,
    pub property_filters: Vec<PropertyFilterArg>,
    pub tag_option_ids: Vec<String>,
    pub match_all_tags: bool,
}

impl From<DocumentSearchArgs> for DocumentQueryBuilder {
    fn from(args: DocumentSearchArgs) -> Self {
        DocumentQueryBuilder::new(args.terms)
            .match_type(&args.match_type)
            .page_size(args.page_size)
            .page(args.page)
            .user_id(&args.user_id)
            .ids(args.document_ids)
            .collapse(args.collapse)
            .ids_only(args.ids_only)
            .sub_types(args.sub_types)
            .mode(args.mode)
            .property_filters(args.property_filters)
            .tag_option_ids(args.tag_option_ids)
            .match_all_tags(args.match_all_tags)
    }
}

// ---------------------------------------------------------------------------
// inner_hits → chunk-level SearchHits
// ---------------------------------------------------------------------------

/// One child chunk as it appears under `inner_hits.<term_name>.hits.hits[]`.
/// The fields we deserialize are exactly what we need to build a
/// `SearchHit` — anything else OpenSearch sends back is dropped.
#[derive(Debug, serde::Deserialize)]
struct ChunkInnerHit {
    #[serde(rename = "_id")]
    id: String,
    #[serde(rename = "_score")]
    score: Option<f64>,
    #[serde(rename = "_source")]
    source: ChunkSource,
    #[serde(default)]
    highlight: Option<HashMap<String, Vec<String>>>,
}

/// The chunk-only `_source` fields we surface in a `SearchHit.goto`.
#[derive(Debug, serde::Deserialize)]
struct ChunkSource {
    #[serde(default)]
    node_id: String,
    #[serde(default)]
    raw_content: Option<String>,
}

/// The shape of one entry under `inner_hits` — keyed by the
/// `has_child` clause name (e.g. `term_0`).
#[derive(Debug, serde::Deserialize)]
struct InnerHitsGroup {
    #[serde(default)]
    hits: InnerHitsList,
}

#[derive(Debug, Default, serde::Deserialize)]
struct InnerHitsList {
    #[serde(default)]
    hits: Vec<ChunkInnerHit>,
}

/// Walk the `inner_hits` block from a join-shape parent hit and emit
/// one `SearchHit` per matching chunk, carrying that chunk's
/// `node_id`, `raw_content`, score, and highlight.
///
/// A chunk matched by multiple `has_child` clauses (multi-term queries)
/// appears once per term in the response; we dedup by chunk `_id` so a
/// single chunk maps to a single `SearchHit` downstream.
///
/// Returns an empty vec if `inner_hits` is malformed or carries no
/// chunks — callers fall back to emitting a single parent hit so the
/// document still surfaces in results.
pub(crate) fn expand_inner_hits_to_search_hits(
    entity_id: uuid::Uuid,
    updated_at: Option<DateTime<Utc>>,
    inner_hits: &serde_json::Value,
) -> Vec<SearchHit> {
    let groups: HashMap<String, InnerHitsGroup> = match serde_json::from_value(inner_hits.clone()) {
        Ok(g) => g,
        Err(_) => return Vec::new(),
    };

    let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
    let mut out: Vec<SearchHit> = Vec::new();
    for group in groups.into_values() {
        for chunk in group.hits.hits {
            if !seen.insert(chunk.id) {
                continue;
            }
            let highlight: Highlight = chunk
                .highlight
                .map(|h| {
                    parse_highlight_hit(
                        h,
                        Keys {
                            title_key: DocumentSearchConfig::TITLE_KEY,
                            content_key: DocumentSearchConfig::CONTENT_KEY,
                        },
                    )
                })
                .unwrap_or_default();
            out.push(SearchHit {
                entity_id,
                entity_type: SearchEntityType::Documents,
                score: chunk.score,
                highlight,
                goto: Some(SearchGotoContent::Documents(SearchGotoDocument {
                    node_id: chunk.source.node_id,
                    raw_content: chunk.source.raw_content,
                })),
                updated_at,
            });
        }
    }
    out
}

#[cfg(test)]
mod test;