Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

16.3 KB
Raw Blame
//! Compact materialized search projection for normalized cache records.
//!
//! Search documents are disposable derivatives of fully merged records. They
//! deliberately contain no GraphQL payloads, so text search never needs to
//! decode the normalized-record postcard blobs.

use crate::codec::encode_record;
use crate::value::{CacheNumber, CacheValue, EntityKey, Record};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::cmp::Ordering;
use std::collections::BTreeSet;
use thiserror::Error;

mod catalog;
mod changes;
mod ranking;
pub(crate) use catalog::SearchCatalogs;
pub(crate) use changes::{collect_search_changes, snapshot_search_fields};
pub(crate) use ranking::rank_documents;

/// Current compact projection profile used by Quick Access, Cmd-K and entity
/// mention pickers. Profile names are persisted and therefore versioned.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum SearchProfile {
    /// First Quick Access projection profile.
    QuickAccessV1,
}

impl SearchProfile {
    /// Stable value persisted in local cache storage.
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::QuickAccessV1 => "quick-access-v1",
        }
    }

    /// Finite bucket set used to fan out indexed empty-query browsing.
    pub const fn buckets(self) -> &'static [&'static str] {
        match self {
            Self::QuickAccessV1 => &[
                "channel",
                "dm",
                "person",
                "document",
                "task",
                "snippet",
                "skill",
                "note",
                "chat",
                "project",
                "email",
                "crm_company",
                "crm_contact",
                "initiative",
            ],
        }
    }
}

/// Exclusive browse cursor ordered by recency and then entity key.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SearchCursor {
    /// Timestamp of the last returned document.
    pub timestamp_ms: i64,
    /// Entity key of the last returned document.
    pub record_key: EntityKey<'static>,
}

/// One compact, transport-neutral search document.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SearchDocument {
    /// Projection profile which produced this row.
    pub profile: SearchProfile,
    /// Key used to fetch/project the full normalized record only after search.
    pub record_key: EntityKey<'static>,
    /// Quick Access bucket (document, note, task, channel, person, ...).
    pub bucket: String,
    /// Lower-cased, whitespace-normalized text used by fuzzy matching.
    pub search_text: String,
    /// Viewed-first recency for Soup entities; activity recency for other rows.
    pub timestamp_ms: i64,
    /// Compact hash of the fully merged source record.
    pub source_hash: String,
}

/// Bounded search request over cached contents.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SearchRequest {
    /// Versioned projection profile.
    pub profile: SearchProfile,
    /// Allowed Quick Access buckets. Empty means every bucket in the profile.
    #[serde(default)]
    pub buckets: Vec<String>,
    /// Fuzzy query. Empty queries use the indexed recent browse path.
    #[serde(default)]
    pub query: String,
    /// Exclusive continuation cursor for empty-query browsing.
    pub cursor: Option<SearchCursor>,
    /// Maximum result count.
    pub limit: usize,
    /// Wall-clock time used for deterministic freshness scoring.
    pub now_ms: i64,
}

/// Bounded search result. It describes cache contents, not corpus completeness.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SearchPage {
    /// Ranked compact documents.
    pub documents: Vec<SearchDocument>,
    /// Empty-query continuation cursor when more cached documents exist.
    pub next_cursor: Option<SearchCursor>,
}

/// Search request validation errors.
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum SearchError {
    /// Search result limits are deliberately bounded at the core boundary.
    #[error("search limit must be between 1 and {MAX_SEARCH_LIMIT}")]
    InvalidLimit,
    /// Bucket names cross storage boundaries and must remain compact.
    #[error("invalid search bucket")]
    InvalidBucket,
    /// Query text is bounded before loading/ranking the catalog.
    #[error("search query is too long")]
    QueryTooLong,
}

/// Revision of the derived Quick Access rows. Storage adapters rebuild existing
/// rows when this changes; normalized records and pending writes remain valid.
pub const QUICK_ACCESS_PROJECTION_VERSION: u32 = 4;

/// Normalized entity types used by the Quick Access projection. Storage rebuilds
/// restrict decoding to this set so unrelated payloads do not delay readiness.
pub const QUICK_ACCESS_TYPENAMES: &[&str] = &[
    "GraphqlSoupDocument",
    "GraphqlSoupChat",
    "GraphqlSoupProject",
    "GraphqlSoupEmailThread",
    "GraphqlSoupChannel",
    "GraphqlSoupCrmCompany",
    "GraphqlSoupCrmContact",
    "GraphqlSoupInitiative",
    "GraphqlUser",
    "User",
];

/// Maximum number of compact documents returned by one RPC.
pub const MAX_SEARCH_LIMIT: usize = 500;
/// Maximum accepted query length in bytes.
pub const MAX_SEARCH_QUERY_BYTES: usize = 512;
/// Restores the Quick Access preference for matching direct messages.
const DM_SEARCH_BOOST: f64 = 1.8;

/// Validates and canonicalizes a request's buckets.
pub fn validate_search_request(request: &SearchRequest) -> Result<Vec<String>, SearchError> {
    if request.limit == 0 || request.limit > MAX_SEARCH_LIMIT {
        return Err(SearchError::InvalidLimit);
    }
    if request.query.len() > MAX_SEARCH_QUERY_BYTES {
        return Err(SearchError::QueryTooLong);
    }
    let mut buckets = BTreeSet::new();
    for bucket in &request.buckets {
        if bucket.is_empty()
            || bucket.len() > 64
            || !bucket
                .bytes()
                .all(|byte| byte.is_ascii_lowercase() || byte == b'_')
        {
            return Err(SearchError::InvalidBucket);
        }
        buckets.insert(bucket.clone());
    }
    Ok(buckets.into_iter().collect())
}

/// Builds every current search projection for a fully merged record.
///
/// A missing result means the record is internal, deleted, hidden, or not part
/// of a current profile. Storage backends delete any old derived row in that
/// case, in the same transaction as the base-record upsert.
pub fn project_search_documents(key: &EntityKey<'static>, record: &Record) -> Vec<SearchDocument> {
    project_quick_access(key, record).into_iter().collect()
}

fn quick_access_fields<'a>(
    key: &'a EntityKey<'_>,
    record: &'a Record,
) -> Option<(&'a str, &'static str, &'static [&'static str])> {
    if key.is_root() || key.as_ref().starts_with("__meta:") || !is_present(record) {
        return None;
    }
    let typename = record
        .typename()
        .or_else(|| key.as_ref().split_once(':').map(|(name, _)| name))?;

    let (bucket, text_fields): (&str, &[&str]) = match typename {
        "GraphqlSoupDocument" => (document_bucket(record), &["documentName", "name"]),
        "GraphqlSoupChat" => ("chat", &["chatName", "name"]),
        "GraphqlSoupProject" => ("project", &["projectName", "name"]),
        "GraphqlSoupEmailThread" => (
            "email",
            &["emailName", "name", "senderName", "senderEmail", "senders"],
        ),
        "GraphqlSoupChannel" => (channel_bucket(record), &["channelName", "name"]),
        "GraphqlSoupCrmCompany" => ("crm_company", &["crmCompanyName", "name", "domains"]),
        "GraphqlSoupCrmContact" => ("crm_contact", &["crmContactName", "name", "email"]),
        "GraphqlSoupInitiative" => ("initiative", &["displayName"]),
        "GraphqlUser" | "User" => ("person", &["name", "email"]),
        _ => return None,
    };

    text_fields
        .iter()
        .any(|field| record.fields.contains_key(*field))
        .then_some((typename, bucket, text_fields))
}

fn project_quick_access(key: &EntityKey<'static>, record: &Record) -> Option<SearchDocument> {
    let (typename, bucket, text_fields) = quick_access_fields(key, record)?;
    let search_text = normalize_search_text(
        text_fields
            .iter()
            .filter_map(|field| record.fields.get(*field))
            .flat_map(search_value_text)
            .collect::<Vec<_>>()
            .join(" | ")
            .as_str(),
    );
    Some(SearchDocument {
        profile: SearchProfile::QuickAccessV1,
        record_key: key.clone(),
        bucket: bucket.to_owned(),
        search_text,
        timestamp_ms: quick_access_timestamp(typename, record),
        source_hash: source_hash(record),
    })
}

fn is_present(record: &Record) -> bool {
    !matches!(record.fields.get("deletedAt"), Some(value) if !matches!(value, CacheValue::Null))
        && !matches!(record.fields.get("hidden"), Some(CacheValue::Bool(true)))
}

fn document_bucket(record: &Record) -> &'static str {
    let subtype = record.fields.get("subType").and_then(value_typename);
    match subtype {
        Some(name) if name.contains("Task") => "task",
        Some(name) if name.contains("Snippet") => "snippet",
        Some(name) if name.contains("Skill") => "skill",
        _ if string_field(record, "fileType")
            .is_some_and(|value| value.eq_ignore_ascii_case("md")) =>
        {
            "note"
        }
        _ => "document",
    }
}

fn channel_bucket(record: &Record) -> &'static str {
    if string_field(record, "channelType").is_some_and(|value| {
        value.eq_ignore_ascii_case("direct_message") || value.eq_ignore_ascii_case("directmessage")
    }) {
        "dm"
    } else {
        "channel"
    }
}

fn value_typename(value: &CacheValue) -> Option<&str> {
    match value {
        CacheValue::Ref(key) => key.as_ref().split_once(':').map(|(name, _)| name),
        CacheValue::Object(fields) => match fields.get("__typename") {
            Some(CacheValue::String(value)) => Some(value),
            _ => None,
        },
        _ => None,
    }
}

fn string_field<'a>(record: &'a Record, field: &str) -> Option<&'a str> {
    match record.fields.get(field) {
        Some(CacheValue::String(value)) => Some(value),
        _ => None,
    }
}

fn search_value_text(value: &CacheValue) -> Vec<&str> {
    match value {
        CacheValue::String(value) => vec![value],
        CacheValue::List(values) => values.iter().flat_map(search_value_text).collect(),
        CacheValue::Object(fields) => fields
            .iter()
            .filter(|(key, _)| matches!(key.as_str(), "name" | "email" | "domain"))
            .flat_map(|(_, value)| search_value_text(value))
            .collect(),
        _ => Vec::new(),
    }
}

fn normalize_search_text(value: &str) -> String {
    value
        .split_whitespace()
        .collect::<Vec<_>>()
        .join(" ")
        .to_lowercase()
}

fn quick_access_timestamp(typename: &str, record: &Record) -> i64 {
    // Initiatives keep their timestamps in an embedded metadata object.
    if typename == "GraphqlSoupInitiative" {
        let Some(CacheValue::Object(metadata)) = record.fields.get("metadata") else {
            return 0;
        };
        return ["viewedAt", "updatedAt", "createdAt"]
            .into_iter()
            .find_map(|field| metadata.get(field).and_then(value_timestamp))
            .unwrap_or(0);
    }
    if typename == "GraphqlSoupCrmContact" {
        return ["lastInteraction", "updatedAt", "createdAt"]
            .into_iter()
            .find_map(|field| record.fields.get(field).and_then(value_timestamp))
            .unwrap_or(0);
    }
    // Match VIEWED_UPDATED and the menu before applying a browse limit. Using
    // max(viewedAt, updatedAt) would let newly updated, long-unopened records
    // displace recently viewed records into later pages.
    if matches!(
        typename,
        "GraphqlSoupDocument"
            | "GraphqlSoupChat"
            | "GraphqlSoupProject"
            | "GraphqlSoupChannel"
            | "GraphqlSoupCrmCompany"
    ) {
        return ["viewedAt", "updatedAt", "createdAt"]
            .into_iter()
            .find_map(|field| record.fields.get(field).and_then(value_timestamp))
            .unwrap_or(0);
    }

    [
        "viewedAt",
        "interactedAt",
        "sortTs",
        "lastInteraction",
        "updatedAt",
        "createdAt",
    ]
    .into_iter()
    .filter_map(|field| record.fields.get(field).and_then(value_timestamp))
    .max()
    .unwrap_or(0)
}

fn value_timestamp(value: &CacheValue) -> Option<i64> {
    match value {
        CacheValue::String(value) => parse_rfc3339_millis(value),
        CacheValue::Number(CacheNumber::PosInt(value)) => i64::try_from(*value).ok(),
        CacheValue::Number(CacheNumber::NegInt(value)) => Some(*value),
        CacheValue::Number(CacheNumber::Float(value)) if value.is_finite() => Some(*value as i64),
        _ => None,
    }
}

fn source_hash(record: &Record) -> String {
    let digest = Sha256::digest(encode_record(record));
    digest[..8]
        .iter()
        .map(|byte| format!("{byte:02x}"))
        .collect()
}

pub(crate) fn compare_record_keys(left: &EntityKey<'_>, right: &EntityKey<'_>) -> Ordering {
    let left = left.as_ref().split_once(':').unwrap_or((left.as_ref(), ""));
    let right = right
        .as_ref()
        .split_once(':')
        .unwrap_or((right.as_ref(), ""));
    left.0.cmp(right.0).then_with(|| left.1.cmp(right.1))
}

/// Sorts recent browse results deterministically.
pub fn compare_recent(left: &SearchDocument, right: &SearchDocument) -> Ordering {
    right
        .timestamp_ms
        .cmp(&left.timestamp_ms)
        .then_with(|| compare_record_keys(&left.record_key, &right.record_key))
}

/// Returns a fuzzy+freshness score, or `None` when every query token fails to
/// match as an ordered subsequence. Fuzzy relevance contributes 70%, freshness
/// contributes 30%, and matching direct messages receive the legacy 1.8x boost.
pub fn fuzzy_freshness_score(document: &SearchDocument, query: &str, now_ms: i64) -> Option<f64> {
    score_normalized_query(document, &normalize_search_text(query), now_ms)
}

fn score_normalized_query(document: &SearchDocument, normalized: &str, now_ms: i64) -> Option<f64> {
    if normalized.is_empty() {
        return Some(1.0);
    }
    let mut score = 0.0;
    let mut tokens = 0usize;
    // Commas separate tokens as in the server's channel name search. Left on a
    // token, a comma would only match a name with one after it.
    for token in normalized
        .split([' ', ','])
        .filter(|token| !token.is_empty())
    {
        tokens += 1;
        score += subsequence_score(&document.search_text, token)?;
    }
    if tokens == 0 {
        return None;
    }
    let fuzzy = score / tokens as f64;
    let age = now_ms.saturating_sub(document.timestamp_ms).max(0) as f64;
    let max_age = 30.0 * 24.0 * 60.0 * 60.0 * 1000.0;
    let freshness = if document.timestamp_ms <= 0 || age >= max_age {
        0.0
    } else {
        (-0.5 * age / max_age).exp()
    };
    let score = 0.7 * fuzzy + 0.3 * freshness;
    Some(if document.bucket == "dm" {
        score * DM_SEARCH_BOOST
    } else {
        score
    })
}

fn subsequence_score(haystack: &str, needle: &str) -> Option<f64> {
    if let Some(start) = haystack.find(needle) {
        let prefix = start == 0
            || haystack[..start]
                .chars()
                .next_back()
                .is_some_and(|character| !character.is_alphanumeric());
        return Some(if prefix { 1.0 } else { 0.9 });
    }
    let mut chars = needle.chars();
    let mut wanted = chars.next()?;
    let mut first = None;
    for (position, character) in haystack.chars().enumerate() {
        if character == wanted {
            first.get_or_insert(position);
            match chars.next() {
                Some(next) => wanted = next,
                None => {
                    let span = position.saturating_sub(first.unwrap_or(position)) + 1;
                    return Some((needle.chars().count() as f64 / span as f64).clamp(0.1, 0.8));
                }
            }
        }
    }
    None
}

fn parse_rfc3339_millis(value: &str) -> Option<i64> {
    chrono::DateTime::parse_from_rfc3339(value)
        .ok()
        .map(|timestamp| timestamp.timestamp_millis())
}

#[cfg(test)]
mod test;