Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(search): index millisecond timestamps in OpenSearch indices (#4891)
gbirman1bb68a99f76c
diff --git a/crates/opensearch_client/src/date_format.rs b/crates/opensearch_client/src/date_format.rsindex b9584e19a1..c6ae20893c 100644--- a/crates/opensearch_client/src/date_format.rs+++ b/crates/opensearch_client/src/date_format.rs@@ -3,6 +3,9 @@ use crate::{Result, error::OpensearchClientError};const MAX_REASONABLE_SECONDS: i64 = 32503680000; // Year ~3000const MIN_REASONABLE_SECONDS: i64 = 946684800; // Year 2000+const MAX_REASONABLE_MILLIS: i64 = MAX_REASONABLE_SECONDS * 1000;+const MIN_REASONABLE_MILLIS: i64 = MIN_REASONABLE_SECONDS * 1000;+#[derive(Debug, Clone, Copy, serde::Serialize)]pub struct EpochSeconds(i64);@@ -32,5 +35,36 @@ impl EpochSeconds {}}+/// A validated epoch-milliseconds timestamp, serialized as a raw `i64` for+/// OpenSearch date fields mapped with `format: epoch_millis`.+#[derive(Debug, Clone, Copy, serde::Serialize)]+pub struct EpochMillis(i64);++impl EpochMillis {+ pub fn new(millis: i64) -> Result<Self> {+ if millis > MAX_REASONABLE_MILLIS {+ return Err(OpensearchClientError::ValidationFailed {+ details: format!(+ "timestamp {} exceeds year 3000. Expected milliseconds since Unix epoch.",+ millis+ ),+ });+ }+ if millis < MIN_REASONABLE_MILLIS {+ return Err(OpensearchClientError::ValidationFailed {+ details: format!(+ "timestamp {} appears to be in seconds (before year 2000). Expected milliseconds since Unix epoch.",+ millis+ ),+ });+ }+ Ok(Self(millis))+ }++ pub fn get(&self) -> i64 {+ self.0+ }+}+#[cfg(test)]mod test;diff --git a/crates/opensearch_client/src/date_format/test.rs b/crates/opensearch_client/src/date_format/test.rsindex 3da7041019..e8df60369c 100644--- a/crates/opensearch_client/src/date_format/test.rs+++ b/crates/opensearch_client/src/date_format/test.rs@@ -34,3 +34,38 @@ fn test_epoch_seconds_serialize() {let serialized = serde_json::to_string(&epoch).unwrap();assert_eq!(serialized, "1704067200");}++#[test]+fn test_epoch_millis_valid() {+ let now_millis = 1704067200123; // 2024-01-01 with sub-second precision+ let epoch = EpochMillis::new(now_millis);+ assert!(epoch.is_ok());+ assert_eq!(epoch.unwrap().get(), now_millis);+}++#[test]+fn test_epoch_millis_seconds() {+ let now_seconds = 1704067200; // 2024-01-01 in seconds+ let result = EpochMillis::new(now_seconds);+ assert!(result.is_err());+ if let Err(OpensearchClientError::ValidationFailed { details }) = result {+ assert!(details.contains("appears to be in seconds"));+ }+}++#[test]+fn test_epoch_millis_too_far_future() {+ let too_far = 32503680000001; // Just after year 3000 in millis+ let result = EpochMillis::new(too_far);+ assert!(result.is_err());+ if let Err(OpensearchClientError::ValidationFailed { details }) = result {+ assert!(details.contains("exceeds year 3000"));+ }+}++#[test]+fn test_epoch_millis_serialize() {+ let epoch = EpochMillis::new(1704067200123).unwrap();+ let serialized = serde_json::to_string(&epoch).unwrap();+ assert_eq!(serialized, "1704067200123");+}diff --git a/crates/opensearch_client/src/search/builder.rs b/crates/opensearch_client/src/search/builder.rsindex eaa19fc07e..7c5cceb1bf 100644--- a/crates/opensearch_client/src/search/builder.rs+++ b/crates/opensearch_client/src/search/builder.rs@@ -47,21 +47,30 @@ macro_rules! delegate_methods {/// Sort mode for the channel content endpoint.#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]pub enum ChannelSortMode {- /// Sort by sent_at_seconds/updated_at_seconds, entity_id as tiebreaker.+ /// Sort by sent_at/updated_at (millis, falling back to seconds), entity_id+ /// as tiebreaker.#[default]Message,/// Sort by thread_id, message_id as tiebreaker.Thread,}-/// Creates sort vec to sort by sent_at_seconds (preferred) or updated_at_seconds (fallback)-/// with entity_id as a tiebreaker. Items without a timestamp are pushed to the end.+/// Creates sort vec to sort by sent_at (preferred) or updated_at (fallback)+/// with entity_id as a tiebreaker. Items without a timestamp are pushed to the+/// end. Prefers the millisecond-resolution `*_millis` fields and falls back to+/// the second-resolution `*_seconds` fields, so docs indexed before the+/// millis backfill still sort correctly. `.toEpochMilli()` normalizes both to+/// milliseconds either way.pub(crate) fn updated_at_sort<'a>() -> Vec<SortType<'a>> {vec![SortType::ScriptSort(ScriptSort::new(Script::new(- r#"if (doc.containsKey('sent_at_seconds') && doc['sent_at_seconds'].size() > 0) {+ r#"if (doc.containsKey('sent_at_millis') && doc['sent_at_millis'].size() > 0) {+ return doc['sent_at_millis'].value.toInstant().toEpochMilli();+ } else if (doc.containsKey('sent_at_seconds') && doc['sent_at_seconds'].size() > 0) {return doc['sent_at_seconds'].value.toInstant().toEpochMilli();+ } else if (doc.containsKey('updated_at_millis') && doc['updated_at_millis'].size() > 0) {+ return doc['updated_at_millis'].value.toInstant().toEpochMilli();} else if (doc.containsKey('updated_at_seconds') && doc['updated_at_seconds'].size() > 0) {return doc['updated_at_seconds'].value.toInstant().toEpochMilli();} else {diff --git a/crates/opensearch_client/src/search/call_records.rs b/crates/opensearch_client/src/search/call_records.rsindex 264925fa5e..58e46ed80d 100644--- a/crates/opensearch_client/src/search/call_records.rs+++ b/crates/opensearch_client/src/search/call_records.rs@@ -9,10 +9,10 @@ use crate::{},properties::build_tag_filter,query::Keys,+ utils::{millis_or_seconds, opt_millis_or_seconds},},};-use chrono::DateTime;use models_opensearch::{OpenSearchEntityType, SearchEntityType};use opensearch_query_builder::{BoolQueryBuilder, HasChildQuery, InnerHits, MatchPhrasePrefixQuery, MatchPhraseQuery,@@ -48,6 +48,10 @@ pub(crate) struct CallRecordIndex {pub started_at_seconds: i64,#[serde(default)]pub ended_at_seconds: Option<i64>,+ #[serde(default)]+ pub started_at_millis: Option<i64>,+ #[serde(default)]+ pub ended_at_millis: Option<i64>,}pub(crate) struct CallRecordSearchConfig;@@ -358,6 +362,10 @@ struct SegmentSource {started_at_seconds: i64,#[serde(default)]ended_at_seconds: Option<i64>,+ #[serde(default)]+ started_at_millis: Option<i64>,+ #[serde(default)]+ ended_at_millis: Option<i64>,}#[derive(Debug, serde::Deserialize)]@@ -415,15 +423,18 @@ pub(crate) fn expand_inner_hits_to_search_hits(transcript_id: seg.source.transcript_id,speaker_id: seg.source.speaker_id.unwrap_or_default(),sequence_num: seg.source.sequence_num.unwrap_or_default(),- started_at: DateTime::from_timestamp(seg.source.started_at_seconds, 0)- .unwrap_or_default(),- ended_at: seg- .source- .ended_at_seconds- .and_then(|s| DateTime::from_timestamp(s, 0)),+ started_at: millis_or_seconds(+ seg.source.started_at_millis,+ seg.source.started_at_seconds,+ )+ .unwrap_or_default(),+ ended_at: opt_millis_or_seconds(+ seg.source.ended_at_millis,+ seg.source.ended_at_seconds,+ ),participant_ids: parent.participant_ids.clone(),})),- updated_at: DateTime::from_timestamp(parent.started_at_seconds, 0),+ updated_at: millis_or_seconds(parent.started_at_millis, parent.started_at_seconds),});}}diff --git a/crates/opensearch_client/src/search/channels.rs b/crates/opensearch_client/src/search/channels.rsindex eb6f7ffd84..22b667c701 100644--- a/crates/opensearch_client/src/search/channels.rs+++ b/crates/opensearch_client/src/search/channels.rs@@ -10,10 +10,11 @@ use crate::{exclude_source_content, inject_fragment_size, parse_highlight_hit,},query::{Keys, TermCombine},+ utils::millis_or_seconds,},};-use chrono::{DateTime, Utc};+use chrono::Utc;use models_opensearch::{OpenSearchEntityType, SearchEntityType, SearchIndex};use models_search_cursor::{SearchCursorOption, SearchMethodCursor};use opensearch_query_builder::*;@@ -31,6 +32,10 @@ pub(crate) struct ChannelMessageIndex {pub mentions: Vec<String>,pub created_at_seconds: i64,pub updated_at_seconds: i64,+ #[serde(default)]+ pub created_at_millis: Option<i64>,+ #[serde(default)]+ pub updated_at_millis: Option<i64>,}#[derive(Default)]@@ -307,10 +312,12 @@ fn channel_hit_to_search_hit(hit: Hit<ChannelMessageIndex>) -> SearchHit {channel_message_id: a.message_id,thread_id: (a.thread_id != a.message_id).then_some(a.thread_id),sender_id: a.sender_id,- created_at: DateTime::from_timestamp(a.created_at_seconds, 0).unwrap_or_default(),- updated_at: DateTime::from_timestamp(a.updated_at_seconds, 0).unwrap_or_default(),+ created_at: millis_or_seconds(a.created_at_millis, a.created_at_seconds)+ .unwrap_or_default(),+ updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds)+ .unwrap_or_default(),})),- updated_at: DateTime::from_timestamp(a.updated_at_seconds, 0),+ updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),}}diff --git a/crates/opensearch_client/src/search/channels/test.rs b/crates/opensearch_client/src/search/channels/test.rsindex 558922086f..945b2895d5 100644--- a/crates/opensearch_client/src/search/channels/test.rs+++ b/crates/opensearch_client/src/search/channels/test.rs@@ -1,6 +1,62 @@use super::*;+use crate::search::model::Hit;use opensearch_query_builder::ToOpenSearchJson;+fn channel_index(updated_at_millis: Option<i64>) -> ChannelMessageIndex {+ let id = uuid::Uuid::new_v4();+ ChannelMessageIndex {+ entity_id: uuid::Uuid::new_v4(),+ channel_type: "team".to_string(),+ org_id: None,+ message_id: id,+ thread_id: id,+ sender_id: "macro|[email protected]".to_string(),+ mentions: vec![],+ created_at_seconds: 1_700_000_000,+ updated_at_seconds: 1_700_000_000,+ created_at_millis: updated_at_millis,+ updated_at_millis,+ }+}++fn hit_for(source: ChannelMessageIndex) -> Hit<ChannelMessageIndex> {+ Hit {+ score: Some(1.0),+ source,+ highlight: None,+ inner_hits: None,+ }+}++/// A millisecond timestamp is preserved at full precision, and two hits in the+/// same second but different milliseconds sort distinctly by `updated_at`.+#[test]+fn millis_timestamps_are_distinct_within_a_second() {+ let earlier = channel_hit_to_search_hit(hit_for(channel_index(Some(1_700_000_000_123))));+ let later = channel_hit_to_search_hit(hit_for(channel_index(Some(1_700_000_000_456))));++ let earlier_ts = earlier.updated_at.expect("updated_at");+ let later_ts = later.updated_at.expect("updated_at");++ assert_eq!(earlier_ts.timestamp_millis(), 1_700_000_000_123);+ assert_eq!(later_ts.timestamp_millis(), 1_700_000_000_456);+ assert!(+ later_ts > earlier_ts,+ "sub-second ordering must be preserved"+ );+}++/// Docs indexed before the millis backfill carry no `*_millis`; the read path+/// falls back to the second-resolution field.+#[test]+fn falls_back_to_seconds_when_millis_absent() {+ let hit = channel_hit_to_search_hit(hit_for(channel_index(None)));+ assert_eq!(+ hit.updated_at.expect("updated_at").timestamp_millis(),+ 1_700_000_000_000+ );+}+#[test]fn test_build_bool_query() -> anyhow::Result<()> {let builder = ChannelMessageQueryBuilder::new(vec!["test".to_string()])diff --git a/crates/opensearch_client/src/search/chats.rs b/crates/opensearch_client/src/search/chats.rsindex 887368c7cd..5ace04aa2e 100644--- a/crates/opensearch_client/src/search/chats.rs+++ b/crates/opensearch_client/src/search/chats.rs@@ -42,6 +42,8 @@ pub(crate) struct ChatIndex {pub user_id: String,pub title: String,pub updated_at_seconds: Option<i64>,+ #[serde(default)]+ pub updated_at_millis: Option<i64>,}pub(crate) struct ChatSearchConfig;diff --git a/crates/opensearch_client/src/search/documents.rs b/crates/opensearch_client/src/search/documents.rsindex 194682e4a7..8d14359144 100644--- a/crates/opensearch_client/src/search/documents.rs+++ b/crates/opensearch_client/src/search/documents.rs@@ -367,6 +367,8 @@ pub(crate) struct DocumentIndex {pub owner_id: String,pub file_type: String,pub updated_at_seconds: Option<i64>,+ #[serde(default)]+ pub updated_at_millis: Option<i64>,}#[derive(Debug)]diff --git a/crates/opensearch_client/src/search/emails.rs b/crates/opensearch_client/src/search/emails.rsindex 0ab3a1de10..eef19a7beb 100644--- a/crates/opensearch_client/src/search/emails.rs+++ b/crates/opensearch_client/src/search/emails.rs@@ -369,6 +369,9 @@ pub(crate) struct EmailIndex {pub subject: Option<String>,/// The sent at time of the email messagepub sent_at_seconds: Option<i64>,+ /// The sent at time of the email message, in milliseconds+ #[serde(default)]+ pub sent_at_millis: Option<i64>,}pub struct EmailSearchArgs {diff --git a/crates/opensearch_client/src/search/projects.rs b/crates/opensearch_client/src/search/projects.rsindex f49c8a326d..6c382c3d3e 100644--- a/crates/opensearch_client/src/search/projects.rs+++ b/crates/opensearch_client/src/search/projects.rs@@ -95,6 +95,8 @@ pub(crate) struct ProjectIndex {#[serde(default)]pub parent_project_id: Option<String>,pub updated_at_seconds: Option<i64>,+ #[serde(default)]+ pub updated_at_millis: Option<i64>,}#[derive(Debug)]diff --git a/crates/opensearch_client/src/search/unified.rs b/crates/opensearch_client/src/search/unified.rsindex c2330ed269..4fffc78cfa 100644--- a/crates/opensearch_client/src/search/unified.rs+++ b/crates/opensearch_client/src/search/unified.rs@@ -26,9 +26,10 @@ use crate::{},projects::{ProjectIndex, ProjectQueryBuilder, ProjectSearchArgs, ProjectSearchConfig},query::Keys,+ utils::{millis_or_seconds, opt_millis_or_seconds},},};-use chrono::{DateTime, Utc};+use chrono::Utc;use models_search_cursor::{SearchCursorOption, SearchMethodCursor};use tracing::Instrument;@@ -365,9 +366,8 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {match &hit.source {UnifiedSearchIndex::Document(parent) => {let entity_id = parent.entity_id;- let updated_at = parent- .updated_at_seconds- .and_then(|s| DateTime::from_timestamp(s, 0));+ let updated_at =+ opt_millis_or_seconds(parent.updated_at_millis, parent.updated_at_seconds);let mut out: Vec<SearchHit> = Vec::new();@@ -412,9 +412,8 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {return vec![hit.into()];};let entity_id = parent.entity_id;- let updated_at = parent- .updated_at_seconds- .and_then(|s| DateTime::from_timestamp(s, 0));+ let updated_at =+ opt_millis_or_seconds(parent.updated_at_millis, parent.updated_at_seconds);let expanded = crate::search::chats::expand_inner_hits_to_search_hits(entity_id, updated_at, inner,);@@ -424,7 +423,7 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {expanded}UnifiedSearchIndex::CallRecord(parent) => {- let updated_at = DateTime::from_timestamp(parent.started_at_seconds, 0);+ let updated_at = millis_or_seconds(parent.started_at_millis, parent.started_at_seconds);let mut out: Vec<SearchHit> = Vec::new();// A name match surfaces as a parent-level hit. Segment content lives@@ -452,9 +451,10 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {speaker_id: String::new(),sequence_num: 0,started_at: updated_at.unwrap_or_default(),- ended_at: parent- .ended_at_seconds- .and_then(|s| DateTime::from_timestamp(s, 0)),+ ended_at: opt_millis_or_seconds(+ parent.ended_at_millis,+ parent.ended_at_seconds,+ ),participant_ids: parent.participant_ids.clone(),})),updated_at,@@ -500,12 +500,12 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {channel_message_id: a.message_id,thread_id: (a.thread_id != a.message_id).then_some(a.thread_id),sender_id: a.sender_id,- created_at: DateTime::from_timestamp(a.created_at_seconds, 0)+ created_at: millis_or_seconds(a.created_at_millis, a.created_at_seconds).unwrap_or_default(),- updated_at: DateTime::from_timestamp(a.updated_at_seconds, 0)+ updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds).unwrap_or_default(),})),- updated_at: DateTime::from_timestamp(a.updated_at_seconds, 0),+ updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),},UnifiedSearchIndex::Document(a) => SearchHit {entity_id: a.entity_id,@@ -524,9 +524,7 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {}).unwrap_or_default(),goto: None,- updated_at: a- .updated_at_seconds- .and_then(|s| DateTime::from_timestamp(s, 0)),+ updated_at: opt_millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),},UnifiedSearchIndex::Email(a) => {let a = *a;@@ -551,15 +549,11 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {bcc: a.bcc,cc: a.cc,labels: a.labels,- sent_at: a- .sent_at_seconds- .and_then(|ts| DateTime::from_timestamp(ts, 0)),+ sent_at: opt_millis_or_seconds(a.sent_at_millis, a.sent_at_seconds),sender: a.sender,recipients: a.recipients,})),- updated_at: a- .sent_at_seconds- .and_then(|s| DateTime::from_timestamp(s, 0)),+ updated_at: opt_millis_or_seconds(a.sent_at_millis, a.sent_at_seconds),}}UnifiedSearchIndex::Chat(a) => SearchHit {@@ -579,9 +573,7 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {}).unwrap_or_default(),goto: None,- updated_at: a- .updated_at_seconds- .and_then(|s| DateTime::from_timestamp(s, 0)),+ updated_at: opt_millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),},UnifiedSearchIndex::Project(a) => SearchHit {entity_id: a.entity_id,@@ -600,9 +592,7 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {}).unwrap_or_default(),goto: None,- updated_at: a- .updated_at_seconds- .and_then(|s| DateTime::from_timestamp(s, 0)),+ updated_at: opt_millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),},UnifiedSearchIndex::CallRecord(a) => SearchHit {entity_id: a.entity_id,@@ -629,14 +619,12 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {transcript_id: uuid::Uuid::nil(),speaker_id: String::new(),sequence_num: 0,- started_at: DateTime::from_timestamp(a.started_at_seconds, 0)+ started_at: millis_or_seconds(a.started_at_millis, a.started_at_seconds).unwrap_or_default(),- ended_at: a- .ended_at_seconds- .and_then(|s| DateTime::from_timestamp(s, 0)),+ ended_at: opt_millis_or_seconds(a.ended_at_millis, a.ended_at_seconds),participant_ids: a.participant_ids,})),- updated_at: DateTime::from_timestamp(a.started_at_seconds, 0),+ updated_at: millis_or_seconds(a.started_at_millis, a.started_at_seconds),},}}diff --git a/crates/opensearch_client/src/search/unified/test.rs b/crates/opensearch_client/src/search/unified/test.rsindex c08178803b..312eb82ca0 100644--- a/crates/opensearch_client/src/search/unified/test.rs+++ b/crates/opensearch_client/src/search/unified/test.rs@@ -23,6 +23,7 @@ fn expand_document_name_highlight_yields_name_hit_without_goto() {owner_id: "alice".to_string(),file_type: "md".to_string(),updated_at_seconds: Some(1_779_000_000),+ updated_at_millis: None,}),highlight: Some(HashMap::from([("document_name".to_string(),@@ -928,6 +929,7 @@ fn doc_hit_with_chunks(owner_id: "alice".to_string(),file_type: "md".to_string(),updated_at_seconds: Some(updated_at_seconds),+ updated_at_millis: None,}),highlight: None,inner_hits: Some(serde_json::json!({ "term_0": { "hits": { "hits": chunks } } })),diff --git a/crates/opensearch_client/src/search/utils.rs b/crates/opensearch_client/src/search/utils.rsindex 20ee336954..7468ad4732 100644--- a/crates/opensearch_client/src/search/utils.rs+++ b/crates/opensearch_client/src/search/utils.rs@@ -1,5 +1,26 @@+use chrono::{DateTime, Utc};use opensearch_query_builder::{BoolQueryBuilder, QueryType, WildcardQuery};+/// Prefer a millisecond epoch timestamp, falling back to a second epoch+/// timestamp. During the seconds→millis migration new/backfilled docs carry a+/// `*_millis` field while older docs only have `*_seconds`.+pub(crate) fn millis_or_seconds(millis: Option<i64>, seconds: i64) -> Option<DateTime<Utc>> {+ millis+ .and_then(DateTime::from_timestamp_millis)+ .or_else(|| DateTime::from_timestamp(seconds, 0))+}++/// Like [`millis_or_seconds`] but both inputs are optional (e.g. email+/// `sent_at`, call `ended_at`).+pub(crate) fn opt_millis_or_seconds(+ millis: Option<i64>,+ seconds: Option<i64>,+) -> Option<DateTime<Utc>> {+ millis+ .and_then(DateTime::from_timestamp_millis)+ .or_else(|| seconds.and_then(|s| DateTime::from_timestamp(s, 0)))+}+pub fn should_wildcard_field_query_builder<'a>(field: &'a str,values: &'a [String],diff --git a/crates/opensearch_client/src/upsert/call_record.rs b/crates/opensearch_client/src/upsert/call_record.rsindex 17b040fc0f..59ec036f04 100644--- a/crates/opensearch_client/src/upsert/call_record.rs+++ b/crates/opensearch_client/src/upsert/call_record.rs@@ -4,7 +4,11 @@ use models_opensearch::SearchIndex;use super::BulkUpsertResult;use crate::upsert::properties::IndexedProperty;-use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};+use crate::{+ Result,+ date_format::{EpochMillis, EpochSeconds},+ error::OpensearchClientError,+};#[cfg(test)]mod test;@@ -34,6 +38,9 @@ pub struct UpsertCallRecordSegmentArgs {pub started_at_seconds: EpochSeconds,#[serde(skip_serializing_if = "Option::is_none")]pub ended_at_seconds: Option<EpochSeconds>,+ pub started_at_millis: EpochMillis,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub ended_at_millis: Option<EpochMillis>,/// Denormalized parent entity properties (tags, custom) used for search/// filtering. Shared across every segment of a call; empty for untagged/// calls. Only written onto the parent doc.@@ -79,6 +86,7 @@ fn parent_doc_body(any_segment: &UpsertCallRecordSegmentArgs) -> serde_json::Val"channel_id": &any_segment.channel_id,"participant_ids": &any_segment.participant_ids,"started_at_seconds": any_segment.started_at_seconds,+ "started_at_millis": any_segment.started_at_millis,"call_relation": PARENT_RELATION,});if let Some(channel_name) = &any_segment.channel_name {@@ -90,6 +98,9 @@ fn parent_doc_body(any_segment: &UpsertCallRecordSegmentArgs) -> serde_json::Valif let Some(ended) = &any_segment.ended_at_seconds {doc["ended_at_seconds"] = serde_json::to_value(ended).unwrap_or(serde_json::Value::Null);}+ if let Some(ended) = &any_segment.ended_at_millis {+ doc["ended_at_millis"] = serde_json::to_value(ended).unwrap_or(serde_json::Value::Null);+ }if !any_segment.properties.is_empty()&& let Ok(properties) = serde_json::to_value(&any_segment.properties){@@ -108,6 +119,8 @@ fn child_doc_body(seg: &UpsertCallRecordSegmentArgs) -> serde_json::Value {"content": &seg.content,"started_at_seconds": seg.started_at_seconds,"ended_at_seconds": &seg.ended_at_seconds,+ "started_at_millis": seg.started_at_millis,+ "ended_at_millis": &seg.ended_at_millis,"call_relation": {"name": CHILD_RELATION,"parent": &seg.call_id,diff --git a/crates/opensearch_client/src/upsert/call_record/test.rs b/crates/opensearch_client/src/upsert/call_record/test.rsindex 692fa084d2..31a1bd76db 100644--- a/crates/opensearch_client/src/upsert/call_record/test.rs+++ b/crates/opensearch_client/src/upsert/call_record/test.rs@@ -13,6 +13,8 @@ fn segment() -> UpsertCallRecordSegmentArgs {content: "segment content".to_string(),started_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),ended_at_seconds: Some(EpochSeconds::new(1_700_000_100).unwrap()),+ started_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),+ ended_at_millis: Some(EpochMillis::new(1_700_000_100_456).unwrap()),properties: vec![],}}@@ -25,6 +27,7 @@ fn parent_doc_body_has_metadata_and_name_no_child_fields() {assert_eq!(doc["channel_id"], "channel1");assert_eq!(doc["channel_name"], "Standup");assert_eq!(doc["name"], "Weekly standup");+ assert_eq!(doc["started_at_millis"], 1_700_000_000_123i64);assert_eq!(doc["call_relation"], "call");// Child-only fields must not be present on the parent.assert!(doc.get("content").is_none());diff --git a/crates/opensearch_client/src/upsert/channel_message.rs b/crates/opensearch_client/src/upsert/channel_message.rsindex 9f9ce1641e..fbe882891f 100644--- a/crates/opensearch_client/src/upsert/channel_message.rs+++ b/crates/opensearch_client/src/upsert/channel_message.rs@@ -1,6 +1,10 @@use models_opensearch::SearchIndex;-use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};+use crate::{+ Result,+ date_format::{EpochMillis, EpochSeconds},+ error::OpensearchClientError,+};/// The arguments for upserting a channel message into the opensearch index./// Threadless messages are indexed with `thread_id == message_id`.@@ -17,6 +21,8 @@ pub struct UpsertChannelMessageArgs {pub content: String,pub created_at_seconds: EpochSeconds,pub updated_at_seconds: EpochSeconds,+ pub created_at_millis: EpochMillis,+ pub updated_at_millis: EpochMillis,}#[tracing::instrument(skip(client))]diff --git a/crates/opensearch_client/src/upsert/chat_message.rs b/crates/opensearch_client/src/upsert/chat_message.rsindex 1c651fa54a..4b48aa0cfc 100644--- a/crates/opensearch_client/src/upsert/chat_message.rs+++ b/crates/opensearch_client/src/upsert/chat_message.rs@@ -1,7 +1,11 @@use models_opensearch::SearchIndex;use crate::upsert::properties::IndexedProperty;-use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};+use crate::{+ Result,+ date_format::{EpochMillis, EpochSeconds},+ error::OpensearchClientError,+};#[cfg(test)]mod test;@@ -28,6 +32,10 @@ pub struct UpsertChatMessageArgs {pub created_at_seconds: EpochSeconds,/// The updated at time of the chat messagepub updated_at_seconds: EpochSeconds,+ /// The created at time of the chat message, in milliseconds+ pub created_at_millis: EpochMillis,+ /// The updated at time of the chat message, in milliseconds+ pub updated_at_millis: EpochMillis,/// The title of the chat messagepub title: String,/// The content of the chat message@@ -51,6 +59,7 @@ fn parent_doc_body(args: &UpsertChatMessageArgs) -> serde_json::Value {"title": &args.title,"user_id": &args.user_id,"updated_at_seconds": args.updated_at_seconds,+ "updated_at_millis": args.updated_at_millis,"chat_relation": PARENT_RELATION,});if !args.properties.is_empty()@@ -70,6 +79,8 @@ fn child_doc_body(args: &UpsertChatMessageArgs) -> serde_json::Value {"role": &args.role,"created_at_seconds": args.created_at_seconds,"updated_at_seconds": args.updated_at_seconds,+ "created_at_millis": args.created_at_millis,+ "updated_at_millis": args.updated_at_millis,"chat_relation": {"name": CHILD_RELATION,"parent": &args.chat_id,diff --git a/crates/opensearch_client/src/upsert/chat_message/test.rs b/crates/opensearch_client/src/upsert/chat_message/test.rsindex 72120d35d5..55f735623a 100644--- a/crates/opensearch_client/src/upsert/chat_message/test.rs+++ b/crates/opensearch_client/src/upsert/chat_message/test.rs@@ -8,6 +8,8 @@ fn args() -> UpsertChatMessageArgs {role: "user".to_string(),created_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),updated_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),+ created_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),+ updated_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),title: "Chat title".to_string(),content: "message content".to_string(),properties: vec![],@@ -21,6 +23,7 @@ fn parent_doc_body_has_metadata_no_child_fields() {assert_eq!(doc["entity_id"], "chat1");assert_eq!(doc["title"], "Chat title");assert_eq!(doc["user_id"], "macro|[email protected]");+ assert_eq!(doc["updated_at_millis"], 1_700_000_000_123i64);assert_eq!(doc["chat_relation"], "chat");// Child-only fields must not be present on the parent.assert!(doc.get("content").is_none());diff --git a/crates/opensearch_client/src/upsert/document.rs b/crates/opensearch_client/src/upsert/document.rsindex 1c5e6eca41..5fa5598260 100644--- a/crates/opensearch_client/src/upsert/document.rs+++ b/crates/opensearch_client/src/upsert/document.rs@@ -4,7 +4,11 @@ use models_opensearch::SearchIndex;use super::BulkUpsertResult;use super::properties::IndexedProperty;-use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};+use crate::{+ Result,+ date_format::{EpochMillis, EpochSeconds},+ error::OpensearchClientError,+};/// Relation name for parent docs in the join field.const PARENT_RELATION: &str = "document";@@ -39,6 +43,8 @@ pub struct UpsertDocumentArgs {pub content: String,/// The updated at time of the documentpub updated_at_seconds: EpochSeconds,+ /// The updated at time of the document, in milliseconds+ pub updated_at_millis: EpochMillis,/// The sub type of the document (e.g. task)#[serde(skip_serializing_if = "Option::is_none")]pub sub_type: Option<String>,@@ -64,6 +70,7 @@ fn parent_doc_body(any_chunk: &UpsertDocumentArgs) -> serde_json::Value {"owner_id": &any_chunk.owner_id,"file_type": &any_chunk.file_type,"updated_at_seconds": any_chunk.updated_at_seconds,+ "updated_at_millis": any_chunk.updated_at_millis,"document_relation": PARENT_RELATION,});if let Some(sub_type) = &any_chunk.sub_type {diff --git a/crates/opensearch_client/src/upsert/document/test.rs b/crates/opensearch_client/src/upsert/document/test.rsindex d312e0d344..83d04543dc 100644--- a/crates/opensearch_client/src/upsert/document/test.rs+++ b/crates/opensearch_client/src/upsert/document/test.rs@@ -1,5 +1,5 @@use super::*;-use crate::date_format::EpochSeconds;+use crate::date_format::{EpochMillis, EpochSeconds};fn args(doc_id: &str, node_id: &str) -> UpsertDocumentArgs {UpsertDocumentArgs {@@ -11,6 +11,7 @@ fn args(doc_id: &str, node_id: &str) -> UpsertDocumentArgs {raw_content: None,content: format!("content {doc_id} {node_id}"),updated_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),+ updated_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),sub_type: None,properties: vec![],}@@ -24,6 +25,7 @@ fn parent_doc_body_has_metadata_no_chunk_fields() {assert_eq!(doc["document_name"], "name-doc1");assert_eq!(doc["owner_id"], "owner-doc1");assert_eq!(doc["file_type"], "md");+ assert_eq!(doc["updated_at_millis"], 1_700_000_000_123i64);assert_eq!(doc["document_relation"], "document");// Child-only fields must not be present on the parent.assert!(doc.get("content").is_none());diff --git a/crates/opensearch_client/src/upsert/email.rs b/crates/opensearch_client/src/upsert/email.rsindex 15db8f7531..1fc6c3ad95 100644--- a/crates/opensearch_client/src/upsert/email.rs+++ b/crates/opensearch_client/src/upsert/email.rs@@ -2,7 +2,11 @@ use models_opensearch::SearchIndex;use super::BulkUpsertResult;use super::properties::IndexedProperty;-use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};+use crate::{+ Result,+ date_format::{EpochMillis, EpochSeconds},+ error::OpensearchClientError,+};/// The arguments for upserting an email message into the opensearch index#[derive(Debug, serde::Serialize)]@@ -40,10 +44,14 @@ pub struct UpsertEmailArgs {pub user_id: String,/// The updated at time of the email messagepub updated_at_seconds: EpochSeconds,+ /// The updated at time of the email message, in milliseconds+ pub updated_at_millis: EpochMillis,/// The subject of the email messagepub subject: Option<String>,/// The sent at time of the email messagepub sent_at_seconds: Option<EpochSeconds>,+ /// The sent at time of the email message, in milliseconds+ pub sent_at_millis: Option<EpochMillis>,/// The content of the email messagepub content: String,/// Denormalized thread properties (e.g. tags) used for search filtering.diff --git a/crates/opensearch_client/src/upsert/project.rs b/crates/opensearch_client/src/upsert/project.rsindex e144ed8b7d..bec2e66b88 100644--- a/crates/opensearch_client/src/upsert/project.rs+++ b/crates/opensearch_client/src/upsert/project.rs@@ -1,7 +1,11 @@use models_opensearch::SearchIndex;use super::properties::IndexedProperty;-use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};+use crate::{+ Result,+ date_format::{EpochMillis, EpochSeconds},+ error::OpensearchClientError,+};/// The arguments for upserting a project into the opensearch index.///@@ -22,6 +26,10 @@ pub struct UpsertProjectArgs {pub created_at_seconds: EpochSeconds,/// The updated at time of the projectpub updated_at_seconds: EpochSeconds,+ /// The created at time of the project, in milliseconds+ pub created_at_millis: EpochMillis,+ /// The updated at time of the project, in milliseconds+ pub updated_at_millis: EpochMillis,/// Entity properties (tags, custom) used for search filtering.pub properties: Vec<IndexedProperty>,}diff --git a/infra/stacks/opensearch/helpers/scripts/copy_timestamp_seconds_to_millis.ts b/infra/stacks/opensearch/helpers/scripts/copy_timestamp_seconds_to_millis.tsnew file mode 100644index 0000000000..c0ab3bb65d--- /dev/null+++ b/infra/stacks/opensearch/helpers/scripts/copy_timestamp_seconds_to_millis.ts@@ -0,0 +1,217 @@+require('dotenv').config();++import type { Client } from '@opensearch-project/opensearch';+import { client } from '../client';+import {+ CALL_RECORDS_ALIAS,+ CHANNEL_INDEX,+ CHAT_INDEX,+ DOCUMENT_INDEX,+ EMAIL_INDEX,+ PROJECTS_ALIAS,+} from '../constants';+import { checkIndexExists } from '../utils/check_index_exists';+import { copyFieldData } from '../utils/copy_field';++// `<base>_seconds` → `<base>_millis` pairs to backfill. call_records only+// backfills the real started_at/ended_at fields; its created_at/updated_at+// are aliases onto started_at and need no separate data.+type FieldPair = { base: string };++interface IndexMigration {+ indexName: string;+ fields: FieldPair[];+}++const MIGRATIONS: IndexMigration[] = [+ {+ indexName: CHANNEL_INDEX,+ fields: [{ base: 'created_at' }, { base: 'updated_at' }],+ },+ {+ indexName: CHAT_INDEX,+ fields: [{ base: 'created_at' }, { base: 'updated_at' }],+ },+ {+ indexName: DOCUMENT_INDEX,+ fields: [{ base: 'updated_at' }],+ },+ {+ indexName: EMAIL_INDEX,+ fields: [{ base: 'updated_at' }, { base: 'sent_at' }],+ },+ {+ indexName: PROJECTS_ALIAS,+ fields: [{ base: 'created_at' }, { base: 'updated_at' }],+ },+ {+ indexName: CALL_RECORDS_ALIAS,+ fields: [{ base: 'started_at' }, { base: 'ended_at' }],+ },+];++/**+ * Sample docs that have both fields and confirm `millis` falls within the same+ * second as `seconds` (i.e. `floor(millis / 1000) === seconds`). This holds for+ * both backfilled docs (millis === seconds * 1000) and dual-written docs that+ * carry genuine sub-second precision (millis === seconds * 1000 + remainder).+ */+async function verifyMillis(+ opensearchClient: Client,+ indexName: string,+ secondsField: string,+ millisField: string+): Promise<void> {+ const response = await opensearchClient.search({+ index: indexName,+ body: {+ size: 50,+ query: {+ bool: {+ must: [+ { exists: { field: secondsField } },+ { exists: { field: millisField } },+ ],+ },+ },+ _source: [secondsField, millisField],+ },+ });++ const hits = response.body.hits.hits;+ if (hits.length === 0) {+ console.log(+ ` ⚠️ No documents with both "${secondsField}" and "${millisField}" to verify`+ );+ return;+ }++ let mismatches = 0;+ for (const hit of hits) {+ const seconds = hit._source?.[secondsField];+ const millis = hit._source?.[millisField];+ if (+ typeof seconds !== 'number' ||+ typeof millis !== 'number' ||+ Math.floor(millis / 1000) !== seconds+ ) {+ mismatches += 1;+ console.log(+ ` ⚠️ Mismatch (ID ${hit._id}): ${secondsField}=${seconds} ${millisField}=${millis}`+ );+ }+ }+ if (mismatches === 0) {+ console.log(+ ` ✓ Verified ${hits.length} sample docs: floor(${millisField} / 1000) === ${secondsField}`+ );+ } else {+ throw new Error(+ `${mismatches}/${hits.length} sample docs failed ${millisField} verification`+ );+ }+}++async function backfillIndex(+ opensearchClient: Client,+ migration: IndexMigration,+ dryRun: boolean+): Promise<void> {+ console.log(`\n${'='.repeat(60)}`);+ console.log(+ `Backfilling millis for index: ${migration.indexName} ${dryRun ? '(DRY-RUN)' : ''}`+ );+ console.log(`${'='.repeat(60)}`);++ const indexExists = await checkIndexExists(+ opensearchClient,+ migration.indexName+ );+ if (!indexExists) {+ console.log(+ `⚠️ Index "${migration.indexName}" does not exist. Skipping...`+ );+ return;+ }++ for (const { base } of migration.fields) {+ const secondsField = `${base}_seconds`;+ const millisField = `${base}_millis`;+ console.log(`\nProcessing ${secondsField} → ${millisField}`);++ await copyFieldData(+ opensearchClient,+ migration.indexName,+ secondsField,+ millisField,+ dryRun,+ false, // only where millis is missing — idempotent, safe to re-run+ `ctx._source.${secondsField} * 1000L`+ );++ if (!dryRun) {+ await verifyMillis(+ opensearchClient,+ migration.indexName,+ secondsField,+ millisField+ );+ }+ }++ console.log(+ `\n✓ Completed millis backfill for index: ${migration.indexName}`+ );+}++async function backfill(dryRun: boolean = true) {+ const opensearchClient = client();++ console.log('\n' + '='.repeat(60));+ console.log(+ `Copy Timestamp Seconds → Millis ${dryRun ? '(DRY-RUN MODE)' : '(LIVE MODE)'}`+ );+ console.log('='.repeat(60));+ console.log(+ '\nThis script derives *_millis fields from *_seconds (× 1000) for existing docs.'+ );+ console.log('It only updates docs where the *_millis field is missing.');+ console.log("\n💡 Safe to run multiple times — it's idempotent.");+ console.log(+ '⚠️ Run create_timestamp_millis_fields.ts first so the mappings exist.'+ );++ if (dryRun) {+ console.log('\n⚠️ DRY-RUN MODE: No changes will be made to the cluster');+ } else {+ console.log('\n🚨 LIVE MODE: Data will be backfilled');+ }++ try {+ for (const migration of MIGRATIONS) {+ await backfillIndex(opensearchClient, migration, dryRun);+ }++ console.log('\n' + '='.repeat(60));+ console.log('Backfill completed successfully!');+ console.log('='.repeat(60));++ if (dryRun) {+ console.log(+ '\nTo run for real, set DRY_RUN=false environment variable\n'+ );+ } else {+ console.log('\n✓ All *_millis fields backfilled from *_seconds.');+ console.log(+ '💡 Re-run after deploying the dual-write code to catch docs written mid-migration.\n'+ );+ }+ } catch (error) {+ console.error('\n❌ Backfill failed:', error);+ throw error;+ }+}++const isDryRun = process.env.DRY_RUN !== 'false';++backfill(isDryRun);diff --git a/infra/stacks/opensearch/helpers/scripts/create_indices.ts b/infra/stacks/opensearch/helpers/scripts/create_indices.tsindex 2f84743a23..8e3e1e6a60 100644--- a/infra/stacks/opensearch/helpers/scripts/create_indices.ts+++ b/infra/stacks/opensearch/helpers/scripts/create_indices.ts@@ -247,6 +247,18 @@ const CHANNEL_BODY = {index: false,doc_values: true,},+ created_at_millis: {+ type: 'date',+ format: 'epoch_millis',+ index: false,+ doc_values: true,+ },+ updated_at_millis: {+ type: 'date',+ format: 'epoch_millis',+ index: false,+ doc_values: true,+ },},},};@@ -298,6 +310,12 @@ const DOCUMENT_BODY = {index: false,doc_values: true,},+ updated_at_millis: {+ type: 'date',+ format: 'epoch_millis',+ index: false,+ doc_values: true,+ },// Parent-only entity properties (status, priority, assignees, custom).// `nested` so a property filter matches definition_id + its value// within the same entry rather than cross-matching across properties.@@ -379,6 +397,18 @@ const PROJECTS_BODY = {index: false,doc_values: true,},+ created_at_millis: {+ type: 'date',+ format: 'epoch_millis',+ index: false,+ doc_values: true,+ },+ updated_at_millis: {+ type: 'date',+ format: 'epoch_millis',+ index: false,+ doc_values: true,+ },// Entity properties (tags, custom). Same nested shape as the documents// index so the shared property/tag query builders apply unchanged.properties: {@@ -472,6 +502,12 @@ const EMAIL_BODY = {index: false,doc_values: true,},+ updated_at_millis: {+ type: 'date',+ format: 'epoch_millis',+ index: false,+ doc_values: true,+ },subject: {type: 'text',fields: {@@ -487,6 +523,12 @@ const EMAIL_BODY = {index: false,doc_values: true,},+ sent_at_millis: {+ type: 'date',+ format: 'epoch_millis',+ index: false,+ doc_values: true,+ },content: {type: 'text',analyzer: 'standard',@@ -535,6 +577,12 @@ const CHATS_V2_BODY = {index: false,doc_values: true,},+ updated_at_millis: {+ type: 'date',+ format: 'epoch_millis',+ index: false,+ doc_values: true,+ },// Parent-only entity properties (tags, custom). Same nested shape as// DOCUMENT_BODY so property filters match definition_id + value within// the same entry rather than cross-matching across properties.@@ -557,6 +605,12 @@ const CHATS_V2_BODY = {index: false,doc_values: true,},+ created_at_millis: {+ type: 'date',+ format: 'epoch_millis',+ index: false,+ doc_values: true,+ },// Join relationshipchat_relation: {type: 'join',@@ -604,6 +658,18 @@ const CALL_RECORDS_V2_BODY = {index: false,doc_values: true,},+ started_at_millis: {+ type: 'date',+ format: 'epoch_millis',+ index: false,+ doc_values: true,+ },+ ended_at_millis: {+ type: 'date',+ format: 'epoch_millis',+ index: false,+ doc_values: true,+ },// Parent-only entity properties (tags, custom). Same nested shape as// DOCUMENT_BODY so property filters match definition_id + value within// the same entry rather than cross-matching across properties.@@ -625,6 +691,8 @@ const CALL_RECORDS_V2_BODY = {// to the parent's call-start timestamp.created_at_seconds: { type: 'alias', path: 'started_at_seconds' },updated_at_seconds: { type: 'alias', path: 'started_at_seconds' },+ created_at_millis: { type: 'alias', path: 'started_at_millis' },+ updated_at_millis: { type: 'alias', path: 'started_at_millis' },// Join relationshipcall_relation: {type: 'join',diff --git a/infra/stacks/opensearch/helpers/scripts/create_timestamp_millis_fields.ts b/infra/stacks/opensearch/helpers/scripts/create_timestamp_millis_fields.tsnew file mode 100644index 0000000000..2ae9a04ed9--- /dev/null+++ b/infra/stacks/opensearch/helpers/scripts/create_timestamp_millis_fields.ts@@ -0,0 +1,201 @@+require('dotenv').config();++import type { Client } from '@opensearch-project/opensearch';+import { client } from '../client';+import {+ CALL_RECORDS_ALIAS,+ CHANNEL_INDEX,+ CHAT_INDEX,+ DOCUMENT_INDEX,+ EMAIL_INDEX,+ PROJECTS_ALIAS,+} from '../constants';+import { checkIndexExists } from '../utils/check_index_exists';++// A real `epoch_millis` date field to add.+type MillisField = { fieldName: string };+// An `alias` field pointing at an already-present real field. Used only by+// call_records, mirroring its existing created_at_seconds/updated_at_seconds+// aliases onto started_at_seconds.+type AliasField = { fieldName: string; aliasFor: string };++interface IndexMigration {+ indexName: string;+ fields: MillisField[];+ aliases?: AliasField[];+}++// Millisecond counterparts of every `*_seconds` field, matching the mappings+// in create_indices.ts. call_records keeps only started_at/ended_at as real+// fields; created_at/updated_at are aliases onto started_at.+const MIGRATIONS: IndexMigration[] = [+ {+ indexName: CHANNEL_INDEX,+ fields: [+ { fieldName: 'created_at_millis' },+ { fieldName: 'updated_at_millis' },+ ],+ },+ {+ indexName: CHAT_INDEX,+ fields: [+ { fieldName: 'created_at_millis' },+ { fieldName: 'updated_at_millis' },+ ],+ },+ {+ indexName: DOCUMENT_INDEX,+ fields: [{ fieldName: 'updated_at_millis' }],+ },+ {+ indexName: EMAIL_INDEX,+ fields: [+ { fieldName: 'updated_at_millis' },+ { fieldName: 'sent_at_millis' },+ ],+ },+ {+ indexName: PROJECTS_ALIAS,+ fields: [+ { fieldName: 'created_at_millis' },+ { fieldName: 'updated_at_millis' },+ ],+ },+ {+ indexName: CALL_RECORDS_ALIAS,+ fields: [+ { fieldName: 'started_at_millis' },+ { fieldName: 'ended_at_millis' },+ ],+ aliases: [+ { fieldName: 'created_at_millis', aliasFor: 'started_at_millis' },+ { fieldName: 'updated_at_millis', aliasFor: 'started_at_millis' },+ ],+ },+];++async function putMapping(+ opensearchClient: Client,+ indexName: string,+ properties: Record<string, unknown>,+ label: string,+ dryRun: boolean+): Promise<void> {+ console.log(+ ` ${dryRun ? '[DRY-RUN] Would add' : 'Adding'} ${label} to index "${indexName}"`+ );+ if (dryRun) {+ return;+ }+ const response = await opensearchClient.indices.putMapping({+ index: indexName,+ body: { properties },+ });+ if (!response.body.acknowledged) {+ throw new Error(`Failed to add ${label} in index "${indexName}"`);+ }+ console.log(` ✓ Successfully added ${label}`);+}++async function addFieldsToIndex(+ opensearchClient: Client,+ migration: IndexMigration,+ dryRun: boolean+): Promise<void> {+ console.log(`\n${'='.repeat(60)}`);+ console.log(+ `Adding millis fields to index: ${migration.indexName} ${dryRun ? '(DRY-RUN)' : ''}`+ );+ console.log(`${'='.repeat(60)}`);++ const indexExists = await checkIndexExists(+ opensearchClient,+ migration.indexName+ );+ if (!indexExists) {+ console.log(+ `⚠️ Index "${migration.indexName}" does not exist. Skipping...`+ );+ return;+ }++ // Real date fields first, so any aliases can reference them.+ for (const field of migration.fields) {+ await putMapping(+ opensearchClient,+ migration.indexName,+ {+ [field.fieldName]: {+ type: 'date',+ format: 'epoch_millis',+ index: false,+ doc_values: true,+ },+ },+ `field "${field.fieldName}" (epoch_millis)`,+ dryRun+ );+ }++ for (const alias of migration.aliases ?? []) {+ await putMapping(+ opensearchClient,+ migration.indexName,+ { [alias.fieldName]: { type: 'alias', path: alias.aliasFor } },+ `alias "${alias.fieldName}" → "${alias.aliasFor}"`,+ dryRun+ );+ }++ console.log(`\n✓ Completed millis fields for index: ${migration.indexName}`);+}++async function createFields(dryRun: boolean = true) {+ const opensearchClient = client();++ console.log('\n' + '='.repeat(60));+ console.log(+ `Create Timestamp Millis Fields ${dryRun ? '(DRY-RUN MODE)' : '(LIVE MODE)'}`+ );+ console.log('='.repeat(60));+ console.log(+ '\nThis script adds *_millis (epoch_millis) timestamp fields to indices.'+ );+ console.log(+ 'Additive and idempotent; safe to run before or after the millis backfill.'+ );++ if (dryRun) {+ console.log('\n⚠️ DRY-RUN MODE: No changes will be made to the cluster');+ } else {+ console.log('\n🚨 LIVE MODE: Fields will be added to indices');+ }++ try {+ for (const migration of MIGRATIONS) {+ await addFieldsToIndex(opensearchClient, migration, dryRun);+ }++ console.log('\n' + '='.repeat(60));+ console.log('Field creation completed successfully!');+ console.log('='.repeat(60));++ if (dryRun) {+ console.log(+ '\nTo run for real, set DRY_RUN=false environment variable\n'+ );+ } else {+ console.log('\n✓ All *_millis fields have been added to indices.');+ console.log(+ '✓ Next step: run copy_timestamp_seconds_to_millis.ts to backfill existing docs.\n'+ );+ }+ } catch (error) {+ console.error('\n❌ Field creation failed:', error);+ throw error;+ }+}++const isDryRun = process.env.DRY_RUN !== 'false';++createFields(isDryRun);diff --git a/infra/stacks/opensearch/helpers/utils/copy_field.ts b/infra/stacks/opensearch/helpers/utils/copy_field.tsindex 4eed30bd3a..0dbde0c098 100644--- a/infra/stacks/opensearch/helpers/utils/copy_field.ts+++ b/infra/stacks/opensearch/helpers/utils/copy_field.ts@@ -10,6 +10,9 @@ import type { Client } from '@opensearch-project/opensearch';* @param includeNonNull A boolean indicating whether to overwrite existing non-null values in newField.* If false (default), only updates documents where newField doesn't exist.* If true, updates all documents regardless of existing newField values.+ * @param valueExpr A painless expression producing the value to assign to newField.+ * Defaults to a verbatim copy (`ctx._source.<oldField>`). Pass a+ * transform (e.g. `ctx._source.<oldField> * 1000L`) to derive newField.* @returns A Promise that resolves to the number of documents updated.*/export async function copyFieldData(@@ -18,11 +21,12 @@ export async function copyFieldData(oldField: string,newField: string,dryRun: boolean,- includeNonNull: boolean = false+ includeNonNull: boolean = false,+ valueExpr: string = `ctx._source.${oldField}`): Promise<number> {const script = {- // If the oldField exists and is not null, copy the value to the new field- source: `if (ctx._source.containsKey('${oldField}') && ctx._source.${oldField} != null) { ctx._source.${newField} = ctx._source.${oldField}; }`,+ // If the oldField exists and is not null, assign the derived value to the new field+ source: `if (ctx._source.containsKey('${oldField}') && ctx._source.${oldField} != null) { ctx._source.${newField} = ${valueExpr}; }`,lang: 'painless',};diff --git a/services/search_processing_service/src/process/call.rs b/services/search_processing_service/src/process/call.rsindex a5b7c4017c..6015c3de4f 100644--- a/services/search_processing_service/src/process/call.rs+++ b/services/search_processing_service/src/process/call.rs@@ -1,7 +1,9 @@use anyhow::Context;use models_properties::EntityType;use opensearch_client::{- OpensearchClient, date_format::EpochSeconds, upsert::call_record::UpsertCallRecordSegmentArgs,+ OpensearchClient,+ date_format::{EpochMillis, EpochSeconds},+ upsert::call_record::UpsertCallRecordSegmentArgs,};use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;use sqlx::PgPool;@@ -60,6 +62,10 @@ pub async fn process_call_record(.ended_at.map(|dt| EpochSeconds::new(dt.timestamp())).transpose()?;+ let ended_at_millis = seg+ .ended_at+ .map(|dt| EpochMillis::new(dt.timestamp_millis()))+ .transpose()?;Ok::<_, anyhow::Error>(UpsertCallRecordSegmentArgs {call_id: call_id_s.clone(),transcript_id: seg.transcript_id.to_string(),@@ -72,6 +78,8 @@ pub async fn process_call_record(content: seg.content,started_at_seconds: EpochSeconds::new(seg.started_at.timestamp())?,ended_at_seconds,+ started_at_millis: EpochMillis::new(seg.started_at.timestamp_millis())?,+ ended_at_millis,properties: properties.clone(),})})diff --git a/services/search_processing_service/src/process/channel.rs b/services/search_processing_service/src/process/channel.rsindex 68c0222858..1e66dd53cc 100644--- a/services/search_processing_service/src/process/channel.rs+++ b/services/search_processing_service/src/process/channel.rs@@ -1,7 +1,9 @@use anyhow::Context;use mention_utils::parse::{ParsedXmlText, PlainTextFormatter, XmlFormatter};use opensearch_client::{- OpensearchClient, date_format::EpochSeconds, upsert::channel_message::UpsertChannelMessageArgs,+ OpensearchClient,+ date_format::{EpochMillis, EpochSeconds},+ upsert::channel_message::UpsertChannelMessageArgs,};use sqlx::{Pool, Postgres};use sqs_client::search::channel::{ChannelMessageUpdate, RemoveChannelMessage};@@ -61,6 +63,18 @@ pub async fn process_channel_message_update(updated_at_seconds: EpochSeconds::new(channel_message_info.channel_message.updated_at.timestamp(),)?,+ created_at_millis: EpochMillis::new(+ channel_message_info+ .channel_message+ .created_at+ .timestamp_millis(),+ )?,+ updated_at_millis: EpochMillis::new(+ channel_message_info+ .channel_message+ .updated_at+ .timestamp_millis(),+ )?,};opensearch_clientdiff --git a/services/search_processing_service/src/process/chat.rs b/services/search_processing_service/src/process/chat.rsindex 5edaeaafbf..7934ebd96b 100644--- a/services/search_processing_service/src/process/chat.rs+++ b/services/search_processing_service/src/process/chat.rs@@ -1,7 +1,9 @@use anyhow::Context;use models_properties::EntityType;use opensearch_client::{- OpensearchClient, date_format::EpochSeconds, upsert::chat_message::UpsertChatMessageArgs,+ OpensearchClient,+ date_format::{EpochMillis, EpochSeconds},+ upsert::chat_message::UpsertChatMessageArgs,};use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;use sqs_client::search::chat::{ChatMessage, RemoveChatMessage};@@ -58,6 +60,8 @@ pub async fn insert_chat_message(user_id: chat_message.user_id.clone(),created_at_seconds: EpochSeconds::new(chat_message.created_at.timestamp())?,updated_at_seconds: EpochSeconds::new(chat_message.updated_at.timestamp())?,+ created_at_millis: EpochMillis::new(chat_message.created_at.timestamp_millis())?,+ updated_at_millis: EpochMillis::new(chat_message.updated_at.timestamp_millis())?,title: info.name,content: info.content,role: info.role,diff --git a/services/search_processing_service/src/process/document/raw_document.rs b/services/search_processing_service/src/process/document/raw_document.rsindex f3aeb08b23..e296483b5c 100644--- a/services/search_processing_service/src/process/document/raw_document.rs+++ b/services/search_processing_service/src/process/document/raw_document.rs@@ -8,7 +8,9 @@ use models_properties::EntityType;use models_search::document::MarkdownParseResult;use models_search::unified::is_searchable_association;use opensearch_client::{- OpensearchClient, date_format::EpochSeconds, upsert::document::UpsertDocumentArgs,+ OpensearchClient,+ date_format::{EpochMillis, EpochSeconds},+ upsert::document::UpsertDocumentArgs,};use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;use s3_key::{@@ -125,6 +127,7 @@ fn generate_parent_only_upsert(owner_id: document_info.owner.to_string(),file_type,updated_at_seconds: EpochSeconds::new(Utc::now().timestamp())?,+ updated_at_millis: EpochMillis::new(Utc::now().timestamp_millis())?,sub_type: document_info.sub_type.map(|st| st.to_string()),properties: vec![],}))@@ -293,6 +296,7 @@ pub async fn update_search_with_raw_document(tracing::trace!("got raw file content");let updated_at = EpochSeconds::new(Utc::now().timestamp())?;+ let updated_at_millis = EpochMillis::new(Utc::now().timestamp_millis())?;let uuid = macro_uuid::generate_uuid_v7().to_string();let mut upserts: Vec<UpsertDocumentArgs> = match file_type {@@ -312,6 +316,7 @@ pub async fn update_search_with_raw_document(owner_id: search_extractor_message.user_id.clone(),file_type: file_type.to_string(),updated_at_seconds: updated_at,+ updated_at_millis,sub_type: sub_type.clone(),properties: vec![],})@@ -336,6 +341,7 @@ pub async fn update_search_with_raw_document(owner_id: search_extractor_message.user_id.clone(),file_type: file_type.to_string(),updated_at_seconds: updated_at,+ updated_at_millis,sub_type: sub_type.clone(),properties: vec![],}]@@ -357,6 +363,7 @@ pub async fn update_search_with_raw_document(owner_id: search_extractor_message.user_id.clone(),file_type: file_type.to_string(),updated_at_seconds: updated_at,+ updated_at_millis,sub_type: sub_type.clone(),properties: vec![],})@@ -376,6 +383,7 @@ pub async fn update_search_with_raw_document(owner_id: search_extractor_message.user_id.clone(),file_type: file_type.to_string(),updated_at_seconds: updated_at,+ updated_at_millis,sub_type: sub_type.clone(),properties: vec![],}]@@ -402,6 +410,7 @@ fn generate_upserts() -> anyhow::Result<Vec<UpsertDocumentArgs>> {let result = markdown_result;let updated_at = EpochSeconds::new(Utc::now().timestamp())?;+ let updated_at_millis = EpochMillis::new(Utc::now().timestamp_millis())?;let file_type = FileType::from_str(document_info.file_type@@ -422,6 +431,7 @@ fn generate_upserts(owner_id: document_info.owner.to_string(),file_type: file_type.to_string(),updated_at_seconds: updated_at,+ updated_at_millis,sub_type: sub_type.clone(),properties: vec![],})diff --git a/services/search_processing_service/src/process/email/upsert.rs b/services/search_processing_service/src/process/email/upsert.rsindex bb5b19cb9c..053f7fe228 100644--- a/services/search_processing_service/src/process/email/upsert.rs+++ b/services/search_processing_service/src/process/email/upsert.rs@@ -3,7 +3,9 @@ use chrono::Utc;use models_email::service::label::system_labels;use models_properties::EntityType;use opensearch_client::{- OpensearchClient, date_format::EpochSeconds, upsert::email::UpsertEmailArgs,+ OpensearchClient,+ date_format::{EpochMillis, EpochSeconds},+ upsert::email::UpsertEmailArgs,};use properties::outbound::entity_properties_get_query::{get_entity_properties_for_index, get_entity_properties_for_index_batch,@@ -53,6 +55,7 @@ pub async fn process_upsert_message(};let now = EpochSeconds::new(Utc::now().timestamp())?;+ let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;let updated_at = message_info.internal_date_ts@@ -60,6 +63,12 @@ pub async fn process_upsert_message(.transpose()?.unwrap_or(now);+ let updated_at_millis = message_info+ .internal_date_ts+ .map(|date| EpochMillis::new(date.timestamp_millis()))+ .transpose()?+ .unwrap_or(now_millis);+// A full index overwrites the doc, so thread properties must ride along// or a reindex would drop them. A fetch failure propagates (retry)// instead of being mistaken for "no properties".@@ -124,10 +133,15 @@ pub async fn process_upsert_message(.collect(),content,updated_at_seconds: updated_at,+ updated_at_millis,sent_at_seconds: message_info.internal_date_ts.map(|date| EpochSeconds::new(date.timestamp())).transpose()?,+ sent_at_millis: message_info+ .internal_date_ts+ .map(|date| EpochMillis::new(date.timestamp_millis()))+ .transpose()?,properties,};@@ -153,6 +167,7 @@ pub async fn process_upsert_thread_message(.context("failed to parse thread_id as UUID")?;let now = EpochSeconds::new(Utc::now().timestamp())?;+ let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;// A full index overwrites each doc, so thread properties must ride along// or a reindex would drop them.@@ -195,12 +210,23 @@ pub async fn process_upsert_thread_message(.map(|date| EpochSeconds::new(date.timestamp())).transpose()?;+ let sent_at_millis = message+ .internal_date_ts+ .map(|date| EpochMillis::new(date.timestamp_millis()))+ .transpose()?;+let updated_at = message.internal_date_ts.map(|date| EpochSeconds::new(date.timestamp())).transpose()?.unwrap_or(now);+ let updated_at_millis = message+ .internal_date_ts+ .map(|date| EpochMillis::new(date.timestamp_millis()))+ .transpose()?+ .unwrap_or(now_millis);+upsert_email_message_args.push(UpsertEmailArgs {message_id: message.db_id.to_string(),link_id: message.link_id.to_string(),@@ -243,7 +269,9 @@ pub async fn process_upsert_thread_message(.collect(),content,updated_at_seconds: updated_at,+ updated_at_millis,sent_at_seconds: sent_at,+ sent_at_millis,properties: properties.clone(),});} else {@@ -287,6 +315,7 @@ pub async fn process_upsert_thread_batch_message(.collect::<anyhow::Result<Vec<_>>>()?;let now = EpochSeconds::new(Utc::now().timestamp())?;+ let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;let messages =email_db_client::messages::get_parsed_search::get_parsed_search_messages_by_thread_ids(@@ -322,12 +351,23 @@ pub async fn process_upsert_thread_batch_message(.map(|date| EpochSeconds::new(date.timestamp())).transpose()?;+ let sent_at_millis = message+ .internal_date_ts+ .map(|date| EpochMillis::new(date.timestamp_millis()))+ .transpose()?;+let updated_at = message.internal_date_ts.map(|date| EpochSeconds::new(date.timestamp())).transpose()?.unwrap_or(now);+ let updated_at_millis = message+ .internal_date_ts+ .map(|date| EpochMillis::new(date.timestamp_millis()))+ .transpose()?+ .unwrap_or(now_millis);+upsert_email_message_args.push(UpsertEmailArgs {message_id: message.db_id.to_string(),link_id: message.link_id.to_string(),@@ -370,7 +410,9 @@ pub async fn process_upsert_thread_batch_message(.collect(),content,updated_at_seconds: updated_at,+ updated_at_millis,sent_at_seconds: sent_at,+ sent_at_millis,properties: properties_by_thread.get(&message.thread_db_id.to_string()).cloned()diff --git a/services/search_processing_service/src/process/project.rs b/services/search_processing_service/src/process/project.rsindex 07b3aba966..607face939 100644--- a/services/search_processing_service/src/process/project.rs+++ b/services/search_processing_service/src/process/project.rs@@ -2,7 +2,7 @@ use anyhow::Context;use models_properties::EntityType;use opensearch_client::{OpensearchClient,- date_format::EpochSeconds,+ date_format::{EpochMillis, EpochSeconds},upsert::{project::UpsertProjectArgs, properties::IndexedProperty},};use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;@@ -77,6 +77,8 @@ pub async fn upsert_project(parent_project_id: project.parent_id,created_at_seconds: EpochSeconds::new(created_at.timestamp())?,updated_at_seconds: EpochSeconds::new(updated_at.timestamp())?,+ created_at_millis: EpochMillis::new(created_at.timestamp_millis())?,+ updated_at_millis: EpochMillis::new(updated_at.timestamp_millis())?,properties,},index_override,