Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(search): index millisecond timestamps in OpenSearch indices (#4891)

gbirman1bb68a99f76c
diff --git a/crates/opensearch_client/src/date_format.rs b/crates/opensearch_client/src/date_format.rs
index b9584e19a1..c6ae20893c 100644
--- a/crates/opensearch_client/src/date_format.rs
+++ b/crates/opensearch_client/src/date_format.rs
@@ -3,6 +3,9 @@ use crate::{Result, error::OpensearchClientError};
const MAX_REASONABLE_SECONDS: i64 = 32503680000; // Year ~3000
const MIN_REASONABLE_SECONDS: i64 = 946684800; // Year 2000
+const MAX_REASONABLE_MILLIS: i64 = MAX_REASONABLE_SECONDS * 1000;
+const MIN_REASONABLE_MILLIS: i64 = MIN_REASONABLE_SECONDS * 1000;
+
#[derive(Debug, Clone, Copy, serde::Serialize)]
pub struct EpochSeconds(i64);
@@ -32,5 +35,36 @@ impl EpochSeconds {
}
}
+/// A validated epoch-milliseconds timestamp, serialized as a raw `i64` for
+/// OpenSearch date fields mapped with `format: epoch_millis`.
+#[derive(Debug, Clone, Copy, serde::Serialize)]
+pub struct EpochMillis(i64);
+
+impl EpochMillis {
+ pub fn new(millis: i64) -> Result<Self> {
+ if millis > MAX_REASONABLE_MILLIS {
+ return Err(OpensearchClientError::ValidationFailed {
+ details: format!(
+ "timestamp {} exceeds year 3000. Expected milliseconds since Unix epoch.",
+ millis
+ ),
+ });
+ }
+ if millis < MIN_REASONABLE_MILLIS {
+ return Err(OpensearchClientError::ValidationFailed {
+ details: format!(
+ "timestamp {} appears to be in seconds (before year 2000). Expected milliseconds since Unix epoch.",
+ millis
+ ),
+ });
+ }
+ Ok(Self(millis))
+ }
+
+ pub fn get(&self) -> i64 {
+ self.0
+ }
+}
+
#[cfg(test)]
mod test;
diff --git a/crates/opensearch_client/src/date_format/test.rs b/crates/opensearch_client/src/date_format/test.rs
index 3da7041019..e8df60369c 100644
--- a/crates/opensearch_client/src/date_format/test.rs
+++ b/crates/opensearch_client/src/date_format/test.rs
@@ -34,3 +34,38 @@ fn test_epoch_seconds_serialize() {
let serialized = serde_json::to_string(&epoch).unwrap();
assert_eq!(serialized, "1704067200");
}
+
+#[test]
+fn test_epoch_millis_valid() {
+ let now_millis = 1704067200123; // 2024-01-01 with sub-second precision
+ let epoch = EpochMillis::new(now_millis);
+ assert!(epoch.is_ok());
+ assert_eq!(epoch.unwrap().get(), now_millis);
+}
+
+#[test]
+fn test_epoch_millis_seconds() {
+ let now_seconds = 1704067200; // 2024-01-01 in seconds
+ let result = EpochMillis::new(now_seconds);
+ assert!(result.is_err());
+ if let Err(OpensearchClientError::ValidationFailed { details }) = result {
+ assert!(details.contains("appears to be in seconds"));
+ }
+}
+
+#[test]
+fn test_epoch_millis_too_far_future() {
+ let too_far = 32503680000001; // Just after year 3000 in millis
+ let result = EpochMillis::new(too_far);
+ assert!(result.is_err());
+ if let Err(OpensearchClientError::ValidationFailed { details }) = result {
+ assert!(details.contains("exceeds year 3000"));
+ }
+}
+
+#[test]
+fn test_epoch_millis_serialize() {
+ let epoch = EpochMillis::new(1704067200123).unwrap();
+ let serialized = serde_json::to_string(&epoch).unwrap();
+ assert_eq!(serialized, "1704067200123");
+}
diff --git a/crates/opensearch_client/src/search/builder.rs b/crates/opensearch_client/src/search/builder.rs
index eaa19fc07e..7c5cceb1bf 100644
--- a/crates/opensearch_client/src/search/builder.rs
+++ b/crates/opensearch_client/src/search/builder.rs
@@ -47,21 +47,30 @@ macro_rules! delegate_methods {
/// Sort mode for the channel content endpoint.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum ChannelSortMode {
- /// Sort by sent_at_seconds/updated_at_seconds, entity_id as tiebreaker.
+ /// Sort by sent_at/updated_at (millis, falling back to seconds), entity_id
+ /// as tiebreaker.
#[default]
Message,
/// Sort by thread_id, message_id as tiebreaker.
Thread,
}
-/// Creates sort vec to sort by sent_at_seconds (preferred) or updated_at_seconds (fallback)
-/// with entity_id as a tiebreaker. Items without a timestamp are pushed to the end.
+/// Creates sort vec to sort by sent_at (preferred) or updated_at (fallback)
+/// with entity_id as a tiebreaker. Items without a timestamp are pushed to the
+/// end. Prefers the millisecond-resolution `*_millis` fields and falls back to
+/// the second-resolution `*_seconds` fields, so docs indexed before the
+/// millis backfill still sort correctly. `.toEpochMilli()` normalizes both to
+/// milliseconds either way.
pub(crate) fn updated_at_sort<'a>() -> Vec<SortType<'a>> {
vec![
SortType::ScriptSort(ScriptSort::new(
Script::new(
- r#"if (doc.containsKey('sent_at_seconds') && doc['sent_at_seconds'].size() > 0) {
+ r#"if (doc.containsKey('sent_at_millis') && doc['sent_at_millis'].size() > 0) {
+ return doc['sent_at_millis'].value.toInstant().toEpochMilli();
+ } else if (doc.containsKey('sent_at_seconds') && doc['sent_at_seconds'].size() > 0) {
return doc['sent_at_seconds'].value.toInstant().toEpochMilli();
+ } else if (doc.containsKey('updated_at_millis') && doc['updated_at_millis'].size() > 0) {
+ return doc['updated_at_millis'].value.toInstant().toEpochMilli();
} else if (doc.containsKey('updated_at_seconds') && doc['updated_at_seconds'].size() > 0) {
return doc['updated_at_seconds'].value.toInstant().toEpochMilli();
} else {
diff --git a/crates/opensearch_client/src/search/call_records.rs b/crates/opensearch_client/src/search/call_records.rs
index 264925fa5e..58e46ed80d 100644
--- a/crates/opensearch_client/src/search/call_records.rs
+++ b/crates/opensearch_client/src/search/call_records.rs
@@ -9,10 +9,10 @@ use crate::{
},
properties::build_tag_filter,
query::Keys,
+ utils::{millis_or_seconds, opt_millis_or_seconds},
},
};
-use chrono::DateTime;
use models_opensearch::{OpenSearchEntityType, SearchEntityType};
use opensearch_query_builder::{
BoolQueryBuilder, HasChildQuery, InnerHits, MatchPhrasePrefixQuery, MatchPhraseQuery,
@@ -48,6 +48,10 @@ pub(crate) struct CallRecordIndex {
pub started_at_seconds: i64,
#[serde(default)]
pub ended_at_seconds: Option<i64>,
+ #[serde(default)]
+ pub started_at_millis: Option<i64>,
+ #[serde(default)]
+ pub ended_at_millis: Option<i64>,
}
pub(crate) struct CallRecordSearchConfig;
@@ -358,6 +362,10 @@ struct SegmentSource {
started_at_seconds: i64,
#[serde(default)]
ended_at_seconds: Option<i64>,
+ #[serde(default)]
+ started_at_millis: Option<i64>,
+ #[serde(default)]
+ ended_at_millis: Option<i64>,
}
#[derive(Debug, serde::Deserialize)]
@@ -415,15 +423,18 @@ pub(crate) fn expand_inner_hits_to_search_hits(
transcript_id: seg.source.transcript_id,
speaker_id: seg.source.speaker_id.unwrap_or_default(),
sequence_num: seg.source.sequence_num.unwrap_or_default(),
- started_at: DateTime::from_timestamp(seg.source.started_at_seconds, 0)
- .unwrap_or_default(),
- ended_at: seg
- .source
- .ended_at_seconds
- .and_then(|s| DateTime::from_timestamp(s, 0)),
+ started_at: millis_or_seconds(
+ seg.source.started_at_millis,
+ seg.source.started_at_seconds,
+ )
+ .unwrap_or_default(),
+ ended_at: opt_millis_or_seconds(
+ seg.source.ended_at_millis,
+ seg.source.ended_at_seconds,
+ ),
participant_ids: parent.participant_ids.clone(),
})),
- updated_at: DateTime::from_timestamp(parent.started_at_seconds, 0),
+ updated_at: millis_or_seconds(parent.started_at_millis, parent.started_at_seconds),
});
}
}
diff --git a/crates/opensearch_client/src/search/channels.rs b/crates/opensearch_client/src/search/channels.rs
index eb6f7ffd84..22b667c701 100644
--- a/crates/opensearch_client/src/search/channels.rs
+++ b/crates/opensearch_client/src/search/channels.rs
@@ -10,10 +10,11 @@ use crate::{
exclude_source_content, inject_fragment_size, parse_highlight_hit,
},
query::{Keys, TermCombine},
+ utils::millis_or_seconds,
},
};
-use chrono::{DateTime, Utc};
+use chrono::Utc;
use models_opensearch::{OpenSearchEntityType, SearchEntityType, SearchIndex};
use models_search_cursor::{SearchCursorOption, SearchMethodCursor};
use opensearch_query_builder::*;
@@ -31,6 +32,10 @@ pub(crate) struct ChannelMessageIndex {
pub mentions: Vec<String>,
pub created_at_seconds: i64,
pub updated_at_seconds: i64,
+ #[serde(default)]
+ pub created_at_millis: Option<i64>,
+ #[serde(default)]
+ pub updated_at_millis: Option<i64>,
}
#[derive(Default)]
@@ -307,10 +312,12 @@ fn channel_hit_to_search_hit(hit: Hit<ChannelMessageIndex>) -> SearchHit {
channel_message_id: a.message_id,
thread_id: (a.thread_id != a.message_id).then_some(a.thread_id),
sender_id: a.sender_id,
- created_at: DateTime::from_timestamp(a.created_at_seconds, 0).unwrap_or_default(),
- updated_at: DateTime::from_timestamp(a.updated_at_seconds, 0).unwrap_or_default(),
+ created_at: millis_or_seconds(a.created_at_millis, a.created_at_seconds)
+ .unwrap_or_default(),
+ updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds)
+ .unwrap_or_default(),
})),
- updated_at: DateTime::from_timestamp(a.updated_at_seconds, 0),
+ updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),
}
}
diff --git a/crates/opensearch_client/src/search/channels/test.rs b/crates/opensearch_client/src/search/channels/test.rs
index 558922086f..945b2895d5 100644
--- a/crates/opensearch_client/src/search/channels/test.rs
+++ b/crates/opensearch_client/src/search/channels/test.rs
@@ -1,6 +1,62 @@
use super::*;
+use crate::search::model::Hit;
use opensearch_query_builder::ToOpenSearchJson;
+fn channel_index(updated_at_millis: Option<i64>) -> ChannelMessageIndex {
+ let id = uuid::Uuid::new_v4();
+ ChannelMessageIndex {
+ entity_id: uuid::Uuid::new_v4(),
+ channel_type: "team".to_string(),
+ org_id: None,
+ message_id: id,
+ thread_id: id,
+ sender_id: "macro|[email protected]".to_string(),
+ mentions: vec![],
+ created_at_seconds: 1_700_000_000,
+ updated_at_seconds: 1_700_000_000,
+ created_at_millis: updated_at_millis,
+ updated_at_millis,
+ }
+}
+
+fn hit_for(source: ChannelMessageIndex) -> Hit<ChannelMessageIndex> {
+ Hit {
+ score: Some(1.0),
+ source,
+ highlight: None,
+ inner_hits: None,
+ }
+}
+
+/// A millisecond timestamp is preserved at full precision, and two hits in the
+/// same second but different milliseconds sort distinctly by `updated_at`.
+#[test]
+fn millis_timestamps_are_distinct_within_a_second() {
+ let earlier = channel_hit_to_search_hit(hit_for(channel_index(Some(1_700_000_000_123))));
+ let later = channel_hit_to_search_hit(hit_for(channel_index(Some(1_700_000_000_456))));
+
+ let earlier_ts = earlier.updated_at.expect("updated_at");
+ let later_ts = later.updated_at.expect("updated_at");
+
+ assert_eq!(earlier_ts.timestamp_millis(), 1_700_000_000_123);
+ assert_eq!(later_ts.timestamp_millis(), 1_700_000_000_456);
+ assert!(
+ later_ts > earlier_ts,
+ "sub-second ordering must be preserved"
+ );
+}
+
+/// Docs indexed before the millis backfill carry no `*_millis`; the read path
+/// falls back to the second-resolution field.
+#[test]
+fn falls_back_to_seconds_when_millis_absent() {
+ let hit = channel_hit_to_search_hit(hit_for(channel_index(None)));
+ assert_eq!(
+ hit.updated_at.expect("updated_at").timestamp_millis(),
+ 1_700_000_000_000
+ );
+}
+
#[test]
fn test_build_bool_query() -> anyhow::Result<()> {
let builder = ChannelMessageQueryBuilder::new(vec!["test".to_string()])
diff --git a/crates/opensearch_client/src/search/chats.rs b/crates/opensearch_client/src/search/chats.rs
index 887368c7cd..5ace04aa2e 100644
--- a/crates/opensearch_client/src/search/chats.rs
+++ b/crates/opensearch_client/src/search/chats.rs
@@ -42,6 +42,8 @@ pub(crate) struct ChatIndex {
pub user_id: String,
pub title: String,
pub updated_at_seconds: Option<i64>,
+ #[serde(default)]
+ pub updated_at_millis: Option<i64>,
}
pub(crate) struct ChatSearchConfig;
diff --git a/crates/opensearch_client/src/search/documents.rs b/crates/opensearch_client/src/search/documents.rs
index 194682e4a7..8d14359144 100644
--- a/crates/opensearch_client/src/search/documents.rs
+++ b/crates/opensearch_client/src/search/documents.rs
@@ -367,6 +367,8 @@ pub(crate) struct DocumentIndex {
pub owner_id: String,
pub file_type: String,
pub updated_at_seconds: Option<i64>,
+ #[serde(default)]
+ pub updated_at_millis: Option<i64>,
}
#[derive(Debug)]
diff --git a/crates/opensearch_client/src/search/emails.rs b/crates/opensearch_client/src/search/emails.rs
index 0ab3a1de10..eef19a7beb 100644
--- a/crates/opensearch_client/src/search/emails.rs
+++ b/crates/opensearch_client/src/search/emails.rs
@@ -369,6 +369,9 @@ pub(crate) struct EmailIndex {
pub subject: Option<String>,
/// The sent at time of the email message
pub sent_at_seconds: Option<i64>,
+ /// The sent at time of the email message, in milliseconds
+ #[serde(default)]
+ pub sent_at_millis: Option<i64>,
}
pub struct EmailSearchArgs {
diff --git a/crates/opensearch_client/src/search/projects.rs b/crates/opensearch_client/src/search/projects.rs
index f49c8a326d..6c382c3d3e 100644
--- a/crates/opensearch_client/src/search/projects.rs
+++ b/crates/opensearch_client/src/search/projects.rs
@@ -95,6 +95,8 @@ pub(crate) struct ProjectIndex {
#[serde(default)]
pub parent_project_id: Option<String>,
pub updated_at_seconds: Option<i64>,
+ #[serde(default)]
+ pub updated_at_millis: Option<i64>,
}
#[derive(Debug)]
diff --git a/crates/opensearch_client/src/search/unified.rs b/crates/opensearch_client/src/search/unified.rs
index c2330ed269..4fffc78cfa 100644
--- a/crates/opensearch_client/src/search/unified.rs
+++ b/crates/opensearch_client/src/search/unified.rs
@@ -26,9 +26,10 @@ use crate::{
},
projects::{ProjectIndex, ProjectQueryBuilder, ProjectSearchArgs, ProjectSearchConfig},
query::Keys,
+ utils::{millis_or_seconds, opt_millis_or_seconds},
},
};
-use chrono::{DateTime, Utc};
+use chrono::Utc;
use models_search_cursor::{SearchCursorOption, SearchMethodCursor};
use tracing::Instrument;
@@ -365,9 +366,8 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {
match &hit.source {
UnifiedSearchIndex::Document(parent) => {
let entity_id = parent.entity_id;
- let updated_at = parent
- .updated_at_seconds
- .and_then(|s| DateTime::from_timestamp(s, 0));
+ let updated_at =
+ opt_millis_or_seconds(parent.updated_at_millis, parent.updated_at_seconds);
let mut out: Vec<SearchHit> = Vec::new();
@@ -412,9 +412,8 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {
return vec![hit.into()];
};
let entity_id = parent.entity_id;
- let updated_at = parent
- .updated_at_seconds
- .and_then(|s| DateTime::from_timestamp(s, 0));
+ let updated_at =
+ opt_millis_or_seconds(parent.updated_at_millis, parent.updated_at_seconds);
let expanded = crate::search::chats::expand_inner_hits_to_search_hits(
entity_id, updated_at, inner,
);
@@ -424,7 +423,7 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {
expanded
}
UnifiedSearchIndex::CallRecord(parent) => {
- let updated_at = DateTime::from_timestamp(parent.started_at_seconds, 0);
+ let updated_at = millis_or_seconds(parent.started_at_millis, parent.started_at_seconds);
let mut out: Vec<SearchHit> = Vec::new();
// A name match surfaces as a parent-level hit. Segment content lives
@@ -452,9 +451,10 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {
speaker_id: String::new(),
sequence_num: 0,
started_at: updated_at.unwrap_or_default(),
- ended_at: parent
- .ended_at_seconds
- .and_then(|s| DateTime::from_timestamp(s, 0)),
+ ended_at: opt_millis_or_seconds(
+ parent.ended_at_millis,
+ parent.ended_at_seconds,
+ ),
participant_ids: parent.participant_ids.clone(),
})),
updated_at,
@@ -500,12 +500,12 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
channel_message_id: a.message_id,
thread_id: (a.thread_id != a.message_id).then_some(a.thread_id),
sender_id: a.sender_id,
- created_at: DateTime::from_timestamp(a.created_at_seconds, 0)
+ created_at: millis_or_seconds(a.created_at_millis, a.created_at_seconds)
.unwrap_or_default(),
- updated_at: DateTime::from_timestamp(a.updated_at_seconds, 0)
+ updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds)
.unwrap_or_default(),
})),
- updated_at: DateTime::from_timestamp(a.updated_at_seconds, 0),
+ updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),
},
UnifiedSearchIndex::Document(a) => SearchHit {
entity_id: a.entity_id,
@@ -524,9 +524,7 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
})
.unwrap_or_default(),
goto: None,
- updated_at: a
- .updated_at_seconds
- .and_then(|s| DateTime::from_timestamp(s, 0)),
+ updated_at: opt_millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),
},
UnifiedSearchIndex::Email(a) => {
let a = *a;
@@ -551,15 +549,11 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
bcc: a.bcc,
cc: a.cc,
labels: a.labels,
- sent_at: a
- .sent_at_seconds
- .and_then(|ts| DateTime::from_timestamp(ts, 0)),
+ sent_at: opt_millis_or_seconds(a.sent_at_millis, a.sent_at_seconds),
sender: a.sender,
recipients: a.recipients,
})),
- updated_at: a
- .sent_at_seconds
- .and_then(|s| DateTime::from_timestamp(s, 0)),
+ updated_at: opt_millis_or_seconds(a.sent_at_millis, a.sent_at_seconds),
}
}
UnifiedSearchIndex::Chat(a) => SearchHit {
@@ -579,9 +573,7 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
})
.unwrap_or_default(),
goto: None,
- updated_at: a
- .updated_at_seconds
- .and_then(|s| DateTime::from_timestamp(s, 0)),
+ updated_at: opt_millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),
},
UnifiedSearchIndex::Project(a) => SearchHit {
entity_id: a.entity_id,
@@ -600,9 +592,7 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
})
.unwrap_or_default(),
goto: None,
- updated_at: a
- .updated_at_seconds
- .and_then(|s| DateTime::from_timestamp(s, 0)),
+ updated_at: opt_millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),
},
UnifiedSearchIndex::CallRecord(a) => SearchHit {
entity_id: a.entity_id,
@@ -629,14 +619,12 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
transcript_id: uuid::Uuid::nil(),
speaker_id: String::new(),
sequence_num: 0,
- started_at: DateTime::from_timestamp(a.started_at_seconds, 0)
+ started_at: millis_or_seconds(a.started_at_millis, a.started_at_seconds)
.unwrap_or_default(),
- ended_at: a
- .ended_at_seconds
- .and_then(|s| DateTime::from_timestamp(s, 0)),
+ ended_at: opt_millis_or_seconds(a.ended_at_millis, a.ended_at_seconds),
participant_ids: a.participant_ids,
})),
- updated_at: DateTime::from_timestamp(a.started_at_seconds, 0),
+ updated_at: millis_or_seconds(a.started_at_millis, a.started_at_seconds),
},
}
}
diff --git a/crates/opensearch_client/src/search/unified/test.rs b/crates/opensearch_client/src/search/unified/test.rs
index c08178803b..312eb82ca0 100644
--- a/crates/opensearch_client/src/search/unified/test.rs
+++ b/crates/opensearch_client/src/search/unified/test.rs
@@ -23,6 +23,7 @@ fn expand_document_name_highlight_yields_name_hit_without_goto() {
owner_id: "alice".to_string(),
file_type: "md".to_string(),
updated_at_seconds: Some(1_779_000_000),
+ updated_at_millis: None,
}),
highlight: Some(HashMap::from([(
"document_name".to_string(),
@@ -928,6 +929,7 @@ fn doc_hit_with_chunks(
owner_id: "alice".to_string(),
file_type: "md".to_string(),
updated_at_seconds: Some(updated_at_seconds),
+ updated_at_millis: None,
}),
highlight: None,
inner_hits: Some(serde_json::json!({ "term_0": { "hits": { "hits": chunks } } })),
diff --git a/crates/opensearch_client/src/search/utils.rs b/crates/opensearch_client/src/search/utils.rs
index 20ee336954..7468ad4732 100644
--- a/crates/opensearch_client/src/search/utils.rs
+++ b/crates/opensearch_client/src/search/utils.rs
@@ -1,5 +1,26 @@
+use chrono::{DateTime, Utc};
use opensearch_query_builder::{BoolQueryBuilder, QueryType, WildcardQuery};
+/// Prefer a millisecond epoch timestamp, falling back to a second epoch
+/// timestamp. During the seconds→millis migration new/backfilled docs carry a
+/// `*_millis` field while older docs only have `*_seconds`.
+pub(crate) fn millis_or_seconds(millis: Option<i64>, seconds: i64) -> Option<DateTime<Utc>> {
+ millis
+ .and_then(DateTime::from_timestamp_millis)
+ .or_else(|| DateTime::from_timestamp(seconds, 0))
+}
+
+/// Like [`millis_or_seconds`] but both inputs are optional (e.g. email
+/// `sent_at`, call `ended_at`).
+pub(crate) fn opt_millis_or_seconds(
+ millis: Option<i64>,
+ seconds: Option<i64>,
+) -> Option<DateTime<Utc>> {
+ millis
+ .and_then(DateTime::from_timestamp_millis)
+ .or_else(|| seconds.and_then(|s| DateTime::from_timestamp(s, 0)))
+}
+
pub fn should_wildcard_field_query_builder<'a>(
field: &'a str,
values: &'a [String],
diff --git a/crates/opensearch_client/src/upsert/call_record.rs b/crates/opensearch_client/src/upsert/call_record.rs
index 17b040fc0f..59ec036f04 100644
--- a/crates/opensearch_client/src/upsert/call_record.rs
+++ b/crates/opensearch_client/src/upsert/call_record.rs
@@ -4,7 +4,11 @@ use models_opensearch::SearchIndex;
use super::BulkUpsertResult;
use crate::upsert::properties::IndexedProperty;
-use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};
+use crate::{
+ Result,
+ date_format::{EpochMillis, EpochSeconds},
+ error::OpensearchClientError,
+};
#[cfg(test)]
mod test;
@@ -34,6 +38,9 @@ pub struct UpsertCallRecordSegmentArgs {
pub started_at_seconds: EpochSeconds,
#[serde(skip_serializing_if = "Option::is_none")]
pub ended_at_seconds: Option<EpochSeconds>,
+ pub started_at_millis: EpochMillis,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub ended_at_millis: Option<EpochMillis>,
/// Denormalized parent entity properties (tags, custom) used for search
/// filtering. Shared across every segment of a call; empty for untagged
/// calls. Only written onto the parent doc.
@@ -79,6 +86,7 @@ fn parent_doc_body(any_segment: &UpsertCallRecordSegmentArgs) -> serde_json::Val
"channel_id": &any_segment.channel_id,
"participant_ids": &any_segment.participant_ids,
"started_at_seconds": any_segment.started_at_seconds,
+ "started_at_millis": any_segment.started_at_millis,
"call_relation": PARENT_RELATION,
});
if let Some(channel_name) = &any_segment.channel_name {
@@ -90,6 +98,9 @@ fn parent_doc_body(any_segment: &UpsertCallRecordSegmentArgs) -> serde_json::Val
if let Some(ended) = &any_segment.ended_at_seconds {
doc["ended_at_seconds"] = serde_json::to_value(ended).unwrap_or(serde_json::Value::Null);
}
+ if let Some(ended) = &any_segment.ended_at_millis {
+ doc["ended_at_millis"] = serde_json::to_value(ended).unwrap_or(serde_json::Value::Null);
+ }
if !any_segment.properties.is_empty()
&& let Ok(properties) = serde_json::to_value(&any_segment.properties)
{
@@ -108,6 +119,8 @@ fn child_doc_body(seg: &UpsertCallRecordSegmentArgs) -> serde_json::Value {
"content": &seg.content,
"started_at_seconds": seg.started_at_seconds,
"ended_at_seconds": &seg.ended_at_seconds,
+ "started_at_millis": seg.started_at_millis,
+ "ended_at_millis": &seg.ended_at_millis,
"call_relation": {
"name": CHILD_RELATION,
"parent": &seg.call_id,
diff --git a/crates/opensearch_client/src/upsert/call_record/test.rs b/crates/opensearch_client/src/upsert/call_record/test.rs
index 692fa084d2..31a1bd76db 100644
--- a/crates/opensearch_client/src/upsert/call_record/test.rs
+++ b/crates/opensearch_client/src/upsert/call_record/test.rs
@@ -13,6 +13,8 @@ fn segment() -> UpsertCallRecordSegmentArgs {
content: "segment content".to_string(),
started_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),
ended_at_seconds: Some(EpochSeconds::new(1_700_000_100).unwrap()),
+ started_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),
+ ended_at_millis: Some(EpochMillis::new(1_700_000_100_456).unwrap()),
properties: vec![],
}
}
@@ -25,6 +27,7 @@ fn parent_doc_body_has_metadata_and_name_no_child_fields() {
assert_eq!(doc["channel_id"], "channel1");
assert_eq!(doc["channel_name"], "Standup");
assert_eq!(doc["name"], "Weekly standup");
+ assert_eq!(doc["started_at_millis"], 1_700_000_000_123i64);
assert_eq!(doc["call_relation"], "call");
// Child-only fields must not be present on the parent.
assert!(doc.get("content").is_none());
diff --git a/crates/opensearch_client/src/upsert/channel_message.rs b/crates/opensearch_client/src/upsert/channel_message.rs
index 9f9ce1641e..fbe882891f 100644
--- a/crates/opensearch_client/src/upsert/channel_message.rs
+++ b/crates/opensearch_client/src/upsert/channel_message.rs
@@ -1,6 +1,10 @@
use models_opensearch::SearchIndex;
-use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};
+use crate::{
+ Result,
+ date_format::{EpochMillis, EpochSeconds},
+ error::OpensearchClientError,
+};
/// The arguments for upserting a channel message into the opensearch index.
/// Threadless messages are indexed with `thread_id == message_id`.
@@ -17,6 +21,8 @@ pub struct UpsertChannelMessageArgs {
pub content: String,
pub created_at_seconds: EpochSeconds,
pub updated_at_seconds: EpochSeconds,
+ pub created_at_millis: EpochMillis,
+ pub updated_at_millis: EpochMillis,
}
#[tracing::instrument(skip(client))]
diff --git a/crates/opensearch_client/src/upsert/chat_message.rs b/crates/opensearch_client/src/upsert/chat_message.rs
index 1c651fa54a..4b48aa0cfc 100644
--- a/crates/opensearch_client/src/upsert/chat_message.rs
+++ b/crates/opensearch_client/src/upsert/chat_message.rs
@@ -1,7 +1,11 @@
use models_opensearch::SearchIndex;
use crate::upsert::properties::IndexedProperty;
-use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};
+use crate::{
+ Result,
+ date_format::{EpochMillis, EpochSeconds},
+ error::OpensearchClientError,
+};
#[cfg(test)]
mod test;
@@ -28,6 +32,10 @@ pub struct UpsertChatMessageArgs {
pub created_at_seconds: EpochSeconds,
/// The updated at time of the chat message
pub updated_at_seconds: EpochSeconds,
+ /// The created at time of the chat message, in milliseconds
+ pub created_at_millis: EpochMillis,
+ /// The updated at time of the chat message, in milliseconds
+ pub updated_at_millis: EpochMillis,
/// The title of the chat message
pub title: String,
/// The content of the chat message
@@ -51,6 +59,7 @@ fn parent_doc_body(args: &UpsertChatMessageArgs) -> serde_json::Value {
"title": &args.title,
"user_id": &args.user_id,
"updated_at_seconds": args.updated_at_seconds,
+ "updated_at_millis": args.updated_at_millis,
"chat_relation": PARENT_RELATION,
});
if !args.properties.is_empty()
@@ -70,6 +79,8 @@ fn child_doc_body(args: &UpsertChatMessageArgs) -> serde_json::Value {
"role": &args.role,
"created_at_seconds": args.created_at_seconds,
"updated_at_seconds": args.updated_at_seconds,
+ "created_at_millis": args.created_at_millis,
+ "updated_at_millis": args.updated_at_millis,
"chat_relation": {
"name": CHILD_RELATION,
"parent": &args.chat_id,
diff --git a/crates/opensearch_client/src/upsert/chat_message/test.rs b/crates/opensearch_client/src/upsert/chat_message/test.rs
index 72120d35d5..55f735623a 100644
--- a/crates/opensearch_client/src/upsert/chat_message/test.rs
+++ b/crates/opensearch_client/src/upsert/chat_message/test.rs
@@ -8,6 +8,8 @@ fn args() -> UpsertChatMessageArgs {
role: "user".to_string(),
created_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),
updated_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),
+ created_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),
+ updated_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),
title: "Chat title".to_string(),
content: "message content".to_string(),
properties: vec![],
@@ -21,6 +23,7 @@ fn parent_doc_body_has_metadata_no_child_fields() {
assert_eq!(doc["entity_id"], "chat1");
assert_eq!(doc["title"], "Chat title");
assert_eq!(doc["user_id"], "macro|[email protected]");
+ assert_eq!(doc["updated_at_millis"], 1_700_000_000_123i64);
assert_eq!(doc["chat_relation"], "chat");
// Child-only fields must not be present on the parent.
assert!(doc.get("content").is_none());
diff --git a/crates/opensearch_client/src/upsert/document.rs b/crates/opensearch_client/src/upsert/document.rs
index 1c5e6eca41..5fa5598260 100644
--- a/crates/opensearch_client/src/upsert/document.rs
+++ b/crates/opensearch_client/src/upsert/document.rs
@@ -4,7 +4,11 @@ use models_opensearch::SearchIndex;
use super::BulkUpsertResult;
use super::properties::IndexedProperty;
-use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};
+use crate::{
+ Result,
+ date_format::{EpochMillis, EpochSeconds},
+ error::OpensearchClientError,
+};
/// Relation name for parent docs in the join field.
const PARENT_RELATION: &str = "document";
@@ -39,6 +43,8 @@ pub struct UpsertDocumentArgs {
pub content: String,
/// The updated at time of the document
pub updated_at_seconds: EpochSeconds,
+ /// The updated at time of the document, in milliseconds
+ pub updated_at_millis: EpochMillis,
/// The sub type of the document (e.g. task)
#[serde(skip_serializing_if = "Option::is_none")]
pub sub_type: Option<String>,
@@ -64,6 +70,7 @@ fn parent_doc_body(any_chunk: &UpsertDocumentArgs) -> serde_json::Value {
"owner_id": &any_chunk.owner_id,
"file_type": &any_chunk.file_type,
"updated_at_seconds": any_chunk.updated_at_seconds,
+ "updated_at_millis": any_chunk.updated_at_millis,
"document_relation": PARENT_RELATION,
});
if let Some(sub_type) = &any_chunk.sub_type {
diff --git a/crates/opensearch_client/src/upsert/document/test.rs b/crates/opensearch_client/src/upsert/document/test.rs
index d312e0d344..83d04543dc 100644
--- a/crates/opensearch_client/src/upsert/document/test.rs
+++ b/crates/opensearch_client/src/upsert/document/test.rs
@@ -1,5 +1,5 @@
use super::*;
-use crate::date_format::EpochSeconds;
+use crate::date_format::{EpochMillis, EpochSeconds};
fn args(doc_id: &str, node_id: &str) -> UpsertDocumentArgs {
UpsertDocumentArgs {
@@ -11,6 +11,7 @@ fn args(doc_id: &str, node_id: &str) -> UpsertDocumentArgs {
raw_content: None,
content: format!("content {doc_id} {node_id}"),
updated_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),
+ updated_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),
sub_type: None,
properties: vec![],
}
@@ -24,6 +25,7 @@ fn parent_doc_body_has_metadata_no_chunk_fields() {
assert_eq!(doc["document_name"], "name-doc1");
assert_eq!(doc["owner_id"], "owner-doc1");
assert_eq!(doc["file_type"], "md");
+ assert_eq!(doc["updated_at_millis"], 1_700_000_000_123i64);
assert_eq!(doc["document_relation"], "document");
// Child-only fields must not be present on the parent.
assert!(doc.get("content").is_none());
diff --git a/crates/opensearch_client/src/upsert/email.rs b/crates/opensearch_client/src/upsert/email.rs
index 15db8f7531..1fc6c3ad95 100644
--- a/crates/opensearch_client/src/upsert/email.rs
+++ b/crates/opensearch_client/src/upsert/email.rs
@@ -2,7 +2,11 @@ use models_opensearch::SearchIndex;
use super::BulkUpsertResult;
use super::properties::IndexedProperty;
-use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};
+use crate::{
+ Result,
+ date_format::{EpochMillis, EpochSeconds},
+ error::OpensearchClientError,
+};
/// The arguments for upserting an email message into the opensearch index
#[derive(Debug, serde::Serialize)]
@@ -40,10 +44,14 @@ pub struct UpsertEmailArgs {
pub user_id: String,
/// The updated at time of the email message
pub updated_at_seconds: EpochSeconds,
+ /// The updated at time of the email message, in milliseconds
+ pub updated_at_millis: EpochMillis,
/// The subject of the email message
pub subject: Option<String>,
/// The sent at time of the email message
pub sent_at_seconds: Option<EpochSeconds>,
+ /// The sent at time of the email message, in milliseconds
+ pub sent_at_millis: Option<EpochMillis>,
/// The content of the email message
pub content: String,
/// Denormalized thread properties (e.g. tags) used for search filtering.
diff --git a/crates/opensearch_client/src/upsert/project.rs b/crates/opensearch_client/src/upsert/project.rs
index e144ed8b7d..bec2e66b88 100644
--- a/crates/opensearch_client/src/upsert/project.rs
+++ b/crates/opensearch_client/src/upsert/project.rs
@@ -1,7 +1,11 @@
use models_opensearch::SearchIndex;
use super::properties::IndexedProperty;
-use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};
+use crate::{
+ Result,
+ date_format::{EpochMillis, EpochSeconds},
+ error::OpensearchClientError,
+};
/// The arguments for upserting a project into the opensearch index.
///
@@ -22,6 +26,10 @@ pub struct UpsertProjectArgs {
pub created_at_seconds: EpochSeconds,
/// The updated at time of the project
pub updated_at_seconds: EpochSeconds,
+ /// The created at time of the project, in milliseconds
+ pub created_at_millis: EpochMillis,
+ /// The updated at time of the project, in milliseconds
+ pub updated_at_millis: EpochMillis,
/// Entity properties (tags, custom) used for search filtering.
pub properties: Vec<IndexedProperty>,
}
diff --git a/infra/stacks/opensearch/helpers/scripts/copy_timestamp_seconds_to_millis.ts b/infra/stacks/opensearch/helpers/scripts/copy_timestamp_seconds_to_millis.ts
new file mode 100644
index 0000000000..c0ab3bb65d
--- /dev/null
+++ b/infra/stacks/opensearch/helpers/scripts/copy_timestamp_seconds_to_millis.ts
@@ -0,0 +1,217 @@
+require('dotenv').config();
+
+import type { Client } from '@opensearch-project/opensearch';
+import { client } from '../client';
+import {
+ CALL_RECORDS_ALIAS,
+ CHANNEL_INDEX,
+ CHAT_INDEX,
+ DOCUMENT_INDEX,
+ EMAIL_INDEX,
+ PROJECTS_ALIAS,
+} from '../constants';
+import { checkIndexExists } from '../utils/check_index_exists';
+import { copyFieldData } from '../utils/copy_field';
+
+// `<base>_seconds` → `<base>_millis` pairs to backfill. call_records only
+// backfills the real started_at/ended_at fields; its created_at/updated_at
+// are aliases onto started_at and need no separate data.
+type FieldPair = { base: string };
+
+interface IndexMigration {
+ indexName: string;
+ fields: FieldPair[];
+}
+
+const MIGRATIONS: IndexMigration[] = [
+ {
+ indexName: CHANNEL_INDEX,
+ fields: [{ base: 'created_at' }, { base: 'updated_at' }],
+ },
+ {
+ indexName: CHAT_INDEX,
+ fields: [{ base: 'created_at' }, { base: 'updated_at' }],
+ },
+ {
+ indexName: DOCUMENT_INDEX,
+ fields: [{ base: 'updated_at' }],
+ },
+ {
+ indexName: EMAIL_INDEX,
+ fields: [{ base: 'updated_at' }, { base: 'sent_at' }],
+ },
+ {
+ indexName: PROJECTS_ALIAS,
+ fields: [{ base: 'created_at' }, { base: 'updated_at' }],
+ },
+ {
+ indexName: CALL_RECORDS_ALIAS,
+ fields: [{ base: 'started_at' }, { base: 'ended_at' }],
+ },
+];
+
+/**
+ * Sample docs that have both fields and confirm `millis` falls within the same
+ * second as `seconds` (i.e. `floor(millis / 1000) === seconds`). This holds for
+ * both backfilled docs (millis === seconds * 1000) and dual-written docs that
+ * carry genuine sub-second precision (millis === seconds * 1000 + remainder).
+ */
+async function verifyMillis(
+ opensearchClient: Client,
+ indexName: string,
+ secondsField: string,
+ millisField: string
+): Promise<void> {
+ const response = await opensearchClient.search({
+ index: indexName,
+ body: {
+ size: 50,
+ query: {
+ bool: {
+ must: [
+ { exists: { field: secondsField } },
+ { exists: { field: millisField } },
+ ],
+ },
+ },
+ _source: [secondsField, millisField],
+ },
+ });
+
+ const hits = response.body.hits.hits;
+ if (hits.length === 0) {
+ console.log(
+ ` ⚠️ No documents with both "${secondsField}" and "${millisField}" to verify`
+ );
+ return;
+ }
+
+ let mismatches = 0;
+ for (const hit of hits) {
+ const seconds = hit._source?.[secondsField];
+ const millis = hit._source?.[millisField];
+ if (
+ typeof seconds !== 'number' ||
+ typeof millis !== 'number' ||
+ Math.floor(millis / 1000) !== seconds
+ ) {
+ mismatches += 1;
+ console.log(
+ ` ⚠️ Mismatch (ID ${hit._id}): ${secondsField}=${seconds} ${millisField}=${millis}`
+ );
+ }
+ }
+ if (mismatches === 0) {
+ console.log(
+ ` ✓ Verified ${hits.length} sample docs: floor(${millisField} / 1000) === ${secondsField}`
+ );
+ } else {
+ throw new Error(
+ `${mismatches}/${hits.length} sample docs failed ${millisField} verification`
+ );
+ }
+}
+
+async function backfillIndex(
+ opensearchClient: Client,
+ migration: IndexMigration,
+ dryRun: boolean
+): Promise<void> {
+ console.log(`\n${'='.repeat(60)}`);
+ console.log(
+ `Backfilling millis for index: ${migration.indexName} ${dryRun ? '(DRY-RUN)' : ''}`
+ );
+ console.log(`${'='.repeat(60)}`);
+
+ const indexExists = await checkIndexExists(
+ opensearchClient,
+ migration.indexName
+ );
+ if (!indexExists) {
+ console.log(
+ `⚠️ Index "${migration.indexName}" does not exist. Skipping...`
+ );
+ return;
+ }
+
+ for (const { base } of migration.fields) {
+ const secondsField = `${base}_seconds`;
+ const millisField = `${base}_millis`;
+ console.log(`\nProcessing ${secondsField} → ${millisField}`);
+
+ await copyFieldData(
+ opensearchClient,
+ migration.indexName,
+ secondsField,
+ millisField,
+ dryRun,
+ false, // only where millis is missing — idempotent, safe to re-run
+ `ctx._source.${secondsField} * 1000L`
+ );
+
+ if (!dryRun) {
+ await verifyMillis(
+ opensearchClient,
+ migration.indexName,
+ secondsField,
+ millisField
+ );
+ }
+ }
+
+ console.log(
+ `\n✓ Completed millis backfill for index: ${migration.indexName}`
+ );
+}
+
+async function backfill(dryRun: boolean = true) {
+ const opensearchClient = client();
+
+ console.log('\n' + '='.repeat(60));
+ console.log(
+ `Copy Timestamp Seconds → Millis ${dryRun ? '(DRY-RUN MODE)' : '(LIVE MODE)'}`
+ );
+ console.log('='.repeat(60));
+ console.log(
+ '\nThis script derives *_millis fields from *_seconds (× 1000) for existing docs.'
+ );
+ console.log('It only updates docs where the *_millis field is missing.');
+ console.log("\n💡 Safe to run multiple times — it's idempotent.");
+ console.log(
+ '⚠️ Run create_timestamp_millis_fields.ts first so the mappings exist.'
+ );
+
+ if (dryRun) {
+ console.log('\n⚠️ DRY-RUN MODE: No changes will be made to the cluster');
+ } else {
+ console.log('\n🚨 LIVE MODE: Data will be backfilled');
+ }
+
+ try {
+ for (const migration of MIGRATIONS) {
+ await backfillIndex(opensearchClient, migration, dryRun);
+ }
+
+ console.log('\n' + '='.repeat(60));
+ console.log('Backfill completed successfully!');
+ console.log('='.repeat(60));
+
+ if (dryRun) {
+ console.log(
+ '\nTo run for real, set DRY_RUN=false environment variable\n'
+ );
+ } else {
+ console.log('\n✓ All *_millis fields backfilled from *_seconds.');
+ console.log(
+ '💡 Re-run after deploying the dual-write code to catch docs written mid-migration.\n'
+ );
+ }
+ } catch (error) {
+ console.error('\n❌ Backfill failed:', error);
+ throw error;
+ }
+}
+
+const isDryRun = process.env.DRY_RUN !== 'false';
+
+backfill(isDryRun);
diff --git a/infra/stacks/opensearch/helpers/scripts/create_indices.ts b/infra/stacks/opensearch/helpers/scripts/create_indices.ts
index 2f84743a23..8e3e1e6a60 100644
--- a/infra/stacks/opensearch/helpers/scripts/create_indices.ts
+++ b/infra/stacks/opensearch/helpers/scripts/create_indices.ts
@@ -247,6 +247,18 @@ const CHANNEL_BODY = {
index: false,
doc_values: true,
},
+ created_at_millis: {
+ type: 'date',
+ format: 'epoch_millis',
+ index: false,
+ doc_values: true,
+ },
+ updated_at_millis: {
+ type: 'date',
+ format: 'epoch_millis',
+ index: false,
+ doc_values: true,
+ },
},
},
};
@@ -298,6 +310,12 @@ const DOCUMENT_BODY = {
index: false,
doc_values: true,
},
+ updated_at_millis: {
+ type: 'date',
+ format: 'epoch_millis',
+ index: false,
+ doc_values: true,
+ },
// Parent-only entity properties (status, priority, assignees, custom).
// `nested` so a property filter matches definition_id + its value
// within the same entry rather than cross-matching across properties.
@@ -379,6 +397,18 @@ const PROJECTS_BODY = {
index: false,
doc_values: true,
},
+ created_at_millis: {
+ type: 'date',
+ format: 'epoch_millis',
+ index: false,
+ doc_values: true,
+ },
+ updated_at_millis: {
+ type: 'date',
+ format: 'epoch_millis',
+ index: false,
+ doc_values: true,
+ },
// Entity properties (tags, custom). Same nested shape as the documents
// index so the shared property/tag query builders apply unchanged.
properties: {
@@ -472,6 +502,12 @@ const EMAIL_BODY = {
index: false,
doc_values: true,
},
+ updated_at_millis: {
+ type: 'date',
+ format: 'epoch_millis',
+ index: false,
+ doc_values: true,
+ },
subject: {
type: 'text',
fields: {
@@ -487,6 +523,12 @@ const EMAIL_BODY = {
index: false,
doc_values: true,
},
+ sent_at_millis: {
+ type: 'date',
+ format: 'epoch_millis',
+ index: false,
+ doc_values: true,
+ },
content: {
type: 'text',
analyzer: 'standard',
@@ -535,6 +577,12 @@ const CHATS_V2_BODY = {
index: false,
doc_values: true,
},
+ updated_at_millis: {
+ type: 'date',
+ format: 'epoch_millis',
+ index: false,
+ doc_values: true,
+ },
// Parent-only entity properties (tags, custom). Same nested shape as
// DOCUMENT_BODY so property filters match definition_id + value within
// the same entry rather than cross-matching across properties.
@@ -557,6 +605,12 @@ const CHATS_V2_BODY = {
index: false,
doc_values: true,
},
+ created_at_millis: {
+ type: 'date',
+ format: 'epoch_millis',
+ index: false,
+ doc_values: true,
+ },
// Join relationship
chat_relation: {
type: 'join',
@@ -604,6 +658,18 @@ const CALL_RECORDS_V2_BODY = {
index: false,
doc_values: true,
},
+ started_at_millis: {
+ type: 'date',
+ format: 'epoch_millis',
+ index: false,
+ doc_values: true,
+ },
+ ended_at_millis: {
+ type: 'date',
+ format: 'epoch_millis',
+ index: false,
+ doc_values: true,
+ },
// Parent-only entity properties (tags, custom). Same nested shape as
// DOCUMENT_BODY so property filters match definition_id + value within
// the same entry rather than cross-matching across properties.
@@ -625,6 +691,8 @@ const CALL_RECORDS_V2_BODY = {
// to the parent's call-start timestamp.
created_at_seconds: { type: 'alias', path: 'started_at_seconds' },
updated_at_seconds: { type: 'alias', path: 'started_at_seconds' },
+ created_at_millis: { type: 'alias', path: 'started_at_millis' },
+ updated_at_millis: { type: 'alias', path: 'started_at_millis' },
// Join relationship
call_relation: {
type: 'join',
diff --git a/infra/stacks/opensearch/helpers/scripts/create_timestamp_millis_fields.ts b/infra/stacks/opensearch/helpers/scripts/create_timestamp_millis_fields.ts
new file mode 100644
index 0000000000..2ae9a04ed9
--- /dev/null
+++ b/infra/stacks/opensearch/helpers/scripts/create_timestamp_millis_fields.ts
@@ -0,0 +1,201 @@
+require('dotenv').config();
+
+import type { Client } from '@opensearch-project/opensearch';
+import { client } from '../client';
+import {
+ CALL_RECORDS_ALIAS,
+ CHANNEL_INDEX,
+ CHAT_INDEX,
+ DOCUMENT_INDEX,
+ EMAIL_INDEX,
+ PROJECTS_ALIAS,
+} from '../constants';
+import { checkIndexExists } from '../utils/check_index_exists';
+
+// A real `epoch_millis` date field to add.
+type MillisField = { fieldName: string };
+// An `alias` field pointing at an already-present real field. Used only by
+// call_records, mirroring its existing created_at_seconds/updated_at_seconds
+// aliases onto started_at_seconds.
+type AliasField = { fieldName: string; aliasFor: string };
+
+interface IndexMigration {
+ indexName: string;
+ fields: MillisField[];
+ aliases?: AliasField[];
+}
+
+// Millisecond counterparts of every `*_seconds` field, matching the mappings
+// in create_indices.ts. call_records keeps only started_at/ended_at as real
+// fields; created_at/updated_at are aliases onto started_at.
+const MIGRATIONS: IndexMigration[] = [
+ {
+ indexName: CHANNEL_INDEX,
+ fields: [
+ { fieldName: 'created_at_millis' },
+ { fieldName: 'updated_at_millis' },
+ ],
+ },
+ {
+ indexName: CHAT_INDEX,
+ fields: [
+ { fieldName: 'created_at_millis' },
+ { fieldName: 'updated_at_millis' },
+ ],
+ },
+ {
+ indexName: DOCUMENT_INDEX,
+ fields: [{ fieldName: 'updated_at_millis' }],
+ },
+ {
+ indexName: EMAIL_INDEX,
+ fields: [
+ { fieldName: 'updated_at_millis' },
+ { fieldName: 'sent_at_millis' },
+ ],
+ },
+ {
+ indexName: PROJECTS_ALIAS,
+ fields: [
+ { fieldName: 'created_at_millis' },
+ { fieldName: 'updated_at_millis' },
+ ],
+ },
+ {
+ indexName: CALL_RECORDS_ALIAS,
+ fields: [
+ { fieldName: 'started_at_millis' },
+ { fieldName: 'ended_at_millis' },
+ ],
+ aliases: [
+ { fieldName: 'created_at_millis', aliasFor: 'started_at_millis' },
+ { fieldName: 'updated_at_millis', aliasFor: 'started_at_millis' },
+ ],
+ },
+];
+
+async function putMapping(
+ opensearchClient: Client,
+ indexName: string,
+ properties: Record<string, unknown>,
+ label: string,
+ dryRun: boolean
+): Promise<void> {
+ console.log(
+ ` ${dryRun ? '[DRY-RUN] Would add' : 'Adding'} ${label} to index "${indexName}"`
+ );
+ if (dryRun) {
+ return;
+ }
+ const response = await opensearchClient.indices.putMapping({
+ index: indexName,
+ body: { properties },
+ });
+ if (!response.body.acknowledged) {
+ throw new Error(`Failed to add ${label} in index "${indexName}"`);
+ }
+ console.log(` ✓ Successfully added ${label}`);
+}
+
+async function addFieldsToIndex(
+ opensearchClient: Client,
+ migration: IndexMigration,
+ dryRun: boolean
+): Promise<void> {
+ console.log(`\n${'='.repeat(60)}`);
+ console.log(
+ `Adding millis fields to index: ${migration.indexName} ${dryRun ? '(DRY-RUN)' : ''}`
+ );
+ console.log(`${'='.repeat(60)}`);
+
+ const indexExists = await checkIndexExists(
+ opensearchClient,
+ migration.indexName
+ );
+ if (!indexExists) {
+ console.log(
+ `⚠️ Index "${migration.indexName}" does not exist. Skipping...`
+ );
+ return;
+ }
+
+ // Real date fields first, so any aliases can reference them.
+ for (const field of migration.fields) {
+ await putMapping(
+ opensearchClient,
+ migration.indexName,
+ {
+ [field.fieldName]: {
+ type: 'date',
+ format: 'epoch_millis',
+ index: false,
+ doc_values: true,
+ },
+ },
+ `field "${field.fieldName}" (epoch_millis)`,
+ dryRun
+ );
+ }
+
+ for (const alias of migration.aliases ?? []) {
+ await putMapping(
+ opensearchClient,
+ migration.indexName,
+ { [alias.fieldName]: { type: 'alias', path: alias.aliasFor } },
+ `alias "${alias.fieldName}" → "${alias.aliasFor}"`,
+ dryRun
+ );
+ }
+
+ console.log(`\n✓ Completed millis fields for index: ${migration.indexName}`);
+}
+
+async function createFields(dryRun: boolean = true) {
+ const opensearchClient = client();
+
+ console.log('\n' + '='.repeat(60));
+ console.log(
+ `Create Timestamp Millis Fields ${dryRun ? '(DRY-RUN MODE)' : '(LIVE MODE)'}`
+ );
+ console.log('='.repeat(60));
+ console.log(
+ '\nThis script adds *_millis (epoch_millis) timestamp fields to indices.'
+ );
+ console.log(
+ 'Additive and idempotent; safe to run before or after the millis backfill.'
+ );
+
+ if (dryRun) {
+ console.log('\n⚠️ DRY-RUN MODE: No changes will be made to the cluster');
+ } else {
+ console.log('\n🚨 LIVE MODE: Fields will be added to indices');
+ }
+
+ try {
+ for (const migration of MIGRATIONS) {
+ await addFieldsToIndex(opensearchClient, migration, dryRun);
+ }
+
+ console.log('\n' + '='.repeat(60));
+ console.log('Field creation completed successfully!');
+ console.log('='.repeat(60));
+
+ if (dryRun) {
+ console.log(
+ '\nTo run for real, set DRY_RUN=false environment variable\n'
+ );
+ } else {
+ console.log('\n✓ All *_millis fields have been added to indices.');
+ console.log(
+ '✓ Next step: run copy_timestamp_seconds_to_millis.ts to backfill existing docs.\n'
+ );
+ }
+ } catch (error) {
+ console.error('\n❌ Field creation failed:', error);
+ throw error;
+ }
+}
+
+const isDryRun = process.env.DRY_RUN !== 'false';
+
+createFields(isDryRun);
diff --git a/infra/stacks/opensearch/helpers/utils/copy_field.ts b/infra/stacks/opensearch/helpers/utils/copy_field.ts
index 4eed30bd3a..0dbde0c098 100644
--- a/infra/stacks/opensearch/helpers/utils/copy_field.ts
+++ b/infra/stacks/opensearch/helpers/utils/copy_field.ts
@@ -10,6 +10,9 @@ import type { Client } from '@opensearch-project/opensearch';
* @param includeNonNull A boolean indicating whether to overwrite existing non-null values in newField.
* If false (default), only updates documents where newField doesn't exist.
* If true, updates all documents regardless of existing newField values.
+ * @param valueExpr A painless expression producing the value to assign to newField.
+ * Defaults to a verbatim copy (`ctx._source.<oldField>`). Pass a
+ * transform (e.g. `ctx._source.<oldField> * 1000L`) to derive newField.
* @returns A Promise that resolves to the number of documents updated.
*/
export async function copyFieldData(
@@ -18,11 +21,12 @@ export async function copyFieldData(
oldField: string,
newField: string,
dryRun: boolean,
- includeNonNull: boolean = false
+ includeNonNull: boolean = false,
+ valueExpr: string = `ctx._source.${oldField}`
): Promise<number> {
const script = {
- // If the oldField exists and is not null, copy the value to the new field
- source: `if (ctx._source.containsKey('${oldField}') && ctx._source.${oldField} != null) { ctx._source.${newField} = ctx._source.${oldField}; }`,
+ // If the oldField exists and is not null, assign the derived value to the new field
+ source: `if (ctx._source.containsKey('${oldField}') && ctx._source.${oldField} != null) { ctx._source.${newField} = ${valueExpr}; }`,
lang: 'painless',
};
diff --git a/services/search_processing_service/src/process/call.rs b/services/search_processing_service/src/process/call.rs
index a5b7c4017c..6015c3de4f 100644
--- a/services/search_processing_service/src/process/call.rs
+++ b/services/search_processing_service/src/process/call.rs
@@ -1,7 +1,9 @@
use anyhow::Context;
use models_properties::EntityType;
use opensearch_client::{
- OpensearchClient, date_format::EpochSeconds, upsert::call_record::UpsertCallRecordSegmentArgs,
+ OpensearchClient,
+ date_format::{EpochMillis, EpochSeconds},
+ upsert::call_record::UpsertCallRecordSegmentArgs,
};
use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;
use sqlx::PgPool;
@@ -60,6 +62,10 @@ pub async fn process_call_record(
.ended_at
.map(|dt| EpochSeconds::new(dt.timestamp()))
.transpose()?;
+ let ended_at_millis = seg
+ .ended_at
+ .map(|dt| EpochMillis::new(dt.timestamp_millis()))
+ .transpose()?;
Ok::<_, anyhow::Error>(UpsertCallRecordSegmentArgs {
call_id: call_id_s.clone(),
transcript_id: seg.transcript_id.to_string(),
@@ -72,6 +78,8 @@ pub async fn process_call_record(
content: seg.content,
started_at_seconds: EpochSeconds::new(seg.started_at.timestamp())?,
ended_at_seconds,
+ started_at_millis: EpochMillis::new(seg.started_at.timestamp_millis())?,
+ ended_at_millis,
properties: properties.clone(),
})
})
diff --git a/services/search_processing_service/src/process/channel.rs b/services/search_processing_service/src/process/channel.rs
index 68c0222858..1e66dd53cc 100644
--- a/services/search_processing_service/src/process/channel.rs
+++ b/services/search_processing_service/src/process/channel.rs
@@ -1,7 +1,9 @@
use anyhow::Context;
use mention_utils::parse::{ParsedXmlText, PlainTextFormatter, XmlFormatter};
use opensearch_client::{
- OpensearchClient, date_format::EpochSeconds, upsert::channel_message::UpsertChannelMessageArgs,
+ OpensearchClient,
+ date_format::{EpochMillis, EpochSeconds},
+ upsert::channel_message::UpsertChannelMessageArgs,
};
use sqlx::{Pool, Postgres};
use sqs_client::search::channel::{ChannelMessageUpdate, RemoveChannelMessage};
@@ -61,6 +63,18 @@ pub async fn process_channel_message_update(
updated_at_seconds: EpochSeconds::new(
channel_message_info.channel_message.updated_at.timestamp(),
)?,
+ created_at_millis: EpochMillis::new(
+ channel_message_info
+ .channel_message
+ .created_at
+ .timestamp_millis(),
+ )?,
+ updated_at_millis: EpochMillis::new(
+ channel_message_info
+ .channel_message
+ .updated_at
+ .timestamp_millis(),
+ )?,
};
opensearch_client
diff --git a/services/search_processing_service/src/process/chat.rs b/services/search_processing_service/src/process/chat.rs
index 5edaeaafbf..7934ebd96b 100644
--- a/services/search_processing_service/src/process/chat.rs
+++ b/services/search_processing_service/src/process/chat.rs
@@ -1,7 +1,9 @@
use anyhow::Context;
use models_properties::EntityType;
use opensearch_client::{
- OpensearchClient, date_format::EpochSeconds, upsert::chat_message::UpsertChatMessageArgs,
+ OpensearchClient,
+ date_format::{EpochMillis, EpochSeconds},
+ upsert::chat_message::UpsertChatMessageArgs,
};
use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;
use sqs_client::search::chat::{ChatMessage, RemoveChatMessage};
@@ -58,6 +60,8 @@ pub async fn insert_chat_message(
user_id: chat_message.user_id.clone(),
created_at_seconds: EpochSeconds::new(chat_message.created_at.timestamp())?,
updated_at_seconds: EpochSeconds::new(chat_message.updated_at.timestamp())?,
+ created_at_millis: EpochMillis::new(chat_message.created_at.timestamp_millis())?,
+ updated_at_millis: EpochMillis::new(chat_message.updated_at.timestamp_millis())?,
title: info.name,
content: info.content,
role: info.role,
diff --git a/services/search_processing_service/src/process/document/raw_document.rs b/services/search_processing_service/src/process/document/raw_document.rs
index f3aeb08b23..e296483b5c 100644
--- a/services/search_processing_service/src/process/document/raw_document.rs
+++ b/services/search_processing_service/src/process/document/raw_document.rs
@@ -8,7 +8,9 @@ use models_properties::EntityType;
use models_search::document::MarkdownParseResult;
use models_search::unified::is_searchable_association;
use opensearch_client::{
- OpensearchClient, date_format::EpochSeconds, upsert::document::UpsertDocumentArgs,
+ OpensearchClient,
+ date_format::{EpochMillis, EpochSeconds},
+ upsert::document::UpsertDocumentArgs,
};
use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;
use s3_key::{
@@ -125,6 +127,7 @@ fn generate_parent_only_upsert(
owner_id: document_info.owner.to_string(),
file_type,
updated_at_seconds: EpochSeconds::new(Utc::now().timestamp())?,
+ updated_at_millis: EpochMillis::new(Utc::now().timestamp_millis())?,
sub_type: document_info.sub_type.map(|st| st.to_string()),
properties: vec![],
}))
@@ -293,6 +296,7 @@ pub async fn update_search_with_raw_document(
tracing::trace!("got raw file content");
let updated_at = EpochSeconds::new(Utc::now().timestamp())?;
+ let updated_at_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
let uuid = macro_uuid::generate_uuid_v7().to_string();
let mut upserts: Vec<UpsertDocumentArgs> = match file_type {
@@ -312,6 +316,7 @@ pub async fn update_search_with_raw_document(
owner_id: search_extractor_message.user_id.clone(),
file_type: file_type.to_string(),
updated_at_seconds: updated_at,
+ updated_at_millis,
sub_type: sub_type.clone(),
properties: vec![],
})
@@ -336,6 +341,7 @@ pub async fn update_search_with_raw_document(
owner_id: search_extractor_message.user_id.clone(),
file_type: file_type.to_string(),
updated_at_seconds: updated_at,
+ updated_at_millis,
sub_type: sub_type.clone(),
properties: vec![],
}]
@@ -357,6 +363,7 @@ pub async fn update_search_with_raw_document(
owner_id: search_extractor_message.user_id.clone(),
file_type: file_type.to_string(),
updated_at_seconds: updated_at,
+ updated_at_millis,
sub_type: sub_type.clone(),
properties: vec![],
})
@@ -376,6 +383,7 @@ pub async fn update_search_with_raw_document(
owner_id: search_extractor_message.user_id.clone(),
file_type: file_type.to_string(),
updated_at_seconds: updated_at,
+ updated_at_millis,
sub_type: sub_type.clone(),
properties: vec![],
}]
@@ -402,6 +410,7 @@ fn generate_upserts(
) -> anyhow::Result<Vec<UpsertDocumentArgs>> {
let result = markdown_result;
let updated_at = EpochSeconds::new(Utc::now().timestamp())?;
+ let updated_at_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
let file_type = FileType::from_str(
document_info
.file_type
@@ -422,6 +431,7 @@ fn generate_upserts(
owner_id: document_info.owner.to_string(),
file_type: file_type.to_string(),
updated_at_seconds: updated_at,
+ updated_at_millis,
sub_type: sub_type.clone(),
properties: vec![],
})
diff --git a/services/search_processing_service/src/process/email/upsert.rs b/services/search_processing_service/src/process/email/upsert.rs
index bb5b19cb9c..053f7fe228 100644
--- a/services/search_processing_service/src/process/email/upsert.rs
+++ b/services/search_processing_service/src/process/email/upsert.rs
@@ -3,7 +3,9 @@ use chrono::Utc;
use models_email::service::label::system_labels;
use models_properties::EntityType;
use opensearch_client::{
- OpensearchClient, date_format::EpochSeconds, upsert::email::UpsertEmailArgs,
+ OpensearchClient,
+ date_format::{EpochMillis, EpochSeconds},
+ upsert::email::UpsertEmailArgs,
};
use properties::outbound::entity_properties_get_query::{
get_entity_properties_for_index, get_entity_properties_for_index_batch,
@@ -53,6 +55,7 @@ pub async fn process_upsert_message(
};
let now = EpochSeconds::new(Utc::now().timestamp())?;
+ let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
let updated_at = message_info
.internal_date_ts
@@ -60,6 +63,12 @@ pub async fn process_upsert_message(
.transpose()?
.unwrap_or(now);
+ let updated_at_millis = message_info
+ .internal_date_ts
+ .map(|date| EpochMillis::new(date.timestamp_millis()))
+ .transpose()?
+ .unwrap_or(now_millis);
+
// A full index overwrites the doc, so thread properties must ride along
// or a reindex would drop them. A fetch failure propagates (retry)
// instead of being mistaken for "no properties".
@@ -124,10 +133,15 @@ pub async fn process_upsert_message(
.collect(),
content,
updated_at_seconds: updated_at,
+ updated_at_millis,
sent_at_seconds: message_info
.internal_date_ts
.map(|date| EpochSeconds::new(date.timestamp()))
.transpose()?,
+ sent_at_millis: message_info
+ .internal_date_ts
+ .map(|date| EpochMillis::new(date.timestamp_millis()))
+ .transpose()?,
properties,
};
@@ -153,6 +167,7 @@ pub async fn process_upsert_thread_message(
.context("failed to parse thread_id as UUID")?;
let now = EpochSeconds::new(Utc::now().timestamp())?;
+ let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
// A full index overwrites each doc, so thread properties must ride along
// or a reindex would drop them.
@@ -195,12 +210,23 @@ pub async fn process_upsert_thread_message(
.map(|date| EpochSeconds::new(date.timestamp()))
.transpose()?;
+ let sent_at_millis = message
+ .internal_date_ts
+ .map(|date| EpochMillis::new(date.timestamp_millis()))
+ .transpose()?;
+
let updated_at = message
.internal_date_ts
.map(|date| EpochSeconds::new(date.timestamp()))
.transpose()?
.unwrap_or(now);
+ let updated_at_millis = message
+ .internal_date_ts
+ .map(|date| EpochMillis::new(date.timestamp_millis()))
+ .transpose()?
+ .unwrap_or(now_millis);
+
upsert_email_message_args.push(UpsertEmailArgs {
message_id: message.db_id.to_string(),
link_id: message.link_id.to_string(),
@@ -243,7 +269,9 @@ pub async fn process_upsert_thread_message(
.collect(),
content,
updated_at_seconds: updated_at,
+ updated_at_millis,
sent_at_seconds: sent_at,
+ sent_at_millis,
properties: properties.clone(),
});
} else {
@@ -287,6 +315,7 @@ pub async fn process_upsert_thread_batch_message(
.collect::<anyhow::Result<Vec<_>>>()?;
let now = EpochSeconds::new(Utc::now().timestamp())?;
+ let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
let messages =
email_db_client::messages::get_parsed_search::get_parsed_search_messages_by_thread_ids(
@@ -322,12 +351,23 @@ pub async fn process_upsert_thread_batch_message(
.map(|date| EpochSeconds::new(date.timestamp()))
.transpose()?;
+ let sent_at_millis = message
+ .internal_date_ts
+ .map(|date| EpochMillis::new(date.timestamp_millis()))
+ .transpose()?;
+
let updated_at = message
.internal_date_ts
.map(|date| EpochSeconds::new(date.timestamp()))
.transpose()?
.unwrap_or(now);
+ let updated_at_millis = message
+ .internal_date_ts
+ .map(|date| EpochMillis::new(date.timestamp_millis()))
+ .transpose()?
+ .unwrap_or(now_millis);
+
upsert_email_message_args.push(UpsertEmailArgs {
message_id: message.db_id.to_string(),
link_id: message.link_id.to_string(),
@@ -370,7 +410,9 @@ pub async fn process_upsert_thread_batch_message(
.collect(),
content,
updated_at_seconds: updated_at,
+ updated_at_millis,
sent_at_seconds: sent_at,
+ sent_at_millis,
properties: properties_by_thread
.get(&message.thread_db_id.to_string())
.cloned()
diff --git a/services/search_processing_service/src/process/project.rs b/services/search_processing_service/src/process/project.rs
index 07b3aba966..607face939 100644
--- a/services/search_processing_service/src/process/project.rs
+++ b/services/search_processing_service/src/process/project.rs
@@ -2,7 +2,7 @@ use anyhow::Context;
use models_properties::EntityType;
use opensearch_client::{
OpensearchClient,
- date_format::EpochSeconds,
+ date_format::{EpochMillis, EpochSeconds},
upsert::{project::UpsertProjectArgs, properties::IndexedProperty},
};
use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;
@@ -77,6 +77,8 @@ pub async fn upsert_project(
parent_project_id: project.parent_id,
created_at_seconds: EpochSeconds::new(created_at.timestamp())?,
updated_at_seconds: EpochSeconds::new(updated_at.timestamp())?,
+ created_at_millis: EpochMillis::new(created_at.timestamp_millis())?,
+ updated_at_millis: EpochMillis::new(updated_at.timestamp_millis())?,
properties,
},
index_override,