Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
refactor(search): make OpenSearch timestamp handling millis-only (#4912)
gbirman4726341fdd42
diff --git a/crates/opensearch_client/src/date_format.rs b/crates/opensearch_client/src/date_format.rsindex c6ae20893c..d996c8a959 100644--- a/crates/opensearch_client/src/date_format.rs+++ b/crates/opensearch_client/src/date_format.rs@@ -6,35 +6,6 @@ const MIN_REASONABLE_SECONDS: i64 = 946684800; // Year 2000const MAX_REASONABLE_MILLIS: i64 = MAX_REASONABLE_SECONDS * 1000;const MIN_REASONABLE_MILLIS: i64 = MIN_REASONABLE_SECONDS * 1000;-#[derive(Debug, Clone, Copy, serde::Serialize)]-pub struct EpochSeconds(i64);--impl EpochSeconds {- pub fn new(seconds: i64) -> Result<Self> {- if seconds > MAX_REASONABLE_SECONDS {- return Err(OpensearchClientError::ValidationFailed {- details: format!(- "timestamp {} appears to be in milliseconds (exceeds year 3000). Expected seconds since Unix epoch.",- seconds- ),- });- }- if seconds < MIN_REASONABLE_SECONDS {- return Err(OpensearchClientError::ValidationFailed {- details: format!(- "timestamp {} is before year 2000. Expected seconds since Unix epoch.",- seconds- ),- });- }- Ok(Self(seconds))- }-- pub fn get(&self) -> i64 {- self.0- }-}-/// A validated epoch-milliseconds timestamp, serialized as a raw `i64` for/// OpenSearch date fields mapped with `format: epoch_millis`.#[derive(Debug, Clone, Copy, serde::Serialize)]diff --git a/crates/opensearch_client/src/date_format/test.rs b/crates/opensearch_client/src/date_format/test.rsindex e8df60369c..11fc8d9cd6 100644--- a/crates/opensearch_client/src/date_format/test.rs+++ b/crates/opensearch_client/src/date_format/test.rs@@ -1,40 +1,5 @@use super::*;-#[test]-fn test_epoch_seconds_valid() {- let now = 1704067200; // 2024-01-01- let epoch = EpochSeconds::new(now);- assert!(epoch.is_ok());- assert_eq!(epoch.unwrap().get(), now);-}--#[test]-fn test_epoch_seconds_milliseconds() {- let now_millis = 1704067200000; // 2024-01-01 in milliseconds- let result = EpochSeconds::new(now_millis);- assert!(result.is_err());- if let Err(OpensearchClientError::ValidationFailed { details }) = result {- assert!(details.contains("appears to be in milliseconds"));- }-}--#[test]-fn test_epoch_seconds_too_old() {- let old = 946684799; // Just before year 2000- let result = EpochSeconds::new(old);- assert!(result.is_err());- if let Err(OpensearchClientError::ValidationFailed { details }) = result {- assert!(details.contains("before year 2000"));- }-}--#[test]-fn test_epoch_seconds_serialize() {- let epoch = EpochSeconds::new(1704067200).unwrap();- let serialized = serde_json::to_string(&epoch).unwrap();- assert_eq!(serialized, "1704067200");-}-#[test]fn test_epoch_millis_valid() {let now_millis = 1704067200123; // 2024-01-01 with sub-second precisiondiff --git a/crates/opensearch_client/src/search/builder.rs b/crates/opensearch_client/src/search/builder.rsindex 7c5cceb1bf..e74a6bbf63 100644--- a/crates/opensearch_client/src/search/builder.rs+++ b/crates/opensearch_client/src/search/builder.rs@@ -47,8 +47,7 @@ macro_rules! delegate_methods {/// Sort mode for the channel content endpoint.#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]pub enum ChannelSortMode {- /// Sort by sent_at/updated_at (millis, falling back to seconds), entity_id- /// as tiebreaker.+ /// Sort by sent_at/updated_at (millis), entity_id as tiebreaker.#[default]Message,/// Sort by thread_id, message_id as tiebreaker.@@ -57,22 +56,16 @@ pub enum ChannelSortMode {/// Creates sort vec to sort by sent_at (preferred) or updated_at (fallback)/// with entity_id as a tiebreaker. Items without a timestamp are pushed to the-/// end. Prefers the millisecond-resolution `*_millis` fields and falls back to-/// the second-resolution `*_seconds` fields, so docs indexed before the-/// millis backfill still sort correctly. `.toEpochMilli()` normalizes both to-/// milliseconds either way.+/// end. Sorts on the millisecond-resolution `*_millis` fields.+/// `.toEpochMilli()` normalizes to milliseconds.pub(crate) fn updated_at_sort<'a>() -> Vec<SortType<'a>> {vec![SortType::ScriptSort(ScriptSort::new(Script::new(r#"if (doc.containsKey('sent_at_millis') && doc['sent_at_millis'].size() > 0) {return doc['sent_at_millis'].value.toInstant().toEpochMilli();- } else if (doc.containsKey('sent_at_seconds') && doc['sent_at_seconds'].size() > 0) {- return doc['sent_at_seconds'].value.toInstant().toEpochMilli();} else if (doc.containsKey('updated_at_millis') && doc['updated_at_millis'].size() > 0) {return doc['updated_at_millis'].value.toInstant().toEpochMilli();- } else if (doc.containsKey('updated_at_seconds') && doc['updated_at_seconds'].size() > 0) {- return doc['updated_at_seconds'].value.toInstant().toEpochMilli();} else {return 0L;}"#,diff --git a/crates/opensearch_client/src/search/call_records.rs b/crates/opensearch_client/src/search/call_records.rsindex 58e46ed80d..68f11099db 100644--- a/crates/opensearch_client/src/search/call_records.rs+++ b/crates/opensearch_client/src/search/call_records.rs@@ -9,7 +9,7 @@ use crate::{},properties::build_tag_filter,query::Keys,- utils::{millis_or_seconds, opt_millis_or_seconds},+ utils::millis_to_datetime,},};@@ -45,9 +45,6 @@ pub(crate) struct CallRecordIndex {pub channel_name: Option<String>,#[serde(default)]pub name: Option<String>,- pub started_at_seconds: i64,- #[serde(default)]- pub ended_at_seconds: Option<i64>,#[serde(default)]pub started_at_millis: Option<i64>,#[serde(default)]@@ -359,9 +356,6 @@ struct SegmentSource {speaker_id: Option<String>,#[serde(default)]sequence_num: Option<i32>,- started_at_seconds: i64,- #[serde(default)]- ended_at_seconds: Option<i64>,#[serde(default)]started_at_millis: Option<i64>,#[serde(default)]@@ -423,18 +417,12 @@ pub(crate) fn expand_inner_hits_to_search_hits(transcript_id: seg.source.transcript_id,speaker_id: seg.source.speaker_id.unwrap_or_default(),sequence_num: seg.source.sequence_num.unwrap_or_default(),- started_at: millis_or_seconds(- seg.source.started_at_millis,- seg.source.started_at_seconds,- )- .unwrap_or_default(),- ended_at: opt_millis_or_seconds(- seg.source.ended_at_millis,- seg.source.ended_at_seconds,- ),+ started_at: millis_to_datetime(seg.source.started_at_millis)+ .unwrap_or_default(),+ ended_at: millis_to_datetime(seg.source.ended_at_millis),participant_ids: parent.participant_ids.clone(),})),- updated_at: millis_or_seconds(parent.started_at_millis, parent.started_at_seconds),+ updated_at: millis_to_datetime(parent.started_at_millis),});}}diff --git a/crates/opensearch_client/src/search/channels.rs b/crates/opensearch_client/src/search/channels.rsindex 22b667c701..254cd31d2d 100644--- a/crates/opensearch_client/src/search/channels.rs+++ b/crates/opensearch_client/src/search/channels.rs@@ -10,7 +10,7 @@ use crate::{exclude_source_content, inject_fragment_size, parse_highlight_hit,},query::{Keys, TermCombine},- utils::millis_or_seconds,+ utils::millis_to_datetime,},};@@ -30,8 +30,6 @@ pub(crate) struct ChannelMessageIndex {pub thread_id: uuid::Uuid,pub sender_id: String,pub mentions: Vec<String>,- pub created_at_seconds: i64,- pub updated_at_seconds: i64,#[serde(default)]pub created_at_millis: Option<i64>,#[serde(default)]@@ -312,12 +310,10 @@ fn channel_hit_to_search_hit(hit: Hit<ChannelMessageIndex>) -> SearchHit {channel_message_id: a.message_id,thread_id: (a.thread_id != a.message_id).then_some(a.thread_id),sender_id: a.sender_id,- created_at: millis_or_seconds(a.created_at_millis, a.created_at_seconds)- .unwrap_or_default(),- updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds)- .unwrap_or_default(),+ created_at: millis_to_datetime(a.created_at_millis).unwrap_or_default(),+ updated_at: millis_to_datetime(a.updated_at_millis).unwrap_or_default(),})),- updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),+ updated_at: millis_to_datetime(a.updated_at_millis),}}diff --git a/crates/opensearch_client/src/search/channels/test.rs b/crates/opensearch_client/src/search/channels/test.rsindex 945b2895d5..e40ddfe74f 100644--- a/crates/opensearch_client/src/search/channels/test.rs+++ b/crates/opensearch_client/src/search/channels/test.rs@@ -12,8 +12,6 @@ fn channel_index(updated_at_millis: Option<i64>) -> ChannelMessageIndex {thread_id: id,sender_id: "macro|[email protected]".to_string(),mentions: vec![],- created_at_seconds: 1_700_000_000,- updated_at_seconds: 1_700_000_000,created_at_millis: updated_at_millis,updated_at_millis,}@@ -46,17 +44,6 @@ fn millis_timestamps_are_distinct_within_a_second() {);}-/// Docs indexed before the millis backfill carry no `*_millis`; the read path-/// falls back to the second-resolution field.-#[test]-fn falls_back_to_seconds_when_millis_absent() {- let hit = channel_hit_to_search_hit(hit_for(channel_index(None)));- assert_eq!(- hit.updated_at.expect("updated_at").timestamp_millis(),- 1_700_000_000_000- );-}-#[test]fn test_build_bool_query() -> anyhow::Result<()> {let builder = ChannelMessageQueryBuilder::new(vec!["test".to_string()])diff --git a/crates/opensearch_client/src/search/chats.rs b/crates/opensearch_client/src/search/chats.rsindex 5ace04aa2e..40116c12bc 100644--- a/crates/opensearch_client/src/search/chats.rs+++ b/crates/opensearch_client/src/search/chats.rs@@ -41,7 +41,6 @@ pub(crate) struct ChatIndex {pub entity_id: uuid::Uuid,pub user_id: String,pub title: String,- pub updated_at_seconds: Option<i64>,#[serde(default)]pub updated_at_millis: Option<i64>,}diff --git a/crates/opensearch_client/src/search/documents.rs b/crates/opensearch_client/src/search/documents.rsindex 8d14359144..9d30a06c05 100644--- a/crates/opensearch_client/src/search/documents.rs+++ b/crates/opensearch_client/src/search/documents.rs@@ -366,7 +366,6 @@ pub(crate) struct DocumentIndex {pub document_name: String,pub owner_id: String,pub file_type: String,- pub updated_at_seconds: Option<i64>,#[serde(default)]pub updated_at_millis: Option<i64>,}diff --git a/crates/opensearch_client/src/search/documents/test.rs b/crates/opensearch_client/src/search/documents/test.rsindex 367c324cb3..3d3706806c 100644--- a/crates/opensearch_client/src/search/documents/test.rs+++ b/crates/opensearch_client/src/search/documents/test.rs@@ -395,7 +395,7 @@ fn document_index_deserializes_parent_shape() {"document_name": "Q3 Planning","owner_id": "macro|[email protected]","file_type": "pdf",- "updated_at_seconds": 1779000000_i64,+ "updated_at_millis": 1779000000000_i64,"document_relation": "document",});diff --git a/crates/opensearch_client/src/search/emails.rs b/crates/opensearch_client/src/search/emails.rsindex eef19a7beb..c8ec2ad070 100644--- a/crates/opensearch_client/src/search/emails.rs+++ b/crates/opensearch_client/src/search/emails.rs@@ -367,8 +367,6 @@ pub(crate) struct EmailIndex {pub user_id: String,/// The subject of the email messagepub subject: Option<String>,- /// The sent at time of the email message- pub sent_at_seconds: Option<i64>,/// The sent at time of the email message, in milliseconds#[serde(default)]pub sent_at_millis: Option<i64>,diff --git a/crates/opensearch_client/src/search/projects.rs b/crates/opensearch_client/src/search/projects.rsindex 6c382c3d3e..ee8d70c9c1 100644--- a/crates/opensearch_client/src/search/projects.rs+++ b/crates/opensearch_client/src/search/projects.rs@@ -94,7 +94,6 @@ pub(crate) struct ProjectIndex {pub owner_id: String,#[serde(default)]pub parent_project_id: Option<String>,- pub updated_at_seconds: Option<i64>,#[serde(default)]pub updated_at_millis: Option<i64>,}diff --git a/crates/opensearch_client/src/search/unified.rs b/crates/opensearch_client/src/search/unified.rsindex 4fffc78cfa..22bda6a627 100644--- a/crates/opensearch_client/src/search/unified.rs+++ b/crates/opensearch_client/src/search/unified.rs@@ -26,7 +26,7 @@ use crate::{},projects::{ProjectIndex, ProjectQueryBuilder, ProjectSearchArgs, ProjectSearchConfig},query::Keys,- utils::{millis_or_seconds, opt_millis_or_seconds},+ utils::millis_to_datetime,},};use chrono::Utc;@@ -366,8 +366,7 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {match &hit.source {UnifiedSearchIndex::Document(parent) => {let entity_id = parent.entity_id;- let updated_at =- opt_millis_or_seconds(parent.updated_at_millis, parent.updated_at_seconds);+ let updated_at = millis_to_datetime(parent.updated_at_millis);let mut out: Vec<SearchHit> = Vec::new();@@ -412,8 +411,7 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {return vec![hit.into()];};let entity_id = parent.entity_id;- let updated_at =- opt_millis_or_seconds(parent.updated_at_millis, parent.updated_at_seconds);+ let updated_at = millis_to_datetime(parent.updated_at_millis);let expanded = crate::search::chats::expand_inner_hits_to_search_hits(entity_id, updated_at, inner,);@@ -423,7 +421,7 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {expanded}UnifiedSearchIndex::CallRecord(parent) => {- let updated_at = millis_or_seconds(parent.started_at_millis, parent.started_at_seconds);+ let updated_at = millis_to_datetime(parent.started_at_millis);let mut out: Vec<SearchHit> = Vec::new();// A name match surfaces as a parent-level hit. Segment content lives@@ -451,10 +449,7 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {speaker_id: String::new(),sequence_num: 0,started_at: updated_at.unwrap_or_default(),- ended_at: opt_millis_or_seconds(- parent.ended_at_millis,- parent.ended_at_seconds,- ),+ ended_at: millis_to_datetime(parent.ended_at_millis),participant_ids: parent.participant_ids.clone(),})),updated_at,@@ -500,12 +495,10 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {channel_message_id: a.message_id,thread_id: (a.thread_id != a.message_id).then_some(a.thread_id),sender_id: a.sender_id,- created_at: millis_or_seconds(a.created_at_millis, a.created_at_seconds)- .unwrap_or_default(),- updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds)- .unwrap_or_default(),+ created_at: millis_to_datetime(a.created_at_millis).unwrap_or_default(),+ updated_at: millis_to_datetime(a.updated_at_millis).unwrap_or_default(),})),- updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),+ updated_at: millis_to_datetime(a.updated_at_millis),},UnifiedSearchIndex::Document(a) => SearchHit {entity_id: a.entity_id,@@ -524,7 +517,7 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {}).unwrap_or_default(),goto: None,- updated_at: opt_millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),+ updated_at: millis_to_datetime(a.updated_at_millis),},UnifiedSearchIndex::Email(a) => {let a = *a;@@ -549,11 +542,11 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {bcc: a.bcc,cc: a.cc,labels: a.labels,- sent_at: opt_millis_or_seconds(a.sent_at_millis, a.sent_at_seconds),+ sent_at: millis_to_datetime(a.sent_at_millis),sender: a.sender,recipients: a.recipients,})),- updated_at: opt_millis_or_seconds(a.sent_at_millis, a.sent_at_seconds),+ updated_at: millis_to_datetime(a.sent_at_millis),}}UnifiedSearchIndex::Chat(a) => SearchHit {@@ -573,7 +566,7 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {}).unwrap_or_default(),goto: None,- updated_at: opt_millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),+ updated_at: millis_to_datetime(a.updated_at_millis),},UnifiedSearchIndex::Project(a) => SearchHit {entity_id: a.entity_id,@@ -592,7 +585,7 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {}).unwrap_or_default(),goto: None,- updated_at: opt_millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),+ updated_at: millis_to_datetime(a.updated_at_millis),},UnifiedSearchIndex::CallRecord(a) => SearchHit {entity_id: a.entity_id,@@ -619,12 +612,11 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {transcript_id: uuid::Uuid::nil(),speaker_id: String::new(),sequence_num: 0,- started_at: millis_or_seconds(a.started_at_millis, a.started_at_seconds)- .unwrap_or_default(),- ended_at: opt_millis_or_seconds(a.ended_at_millis, a.ended_at_seconds),+ started_at: millis_to_datetime(a.started_at_millis).unwrap_or_default(),+ ended_at: millis_to_datetime(a.ended_at_millis),participant_ids: a.participant_ids,})),- updated_at: millis_or_seconds(a.started_at_millis, a.started_at_seconds),+ updated_at: millis_to_datetime(a.started_at_millis),},}}diff --git a/crates/opensearch_client/src/search/unified/test.rs b/crates/opensearch_client/src/search/unified/test.rsindex 312eb82ca0..9d39b0f225 100644--- a/crates/opensearch_client/src/search/unified/test.rs+++ b/crates/opensearch_client/src/search/unified/test.rs@@ -22,8 +22,7 @@ fn expand_document_name_highlight_yields_name_hit_without_goto() {document_name: "Q3 Report".to_string(),owner_id: "alice".to_string(),file_type: "md".to_string(),- updated_at_seconds: Some(1_779_000_000),- updated_at_millis: None,+ updated_at_millis: Some(1_779_000_000_000),}),highlight: Some(HashMap::from([("document_name".to_string(),@@ -85,10 +84,10 @@ fn test_deserialization() -> anyhow::Result<()> {"IMPORTANT"],"thread_id": "019a5a07-a1ed-7342-93a7-861582b59a55",- "sent_at_seconds": 1762447040,+ "sent_at_millis": 1762447040000_i64,"sender": "[email protected]","user_id": "macro|[email protected]",- "updated_at_seconds": 1762447044,+ "updated_at_millis": 1762447044000_i64,"recipients": []@@ -123,10 +122,10 @@ fn test_deserialization() -> anyhow::Result<()> {"sent_at": 1759241406,"thread_id": "01999c74-fb30-7c8a-b334-16035ee46783","updated_at": 1759266586,- "sent_at_seconds": 1759241406,+ "sent_at_millis": 1759241406000_i64,"sender": "[email protected]","user_id": "macro|[email protected]",- "updated_at_seconds": 1759266586,+ "updated_at_millis": 1759266586000_i64,"recipients": []@@ -146,7 +145,7 @@ fn test_deserialization() -> anyhow::Result<()> {"_source": {"document_name": "Rust for Rustaceans Idiomatic Programming for Experienced Developers (1)","updated_at": 1752523635,- "updated_at_seconds": 1752523635,+ "updated_at_millis": 1752523635000_i64,"file_type": "pdf","owner_id": "macro|[email protected]","document_id": "013021c7-5e14-4aab-855e-01413fff85dc",@@ -183,10 +182,10 @@ fn test_deserialization() -> anyhow::Result<()> {"sent_at": 1761742081,"thread_id": "019a3002-d9ff-79ad-9a5f-c3116b5b229a","updated_at": 1761742084,- "sent_at_seconds": 1761742081,+ "sent_at_millis": 1761742081000_i64,"sender": "[email protected]","user_id": "macro|[email protected]",- "updated_at_seconds": 1761742084,+ "updated_at_millis": 1761742084000_i64,"recipients": []@@ -207,7 +206,7 @@ fn test_deserialization() -> anyhow::Result<()> {"_source": {"document_name": "Rust for Rustaceans Idiomatic Programming for Experienced Developers (1)","updated_at": 1752522588,- "updated_at_seconds": 1752522588,+ "updated_at_millis": 1752522588000_i64,"file_type": "pdf","owner_id": "macro|[email protected]","document_id": "71ff8822-b421-4cab-b23e-f65355bf90d4",@@ -248,10 +247,10 @@ fn test_deserialization() -> anyhow::Result<()> {"sent_at": 1761313387,"thread_id": "019a1675-844c-7dcc-bd45-109c25b94fa0","updated_at": 1761313391,- "sent_at_seconds": 1761313387,+ "sent_at_millis": 1761313387000_i64,"sender": "[email protected]","user_id": "macro|[email protected]",- "updated_at_seconds": 1761313391,+ "updated_at_millis": 1761313391000_i64,"recipients": []@@ -290,10 +289,10 @@ fn test_deserialization() -> anyhow::Result<()> {"sent_at": 1761848050,"thread_id": "019a3653-c914-769b-9454-3b70dc2ad966","updated_at": 1761848053,- "sent_at_seconds": 1761848050,+ "sent_at_millis": 1761848050000_i64,"sender": "[email protected]","user_id": "macro|[email protected]",- "updated_at_seconds": 1761848053,+ "updated_at_millis": 1761848053000_i64,"recipients": []@@ -332,10 +331,10 @@ fn test_deserialization() -> anyhow::Result<()> {"sent_at": 1761252648,"thread_id": "019a12cf-ed6d-7cbc-b705-a78bbbe1bba0","updated_at": 1761252711,- "sent_at_seconds": 1761252648,+ "sent_at_millis": 1761252648000_i64,"sender": "[email protected]","user_id": "macro|[email protected]",- "updated_at_seconds": 1761252711,+ "updated_at_millis": 1761252711000_i64,"recipients": []@@ -371,10 +370,10 @@ fn test_deserialization() -> anyhow::Result<()> {"sent_at": 1761833460,"thread_id": "019a3575-3540-7231-8b83-51279a1bf623","updated_at": 1761833465,- "sent_at_seconds": 1761833460,+ "sent_at_millis": 1761833460000_i64,"sender": "[email protected]","user_id": "macro|[email protected]",- "updated_at_seconds": 1761833465,+ "updated_at_millis": 1761833465000_i64,"recipients": []@@ -413,10 +412,10 @@ fn test_deserialization() -> anyhow::Result<()> {"sent_at": 1761753544,"thread_id": "019a30ae-2a24-78b3-9eef-e3aa7594a1e4","updated_at": 1761753548,- "sent_at_seconds": 1761753544,+ "sent_at_millis": 1761753544000_i64,"sender": "[email protected]","user_id": "macro|[email protected]",- "updated_at_seconds": 1761753548,+ "updated_at_millis": 1761753548000_i64,"recipients": []@@ -907,7 +906,7 @@ fn test_thread_sort_is_thread_id_then_message_id_field_sorts() {/// content query.fn doc_hit_with_chunks(entity_id: uuid::Uuid,- updated_at_seconds: i64,+ updated_at_millis: i64,chunk_count: usize,) -> Hit<UnifiedSearchIndex> {let chunks: Vec<serde_json::Value> = (0..chunk_count)@@ -928,8 +927,7 @@ fn doc_hit_with_chunks(document_name: "Doc".to_string(),owner_id: "alice".to_string(),file_type: "md".to_string(),- updated_at_seconds: Some(updated_at_seconds),- updated_at_millis: None,+ updated_at_millis: Some(updated_at_millis),}),highlight: None,inner_hits: Some(serde_json::json!({ "term_0": { "hits": { "hits": chunks } } })),@@ -948,8 +946,8 @@ fn paginate_counts_parent_entities_not_inner_hits() {.parse::<uuid::Uuid>().unwrap();let hits = vec![- doc_hit_with_chunks(d1, 1_779_000_100, 30),- doc_hit_with_chunks(d2, 1_779_000_050, 30),+ doc_hit_with_chunks(d1, 1_779_000_100_000, 30),+ doc_hit_with_chunks(d2, 1_779_000_050_000, 30),];let (results, cursor) = paginate_unified_hits(hits, 10);@@ -979,9 +977,9 @@ fn paginate_has_more_when_parent_count_exceeds_page_size() {.parse::<uuid::Uuid>().unwrap();let hits = vec![- doc_hit_with_chunks(d1, 1_779_000_300, 5),- doc_hit_with_chunks(d2, 1_779_000_200, 5),- doc_hit_with_chunks(d3, 1_779_000_100, 5),+ doc_hit_with_chunks(d1, 1_779_000_300_000, 5),+ doc_hit_with_chunks(d2, 1_779_000_200_000, 5),+ doc_hit_with_chunks(d3, 1_779_000_100_000, 5),];let (results, cursor) = paginate_unified_hits(hits, 2);@@ -1014,7 +1012,7 @@ fn paginate_page_size_zero_terminates() {let d1 = "00000000-0000-0000-0000-000000000001".parse::<uuid::Uuid>().unwrap();- let hits = vec![doc_hit_with_chunks(d1, 1_779_000_100, 5)];+ let hits = vec![doc_hit_with_chunks(d1, 1_779_000_100_000, 5)];let (results, cursor) = paginate_unified_hits(hits, 0);@@ -1048,8 +1046,8 @@ fn project_hit_deserializes_and_converts() -> anyhow::Result<()> {"name": "Mobile Redesign","owner_id": "macro|[email protected]","parent_project_id": "0197a863-0000-7000-8000-000000000002",- "created_at_seconds": 1770000000,- "updated_at_seconds": 1783000000,+ "created_at_millis": 1770000000000_i64,+ "updated_at_millis": 1783000000000_i64,"properties": [{ "definition_id": "0197a863-0000-7000-8000-00000000000a", "values": ["opt-1"] }]diff --git a/crates/opensearch_client/src/search/utils.rs b/crates/opensearch_client/src/search/utils.rsindex 7468ad4732..bd38486991 100644--- a/crates/opensearch_client/src/search/utils.rs+++ b/crates/opensearch_client/src/search/utils.rs@@ -1,24 +1,9 @@use chrono::{DateTime, Utc};use opensearch_query_builder::{BoolQueryBuilder, QueryType, WildcardQuery};-/// Prefer a millisecond epoch timestamp, falling back to a second epoch-/// timestamp. During the seconds→millis migration new/backfilled docs carry a-/// `*_millis` field while older docs only have `*_seconds`.-pub(crate) fn millis_or_seconds(millis: Option<i64>, seconds: i64) -> Option<DateTime<Utc>> {- millis- .and_then(DateTime::from_timestamp_millis)- .or_else(|| DateTime::from_timestamp(seconds, 0))-}--/// Like [`millis_or_seconds`] but both inputs are optional (e.g. email-/// `sent_at`, call `ended_at`).-pub(crate) fn opt_millis_or_seconds(- millis: Option<i64>,- seconds: Option<i64>,-) -> Option<DateTime<Utc>> {- millis- .and_then(DateTime::from_timestamp_millis)- .or_else(|| seconds.and_then(|s| DateTime::from_timestamp(s, 0)))+/// Convert an epoch-millisecond timestamp to a `DateTime<Utc>`.+pub(crate) fn millis_to_datetime(millis: Option<i64>) -> Option<DateTime<Utc>> {+ millis.and_then(DateTime::from_timestamp_millis)}pub fn should_wildcard_field_query_builder<'a>(diff --git a/crates/opensearch_client/src/upsert/call_record.rs b/crates/opensearch_client/src/upsert/call_record.rsindex 59ec036f04..90a0202ec8 100644--- a/crates/opensearch_client/src/upsert/call_record.rs+++ b/crates/opensearch_client/src/upsert/call_record.rs@@ -4,11 +4,7 @@ use models_opensearch::SearchIndex;use super::BulkUpsertResult;use crate::upsert::properties::IndexedProperty;-use crate::{- Result,- date_format::{EpochMillis, EpochSeconds},- error::OpensearchClientError,-};+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};#[cfg(test)]mod test;@@ -35,9 +31,6 @@ pub struct UpsertCallRecordSegmentArgs {pub speaker_id: String,pub sequence_num: i32,pub content: String,- pub started_at_seconds: EpochSeconds,- #[serde(skip_serializing_if = "Option::is_none")]- pub ended_at_seconds: Option<EpochSeconds>,pub started_at_millis: EpochMillis,#[serde(skip_serializing_if = "Option::is_none")]pub ended_at_millis: Option<EpochMillis>,@@ -85,7 +78,6 @@ fn parent_doc_body(any_segment: &UpsertCallRecordSegmentArgs) -> serde_json::Val"entity_id": &any_segment.call_id,"channel_id": &any_segment.channel_id,"participant_ids": &any_segment.participant_ids,- "started_at_seconds": any_segment.started_at_seconds,"started_at_millis": any_segment.started_at_millis,"call_relation": PARENT_RELATION,});@@ -95,9 +87,6 @@ fn parent_doc_body(any_segment: &UpsertCallRecordSegmentArgs) -> serde_json::Valif let Some(name) = &any_segment.name {doc["name"] = serde_json::Value::String(name.clone());}- if let Some(ended) = &any_segment.ended_at_seconds {- doc["ended_at_seconds"] = serde_json::to_value(ended).unwrap_or(serde_json::Value::Null);- }if let Some(ended) = &any_segment.ended_at_millis {doc["ended_at_millis"] = serde_json::to_value(ended).unwrap_or(serde_json::Value::Null);}@@ -117,8 +106,6 @@ fn child_doc_body(seg: &UpsertCallRecordSegmentArgs) -> serde_json::Value {"speaker_id": &seg.speaker_id,"sequence_num": seg.sequence_num,"content": &seg.content,- "started_at_seconds": seg.started_at_seconds,- "ended_at_seconds": &seg.ended_at_seconds,"started_at_millis": seg.started_at_millis,"ended_at_millis": &seg.ended_at_millis,"call_relation": {diff --git a/crates/opensearch_client/src/upsert/call_record/test.rs b/crates/opensearch_client/src/upsert/call_record/test.rsindex 31a1bd76db..85f039c095 100644--- a/crates/opensearch_client/src/upsert/call_record/test.rs+++ b/crates/opensearch_client/src/upsert/call_record/test.rs@@ -11,8 +11,6 @@ fn segment() -> UpsertCallRecordSegmentArgs {speaker_id: "macro|[email protected]".to_string(),sequence_num: 0,content: "segment content".to_string(),- started_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),- ended_at_seconds: Some(EpochSeconds::new(1_700_000_100).unwrap()),started_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),ended_at_millis: Some(EpochMillis::new(1_700_000_100_456).unwrap()),properties: vec![],diff --git a/crates/opensearch_client/src/upsert/channel_message.rs b/crates/opensearch_client/src/upsert/channel_message.rsindex fbe882891f..e8ba14a271 100644--- a/crates/opensearch_client/src/upsert/channel_message.rs+++ b/crates/opensearch_client/src/upsert/channel_message.rs@@ -1,10 +1,6 @@use models_opensearch::SearchIndex;-use crate::{- Result,- date_format::{EpochMillis, EpochSeconds},- error::OpensearchClientError,-};+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};/// The arguments for upserting a channel message into the opensearch index./// Threadless messages are indexed with `thread_id == message_id`.@@ -19,8 +15,6 @@ pub struct UpsertChannelMessageArgs {pub sender_id: String,pub mentions: Vec<String>,pub content: String,- pub created_at_seconds: EpochSeconds,- pub updated_at_seconds: EpochSeconds,pub created_at_millis: EpochMillis,pub updated_at_millis: EpochMillis,}diff --git a/crates/opensearch_client/src/upsert/chat_message.rs b/crates/opensearch_client/src/upsert/chat_message.rsindex 4b48aa0cfc..b960fb5d21 100644--- a/crates/opensearch_client/src/upsert/chat_message.rs+++ b/crates/opensearch_client/src/upsert/chat_message.rs@@ -1,11 +1,7 @@use models_opensearch::SearchIndex;use crate::upsert::properties::IndexedProperty;-use crate::{- Result,- date_format::{EpochMillis, EpochSeconds},- error::OpensearchClientError,-};+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};#[cfg(test)]mod test;@@ -28,10 +24,6 @@ pub struct UpsertChatMessageArgs {pub user_id: String,/// The role of the chat messagepub role: String,- /// The created at time of the chat message- pub created_at_seconds: EpochSeconds,- /// The updated at time of the chat message- pub updated_at_seconds: EpochSeconds,/// The created at time of the chat message, in millisecondspub created_at_millis: EpochMillis,/// The updated at time of the chat message, in milliseconds@@ -58,7 +50,6 @@ fn parent_doc_body(args: &UpsertChatMessageArgs) -> serde_json::Value {"entity_id": &args.chat_id,"title": &args.title,"user_id": &args.user_id,- "updated_at_seconds": args.updated_at_seconds,"updated_at_millis": args.updated_at_millis,"chat_relation": PARENT_RELATION,});@@ -77,8 +68,6 @@ fn child_doc_body(args: &UpsertChatMessageArgs) -> serde_json::Value {"chat_message_id": &args.chat_message_id,"content": &args.content,"role": &args.role,- "created_at_seconds": args.created_at_seconds,- "updated_at_seconds": args.updated_at_seconds,"created_at_millis": args.created_at_millis,"updated_at_millis": args.updated_at_millis,"chat_relation": {diff --git a/crates/opensearch_client/src/upsert/chat_message/test.rs b/crates/opensearch_client/src/upsert/chat_message/test.rsindex 55f735623a..32250a9e61 100644--- a/crates/opensearch_client/src/upsert/chat_message/test.rs+++ b/crates/opensearch_client/src/upsert/chat_message/test.rs@@ -6,8 +6,6 @@ fn args() -> UpsertChatMessageArgs {chat_message_id: "msg1".to_string(),user_id: "macro|[email protected]".to_string(),role: "user".to_string(),- created_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),- updated_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),created_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),updated_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),title: "Chat title".to_string(),diff --git a/crates/opensearch_client/src/upsert/document.rs b/crates/opensearch_client/src/upsert/document.rsindex 5fa5598260..28ef44745b 100644--- a/crates/opensearch_client/src/upsert/document.rs+++ b/crates/opensearch_client/src/upsert/document.rs@@ -4,11 +4,7 @@ use models_opensearch::SearchIndex;use super::BulkUpsertResult;use super::properties::IndexedProperty;-use crate::{- Result,- date_format::{EpochMillis, EpochSeconds},- error::OpensearchClientError,-};+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};/// Relation name for parent docs in the join field.const PARENT_RELATION: &str = "document";@@ -41,8 +37,6 @@ pub struct UpsertDocumentArgs {pub raw_content: Option<String>,/// The content of the documentpub content: String,- /// The updated at time of the document- pub updated_at_seconds: EpochSeconds,/// The updated at time of the document, in millisecondspub updated_at_millis: EpochMillis,/// The sub type of the document (e.g. task)@@ -69,7 +63,6 @@ fn parent_doc_body(any_chunk: &UpsertDocumentArgs) -> serde_json::Value {"document_name": &any_chunk.document_name,"owner_id": &any_chunk.owner_id,"file_type": &any_chunk.file_type,- "updated_at_seconds": any_chunk.updated_at_seconds,"updated_at_millis": any_chunk.updated_at_millis,"document_relation": PARENT_RELATION,});diff --git a/crates/opensearch_client/src/upsert/document/test.rs b/crates/opensearch_client/src/upsert/document/test.rsindex 83d04543dc..bded2aa8b6 100644--- a/crates/opensearch_client/src/upsert/document/test.rs+++ b/crates/opensearch_client/src/upsert/document/test.rs@@ -1,5 +1,5 @@use super::*;-use crate::date_format::{EpochMillis, EpochSeconds};+use crate::date_format::EpochMillis;fn args(doc_id: &str, node_id: &str) -> UpsertDocumentArgs {UpsertDocumentArgs {@@ -10,7 +10,6 @@ fn args(doc_id: &str, node_id: &str) -> UpsertDocumentArgs {owner_id: format!("owner-{doc_id}"),raw_content: None,content: format!("content {doc_id} {node_id}"),- updated_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),updated_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),sub_type: None,properties: vec![],diff --git a/crates/opensearch_client/src/upsert/email.rs b/crates/opensearch_client/src/upsert/email.rsindex 1fc6c3ad95..1e41ac35bd 100644--- a/crates/opensearch_client/src/upsert/email.rs+++ b/crates/opensearch_client/src/upsert/email.rs@@ -2,11 +2,7 @@ use models_opensearch::SearchIndex;use super::BulkUpsertResult;use super::properties::IndexedProperty;-use crate::{- Result,- date_format::{EpochMillis, EpochSeconds},- error::OpensearchClientError,-};+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};/// The arguments for upserting an email message into the opensearch index#[derive(Debug, serde::Serialize)]@@ -42,14 +38,10 @@ pub struct UpsertEmailArgs {pub link_id: String,/// The user id of the email messagepub user_id: String,- /// The updated at time of the email message- pub updated_at_seconds: EpochSeconds,/// The updated at time of the email message, in millisecondspub updated_at_millis: EpochMillis,/// The subject of the email messagepub subject: Option<String>,- /// The sent at time of the email message- pub sent_at_seconds: Option<EpochSeconds>,/// The sent at time of the email message, in millisecondspub sent_at_millis: Option<EpochMillis>,/// The content of the email messagediff --git a/crates/opensearch_client/src/upsert/project.rs b/crates/opensearch_client/src/upsert/project.rsindex bec2e66b88..859e465183 100644--- a/crates/opensearch_client/src/upsert/project.rs+++ b/crates/opensearch_client/src/upsert/project.rs@@ -1,11 +1,7 @@use models_opensearch::SearchIndex;use super::properties::IndexedProperty;-use crate::{- Result,- date_format::{EpochMillis, EpochSeconds},- error::OpensearchClientError,-};+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};/// The arguments for upserting a project into the opensearch index.///@@ -22,10 +18,6 @@ pub struct UpsertProjectArgs {/// The parent project id, if nested#[serde(skip_serializing_if = "Option::is_none")]pub parent_project_id: Option<String>,- /// The created at time of the project- pub created_at_seconds: EpochSeconds,- /// The updated at time of the project- pub updated_at_seconds: EpochSeconds,/// The created at time of the project, in millisecondspub created_at_millis: EpochMillis,/// The updated at time of the project, in millisecondsdiff --git a/infra/stacks/opensearch/helpers/scripts/copy_timestamp_seconds_to_millis.ts b/infra/stacks/opensearch/helpers/scripts/copy_timestamp_seconds_to_millis.tsdeleted file mode 100644index c0ab3bb65d..0000000000--- a/infra/stacks/opensearch/helpers/scripts/copy_timestamp_seconds_to_millis.ts+++ /dev/null@@ -1,217 +0,0 @@-require('dotenv').config();--import type { Client } from '@opensearch-project/opensearch';-import { client } from '../client';-import {- CALL_RECORDS_ALIAS,- CHANNEL_INDEX,- CHAT_INDEX,- DOCUMENT_INDEX,- EMAIL_INDEX,- PROJECTS_ALIAS,-} from '../constants';-import { checkIndexExists } from '../utils/check_index_exists';-import { copyFieldData } from '../utils/copy_field';--// `<base>_seconds` → `<base>_millis` pairs to backfill. call_records only-// backfills the real started_at/ended_at fields; its created_at/updated_at-// are aliases onto started_at and need no separate data.-type FieldPair = { base: string };--interface IndexMigration {- indexName: string;- fields: FieldPair[];-}--const MIGRATIONS: IndexMigration[] = [- {- indexName: CHANNEL_INDEX,- fields: [{ base: 'created_at' }, { base: 'updated_at' }],- },- {- indexName: CHAT_INDEX,- fields: [{ base: 'created_at' }, { base: 'updated_at' }],- },- {- indexName: DOCUMENT_INDEX,- fields: [{ base: 'updated_at' }],- },- {- indexName: EMAIL_INDEX,- fields: [{ base: 'updated_at' }, { base: 'sent_at' }],- },- {- indexName: PROJECTS_ALIAS,- fields: [{ base: 'created_at' }, { base: 'updated_at' }],- },- {- indexName: CALL_RECORDS_ALIAS,- fields: [{ base: 'started_at' }, { base: 'ended_at' }],- },-];--/**- * Sample docs that have both fields and confirm `millis` falls within the same- * second as `seconds` (i.e. `floor(millis / 1000) === seconds`). This holds for- * both backfilled docs (millis === seconds * 1000) and dual-written docs that- * carry genuine sub-second precision (millis === seconds * 1000 + remainder).- */-async function verifyMillis(- opensearchClient: Client,- indexName: string,- secondsField: string,- millisField: string-): Promise<void> {- const response = await opensearchClient.search({- index: indexName,- body: {- size: 50,- query: {- bool: {- must: [- { exists: { field: secondsField } },- { exists: { field: millisField } },- ],- },- },- _source: [secondsField, millisField],- },- });-- const hits = response.body.hits.hits;- if (hits.length === 0) {- console.log(- ` ⚠️ No documents with both "${secondsField}" and "${millisField}" to verify`- );- return;- }-- let mismatches = 0;- for (const hit of hits) {- const seconds = hit._source?.[secondsField];- const millis = hit._source?.[millisField];- if (- typeof seconds !== 'number' ||- typeof millis !== 'number' ||- Math.floor(millis / 1000) !== seconds- ) {- mismatches += 1;- console.log(- ` ⚠️ Mismatch (ID ${hit._id}): ${secondsField}=${seconds} ${millisField}=${millis}`- );- }- }- if (mismatches === 0) {- console.log(- ` ✓ Verified ${hits.length} sample docs: floor(${millisField} / 1000) === ${secondsField}`- );- } else {- throw new Error(- `${mismatches}/${hits.length} sample docs failed ${millisField} verification`- );- }-}--async function backfillIndex(- opensearchClient: Client,- migration: IndexMigration,- dryRun: boolean-): Promise<void> {- console.log(`\n${'='.repeat(60)}`);- console.log(- `Backfilling millis for index: ${migration.indexName} ${dryRun ? '(DRY-RUN)' : ''}`- );- console.log(`${'='.repeat(60)}`);-- const indexExists = await checkIndexExists(- opensearchClient,- migration.indexName- );- if (!indexExists) {- console.log(- `⚠️ Index "${migration.indexName}" does not exist. Skipping...`- );- return;- }-- for (const { base } of migration.fields) {- const secondsField = `${base}_seconds`;- const millisField = `${base}_millis`;- console.log(`\nProcessing ${secondsField} → ${millisField}`);-- await copyFieldData(- opensearchClient,- migration.indexName,- secondsField,- millisField,- dryRun,- false, // only where millis is missing — idempotent, safe to re-run- `ctx._source.${secondsField} * 1000L`- );-- if (!dryRun) {- await verifyMillis(- opensearchClient,- migration.indexName,- secondsField,- millisField- );- }- }-- console.log(- `\n✓ Completed millis backfill for index: ${migration.indexName}`- );-}--async function backfill(dryRun: boolean = true) {- const opensearchClient = client();-- console.log('\n' + '='.repeat(60));- console.log(- `Copy Timestamp Seconds → Millis ${dryRun ? '(DRY-RUN MODE)' : '(LIVE MODE)'}`- );- console.log('='.repeat(60));- console.log(- '\nThis script derives *_millis fields from *_seconds (× 1000) for existing docs.'- );- console.log('It only updates docs where the *_millis field is missing.');- console.log("\n💡 Safe to run multiple times — it's idempotent.");- console.log(- '⚠️ Run create_timestamp_millis_fields.ts first so the mappings exist.'- );-- if (dryRun) {- console.log('\n⚠️ DRY-RUN MODE: No changes will be made to the cluster');- } else {- console.log('\n🚨 LIVE MODE: Data will be backfilled');- }-- try {- for (const migration of MIGRATIONS) {- await backfillIndex(opensearchClient, migration, dryRun);- }-- console.log('\n' + '='.repeat(60));- console.log('Backfill completed successfully!');- console.log('='.repeat(60));-- if (dryRun) {- console.log(- '\nTo run for real, set DRY_RUN=false environment variable\n'- );- } else {- console.log('\n✓ All *_millis fields backfilled from *_seconds.');- console.log(- '💡 Re-run after deploying the dual-write code to catch docs written mid-migration.\n'- );- }- } catch (error) {- console.error('\n❌ Backfill failed:', error);- throw error;- }-}--const isDryRun = process.env.DRY_RUN !== 'false';--backfill(isDryRun);diff --git a/infra/stacks/opensearch/helpers/scripts/create_timestamp_millis_fields.ts b/infra/stacks/opensearch/helpers/scripts/create_timestamp_millis_fields.tsdeleted file mode 100644index 2ae9a04ed9..0000000000--- a/infra/stacks/opensearch/helpers/scripts/create_timestamp_millis_fields.ts+++ /dev/null@@ -1,201 +0,0 @@-require('dotenv').config();--import type { Client } from '@opensearch-project/opensearch';-import { client } from '../client';-import {- CALL_RECORDS_ALIAS,- CHANNEL_INDEX,- CHAT_INDEX,- DOCUMENT_INDEX,- EMAIL_INDEX,- PROJECTS_ALIAS,-} from '../constants';-import { checkIndexExists } from '../utils/check_index_exists';--// A real `epoch_millis` date field to add.-type MillisField = { fieldName: string };-// An `alias` field pointing at an already-present real field. Used only by-// call_records, mirroring its existing created_at_seconds/updated_at_seconds-// aliases onto started_at_seconds.-type AliasField = { fieldName: string; aliasFor: string };--interface IndexMigration {- indexName: string;- fields: MillisField[];- aliases?: AliasField[];-}--// Millisecond counterparts of every `*_seconds` field, matching the mappings-// in create_indices.ts. call_records keeps only started_at/ended_at as real-// fields; created_at/updated_at are aliases onto started_at.-const MIGRATIONS: IndexMigration[] = [- {- indexName: CHANNEL_INDEX,- fields: [- { fieldName: 'created_at_millis' },- { fieldName: 'updated_at_millis' },- ],- },- {- indexName: CHAT_INDEX,- fields: [- { fieldName: 'created_at_millis' },- { fieldName: 'updated_at_millis' },- ],- },- {- indexName: DOCUMENT_INDEX,- fields: [{ fieldName: 'updated_at_millis' }],- },- {- indexName: EMAIL_INDEX,- fields: [- { fieldName: 'updated_at_millis' },- { fieldName: 'sent_at_millis' },- ],- },- {- indexName: PROJECTS_ALIAS,- fields: [- { fieldName: 'created_at_millis' },- { fieldName: 'updated_at_millis' },- ],- },- {- indexName: CALL_RECORDS_ALIAS,- fields: [- { fieldName: 'started_at_millis' },- { fieldName: 'ended_at_millis' },- ],- aliases: [- { fieldName: 'created_at_millis', aliasFor: 'started_at_millis' },- { fieldName: 'updated_at_millis', aliasFor: 'started_at_millis' },- ],- },-];--async function putMapping(- opensearchClient: Client,- indexName: string,- properties: Record<string, unknown>,- label: string,- dryRun: boolean-): Promise<void> {- console.log(- ` ${dryRun ? '[DRY-RUN] Would add' : 'Adding'} ${label} to index "${indexName}"`- );- if (dryRun) {- return;- }- const response = await opensearchClient.indices.putMapping({- index: indexName,- body: { properties },- });- if (!response.body.acknowledged) {- throw new Error(`Failed to add ${label} in index "${indexName}"`);- }- console.log(` ✓ Successfully added ${label}`);-}--async function addFieldsToIndex(- opensearchClient: Client,- migration: IndexMigration,- dryRun: boolean-): Promise<void> {- console.log(`\n${'='.repeat(60)}`);- console.log(- `Adding millis fields to index: ${migration.indexName} ${dryRun ? '(DRY-RUN)' : ''}`- );- console.log(`${'='.repeat(60)}`);-- const indexExists = await checkIndexExists(- opensearchClient,- migration.indexName- );- if (!indexExists) {- console.log(- `⚠️ Index "${migration.indexName}" does not exist. Skipping...`- );- return;- }-- // Real date fields first, so any aliases can reference them.- for (const field of migration.fields) {- await putMapping(- opensearchClient,- migration.indexName,- {- [field.fieldName]: {- type: 'date',- format: 'epoch_millis',- index: false,- doc_values: true,- },- },- `field "${field.fieldName}" (epoch_millis)`,- dryRun- );- }-- for (const alias of migration.aliases ?? []) {- await putMapping(- opensearchClient,- migration.indexName,- { [alias.fieldName]: { type: 'alias', path: alias.aliasFor } },- `alias "${alias.fieldName}" → "${alias.aliasFor}"`,- dryRun- );- }-- console.log(`\n✓ Completed millis fields for index: ${migration.indexName}`);-}--async function createFields(dryRun: boolean = true) {- const opensearchClient = client();-- console.log('\n' + '='.repeat(60));- console.log(- `Create Timestamp Millis Fields ${dryRun ? '(DRY-RUN MODE)' : '(LIVE MODE)'}`- );- console.log('='.repeat(60));- console.log(- '\nThis script adds *_millis (epoch_millis) timestamp fields to indices.'- );- console.log(- 'Additive and idempotent; safe to run before or after the millis backfill.'- );-- if (dryRun) {- console.log('\n⚠️ DRY-RUN MODE: No changes will be made to the cluster');- } else {- console.log('\n🚨 LIVE MODE: Fields will be added to indices');- }-- try {- for (const migration of MIGRATIONS) {- await addFieldsToIndex(opensearchClient, migration, dryRun);- }-- console.log('\n' + '='.repeat(60));- console.log('Field creation completed successfully!');- console.log('='.repeat(60));-- if (dryRun) {- console.log(- '\nTo run for real, set DRY_RUN=false environment variable\n'- );- } else {- console.log('\n✓ All *_millis fields have been added to indices.');- console.log(- '✓ Next step: run copy_timestamp_seconds_to_millis.ts to backfill existing docs.\n'- );- }- } catch (error) {- console.error('\n❌ Field creation failed:', error);- throw error;- }-}--const isDryRun = process.env.DRY_RUN !== 'false';--createFields(isDryRun);diff --git a/infra/stacks/opensearch/helpers/utils/copy_field.ts b/infra/stacks/opensearch/helpers/utils/copy_field.tsindex 0dbde0c098..4eed30bd3a 100644--- a/infra/stacks/opensearch/helpers/utils/copy_field.ts+++ b/infra/stacks/opensearch/helpers/utils/copy_field.ts@@ -10,9 +10,6 @@ import type { Client } from '@opensearch-project/opensearch';* @param includeNonNull A boolean indicating whether to overwrite existing non-null values in newField.* If false (default), only updates documents where newField doesn't exist.* If true, updates all documents regardless of existing newField values.- * @param valueExpr A painless expression producing the value to assign to newField.- * Defaults to a verbatim copy (`ctx._source.<oldField>`). Pass a- * transform (e.g. `ctx._source.<oldField> * 1000L`) to derive newField.* @returns A Promise that resolves to the number of documents updated.*/export async function copyFieldData(@@ -21,12 +18,11 @@ export async function copyFieldData(oldField: string,newField: string,dryRun: boolean,- includeNonNull: boolean = false,- valueExpr: string = `ctx._source.${oldField}`+ includeNonNull: boolean = false): Promise<number> {const script = {- // If the oldField exists and is not null, assign the derived value to the new field- source: `if (ctx._source.containsKey('${oldField}') && ctx._source.${oldField} != null) { ctx._source.${newField} = ${valueExpr}; }`,+ // If the oldField exists and is not null, copy the value to the new field+ source: `if (ctx._source.containsKey('${oldField}') && ctx._source.${oldField} != null) { ctx._source.${newField} = ctx._source.${oldField}; }`,lang: 'painless',};diff --git a/services/search_processing_service/src/process/call.rs b/services/search_processing_service/src/process/call.rsindex 6015c3de4f..0929b82f02 100644--- a/services/search_processing_service/src/process/call.rs+++ b/services/search_processing_service/src/process/call.rs@@ -1,9 +1,7 @@use anyhow::Context;use models_properties::EntityType;use opensearch_client::{- OpensearchClient,- date_format::{EpochMillis, EpochSeconds},- upsert::call_record::UpsertCallRecordSegmentArgs,+ OpensearchClient, date_format::EpochMillis, upsert::call_record::UpsertCallRecordSegmentArgs,};use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;use sqlx::PgPool;@@ -58,10 +56,6 @@ pub async fn process_call_record(.segments.into_iter().map(|seg| {- let ended_at_seconds = seg- .ended_at- .map(|dt| EpochSeconds::new(dt.timestamp()))- .transpose()?;let ended_at_millis = seg.ended_at.map(|dt| EpochMillis::new(dt.timestamp_millis()))@@ -76,8 +70,6 @@ pub async fn process_call_record(speaker_id: seg.speaker_id,sequence_num: seg.sequence_num,content: seg.content,- started_at_seconds: EpochSeconds::new(seg.started_at.timestamp())?,- ended_at_seconds,started_at_millis: EpochMillis::new(seg.started_at.timestamp_millis())?,ended_at_millis,properties: properties.clone(),diff --git a/services/search_processing_service/src/process/channel.rs b/services/search_processing_service/src/process/channel.rsindex 1e66dd53cc..6470dac707 100644--- a/services/search_processing_service/src/process/channel.rs+++ b/services/search_processing_service/src/process/channel.rs@@ -1,9 +1,7 @@use anyhow::Context;use mention_utils::parse::{ParsedXmlText, PlainTextFormatter, XmlFormatter};use opensearch_client::{- OpensearchClient,- date_format::{EpochMillis, EpochSeconds},- upsert::channel_message::UpsertChannelMessageArgs,+ OpensearchClient, date_format::EpochMillis, upsert::channel_message::UpsertChannelMessageArgs,};use sqlx::{Pool, Postgres};use sqs_client::search::channel::{ChannelMessageUpdate, RemoveChannelMessage};@@ -57,12 +55,6 @@ pub async fn process_channel_message_update(sender_id: channel_message_info.channel_message.sender_id,mentions: channel_message_info.channel_message.mentions,content: transformed_content.trim().to_string(),- created_at_seconds: EpochSeconds::new(- channel_message_info.channel_message.created_at.timestamp(),- )?,- updated_at_seconds: EpochSeconds::new(- channel_message_info.channel_message.updated_at.timestamp(),- )?,created_at_millis: EpochMillis::new(channel_message_info.channel_messagediff --git a/services/search_processing_service/src/process/chat.rs b/services/search_processing_service/src/process/chat.rsindex 7934ebd96b..fb33b30bd0 100644--- a/services/search_processing_service/src/process/chat.rs+++ b/services/search_processing_service/src/process/chat.rs@@ -1,9 +1,7 @@use anyhow::Context;use models_properties::EntityType;use opensearch_client::{- OpensearchClient,- date_format::{EpochMillis, EpochSeconds},- upsert::chat_message::UpsertChatMessageArgs,+ OpensearchClient, date_format::EpochMillis, upsert::chat_message::UpsertChatMessageArgs,};use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;use sqs_client::search::chat::{ChatMessage, RemoveChatMessage};@@ -58,8 +56,6 @@ pub async fn insert_chat_message(chat_id: chat_message.chat_id.clone(),chat_message_id: chat_message.message_id.clone(),user_id: chat_message.user_id.clone(),- created_at_seconds: EpochSeconds::new(chat_message.created_at.timestamp())?,- updated_at_seconds: EpochSeconds::new(chat_message.updated_at.timestamp())?,created_at_millis: EpochMillis::new(chat_message.created_at.timestamp_millis())?,updated_at_millis: EpochMillis::new(chat_message.updated_at.timestamp_millis())?,title: info.name,diff --git a/services/search_processing_service/src/process/document/raw_document.rs b/services/search_processing_service/src/process/document/raw_document.rsindex e296483b5c..937b2dcefd 100644--- a/services/search_processing_service/src/process/document/raw_document.rs+++ b/services/search_processing_service/src/process/document/raw_document.rs@@ -8,9 +8,7 @@ use models_properties::EntityType;use models_search::document::MarkdownParseResult;use models_search::unified::is_searchable_association;use opensearch_client::{- OpensearchClient,- date_format::{EpochMillis, EpochSeconds},- upsert::document::UpsertDocumentArgs,+ OpensearchClient, date_format::EpochMillis, upsert::document::UpsertDocumentArgs,};use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;use s3_key::{@@ -126,7 +124,6 @@ fn generate_parent_only_upsert(content: String::new(),owner_id: document_info.owner.to_string(),file_type,- updated_at_seconds: EpochSeconds::new(Utc::now().timestamp())?,updated_at_millis: EpochMillis::new(Utc::now().timestamp_millis())?,sub_type: document_info.sub_type.map(|st| st.to_string()),properties: vec![],@@ -295,7 +292,6 @@ pub async fn update_search_with_raw_document(tracing::trace!("got raw file content");- let updated_at = EpochSeconds::new(Utc::now().timestamp())?;let updated_at_millis = EpochMillis::new(Utc::now().timestamp_millis())?;let uuid = macro_uuid::generate_uuid_v7().to_string();@@ -315,7 +311,6 @@ pub async fn update_search_with_raw_document(content: page_content.clone(),owner_id: search_extractor_message.user_id.clone(),file_type: file_type.to_string(),- updated_at_seconds: updated_at,updated_at_millis,sub_type: sub_type.clone(),properties: vec![],@@ -340,7 +335,6 @@ pub async fn update_search_with_raw_document(content: content.clone(),owner_id: search_extractor_message.user_id.clone(),file_type: file_type.to_string(),- updated_at_seconds: updated_at,updated_at_millis,sub_type: sub_type.clone(),properties: vec![],@@ -362,7 +356,6 @@ pub async fn update_search_with_raw_document(content: result.content,owner_id: search_extractor_message.user_id.clone(),file_type: file_type.to_string(),- updated_at_seconds: updated_at,updated_at_millis,sub_type: sub_type.clone(),properties: vec![],@@ -382,7 +375,6 @@ pub async fn update_search_with_raw_document(content: content.clone(),owner_id: search_extractor_message.user_id.clone(),file_type: file_type.to_string(),- updated_at_seconds: updated_at,updated_at_millis,sub_type: sub_type.clone(),properties: vec![],@@ -409,7 +401,6 @@ fn generate_upserts(markdown_result: Vec<MarkdownParseResult>,) -> anyhow::Result<Vec<UpsertDocumentArgs>> {let result = markdown_result;- let updated_at = EpochSeconds::new(Utc::now().timestamp())?;let updated_at_millis = EpochMillis::new(Utc::now().timestamp_millis())?;let file_type = FileType::from_str(document_info@@ -430,7 +421,6 @@ fn generate_upserts(content: result.content,owner_id: document_info.owner.to_string(),file_type: file_type.to_string(),- updated_at_seconds: updated_at,updated_at_millis,sub_type: sub_type.clone(),properties: vec![],diff --git a/services/search_processing_service/src/process/email/upsert.rs b/services/search_processing_service/src/process/email/upsert.rsindex 053f7fe228..97a32fb232 100644--- a/services/search_processing_service/src/process/email/upsert.rs+++ b/services/search_processing_service/src/process/email/upsert.rs@@ -3,9 +3,7 @@ use chrono::Utc;use models_email::service::label::system_labels;use models_properties::EntityType;use opensearch_client::{- OpensearchClient,- date_format::{EpochMillis, EpochSeconds},- upsert::email::UpsertEmailArgs,+ OpensearchClient, date_format::EpochMillis, upsert::email::UpsertEmailArgs,};use properties::outbound::entity_properties_get_query::{get_entity_properties_for_index, get_entity_properties_for_index_batch,@@ -54,15 +52,8 @@ pub async fn process_upsert_message(return Ok(());};- let now = EpochSeconds::new(Utc::now().timestamp())?;let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;- let updated_at = message_info- .internal_date_ts- .map(|date| EpochSeconds::new(date.timestamp()))- .transpose()?- .unwrap_or(now);-let updated_at_millis = message_info.internal_date_ts.map(|date| EpochMillis::new(date.timestamp_millis()))@@ -132,12 +123,7 @@ pub async fn process_upsert_message(.map(|label| label.name.clone()).collect(),content,- updated_at_seconds: updated_at,updated_at_millis,- sent_at_seconds: message_info- .internal_date_ts- .map(|date| EpochSeconds::new(date.timestamp()))- .transpose()?,sent_at_millis: message_info.internal_date_ts.map(|date| EpochMillis::new(date.timestamp_millis()))@@ -166,7 +152,6 @@ pub async fn process_upsert_thread_message(.parse().context("failed to parse thread_id as UUID")?;- let now = EpochSeconds::new(Utc::now().timestamp())?;let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;// A full index overwrites each doc, so thread properties must ride along@@ -205,22 +190,11 @@ pub async fn process_upsert_thread_message(}if let Some(content) = message.body_parsed_linkless {- let sent_at = message- .internal_date_ts- .map(|date| EpochSeconds::new(date.timestamp()))- .transpose()?;-let sent_at_millis = message.internal_date_ts.map(|date| EpochMillis::new(date.timestamp_millis())).transpose()?;- let updated_at = message- .internal_date_ts- .map(|date| EpochSeconds::new(date.timestamp()))- .transpose()?- .unwrap_or(now);-let updated_at_millis = message.internal_date_ts.map(|date| EpochMillis::new(date.timestamp_millis()))@@ -268,9 +242,7 @@ pub async fn process_upsert_thread_message(.map(|label| label.name.clone()).collect(),content,- updated_at_seconds: updated_at,updated_at_millis,- sent_at_seconds: sent_at,sent_at_millis,properties: properties.clone(),});@@ -314,7 +286,6 @@ pub async fn process_upsert_thread_batch_message(.map(|id| id.parse().context("failed to parse thread_id as UUID")).collect::<anyhow::Result<Vec<_>>>()?;- let now = EpochSeconds::new(Utc::now().timestamp())?;let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;let messages =@@ -346,22 +317,11 @@ pub async fn process_upsert_thread_batch_message(}if let Some(content) = message.body_parsed_linkless {- let sent_at = message- .internal_date_ts- .map(|date| EpochSeconds::new(date.timestamp()))- .transpose()?;-let sent_at_millis = message.internal_date_ts.map(|date| EpochMillis::new(date.timestamp_millis())).transpose()?;- let updated_at = message- .internal_date_ts- .map(|date| EpochSeconds::new(date.timestamp()))- .transpose()?- .unwrap_or(now);-let updated_at_millis = message.internal_date_ts.map(|date| EpochMillis::new(date.timestamp_millis()))@@ -409,9 +369,7 @@ pub async fn process_upsert_thread_batch_message(.map(|label| label.name.clone()).collect(),content,- updated_at_seconds: updated_at,updated_at_millis,- sent_at_seconds: sent_at,sent_at_millis,properties: properties_by_thread.get(&message.thread_db_id.to_string())diff --git a/services/search_processing_service/src/process/project.rs b/services/search_processing_service/src/process/project.rsindex 607face939..8bc0e23a34 100644--- a/services/search_processing_service/src/process/project.rs+++ b/services/search_processing_service/src/process/project.rs@@ -2,7 +2,7 @@ use anyhow::Context;use models_properties::EntityType;use opensearch_client::{OpensearchClient,- date_format::{EpochMillis, EpochSeconds},+ date_format::EpochMillis,upsert::{project::UpsertProjectArgs, properties::IndexedProperty},};use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;@@ -75,8 +75,6 @@ pub async fn upsert_project(name: project.name,owner_id: project.user_id,parent_project_id: project.parent_id,- created_at_seconds: EpochSeconds::new(created_at.timestamp())?,- updated_at_seconds: EpochSeconds::new(updated_at.timestamp())?,created_at_millis: EpochMillis::new(created_at.timestamp_millis())?,updated_at_millis: EpochMillis::new(updated_at.timestamp_millis())?,properties,