Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

refactor(search): make OpenSearch timestamp handling millis-only (#4912)

gbirman4726341fdd42
diff --git a/crates/opensearch_client/src/date_format.rs b/crates/opensearch_client/src/date_format.rs
index c6ae20893c..d996c8a959 100644
--- a/crates/opensearch_client/src/date_format.rs
+++ b/crates/opensearch_client/src/date_format.rs
@@ -6,35 +6,6 @@ const MIN_REASONABLE_SECONDS: i64 = 946684800; // Year 2000
const MAX_REASONABLE_MILLIS: i64 = MAX_REASONABLE_SECONDS * 1000;
const MIN_REASONABLE_MILLIS: i64 = MIN_REASONABLE_SECONDS * 1000;
-#[derive(Debug, Clone, Copy, serde::Serialize)]
-pub struct EpochSeconds(i64);
-
-impl EpochSeconds {
- pub fn new(seconds: i64) -> Result<Self> {
- if seconds > MAX_REASONABLE_SECONDS {
- return Err(OpensearchClientError::ValidationFailed {
- details: format!(
- "timestamp {} appears to be in milliseconds (exceeds year 3000). Expected seconds since Unix epoch.",
- seconds
- ),
- });
- }
- if seconds < MIN_REASONABLE_SECONDS {
- return Err(OpensearchClientError::ValidationFailed {
- details: format!(
- "timestamp {} is before year 2000. Expected seconds since Unix epoch.",
- seconds
- ),
- });
- }
- Ok(Self(seconds))
- }
-
- pub fn get(&self) -> i64 {
- self.0
- }
-}
-
/// A validated epoch-milliseconds timestamp, serialized as a raw `i64` for
/// OpenSearch date fields mapped with `format: epoch_millis`.
#[derive(Debug, Clone, Copy, serde::Serialize)]
diff --git a/crates/opensearch_client/src/date_format/test.rs b/crates/opensearch_client/src/date_format/test.rs
index e8df60369c..11fc8d9cd6 100644
--- a/crates/opensearch_client/src/date_format/test.rs
+++ b/crates/opensearch_client/src/date_format/test.rs
@@ -1,40 +1,5 @@
use super::*;
-#[test]
-fn test_epoch_seconds_valid() {
- let now = 1704067200; // 2024-01-01
- let epoch = EpochSeconds::new(now);
- assert!(epoch.is_ok());
- assert_eq!(epoch.unwrap().get(), now);
-}
-
-#[test]
-fn test_epoch_seconds_milliseconds() {
- let now_millis = 1704067200000; // 2024-01-01 in milliseconds
- let result = EpochSeconds::new(now_millis);
- assert!(result.is_err());
- if let Err(OpensearchClientError::ValidationFailed { details }) = result {
- assert!(details.contains("appears to be in milliseconds"));
- }
-}
-
-#[test]
-fn test_epoch_seconds_too_old() {
- let old = 946684799; // Just before year 2000
- let result = EpochSeconds::new(old);
- assert!(result.is_err());
- if let Err(OpensearchClientError::ValidationFailed { details }) = result {
- assert!(details.contains("before year 2000"));
- }
-}
-
-#[test]
-fn test_epoch_seconds_serialize() {
- let epoch = EpochSeconds::new(1704067200).unwrap();
- let serialized = serde_json::to_string(&epoch).unwrap();
- assert_eq!(serialized, "1704067200");
-}
-
#[test]
fn test_epoch_millis_valid() {
let now_millis = 1704067200123; // 2024-01-01 with sub-second precision
diff --git a/crates/opensearch_client/src/search/builder.rs b/crates/opensearch_client/src/search/builder.rs
index 7c5cceb1bf..e74a6bbf63 100644
--- a/crates/opensearch_client/src/search/builder.rs
+++ b/crates/opensearch_client/src/search/builder.rs
@@ -47,8 +47,7 @@ macro_rules! delegate_methods {
/// Sort mode for the channel content endpoint.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum ChannelSortMode {
- /// Sort by sent_at/updated_at (millis, falling back to seconds), entity_id
- /// as tiebreaker.
+ /// Sort by sent_at/updated_at (millis), entity_id as tiebreaker.
#[default]
Message,
/// Sort by thread_id, message_id as tiebreaker.
@@ -57,22 +56,16 @@ pub enum ChannelSortMode {
/// Creates sort vec to sort by sent_at (preferred) or updated_at (fallback)
/// with entity_id as a tiebreaker. Items without a timestamp are pushed to the
-/// end. Prefers the millisecond-resolution `*_millis` fields and falls back to
-/// the second-resolution `*_seconds` fields, so docs indexed before the
-/// millis backfill still sort correctly. `.toEpochMilli()` normalizes both to
-/// milliseconds either way.
+/// end. Sorts on the millisecond-resolution `*_millis` fields.
+/// `.toEpochMilli()` normalizes to milliseconds.
pub(crate) fn updated_at_sort<'a>() -> Vec<SortType<'a>> {
vec![
SortType::ScriptSort(ScriptSort::new(
Script::new(
r#"if (doc.containsKey('sent_at_millis') && doc['sent_at_millis'].size() > 0) {
return doc['sent_at_millis'].value.toInstant().toEpochMilli();
- } else if (doc.containsKey('sent_at_seconds') && doc['sent_at_seconds'].size() > 0) {
- return doc['sent_at_seconds'].value.toInstant().toEpochMilli();
} else if (doc.containsKey('updated_at_millis') && doc['updated_at_millis'].size() > 0) {
return doc['updated_at_millis'].value.toInstant().toEpochMilli();
- } else if (doc.containsKey('updated_at_seconds') && doc['updated_at_seconds'].size() > 0) {
- return doc['updated_at_seconds'].value.toInstant().toEpochMilli();
} else {
return 0L;
}"#,
diff --git a/crates/opensearch_client/src/search/call_records.rs b/crates/opensearch_client/src/search/call_records.rs
index 58e46ed80d..68f11099db 100644
--- a/crates/opensearch_client/src/search/call_records.rs
+++ b/crates/opensearch_client/src/search/call_records.rs
@@ -9,7 +9,7 @@ use crate::{
},
properties::build_tag_filter,
query::Keys,
- utils::{millis_or_seconds, opt_millis_or_seconds},
+ utils::millis_to_datetime,
},
};
@@ -45,9 +45,6 @@ pub(crate) struct CallRecordIndex {
pub channel_name: Option<String>,
#[serde(default)]
pub name: Option<String>,
- pub started_at_seconds: i64,
- #[serde(default)]
- pub ended_at_seconds: Option<i64>,
#[serde(default)]
pub started_at_millis: Option<i64>,
#[serde(default)]
@@ -359,9 +356,6 @@ struct SegmentSource {
speaker_id: Option<String>,
#[serde(default)]
sequence_num: Option<i32>,
- started_at_seconds: i64,
- #[serde(default)]
- ended_at_seconds: Option<i64>,
#[serde(default)]
started_at_millis: Option<i64>,
#[serde(default)]
@@ -423,18 +417,12 @@ pub(crate) fn expand_inner_hits_to_search_hits(
transcript_id: seg.source.transcript_id,
speaker_id: seg.source.speaker_id.unwrap_or_default(),
sequence_num: seg.source.sequence_num.unwrap_or_default(),
- started_at: millis_or_seconds(
- seg.source.started_at_millis,
- seg.source.started_at_seconds,
- )
- .unwrap_or_default(),
- ended_at: opt_millis_or_seconds(
- seg.source.ended_at_millis,
- seg.source.ended_at_seconds,
- ),
+ started_at: millis_to_datetime(seg.source.started_at_millis)
+ .unwrap_or_default(),
+ ended_at: millis_to_datetime(seg.source.ended_at_millis),
participant_ids: parent.participant_ids.clone(),
})),
- updated_at: millis_or_seconds(parent.started_at_millis, parent.started_at_seconds),
+ updated_at: millis_to_datetime(parent.started_at_millis),
});
}
}
diff --git a/crates/opensearch_client/src/search/channels.rs b/crates/opensearch_client/src/search/channels.rs
index 22b667c701..254cd31d2d 100644
--- a/crates/opensearch_client/src/search/channels.rs
+++ b/crates/opensearch_client/src/search/channels.rs
@@ -10,7 +10,7 @@ use crate::{
exclude_source_content, inject_fragment_size, parse_highlight_hit,
},
query::{Keys, TermCombine},
- utils::millis_or_seconds,
+ utils::millis_to_datetime,
},
};
@@ -30,8 +30,6 @@ pub(crate) struct ChannelMessageIndex {
pub thread_id: uuid::Uuid,
pub sender_id: String,
pub mentions: Vec<String>,
- pub created_at_seconds: i64,
- pub updated_at_seconds: i64,
#[serde(default)]
pub created_at_millis: Option<i64>,
#[serde(default)]
@@ -312,12 +310,10 @@ fn channel_hit_to_search_hit(hit: Hit<ChannelMessageIndex>) -> SearchHit {
channel_message_id: a.message_id,
thread_id: (a.thread_id != a.message_id).then_some(a.thread_id),
sender_id: a.sender_id,
- created_at: millis_or_seconds(a.created_at_millis, a.created_at_seconds)
- .unwrap_or_default(),
- updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds)
- .unwrap_or_default(),
+ created_at: millis_to_datetime(a.created_at_millis).unwrap_or_default(),
+ updated_at: millis_to_datetime(a.updated_at_millis).unwrap_or_default(),
})),
- updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),
+ updated_at: millis_to_datetime(a.updated_at_millis),
}
}
diff --git a/crates/opensearch_client/src/search/channels/test.rs b/crates/opensearch_client/src/search/channels/test.rs
index 945b2895d5..e40ddfe74f 100644
--- a/crates/opensearch_client/src/search/channels/test.rs
+++ b/crates/opensearch_client/src/search/channels/test.rs
@@ -12,8 +12,6 @@ fn channel_index(updated_at_millis: Option<i64>) -> ChannelMessageIndex {
thread_id: id,
sender_id: "macro|[email protected]".to_string(),
mentions: vec![],
- created_at_seconds: 1_700_000_000,
- updated_at_seconds: 1_700_000_000,
created_at_millis: updated_at_millis,
updated_at_millis,
}
@@ -46,17 +44,6 @@ fn millis_timestamps_are_distinct_within_a_second() {
);
}
-/// Docs indexed before the millis backfill carry no `*_millis`; the read path
-/// falls back to the second-resolution field.
-#[test]
-fn falls_back_to_seconds_when_millis_absent() {
- let hit = channel_hit_to_search_hit(hit_for(channel_index(None)));
- assert_eq!(
- hit.updated_at.expect("updated_at").timestamp_millis(),
- 1_700_000_000_000
- );
-}
-
#[test]
fn test_build_bool_query() -> anyhow::Result<()> {
let builder = ChannelMessageQueryBuilder::new(vec!["test".to_string()])
diff --git a/crates/opensearch_client/src/search/chats.rs b/crates/opensearch_client/src/search/chats.rs
index 5ace04aa2e..40116c12bc 100644
--- a/crates/opensearch_client/src/search/chats.rs
+++ b/crates/opensearch_client/src/search/chats.rs
@@ -41,7 +41,6 @@ pub(crate) struct ChatIndex {
pub entity_id: uuid::Uuid,
pub user_id: String,
pub title: String,
- pub updated_at_seconds: Option<i64>,
#[serde(default)]
pub updated_at_millis: Option<i64>,
}
diff --git a/crates/opensearch_client/src/search/documents.rs b/crates/opensearch_client/src/search/documents.rs
index 8d14359144..9d30a06c05 100644
--- a/crates/opensearch_client/src/search/documents.rs
+++ b/crates/opensearch_client/src/search/documents.rs
@@ -366,7 +366,6 @@ pub(crate) struct DocumentIndex {
pub document_name: String,
pub owner_id: String,
pub file_type: String,
- pub updated_at_seconds: Option<i64>,
#[serde(default)]
pub updated_at_millis: Option<i64>,
}
diff --git a/crates/opensearch_client/src/search/documents/test.rs b/crates/opensearch_client/src/search/documents/test.rs
index 367c324cb3..3d3706806c 100644
--- a/crates/opensearch_client/src/search/documents/test.rs
+++ b/crates/opensearch_client/src/search/documents/test.rs
@@ -395,7 +395,7 @@ fn document_index_deserializes_parent_shape() {
"document_name": "Q3 Planning",
"owner_id": "macro|[email protected]",
"file_type": "pdf",
- "updated_at_seconds": 1779000000_i64,
+ "updated_at_millis": 1779000000000_i64,
"document_relation": "document",
});
diff --git a/crates/opensearch_client/src/search/emails.rs b/crates/opensearch_client/src/search/emails.rs
index eef19a7beb..c8ec2ad070 100644
--- a/crates/opensearch_client/src/search/emails.rs
+++ b/crates/opensearch_client/src/search/emails.rs
@@ -367,8 +367,6 @@ pub(crate) struct EmailIndex {
pub user_id: String,
/// The subject of the email message
pub subject: Option<String>,
- /// The sent at time of the email message
- pub sent_at_seconds: Option<i64>,
/// The sent at time of the email message, in milliseconds
#[serde(default)]
pub sent_at_millis: Option<i64>,
diff --git a/crates/opensearch_client/src/search/projects.rs b/crates/opensearch_client/src/search/projects.rs
index 6c382c3d3e..ee8d70c9c1 100644
--- a/crates/opensearch_client/src/search/projects.rs
+++ b/crates/opensearch_client/src/search/projects.rs
@@ -94,7 +94,6 @@ pub(crate) struct ProjectIndex {
pub owner_id: String,
#[serde(default)]
pub parent_project_id: Option<String>,
- pub updated_at_seconds: Option<i64>,
#[serde(default)]
pub updated_at_millis: Option<i64>,
}
diff --git a/crates/opensearch_client/src/search/unified.rs b/crates/opensearch_client/src/search/unified.rs
index 4fffc78cfa..22bda6a627 100644
--- a/crates/opensearch_client/src/search/unified.rs
+++ b/crates/opensearch_client/src/search/unified.rs
@@ -26,7 +26,7 @@ use crate::{
},
projects::{ProjectIndex, ProjectQueryBuilder, ProjectSearchArgs, ProjectSearchConfig},
query::Keys,
- utils::{millis_or_seconds, opt_millis_or_seconds},
+ utils::millis_to_datetime,
},
};
use chrono::Utc;
@@ -366,8 +366,7 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {
match &hit.source {
UnifiedSearchIndex::Document(parent) => {
let entity_id = parent.entity_id;
- let updated_at =
- opt_millis_or_seconds(parent.updated_at_millis, parent.updated_at_seconds);
+ let updated_at = millis_to_datetime(parent.updated_at_millis);
let mut out: Vec<SearchHit> = Vec::new();
@@ -412,8 +411,7 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {
return vec![hit.into()];
};
let entity_id = parent.entity_id;
- let updated_at =
- opt_millis_or_seconds(parent.updated_at_millis, parent.updated_at_seconds);
+ let updated_at = millis_to_datetime(parent.updated_at_millis);
let expanded = crate::search::chats::expand_inner_hits_to_search_hits(
entity_id, updated_at, inner,
);
@@ -423,7 +421,7 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {
expanded
}
UnifiedSearchIndex::CallRecord(parent) => {
- let updated_at = millis_or_seconds(parent.started_at_millis, parent.started_at_seconds);
+ let updated_at = millis_to_datetime(parent.started_at_millis);
let mut out: Vec<SearchHit> = Vec::new();
// A name match surfaces as a parent-level hit. Segment content lives
@@ -451,10 +449,7 @@ fn expand_hit_into_search_hits(hit: Hit<UnifiedSearchIndex>) -> Vec<SearchHit> {
speaker_id: String::new(),
sequence_num: 0,
started_at: updated_at.unwrap_or_default(),
- ended_at: opt_millis_or_seconds(
- parent.ended_at_millis,
- parent.ended_at_seconds,
- ),
+ ended_at: millis_to_datetime(parent.ended_at_millis),
participant_ids: parent.participant_ids.clone(),
})),
updated_at,
@@ -500,12 +495,10 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
channel_message_id: a.message_id,
thread_id: (a.thread_id != a.message_id).then_some(a.thread_id),
sender_id: a.sender_id,
- created_at: millis_or_seconds(a.created_at_millis, a.created_at_seconds)
- .unwrap_or_default(),
- updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds)
- .unwrap_or_default(),
+ created_at: millis_to_datetime(a.created_at_millis).unwrap_or_default(),
+ updated_at: millis_to_datetime(a.updated_at_millis).unwrap_or_default(),
})),
- updated_at: millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),
+ updated_at: millis_to_datetime(a.updated_at_millis),
},
UnifiedSearchIndex::Document(a) => SearchHit {
entity_id: a.entity_id,
@@ -524,7 +517,7 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
})
.unwrap_or_default(),
goto: None,
- updated_at: opt_millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),
+ updated_at: millis_to_datetime(a.updated_at_millis),
},
UnifiedSearchIndex::Email(a) => {
let a = *a;
@@ -549,11 +542,11 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
bcc: a.bcc,
cc: a.cc,
labels: a.labels,
- sent_at: opt_millis_or_seconds(a.sent_at_millis, a.sent_at_seconds),
+ sent_at: millis_to_datetime(a.sent_at_millis),
sender: a.sender,
recipients: a.recipients,
})),
- updated_at: opt_millis_or_seconds(a.sent_at_millis, a.sent_at_seconds),
+ updated_at: millis_to_datetime(a.sent_at_millis),
}
}
UnifiedSearchIndex::Chat(a) => SearchHit {
@@ -573,7 +566,7 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
})
.unwrap_or_default(),
goto: None,
- updated_at: opt_millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),
+ updated_at: millis_to_datetime(a.updated_at_millis),
},
UnifiedSearchIndex::Project(a) => SearchHit {
entity_id: a.entity_id,
@@ -592,7 +585,7 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
})
.unwrap_or_default(),
goto: None,
- updated_at: opt_millis_or_seconds(a.updated_at_millis, a.updated_at_seconds),
+ updated_at: millis_to_datetime(a.updated_at_millis),
},
UnifiedSearchIndex::CallRecord(a) => SearchHit {
entity_id: a.entity_id,
@@ -619,12 +612,11 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
transcript_id: uuid::Uuid::nil(),
speaker_id: String::new(),
sequence_num: 0,
- started_at: millis_or_seconds(a.started_at_millis, a.started_at_seconds)
- .unwrap_or_default(),
- ended_at: opt_millis_or_seconds(a.ended_at_millis, a.ended_at_seconds),
+ started_at: millis_to_datetime(a.started_at_millis).unwrap_or_default(),
+ ended_at: millis_to_datetime(a.ended_at_millis),
participant_ids: a.participant_ids,
})),
- updated_at: millis_or_seconds(a.started_at_millis, a.started_at_seconds),
+ updated_at: millis_to_datetime(a.started_at_millis),
},
}
}
diff --git a/crates/opensearch_client/src/search/unified/test.rs b/crates/opensearch_client/src/search/unified/test.rs
index 312eb82ca0..9d39b0f225 100644
--- a/crates/opensearch_client/src/search/unified/test.rs
+++ b/crates/opensearch_client/src/search/unified/test.rs
@@ -22,8 +22,7 @@ fn expand_document_name_highlight_yields_name_hit_without_goto() {
document_name: "Q3 Report".to_string(),
owner_id: "alice".to_string(),
file_type: "md".to_string(),
- updated_at_seconds: Some(1_779_000_000),
- updated_at_millis: None,
+ updated_at_millis: Some(1_779_000_000_000),
}),
highlight: Some(HashMap::from([(
"document_name".to_string(),
@@ -85,10 +84,10 @@ fn test_deserialization() -> anyhow::Result<()> {
"IMPORTANT"
],
"thread_id": "019a5a07-a1ed-7342-93a7-861582b59a55",
- "sent_at_seconds": 1762447040,
+ "sent_at_millis": 1762447040000_i64,
"sender": "[email protected]",
"user_id": "macro|[email protected]",
- "updated_at_seconds": 1762447044,
+ "updated_at_millis": 1762447044000_i64,
"recipients": [
]
@@ -123,10 +122,10 @@ fn test_deserialization() -> anyhow::Result<()> {
"sent_at": 1759241406,
"thread_id": "01999c74-fb30-7c8a-b334-16035ee46783",
"updated_at": 1759266586,
- "sent_at_seconds": 1759241406,
+ "sent_at_millis": 1759241406000_i64,
"sender": "[email protected]",
"user_id": "macro|[email protected]",
- "updated_at_seconds": 1759266586,
+ "updated_at_millis": 1759266586000_i64,
"recipients": [
]
@@ -146,7 +145,7 @@ fn test_deserialization() -> anyhow::Result<()> {
"_source": {
"document_name": "Rust for Rustaceans Idiomatic Programming for Experienced Developers (1)",
"updated_at": 1752523635,
- "updated_at_seconds": 1752523635,
+ "updated_at_millis": 1752523635000_i64,
"file_type": "pdf",
"owner_id": "macro|[email protected]",
"document_id": "013021c7-5e14-4aab-855e-01413fff85dc",
@@ -183,10 +182,10 @@ fn test_deserialization() -> anyhow::Result<()> {
"sent_at": 1761742081,
"thread_id": "019a3002-d9ff-79ad-9a5f-c3116b5b229a",
"updated_at": 1761742084,
- "sent_at_seconds": 1761742081,
+ "sent_at_millis": 1761742081000_i64,
"sender": "[email protected]",
"user_id": "macro|[email protected]",
- "updated_at_seconds": 1761742084,
+ "updated_at_millis": 1761742084000_i64,
"recipients": [
]
@@ -207,7 +206,7 @@ fn test_deserialization() -> anyhow::Result<()> {
"_source": {
"document_name": "Rust for Rustaceans Idiomatic Programming for Experienced Developers (1)",
"updated_at": 1752522588,
- "updated_at_seconds": 1752522588,
+ "updated_at_millis": 1752522588000_i64,
"file_type": "pdf",
"owner_id": "macro|[email protected]",
"document_id": "71ff8822-b421-4cab-b23e-f65355bf90d4",
@@ -248,10 +247,10 @@ fn test_deserialization() -> anyhow::Result<()> {
"sent_at": 1761313387,
"thread_id": "019a1675-844c-7dcc-bd45-109c25b94fa0",
"updated_at": 1761313391,
- "sent_at_seconds": 1761313387,
+ "sent_at_millis": 1761313387000_i64,
"sender": "[email protected]",
"user_id": "macro|[email protected]",
- "updated_at_seconds": 1761313391,
+ "updated_at_millis": 1761313391000_i64,
"recipients": [
]
@@ -290,10 +289,10 @@ fn test_deserialization() -> anyhow::Result<()> {
"sent_at": 1761848050,
"thread_id": "019a3653-c914-769b-9454-3b70dc2ad966",
"updated_at": 1761848053,
- "sent_at_seconds": 1761848050,
+ "sent_at_millis": 1761848050000_i64,
"sender": "[email protected]",
"user_id": "macro|[email protected]",
- "updated_at_seconds": 1761848053,
+ "updated_at_millis": 1761848053000_i64,
"recipients": [
]
@@ -332,10 +331,10 @@ fn test_deserialization() -> anyhow::Result<()> {
"sent_at": 1761252648,
"thread_id": "019a12cf-ed6d-7cbc-b705-a78bbbe1bba0",
"updated_at": 1761252711,
- "sent_at_seconds": 1761252648,
+ "sent_at_millis": 1761252648000_i64,
"sender": "[email protected]",
"user_id": "macro|[email protected]",
- "updated_at_seconds": 1761252711,
+ "updated_at_millis": 1761252711000_i64,
"recipients": [
]
@@ -371,10 +370,10 @@ fn test_deserialization() -> anyhow::Result<()> {
"sent_at": 1761833460,
"thread_id": "019a3575-3540-7231-8b83-51279a1bf623",
"updated_at": 1761833465,
- "sent_at_seconds": 1761833460,
+ "sent_at_millis": 1761833460000_i64,
"sender": "[email protected]",
"user_id": "macro|[email protected]",
- "updated_at_seconds": 1761833465,
+ "updated_at_millis": 1761833465000_i64,
"recipients": [
]
@@ -413,10 +412,10 @@ fn test_deserialization() -> anyhow::Result<()> {
"sent_at": 1761753544,
"thread_id": "019a30ae-2a24-78b3-9eef-e3aa7594a1e4",
"updated_at": 1761753548,
- "sent_at_seconds": 1761753544,
+ "sent_at_millis": 1761753544000_i64,
"sender": "[email protected]",
"user_id": "macro|[email protected]",
- "updated_at_seconds": 1761753548,
+ "updated_at_millis": 1761753548000_i64,
"recipients": [
]
@@ -907,7 +906,7 @@ fn test_thread_sort_is_thread_id_then_message_id_field_sorts() {
/// content query.
fn doc_hit_with_chunks(
entity_id: uuid::Uuid,
- updated_at_seconds: i64,
+ updated_at_millis: i64,
chunk_count: usize,
) -> Hit<UnifiedSearchIndex> {
let chunks: Vec<serde_json::Value> = (0..chunk_count)
@@ -928,8 +927,7 @@ fn doc_hit_with_chunks(
document_name: "Doc".to_string(),
owner_id: "alice".to_string(),
file_type: "md".to_string(),
- updated_at_seconds: Some(updated_at_seconds),
- updated_at_millis: None,
+ updated_at_millis: Some(updated_at_millis),
}),
highlight: None,
inner_hits: Some(serde_json::json!({ "term_0": { "hits": { "hits": chunks } } })),
@@ -948,8 +946,8 @@ fn paginate_counts_parent_entities_not_inner_hits() {
.parse::<uuid::Uuid>()
.unwrap();
let hits = vec![
- doc_hit_with_chunks(d1, 1_779_000_100, 30),
- doc_hit_with_chunks(d2, 1_779_000_050, 30),
+ doc_hit_with_chunks(d1, 1_779_000_100_000, 30),
+ doc_hit_with_chunks(d2, 1_779_000_050_000, 30),
];
let (results, cursor) = paginate_unified_hits(hits, 10);
@@ -979,9 +977,9 @@ fn paginate_has_more_when_parent_count_exceeds_page_size() {
.parse::<uuid::Uuid>()
.unwrap();
let hits = vec![
- doc_hit_with_chunks(d1, 1_779_000_300, 5),
- doc_hit_with_chunks(d2, 1_779_000_200, 5),
- doc_hit_with_chunks(d3, 1_779_000_100, 5),
+ doc_hit_with_chunks(d1, 1_779_000_300_000, 5),
+ doc_hit_with_chunks(d2, 1_779_000_200_000, 5),
+ doc_hit_with_chunks(d3, 1_779_000_100_000, 5),
];
let (results, cursor) = paginate_unified_hits(hits, 2);
@@ -1014,7 +1012,7 @@ fn paginate_page_size_zero_terminates() {
let d1 = "00000000-0000-0000-0000-000000000001"
.parse::<uuid::Uuid>()
.unwrap();
- let hits = vec![doc_hit_with_chunks(d1, 1_779_000_100, 5)];
+ let hits = vec![doc_hit_with_chunks(d1, 1_779_000_100_000, 5)];
let (results, cursor) = paginate_unified_hits(hits, 0);
@@ -1048,8 +1046,8 @@ fn project_hit_deserializes_and_converts() -> anyhow::Result<()> {
"name": "Mobile Redesign",
"owner_id": "macro|[email protected]",
"parent_project_id": "0197a863-0000-7000-8000-000000000002",
- "created_at_seconds": 1770000000,
- "updated_at_seconds": 1783000000,
+ "created_at_millis": 1770000000000_i64,
+ "updated_at_millis": 1783000000000_i64,
"properties": [
{ "definition_id": "0197a863-0000-7000-8000-00000000000a", "values": ["opt-1"] }
]
diff --git a/crates/opensearch_client/src/search/utils.rs b/crates/opensearch_client/src/search/utils.rs
index 7468ad4732..bd38486991 100644
--- a/crates/opensearch_client/src/search/utils.rs
+++ b/crates/opensearch_client/src/search/utils.rs
@@ -1,24 +1,9 @@
use chrono::{DateTime, Utc};
use opensearch_query_builder::{BoolQueryBuilder, QueryType, WildcardQuery};
-/// Prefer a millisecond epoch timestamp, falling back to a second epoch
-/// timestamp. During the seconds→millis migration new/backfilled docs carry a
-/// `*_millis` field while older docs only have `*_seconds`.
-pub(crate) fn millis_or_seconds(millis: Option<i64>, seconds: i64) -> Option<DateTime<Utc>> {
- millis
- .and_then(DateTime::from_timestamp_millis)
- .or_else(|| DateTime::from_timestamp(seconds, 0))
-}
-
-/// Like [`millis_or_seconds`] but both inputs are optional (e.g. email
-/// `sent_at`, call `ended_at`).
-pub(crate) fn opt_millis_or_seconds(
- millis: Option<i64>,
- seconds: Option<i64>,
-) -> Option<DateTime<Utc>> {
- millis
- .and_then(DateTime::from_timestamp_millis)
- .or_else(|| seconds.and_then(|s| DateTime::from_timestamp(s, 0)))
+/// Convert an epoch-millisecond timestamp to a `DateTime<Utc>`.
+pub(crate) fn millis_to_datetime(millis: Option<i64>) -> Option<DateTime<Utc>> {
+ millis.and_then(DateTime::from_timestamp_millis)
}
pub fn should_wildcard_field_query_builder<'a>(
diff --git a/crates/opensearch_client/src/upsert/call_record.rs b/crates/opensearch_client/src/upsert/call_record.rs
index 59ec036f04..90a0202ec8 100644
--- a/crates/opensearch_client/src/upsert/call_record.rs
+++ b/crates/opensearch_client/src/upsert/call_record.rs
@@ -4,11 +4,7 @@ use models_opensearch::SearchIndex;
use super::BulkUpsertResult;
use crate::upsert::properties::IndexedProperty;
-use crate::{
- Result,
- date_format::{EpochMillis, EpochSeconds},
- error::OpensearchClientError,
-};
+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};
#[cfg(test)]
mod test;
@@ -35,9 +31,6 @@ pub struct UpsertCallRecordSegmentArgs {
pub speaker_id: String,
pub sequence_num: i32,
pub content: String,
- pub started_at_seconds: EpochSeconds,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub ended_at_seconds: Option<EpochSeconds>,
pub started_at_millis: EpochMillis,
#[serde(skip_serializing_if = "Option::is_none")]
pub ended_at_millis: Option<EpochMillis>,
@@ -85,7 +78,6 @@ fn parent_doc_body(any_segment: &UpsertCallRecordSegmentArgs) -> serde_json::Val
"entity_id": &any_segment.call_id,
"channel_id": &any_segment.channel_id,
"participant_ids": &any_segment.participant_ids,
- "started_at_seconds": any_segment.started_at_seconds,
"started_at_millis": any_segment.started_at_millis,
"call_relation": PARENT_RELATION,
});
@@ -95,9 +87,6 @@ fn parent_doc_body(any_segment: &UpsertCallRecordSegmentArgs) -> serde_json::Val
if let Some(name) = &any_segment.name {
doc["name"] = serde_json::Value::String(name.clone());
}
- if let Some(ended) = &any_segment.ended_at_seconds {
- doc["ended_at_seconds"] = serde_json::to_value(ended).unwrap_or(serde_json::Value::Null);
- }
if let Some(ended) = &any_segment.ended_at_millis {
doc["ended_at_millis"] = serde_json::to_value(ended).unwrap_or(serde_json::Value::Null);
}
@@ -117,8 +106,6 @@ fn child_doc_body(seg: &UpsertCallRecordSegmentArgs) -> serde_json::Value {
"speaker_id": &seg.speaker_id,
"sequence_num": seg.sequence_num,
"content": &seg.content,
- "started_at_seconds": seg.started_at_seconds,
- "ended_at_seconds": &seg.ended_at_seconds,
"started_at_millis": seg.started_at_millis,
"ended_at_millis": &seg.ended_at_millis,
"call_relation": {
diff --git a/crates/opensearch_client/src/upsert/call_record/test.rs b/crates/opensearch_client/src/upsert/call_record/test.rs
index 31a1bd76db..85f039c095 100644
--- a/crates/opensearch_client/src/upsert/call_record/test.rs
+++ b/crates/opensearch_client/src/upsert/call_record/test.rs
@@ -11,8 +11,6 @@ fn segment() -> UpsertCallRecordSegmentArgs {
speaker_id: "macro|[email protected]".to_string(),
sequence_num: 0,
content: "segment content".to_string(),
- started_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),
- ended_at_seconds: Some(EpochSeconds::new(1_700_000_100).unwrap()),
started_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),
ended_at_millis: Some(EpochMillis::new(1_700_000_100_456).unwrap()),
properties: vec![],
diff --git a/crates/opensearch_client/src/upsert/channel_message.rs b/crates/opensearch_client/src/upsert/channel_message.rs
index fbe882891f..e8ba14a271 100644
--- a/crates/opensearch_client/src/upsert/channel_message.rs
+++ b/crates/opensearch_client/src/upsert/channel_message.rs
@@ -1,10 +1,6 @@
use models_opensearch::SearchIndex;
-use crate::{
- Result,
- date_format::{EpochMillis, EpochSeconds},
- error::OpensearchClientError,
-};
+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};
/// The arguments for upserting a channel message into the opensearch index.
/// Threadless messages are indexed with `thread_id == message_id`.
@@ -19,8 +15,6 @@ pub struct UpsertChannelMessageArgs {
pub sender_id: String,
pub mentions: Vec<String>,
pub content: String,
- pub created_at_seconds: EpochSeconds,
- pub updated_at_seconds: EpochSeconds,
pub created_at_millis: EpochMillis,
pub updated_at_millis: EpochMillis,
}
diff --git a/crates/opensearch_client/src/upsert/chat_message.rs b/crates/opensearch_client/src/upsert/chat_message.rs
index 4b48aa0cfc..b960fb5d21 100644
--- a/crates/opensearch_client/src/upsert/chat_message.rs
+++ b/crates/opensearch_client/src/upsert/chat_message.rs
@@ -1,11 +1,7 @@
use models_opensearch::SearchIndex;
use crate::upsert::properties::IndexedProperty;
-use crate::{
- Result,
- date_format::{EpochMillis, EpochSeconds},
- error::OpensearchClientError,
-};
+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};
#[cfg(test)]
mod test;
@@ -28,10 +24,6 @@ pub struct UpsertChatMessageArgs {
pub user_id: String,
/// The role of the chat message
pub role: String,
- /// The created at time of the chat message
- pub created_at_seconds: EpochSeconds,
- /// The updated at time of the chat message
- pub updated_at_seconds: EpochSeconds,
/// The created at time of the chat message, in milliseconds
pub created_at_millis: EpochMillis,
/// The updated at time of the chat message, in milliseconds
@@ -58,7 +50,6 @@ fn parent_doc_body(args: &UpsertChatMessageArgs) -> serde_json::Value {
"entity_id": &args.chat_id,
"title": &args.title,
"user_id": &args.user_id,
- "updated_at_seconds": args.updated_at_seconds,
"updated_at_millis": args.updated_at_millis,
"chat_relation": PARENT_RELATION,
});
@@ -77,8 +68,6 @@ fn child_doc_body(args: &UpsertChatMessageArgs) -> serde_json::Value {
"chat_message_id": &args.chat_message_id,
"content": &args.content,
"role": &args.role,
- "created_at_seconds": args.created_at_seconds,
- "updated_at_seconds": args.updated_at_seconds,
"created_at_millis": args.created_at_millis,
"updated_at_millis": args.updated_at_millis,
"chat_relation": {
diff --git a/crates/opensearch_client/src/upsert/chat_message/test.rs b/crates/opensearch_client/src/upsert/chat_message/test.rs
index 55f735623a..32250a9e61 100644
--- a/crates/opensearch_client/src/upsert/chat_message/test.rs
+++ b/crates/opensearch_client/src/upsert/chat_message/test.rs
@@ -6,8 +6,6 @@ fn args() -> UpsertChatMessageArgs {
chat_message_id: "msg1".to_string(),
user_id: "macro|[email protected]".to_string(),
role: "user".to_string(),
- created_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),
- updated_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),
created_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),
updated_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),
title: "Chat title".to_string(),
diff --git a/crates/opensearch_client/src/upsert/document.rs b/crates/opensearch_client/src/upsert/document.rs
index 5fa5598260..28ef44745b 100644
--- a/crates/opensearch_client/src/upsert/document.rs
+++ b/crates/opensearch_client/src/upsert/document.rs
@@ -4,11 +4,7 @@ use models_opensearch::SearchIndex;
use super::BulkUpsertResult;
use super::properties::IndexedProperty;
-use crate::{
- Result,
- date_format::{EpochMillis, EpochSeconds},
- error::OpensearchClientError,
-};
+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};
/// Relation name for parent docs in the join field.
const PARENT_RELATION: &str = "document";
@@ -41,8 +37,6 @@ pub struct UpsertDocumentArgs {
pub raw_content: Option<String>,
/// The content of the document
pub content: String,
- /// The updated at time of the document
- pub updated_at_seconds: EpochSeconds,
/// The updated at time of the document, in milliseconds
pub updated_at_millis: EpochMillis,
/// The sub type of the document (e.g. task)
@@ -69,7 +63,6 @@ fn parent_doc_body(any_chunk: &UpsertDocumentArgs) -> serde_json::Value {
"document_name": &any_chunk.document_name,
"owner_id": &any_chunk.owner_id,
"file_type": &any_chunk.file_type,
- "updated_at_seconds": any_chunk.updated_at_seconds,
"updated_at_millis": any_chunk.updated_at_millis,
"document_relation": PARENT_RELATION,
});
diff --git a/crates/opensearch_client/src/upsert/document/test.rs b/crates/opensearch_client/src/upsert/document/test.rs
index 83d04543dc..bded2aa8b6 100644
--- a/crates/opensearch_client/src/upsert/document/test.rs
+++ b/crates/opensearch_client/src/upsert/document/test.rs
@@ -1,5 +1,5 @@
use super::*;
-use crate::date_format::{EpochMillis, EpochSeconds};
+use crate::date_format::EpochMillis;
fn args(doc_id: &str, node_id: &str) -> UpsertDocumentArgs {
UpsertDocumentArgs {
@@ -10,7 +10,6 @@ fn args(doc_id: &str, node_id: &str) -> UpsertDocumentArgs {
owner_id: format!("owner-{doc_id}"),
raw_content: None,
content: format!("content {doc_id} {node_id}"),
- updated_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),
updated_at_millis: EpochMillis::new(1_700_000_000_123).unwrap(),
sub_type: None,
properties: vec![],
diff --git a/crates/opensearch_client/src/upsert/email.rs b/crates/opensearch_client/src/upsert/email.rs
index 1fc6c3ad95..1e41ac35bd 100644
--- a/crates/opensearch_client/src/upsert/email.rs
+++ b/crates/opensearch_client/src/upsert/email.rs
@@ -2,11 +2,7 @@ use models_opensearch::SearchIndex;
use super::BulkUpsertResult;
use super::properties::IndexedProperty;
-use crate::{
- Result,
- date_format::{EpochMillis, EpochSeconds},
- error::OpensearchClientError,
-};
+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};
/// The arguments for upserting an email message into the opensearch index
#[derive(Debug, serde::Serialize)]
@@ -42,14 +38,10 @@ pub struct UpsertEmailArgs {
pub link_id: String,
/// The user id of the email message
pub user_id: String,
- /// The updated at time of the email message
- pub updated_at_seconds: EpochSeconds,
/// The updated at time of the email message, in milliseconds
pub updated_at_millis: EpochMillis,
/// The subject of the email message
pub subject: Option<String>,
- /// The sent at time of the email message
- pub sent_at_seconds: Option<EpochSeconds>,
/// The sent at time of the email message, in milliseconds
pub sent_at_millis: Option<EpochMillis>,
/// The content of the email message
diff --git a/crates/opensearch_client/src/upsert/project.rs b/crates/opensearch_client/src/upsert/project.rs
index bec2e66b88..859e465183 100644
--- a/crates/opensearch_client/src/upsert/project.rs
+++ b/crates/opensearch_client/src/upsert/project.rs
@@ -1,11 +1,7 @@
use models_opensearch::SearchIndex;
use super::properties::IndexedProperty;
-use crate::{
- Result,
- date_format::{EpochMillis, EpochSeconds},
- error::OpensearchClientError,
-};
+use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};
/// The arguments for upserting a project into the opensearch index.
///
@@ -22,10 +18,6 @@ pub struct UpsertProjectArgs {
/// The parent project id, if nested
#[serde(skip_serializing_if = "Option::is_none")]
pub parent_project_id: Option<String>,
- /// The created at time of the project
- pub created_at_seconds: EpochSeconds,
- /// The updated at time of the project
- pub updated_at_seconds: EpochSeconds,
/// The created at time of the project, in milliseconds
pub created_at_millis: EpochMillis,
/// The updated at time of the project, in milliseconds
diff --git a/infra/stacks/opensearch/helpers/scripts/copy_timestamp_seconds_to_millis.ts b/infra/stacks/opensearch/helpers/scripts/copy_timestamp_seconds_to_millis.ts
deleted file mode 100644
index c0ab3bb65d..0000000000
--- a/infra/stacks/opensearch/helpers/scripts/copy_timestamp_seconds_to_millis.ts
+++ /dev/null
@@ -1,217 +0,0 @@
-require('dotenv').config();
-
-import type { Client } from '@opensearch-project/opensearch';
-import { client } from '../client';
-import {
- CALL_RECORDS_ALIAS,
- CHANNEL_INDEX,
- CHAT_INDEX,
- DOCUMENT_INDEX,
- EMAIL_INDEX,
- PROJECTS_ALIAS,
-} from '../constants';
-import { checkIndexExists } from '../utils/check_index_exists';
-import { copyFieldData } from '../utils/copy_field';
-
-// `<base>_seconds` → `<base>_millis` pairs to backfill. call_records only
-// backfills the real started_at/ended_at fields; its created_at/updated_at
-// are aliases onto started_at and need no separate data.
-type FieldPair = { base: string };
-
-interface IndexMigration {
- indexName: string;
- fields: FieldPair[];
-}
-
-const MIGRATIONS: IndexMigration[] = [
- {
- indexName: CHANNEL_INDEX,
- fields: [{ base: 'created_at' }, { base: 'updated_at' }],
- },
- {
- indexName: CHAT_INDEX,
- fields: [{ base: 'created_at' }, { base: 'updated_at' }],
- },
- {
- indexName: DOCUMENT_INDEX,
- fields: [{ base: 'updated_at' }],
- },
- {
- indexName: EMAIL_INDEX,
- fields: [{ base: 'updated_at' }, { base: 'sent_at' }],
- },
- {
- indexName: PROJECTS_ALIAS,
- fields: [{ base: 'created_at' }, { base: 'updated_at' }],
- },
- {
- indexName: CALL_RECORDS_ALIAS,
- fields: [{ base: 'started_at' }, { base: 'ended_at' }],
- },
-];
-
-/**
- * Sample docs that have both fields and confirm `millis` falls within the same
- * second as `seconds` (i.e. `floor(millis / 1000) === seconds`). This holds for
- * both backfilled docs (millis === seconds * 1000) and dual-written docs that
- * carry genuine sub-second precision (millis === seconds * 1000 + remainder).
- */
-async function verifyMillis(
- opensearchClient: Client,
- indexName: string,
- secondsField: string,
- millisField: string
-): Promise<void> {
- const response = await opensearchClient.search({
- index: indexName,
- body: {
- size: 50,
- query: {
- bool: {
- must: [
- { exists: { field: secondsField } },
- { exists: { field: millisField } },
- ],
- },
- },
- _source: [secondsField, millisField],
- },
- });
-
- const hits = response.body.hits.hits;
- if (hits.length === 0) {
- console.log(
- ` ⚠️ No documents with both "${secondsField}" and "${millisField}" to verify`
- );
- return;
- }
-
- let mismatches = 0;
- for (const hit of hits) {
- const seconds = hit._source?.[secondsField];
- const millis = hit._source?.[millisField];
- if (
- typeof seconds !== 'number' ||
- typeof millis !== 'number' ||
- Math.floor(millis / 1000) !== seconds
- ) {
- mismatches += 1;
- console.log(
- ` ⚠️ Mismatch (ID ${hit._id}): ${secondsField}=${seconds} ${millisField}=${millis}`
- );
- }
- }
- if (mismatches === 0) {
- console.log(
- ` ✓ Verified ${hits.length} sample docs: floor(${millisField} / 1000) === ${secondsField}`
- );
- } else {
- throw new Error(
- `${mismatches}/${hits.length} sample docs failed ${millisField} verification`
- );
- }
-}
-
-async function backfillIndex(
- opensearchClient: Client,
- migration: IndexMigration,
- dryRun: boolean
-): Promise<void> {
- console.log(`\n${'='.repeat(60)}`);
- console.log(
- `Backfilling millis for index: ${migration.indexName} ${dryRun ? '(DRY-RUN)' : ''}`
- );
- console.log(`${'='.repeat(60)}`);
-
- const indexExists = await checkIndexExists(
- opensearchClient,
- migration.indexName
- );
- if (!indexExists) {
- console.log(
- `⚠️ Index "${migration.indexName}" does not exist. Skipping...`
- );
- return;
- }
-
- for (const { base } of migration.fields) {
- const secondsField = `${base}_seconds`;
- const millisField = `${base}_millis`;
- console.log(`\nProcessing ${secondsField} → ${millisField}`);
-
- await copyFieldData(
- opensearchClient,
- migration.indexName,
- secondsField,
- millisField,
- dryRun,
- false, // only where millis is missing — idempotent, safe to re-run
- `ctx._source.${secondsField} * 1000L`
- );
-
- if (!dryRun) {
- await verifyMillis(
- opensearchClient,
- migration.indexName,
- secondsField,
- millisField
- );
- }
- }
-
- console.log(
- `\n✓ Completed millis backfill for index: ${migration.indexName}`
- );
-}
-
-async function backfill(dryRun: boolean = true) {
- const opensearchClient = client();
-
- console.log('\n' + '='.repeat(60));
- console.log(
- `Copy Timestamp Seconds → Millis ${dryRun ? '(DRY-RUN MODE)' : '(LIVE MODE)'}`
- );
- console.log('='.repeat(60));
- console.log(
- '\nThis script derives *_millis fields from *_seconds (× 1000) for existing docs.'
- );
- console.log('It only updates docs where the *_millis field is missing.');
- console.log("\n💡 Safe to run multiple times — it's idempotent.");
- console.log(
- '⚠️ Run create_timestamp_millis_fields.ts first so the mappings exist.'
- );
-
- if (dryRun) {
- console.log('\n⚠️ DRY-RUN MODE: No changes will be made to the cluster');
- } else {
- console.log('\n🚨 LIVE MODE: Data will be backfilled');
- }
-
- try {
- for (const migration of MIGRATIONS) {
- await backfillIndex(opensearchClient, migration, dryRun);
- }
-
- console.log('\n' + '='.repeat(60));
- console.log('Backfill completed successfully!');
- console.log('='.repeat(60));
-
- if (dryRun) {
- console.log(
- '\nTo run for real, set DRY_RUN=false environment variable\n'
- );
- } else {
- console.log('\n✓ All *_millis fields backfilled from *_seconds.');
- console.log(
- '💡 Re-run after deploying the dual-write code to catch docs written mid-migration.\n'
- );
- }
- } catch (error) {
- console.error('\n❌ Backfill failed:', error);
- throw error;
- }
-}
-
-const isDryRun = process.env.DRY_RUN !== 'false';
-
-backfill(isDryRun);
diff --git a/infra/stacks/opensearch/helpers/scripts/create_timestamp_millis_fields.ts b/infra/stacks/opensearch/helpers/scripts/create_timestamp_millis_fields.ts
deleted file mode 100644
index 2ae9a04ed9..0000000000
--- a/infra/stacks/opensearch/helpers/scripts/create_timestamp_millis_fields.ts
+++ /dev/null
@@ -1,201 +0,0 @@
-require('dotenv').config();
-
-import type { Client } from '@opensearch-project/opensearch';
-import { client } from '../client';
-import {
- CALL_RECORDS_ALIAS,
- CHANNEL_INDEX,
- CHAT_INDEX,
- DOCUMENT_INDEX,
- EMAIL_INDEX,
- PROJECTS_ALIAS,
-} from '../constants';
-import { checkIndexExists } from '../utils/check_index_exists';
-
-// A real `epoch_millis` date field to add.
-type MillisField = { fieldName: string };
-// An `alias` field pointing at an already-present real field. Used only by
-// call_records, mirroring its existing created_at_seconds/updated_at_seconds
-// aliases onto started_at_seconds.
-type AliasField = { fieldName: string; aliasFor: string };
-
-interface IndexMigration {
- indexName: string;
- fields: MillisField[];
- aliases?: AliasField[];
-}
-
-// Millisecond counterparts of every `*_seconds` field, matching the mappings
-// in create_indices.ts. call_records keeps only started_at/ended_at as real
-// fields; created_at/updated_at are aliases onto started_at.
-const MIGRATIONS: IndexMigration[] = [
- {
- indexName: CHANNEL_INDEX,
- fields: [
- { fieldName: 'created_at_millis' },
- { fieldName: 'updated_at_millis' },
- ],
- },
- {
- indexName: CHAT_INDEX,
- fields: [
- { fieldName: 'created_at_millis' },
- { fieldName: 'updated_at_millis' },
- ],
- },
- {
- indexName: DOCUMENT_INDEX,
- fields: [{ fieldName: 'updated_at_millis' }],
- },
- {
- indexName: EMAIL_INDEX,
- fields: [
- { fieldName: 'updated_at_millis' },
- { fieldName: 'sent_at_millis' },
- ],
- },
- {
- indexName: PROJECTS_ALIAS,
- fields: [
- { fieldName: 'created_at_millis' },
- { fieldName: 'updated_at_millis' },
- ],
- },
- {
- indexName: CALL_RECORDS_ALIAS,
- fields: [
- { fieldName: 'started_at_millis' },
- { fieldName: 'ended_at_millis' },
- ],
- aliases: [
- { fieldName: 'created_at_millis', aliasFor: 'started_at_millis' },
- { fieldName: 'updated_at_millis', aliasFor: 'started_at_millis' },
- ],
- },
-];
-
-async function putMapping(
- opensearchClient: Client,
- indexName: string,
- properties: Record<string, unknown>,
- label: string,
- dryRun: boolean
-): Promise<void> {
- console.log(
- ` ${dryRun ? '[DRY-RUN] Would add' : 'Adding'} ${label} to index "${indexName}"`
- );
- if (dryRun) {
- return;
- }
- const response = await opensearchClient.indices.putMapping({
- index: indexName,
- body: { properties },
- });
- if (!response.body.acknowledged) {
- throw new Error(`Failed to add ${label} in index "${indexName}"`);
- }
- console.log(` ✓ Successfully added ${label}`);
-}
-
-async function addFieldsToIndex(
- opensearchClient: Client,
- migration: IndexMigration,
- dryRun: boolean
-): Promise<void> {
- console.log(`\n${'='.repeat(60)}`);
- console.log(
- `Adding millis fields to index: ${migration.indexName} ${dryRun ? '(DRY-RUN)' : ''}`
- );
- console.log(`${'='.repeat(60)}`);
-
- const indexExists = await checkIndexExists(
- opensearchClient,
- migration.indexName
- );
- if (!indexExists) {
- console.log(
- `⚠️ Index "${migration.indexName}" does not exist. Skipping...`
- );
- return;
- }
-
- // Real date fields first, so any aliases can reference them.
- for (const field of migration.fields) {
- await putMapping(
- opensearchClient,
- migration.indexName,
- {
- [field.fieldName]: {
- type: 'date',
- format: 'epoch_millis',
- index: false,
- doc_values: true,
- },
- },
- `field "${field.fieldName}" (epoch_millis)`,
- dryRun
- );
- }
-
- for (const alias of migration.aliases ?? []) {
- await putMapping(
- opensearchClient,
- migration.indexName,
- { [alias.fieldName]: { type: 'alias', path: alias.aliasFor } },
- `alias "${alias.fieldName}" → "${alias.aliasFor}"`,
- dryRun
- );
- }
-
- console.log(`\n✓ Completed millis fields for index: ${migration.indexName}`);
-}
-
-async function createFields(dryRun: boolean = true) {
- const opensearchClient = client();
-
- console.log('\n' + '='.repeat(60));
- console.log(
- `Create Timestamp Millis Fields ${dryRun ? '(DRY-RUN MODE)' : '(LIVE MODE)'}`
- );
- console.log('='.repeat(60));
- console.log(
- '\nThis script adds *_millis (epoch_millis) timestamp fields to indices.'
- );
- console.log(
- 'Additive and idempotent; safe to run before or after the millis backfill.'
- );
-
- if (dryRun) {
- console.log('\n⚠️ DRY-RUN MODE: No changes will be made to the cluster');
- } else {
- console.log('\n🚨 LIVE MODE: Fields will be added to indices');
- }
-
- try {
- for (const migration of MIGRATIONS) {
- await addFieldsToIndex(opensearchClient, migration, dryRun);
- }
-
- console.log('\n' + '='.repeat(60));
- console.log('Field creation completed successfully!');
- console.log('='.repeat(60));
-
- if (dryRun) {
- console.log(
- '\nTo run for real, set DRY_RUN=false environment variable\n'
- );
- } else {
- console.log('\n✓ All *_millis fields have been added to indices.');
- console.log(
- '✓ Next step: run copy_timestamp_seconds_to_millis.ts to backfill existing docs.\n'
- );
- }
- } catch (error) {
- console.error('\n❌ Field creation failed:', error);
- throw error;
- }
-}
-
-const isDryRun = process.env.DRY_RUN !== 'false';
-
-createFields(isDryRun);
diff --git a/infra/stacks/opensearch/helpers/utils/copy_field.ts b/infra/stacks/opensearch/helpers/utils/copy_field.ts
index 0dbde0c098..4eed30bd3a 100644
--- a/infra/stacks/opensearch/helpers/utils/copy_field.ts
+++ b/infra/stacks/opensearch/helpers/utils/copy_field.ts
@@ -10,9 +10,6 @@ import type { Client } from '@opensearch-project/opensearch';
* @param includeNonNull A boolean indicating whether to overwrite existing non-null values in newField.
* If false (default), only updates documents where newField doesn't exist.
* If true, updates all documents regardless of existing newField values.
- * @param valueExpr A painless expression producing the value to assign to newField.
- * Defaults to a verbatim copy (`ctx._source.<oldField>`). Pass a
- * transform (e.g. `ctx._source.<oldField> * 1000L`) to derive newField.
* @returns A Promise that resolves to the number of documents updated.
*/
export async function copyFieldData(
@@ -21,12 +18,11 @@ export async function copyFieldData(
oldField: string,
newField: string,
dryRun: boolean,
- includeNonNull: boolean = false,
- valueExpr: string = `ctx._source.${oldField}`
+ includeNonNull: boolean = false
): Promise<number> {
const script = {
- // If the oldField exists and is not null, assign the derived value to the new field
- source: `if (ctx._source.containsKey('${oldField}') && ctx._source.${oldField} != null) { ctx._source.${newField} = ${valueExpr}; }`,
+ // If the oldField exists and is not null, copy the value to the new field
+ source: `if (ctx._source.containsKey('${oldField}') && ctx._source.${oldField} != null) { ctx._source.${newField} = ctx._source.${oldField}; }`,
lang: 'painless',
};
diff --git a/services/search_processing_service/src/process/call.rs b/services/search_processing_service/src/process/call.rs
index 6015c3de4f..0929b82f02 100644
--- a/services/search_processing_service/src/process/call.rs
+++ b/services/search_processing_service/src/process/call.rs
@@ -1,9 +1,7 @@
use anyhow::Context;
use models_properties::EntityType;
use opensearch_client::{
- OpensearchClient,
- date_format::{EpochMillis, EpochSeconds},
- upsert::call_record::UpsertCallRecordSegmentArgs,
+ OpensearchClient, date_format::EpochMillis, upsert::call_record::UpsertCallRecordSegmentArgs,
};
use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;
use sqlx::PgPool;
@@ -58,10 +56,6 @@ pub async fn process_call_record(
.segments
.into_iter()
.map(|seg| {
- let ended_at_seconds = seg
- .ended_at
- .map(|dt| EpochSeconds::new(dt.timestamp()))
- .transpose()?;
let ended_at_millis = seg
.ended_at
.map(|dt| EpochMillis::new(dt.timestamp_millis()))
@@ -76,8 +70,6 @@ pub async fn process_call_record(
speaker_id: seg.speaker_id,
sequence_num: seg.sequence_num,
content: seg.content,
- started_at_seconds: EpochSeconds::new(seg.started_at.timestamp())?,
- ended_at_seconds,
started_at_millis: EpochMillis::new(seg.started_at.timestamp_millis())?,
ended_at_millis,
properties: properties.clone(),
diff --git a/services/search_processing_service/src/process/channel.rs b/services/search_processing_service/src/process/channel.rs
index 1e66dd53cc..6470dac707 100644
--- a/services/search_processing_service/src/process/channel.rs
+++ b/services/search_processing_service/src/process/channel.rs
@@ -1,9 +1,7 @@
use anyhow::Context;
use mention_utils::parse::{ParsedXmlText, PlainTextFormatter, XmlFormatter};
use opensearch_client::{
- OpensearchClient,
- date_format::{EpochMillis, EpochSeconds},
- upsert::channel_message::UpsertChannelMessageArgs,
+ OpensearchClient, date_format::EpochMillis, upsert::channel_message::UpsertChannelMessageArgs,
};
use sqlx::{Pool, Postgres};
use sqs_client::search::channel::{ChannelMessageUpdate, RemoveChannelMessage};
@@ -57,12 +55,6 @@ pub async fn process_channel_message_update(
sender_id: channel_message_info.channel_message.sender_id,
mentions: channel_message_info.channel_message.mentions,
content: transformed_content.trim().to_string(),
- created_at_seconds: EpochSeconds::new(
- channel_message_info.channel_message.created_at.timestamp(),
- )?,
- updated_at_seconds: EpochSeconds::new(
- channel_message_info.channel_message.updated_at.timestamp(),
- )?,
created_at_millis: EpochMillis::new(
channel_message_info
.channel_message
diff --git a/services/search_processing_service/src/process/chat.rs b/services/search_processing_service/src/process/chat.rs
index 7934ebd96b..fb33b30bd0 100644
--- a/services/search_processing_service/src/process/chat.rs
+++ b/services/search_processing_service/src/process/chat.rs
@@ -1,9 +1,7 @@
use anyhow::Context;
use models_properties::EntityType;
use opensearch_client::{
- OpensearchClient,
- date_format::{EpochMillis, EpochSeconds},
- upsert::chat_message::UpsertChatMessageArgs,
+ OpensearchClient, date_format::EpochMillis, upsert::chat_message::UpsertChatMessageArgs,
};
use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;
use sqs_client::search::chat::{ChatMessage, RemoveChatMessage};
@@ -58,8 +56,6 @@ pub async fn insert_chat_message(
chat_id: chat_message.chat_id.clone(),
chat_message_id: chat_message.message_id.clone(),
user_id: chat_message.user_id.clone(),
- created_at_seconds: EpochSeconds::new(chat_message.created_at.timestamp())?,
- updated_at_seconds: EpochSeconds::new(chat_message.updated_at.timestamp())?,
created_at_millis: EpochMillis::new(chat_message.created_at.timestamp_millis())?,
updated_at_millis: EpochMillis::new(chat_message.updated_at.timestamp_millis())?,
title: info.name,
diff --git a/services/search_processing_service/src/process/document/raw_document.rs b/services/search_processing_service/src/process/document/raw_document.rs
index e296483b5c..937b2dcefd 100644
--- a/services/search_processing_service/src/process/document/raw_document.rs
+++ b/services/search_processing_service/src/process/document/raw_document.rs
@@ -8,9 +8,7 @@ use models_properties::EntityType;
use models_search::document::MarkdownParseResult;
use models_search::unified::is_searchable_association;
use opensearch_client::{
- OpensearchClient,
- date_format::{EpochMillis, EpochSeconds},
- upsert::document::UpsertDocumentArgs,
+ OpensearchClient, date_format::EpochMillis, upsert::document::UpsertDocumentArgs,
};
use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;
use s3_key::{
@@ -126,7 +124,6 @@ fn generate_parent_only_upsert(
content: String::new(),
owner_id: document_info.owner.to_string(),
file_type,
- updated_at_seconds: EpochSeconds::new(Utc::now().timestamp())?,
updated_at_millis: EpochMillis::new(Utc::now().timestamp_millis())?,
sub_type: document_info.sub_type.map(|st| st.to_string()),
properties: vec![],
@@ -295,7 +292,6 @@ pub async fn update_search_with_raw_document(
tracing::trace!("got raw file content");
- let updated_at = EpochSeconds::new(Utc::now().timestamp())?;
let updated_at_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
let uuid = macro_uuid::generate_uuid_v7().to_string();
@@ -315,7 +311,6 @@ pub async fn update_search_with_raw_document(
content: page_content.clone(),
owner_id: search_extractor_message.user_id.clone(),
file_type: file_type.to_string(),
- updated_at_seconds: updated_at,
updated_at_millis,
sub_type: sub_type.clone(),
properties: vec![],
@@ -340,7 +335,6 @@ pub async fn update_search_with_raw_document(
content: content.clone(),
owner_id: search_extractor_message.user_id.clone(),
file_type: file_type.to_string(),
- updated_at_seconds: updated_at,
updated_at_millis,
sub_type: sub_type.clone(),
properties: vec![],
@@ -362,7 +356,6 @@ pub async fn update_search_with_raw_document(
content: result.content,
owner_id: search_extractor_message.user_id.clone(),
file_type: file_type.to_string(),
- updated_at_seconds: updated_at,
updated_at_millis,
sub_type: sub_type.clone(),
properties: vec![],
@@ -382,7 +375,6 @@ pub async fn update_search_with_raw_document(
content: content.clone(),
owner_id: search_extractor_message.user_id.clone(),
file_type: file_type.to_string(),
- updated_at_seconds: updated_at,
updated_at_millis,
sub_type: sub_type.clone(),
properties: vec![],
@@ -409,7 +401,6 @@ fn generate_upserts(
markdown_result: Vec<MarkdownParseResult>,
) -> anyhow::Result<Vec<UpsertDocumentArgs>> {
let result = markdown_result;
- let updated_at = EpochSeconds::new(Utc::now().timestamp())?;
let updated_at_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
let file_type = FileType::from_str(
document_info
@@ -430,7 +421,6 @@ fn generate_upserts(
content: result.content,
owner_id: document_info.owner.to_string(),
file_type: file_type.to_string(),
- updated_at_seconds: updated_at,
updated_at_millis,
sub_type: sub_type.clone(),
properties: vec![],
diff --git a/services/search_processing_service/src/process/email/upsert.rs b/services/search_processing_service/src/process/email/upsert.rs
index 053f7fe228..97a32fb232 100644
--- a/services/search_processing_service/src/process/email/upsert.rs
+++ b/services/search_processing_service/src/process/email/upsert.rs
@@ -3,9 +3,7 @@ use chrono::Utc;
use models_email::service::label::system_labels;
use models_properties::EntityType;
use opensearch_client::{
- OpensearchClient,
- date_format::{EpochMillis, EpochSeconds},
- upsert::email::UpsertEmailArgs,
+ OpensearchClient, date_format::EpochMillis, upsert::email::UpsertEmailArgs,
};
use properties::outbound::entity_properties_get_query::{
get_entity_properties_for_index, get_entity_properties_for_index_batch,
@@ -54,15 +52,8 @@ pub async fn process_upsert_message(
return Ok(());
};
- let now = EpochSeconds::new(Utc::now().timestamp())?;
let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
- let updated_at = message_info
- .internal_date_ts
- .map(|date| EpochSeconds::new(date.timestamp()))
- .transpose()?
- .unwrap_or(now);
-
let updated_at_millis = message_info
.internal_date_ts
.map(|date| EpochMillis::new(date.timestamp_millis()))
@@ -132,12 +123,7 @@ pub async fn process_upsert_message(
.map(|label| label.name.clone())
.collect(),
content,
- updated_at_seconds: updated_at,
updated_at_millis,
- sent_at_seconds: message_info
- .internal_date_ts
- .map(|date| EpochSeconds::new(date.timestamp()))
- .transpose()?,
sent_at_millis: message_info
.internal_date_ts
.map(|date| EpochMillis::new(date.timestamp_millis()))
@@ -166,7 +152,6 @@ pub async fn process_upsert_thread_message(
.parse()
.context("failed to parse thread_id as UUID")?;
- let now = EpochSeconds::new(Utc::now().timestamp())?;
let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
// A full index overwrites each doc, so thread properties must ride along
@@ -205,22 +190,11 @@ pub async fn process_upsert_thread_message(
}
if let Some(content) = message.body_parsed_linkless {
- let sent_at = message
- .internal_date_ts
- .map(|date| EpochSeconds::new(date.timestamp()))
- .transpose()?;
-
let sent_at_millis = message
.internal_date_ts
.map(|date| EpochMillis::new(date.timestamp_millis()))
.transpose()?;
- let updated_at = message
- .internal_date_ts
- .map(|date| EpochSeconds::new(date.timestamp()))
- .transpose()?
- .unwrap_or(now);
-
let updated_at_millis = message
.internal_date_ts
.map(|date| EpochMillis::new(date.timestamp_millis()))
@@ -268,9 +242,7 @@ pub async fn process_upsert_thread_message(
.map(|label| label.name.clone())
.collect(),
content,
- updated_at_seconds: updated_at,
updated_at_millis,
- sent_at_seconds: sent_at,
sent_at_millis,
properties: properties.clone(),
});
@@ -314,7 +286,6 @@ pub async fn process_upsert_thread_batch_message(
.map(|id| id.parse().context("failed to parse thread_id as UUID"))
.collect::<anyhow::Result<Vec<_>>>()?;
- let now = EpochSeconds::new(Utc::now().timestamp())?;
let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
let messages =
@@ -346,22 +317,11 @@ pub async fn process_upsert_thread_batch_message(
}
if let Some(content) = message.body_parsed_linkless {
- let sent_at = message
- .internal_date_ts
- .map(|date| EpochSeconds::new(date.timestamp()))
- .transpose()?;
-
let sent_at_millis = message
.internal_date_ts
.map(|date| EpochMillis::new(date.timestamp_millis()))
.transpose()?;
- let updated_at = message
- .internal_date_ts
- .map(|date| EpochSeconds::new(date.timestamp()))
- .transpose()?
- .unwrap_or(now);
-
let updated_at_millis = message
.internal_date_ts
.map(|date| EpochMillis::new(date.timestamp_millis()))
@@ -409,9 +369,7 @@ pub async fn process_upsert_thread_batch_message(
.map(|label| label.name.clone())
.collect(),
content,
- updated_at_seconds: updated_at,
updated_at_millis,
- sent_at_seconds: sent_at,
sent_at_millis,
properties: properties_by_thread
.get(&message.thread_db_id.to_string())
diff --git a/services/search_processing_service/src/process/project.rs b/services/search_processing_service/src/process/project.rs
index 607face939..8bc0e23a34 100644
--- a/services/search_processing_service/src/process/project.rs
+++ b/services/search_processing_service/src/process/project.rs
@@ -2,7 +2,7 @@ use anyhow::Context;
use models_properties::EntityType;
use opensearch_client::{
OpensearchClient,
- date_format::{EpochMillis, EpochSeconds},
+ date_format::EpochMillis,
upsert::{project::UpsertProjectArgs, properties::IndexedProperty},
};
use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;
@@ -75,8 +75,6 @@ pub async fn upsert_project(
name: project.name,
owner_id: project.user_id,
parent_project_id: project.parent_id,
- created_at_seconds: EpochSeconds::new(created_at.timestamp())?,
- updated_at_seconds: EpochSeconds::new(updated_at.timestamp())?,
created_at_millis: EpochMillis::new(created_at.timestamp_millis())?,
updated_at_millis: EpochMillis::new(updated_at.timestamp_millis())?,
properties,