Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(search-processing): resend OpenSearch requests on dropped connections (alert 701) (#9719)

teoe96601ccdb22
diff --git a/Cargo.lock b/Cargo.lock
index 2c9b293488..63170703a9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -14611,19 +14611,25 @@ name = "opensearch_client"
version = "0.1.0"
dependencies = [
"anyhow",
+ "bytes",
"chrono",
+ "h2 0.4.16",
"hyper 1.9.0",
+ "hyper-util",
"macro_entrypoint",
"models_opensearch",
"models_search_cursor",
"opensearch",
"opensearch_query_builder",
"reqwest 0.13.4",
+ "rustls",
"serde",
"serde_json",
"strum 0.27.2",
"thiserror 2.0.18",
"tokio",
+ "tokio-retry",
+ "tokio-rustls",
"tracing",
"uuid",
"workspace-hack",
diff --git a/Cargo.toml b/Cargo.toml
index 1429646eb1..32c0e96847 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -290,6 +290,7 @@ glob = "0.3.3"
# CI always run the identical generator; guppy is pinned to the version the
# hakari release is built against.
guppy = "0.17.24"
+h2 = "0.4"
hex = "0.4.3"
hkdf = "0.12.4"
hmac = "0.12.1"
@@ -435,6 +436,7 @@ time = "0.3.37"
tiny-skia = { version = "0.11.4", default-features = false, features = ["simd", "std"] }
tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread"] }
tokio-retry = { version = "0.3.2", default-features = false }
+tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs"] }
tokio-stream = { version = "0.1.17" }
tokio-tungstenite = { version = "0.28", default-features = false, features = [
"connect",
diff --git a/crates/opensearch_client/AGENTS.md b/crates/opensearch_client/AGENTS.md
index b688491ffa..f1496623ec 100644
--- a/crates/opensearch_client/AGENTS.md
+++ b/crates/opensearch_client/AGENTS.md
@@ -51,7 +51,19 @@ projects.
`OpensearchOperation`: add a variant for a new request rather than passing a
name. `request_failed` classifies the cause as a `TransportFailure` (timeout,
connect, connection closed, other) and keeps the full source chain in
- `details`; reqwest's `Display` alone drops the cause.
+ `details`; reqwest's `Display` alone drops the cause. A body cut off by a
+ transport failure also becomes `RequestFailed`, not `DeserializationFailed`.
+- Workspace feature unification enables reqwest's HTTP/2, so every request
+ from one process multiplexes over one connection, and a recycled connection
+ fails them all at once. Wrap each public `OpensearchClient` method's request
+ in `retry::resend_on_lost_connection`, which resends twice on a connection
+ closed or refused before a complete response, never on a timeout or an
+ error status. Every request must stay safe to resend: write by explicit id
+ or by query.
+- Put `instrument(err)` on the public method, not the inner function it
+ retries, so a recovered attempt does not record an error span.
+- Requests time out after five minutes (`REQUEST_TIMEOUT`); delete-by-query
+ across a large mailbox runs for minutes.
## Related
diff --git a/crates/opensearch_client/Cargo.toml b/crates/opensearch_client/Cargo.toml
index 35316ab448..6a07840f9a 100644
--- a/crates/opensearch_client/Cargo.toml
+++ b/crates/opensearch_client/Cargo.toml
@@ -6,14 +6,20 @@ publish = false
version = "0.1.0"
[dev-dependencies]
+bytes = { workspace = true }
+hyper = { workspace = true, features = ["http2", "server"] }
+hyper-util = { workspace = true, features = ["http2", "server"] }
macro_entrypoint = { path = "../macro_entrypoint" }
-tokio = { workspace = true }
+rustls = { workspace = true }
+tokio = { workspace = true, features = ["net"] }
+tokio-rustls = { workspace = true }
uuid = { workspace = true }
[dependencies]
anyhow = { workspace = true }
chrono = { workspace = true }
+h2 = { workspace = true }
hyper = { workspace = true }
models_opensearch = { path = "../models_opensearch" }
models_search_cursor = { path = "../models_search_cursor" }
@@ -24,6 +30,8 @@ serde = { workspace = true }
serde_json = { workspace = true }
strum = { workspace = true }
thiserror = { workspace = true }
+tokio = { workspace = true, features = ["time"] }
+tokio-retry = { workspace = true }
tracing = { workspace = true }
uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/crates/opensearch_client/src/agent_session.rs b/crates/opensearch_client/src/agent_session.rs
index 628315d096..bffc045391 100644
--- a/crates/opensearch_client/src/agent_session.rs
+++ b/crates/opensearch_client/src/agent_session.rs
@@ -1,6 +1,7 @@
use crate::{
OpensearchClient, Result, delete,
error::{OpensearchClientError, OpensearchOperation},
+ retry::resend_on_lost_connection,
upsert,
};
@@ -71,7 +72,10 @@ impl OpensearchClient {
args: &ReconcileAgentSessionArgs,
index_override: Option<&str>,
) -> Result<()> {
- upsert::agent_session::reconcile_agent_session(&self.inner, args, index_override).await
+ resend_on_lost_connection(|| {
+ upsert::agent_session::reconcile_agent_session(&self.inner, args, index_override)
+ })
+ .await
}
/// Delete the parent and every folded-message child for an agent session.
@@ -81,7 +85,13 @@ impl OpensearchClient {
agent_session_id: &str,
index_override: Option<&str>,
) -> Result<()> {
- delete::agent_session::delete_agent_session(&self.inner, agent_session_id, index_override)
- .await
+ resend_on_lost_connection(|| {
+ delete::agent_session::delete_agent_session(
+ &self.inner,
+ agent_session_id,
+ index_override,
+ )
+ })
+ .await
}
}
diff --git a/crates/opensearch_client/src/calendar_event.rs b/crates/opensearch_client/src/calendar_event.rs
index 68e6246d13..59a60d25f6 100644
--- a/crates/opensearch_client/src/calendar_event.rs
+++ b/crates/opensearch_client/src/calendar_event.rs
@@ -1,5 +1,6 @@
use crate::{
OpensearchClient, Result, delete,
+ retry::resend_on_lost_connection,
upsert::{self, calendar_event::UpsertCalendarEventArgs, properties::IndexedProperty},
};
@@ -12,7 +13,10 @@ impl OpensearchClient {
args: &UpsertCalendarEventArgs,
index_override: Option<&str>,
) -> Result<()> {
- upsert::calendar_event::upsert_calendar_event(&self.inner, args, index_override).await
+ resend_on_lost_connection(|| {
+ upsert::calendar_event::upsert_calendar_event(&self.inner, args, index_override)
+ })
+ .await
}
/// Deletes a calendar event from the opensearch calendar_events index
@@ -22,8 +26,14 @@ impl OpensearchClient {
event_id: &str,
index_override: Option<&str>,
) -> Result<()> {
- delete::calendar_event::delete_calendar_event_by_id(&self.inner, event_id, index_override)
- .await
+ resend_on_lost_connection(|| {
+ delete::calendar_event::delete_calendar_event_by_id(
+ &self.inner,
+ event_id,
+ index_override,
+ )
+ })
+ .await
}
/// Updates only the denormalized `properties` on an indexed calendar event
@@ -33,12 +43,14 @@ impl OpensearchClient {
event_id: &str,
properties: &[IndexedProperty],
) -> Result<()> {
- upsert::calendar_event::update_calendar_event_properties(
- &self.inner,
- event_id,
- properties,
- None,
- )
+ resend_on_lost_connection(|| {
+ upsert::calendar_event::update_calendar_event_properties(
+ &self.inner,
+ event_id,
+ properties,
+ None,
+ )
+ })
.await
}
}
diff --git a/crates/opensearch_client/src/call_record.rs b/crates/opensearch_client/src/call_record.rs
index 874f607a33..612c6f356c 100644
--- a/crates/opensearch_client/src/call_record.rs
+++ b/crates/opensearch_client/src/call_record.rs
@@ -1,5 +1,6 @@
use crate::{
OpensearchClient, Result, delete,
+ retry::resend_on_lost_connection,
upsert::{
self, BulkUpsertResult,
call_record::{UpsertCallRecordSegmentArgs, bulk_upsert_call_record_segments},
@@ -8,22 +9,28 @@ use crate::{
};
impl OpensearchClient {
- #[tracing::instrument(skip(self))]
+ #[tracing::instrument(skip(self), err)]
pub async fn upsert_call_record_segment(
&self,
args: &UpsertCallRecordSegmentArgs,
index_override: Option<&str>,
) -> Result<()> {
- upsert::call_record::upsert_call_record_segment(&self.inner, args, index_override).await
+ resend_on_lost_connection(|| {
+ upsert::call_record::upsert_call_record_segment(&self.inner, args, index_override)
+ })
+ .await
}
- #[tracing::instrument(skip(self, segments))]
+ #[tracing::instrument(skip(self, segments), err)]
pub async fn bulk_upsert_call_record_segments(
&self,
segments: &[UpsertCallRecordSegmentArgs],
index_override: Option<&str>,
) -> Result<BulkUpsertResult> {
- bulk_upsert_call_record_segments(&self.inner, segments, index_override).await
+ resend_on_lost_connection(|| {
+ bulk_upsert_call_record_segments(&self.inner, segments, index_override)
+ })
+ .await
}
/// Refresh only the denormalized `properties` on an existing parent call doc.
@@ -33,7 +40,10 @@ impl OpensearchClient {
call_id: &str,
properties: &[IndexedProperty],
) -> Result<()> {
- upsert::call_record::update_call_record_properties(&self.inner, call_id, properties).await
+ resend_on_lost_connection(|| {
+ upsert::call_record::update_call_record_properties(&self.inner, call_id, properties)
+ })
+ .await
}
#[tracing::instrument(skip(self))]
@@ -42,7 +52,10 @@ impl OpensearchClient {
call_id: &str,
index_override: Option<&str>,
) -> Result<()> {
- delete::call_record::delete_call_record_by_id(&self.inner, call_id, index_override).await
+ resend_on_lost_connection(|| {
+ delete::call_record::delete_call_record_by_id(&self.inner, call_id, index_override)
+ })
+ .await
}
#[tracing::instrument(skip(self))]
@@ -51,11 +64,13 @@ impl OpensearchClient {
channel_id: &str,
index_override: Option<&str>,
) -> Result<()> {
- delete::call_record::delete_call_records_by_channel_id(
- &self.inner,
- channel_id,
- index_override,
- )
+ resend_on_lost_connection(|| {
+ delete::call_record::delete_call_records_by_channel_id(
+ &self.inner,
+ channel_id,
+ index_override,
+ )
+ })
.await
}
}
diff --git a/crates/opensearch_client/src/channel_message.rs b/crates/opensearch_client/src/channel_message.rs
index 28dbd93a4c..21e4d92b15 100644
--- a/crates/opensearch_client/src/channel_message.rs
+++ b/crates/opensearch_client/src/channel_message.rs
@@ -1,5 +1,6 @@
use crate::{
OpensearchClient, Result, delete,
+ retry::resend_on_lost_connection,
search::{
self,
channels::{ChannelSearchArgs, ChannelSearchResults},
@@ -9,9 +10,10 @@ use crate::{
impl OpensearchClient {
/// Performs a channel content search.
- #[tracing::instrument(skip(self, args))]
+ #[tracing::instrument(skip(self, args), err)]
pub async fn search_channel(&self, args: ChannelSearchArgs) -> Result<ChannelSearchResults> {
- search::channels::search_channel(&self.inner, args).await
+ resend_on_lost_connection(|| search::channels::search_channel(&self.inner, args.clone()))
+ .await
}
}
@@ -23,11 +25,13 @@ impl OpensearchClient {
upsert_channel_message_args: &UpsertChannelMessageArgs,
index_override: Option<&str>,
) -> Result<()> {
- upsert::channel_message::upsert_channel_message(
- &self.inner,
- upsert_channel_message_args,
- index_override,
- )
+ resend_on_lost_connection(|| {
+ upsert::channel_message::upsert_channel_message(
+ &self.inner,
+ upsert_channel_message_args,
+ index_override,
+ )
+ })
.await
}
@@ -39,7 +43,10 @@ impl OpensearchClient {
channel_id: &str,
index_override: Option<&str>,
) -> Result<()> {
- delete::channel::delete_channel_by_id(&self.inner, channel_id, index_override).await
+ resend_on_lost_connection(|| {
+ delete::channel::delete_channel_by_id(&self.inner, channel_id, index_override)
+ })
+ .await
}
/// Deletes a channel message from the opensearch channel index
@@ -51,12 +58,14 @@ impl OpensearchClient {
channel_message_id: &str,
index_override: Option<&str>,
) -> Result<()> {
- delete::channel::delete_channel_message_by_id(
- &self.inner,
- channel_id,
- channel_message_id,
- index_override,
- )
+ resend_on_lost_connection(|| {
+ delete::channel::delete_channel_message_by_id(
+ &self.inner,
+ channel_id,
+ channel_message_id,
+ index_override,
+ )
+ })
.await
}
}
diff --git a/crates/opensearch_client/src/chat.rs b/crates/opensearch_client/src/chat.rs
index 682e8ac83e..52431c957e 100644
--- a/crates/opensearch_client/src/chat.rs
+++ b/crates/opensearch_client/src/chat.rs
@@ -1,5 +1,6 @@
use crate::{
OpensearchClient, Result, delete,
+ retry::resend_on_lost_connection,
upsert::{self, chat_message::UpsertChatMessageArgs, properties::IndexedProperty},
};
@@ -10,11 +11,13 @@ impl OpensearchClient {
upsert_chat_message_args: &UpsertChatMessageArgs,
index_override: Option<&str>,
) -> Result<()> {
- upsert::chat_message::upsert_chat_message(
- &self.inner,
- upsert_chat_message_args,
- index_override,
- )
+ resend_on_lost_connection(|| {
+ upsert::chat_message::upsert_chat_message(
+ &self.inner,
+ upsert_chat_message_args,
+ index_override,
+ )
+ })
.await
}
@@ -22,7 +25,10 @@ impl OpensearchClient {
/// This will remove all messages for a given chat
#[tracing::instrument(skip(self))]
pub async fn delete_chat(&self, chat_id: &str, index_override: Option<&str>) -> Result<()> {
- delete::chat::delete_chat_by_id(&self.inner, chat_id, index_override).await
+ resend_on_lost_connection(|| {
+ delete::chat::delete_chat_by_id(&self.inner, chat_id, index_override)
+ })
+ .await
}
/// Deletes a chat message from the opensearch chat index
@@ -34,17 +40,22 @@ impl OpensearchClient {
chat_message_id: &str,
index_override: Option<&str>,
) -> Result<()> {
- delete::chat::delete_chat_message_by_id(
- &self.inner,
- chat_id,
- chat_message_id,
- index_override,
- )
+ resend_on_lost_connection(|| {
+ delete::chat::delete_chat_message_by_id(
+ &self.inner,
+ chat_id,
+ chat_message_id,
+ index_override,
+ )
+ })
.await
}
pub async fn update_chat_metadata(&self, chat_id: &str, title: &str) -> Result<()> {
- upsert::chat_message::update_chat_metadata(&self.inner, chat_id, title).await
+ resend_on_lost_connection(|| {
+ upsert::chat_message::update_chat_metadata(&self.inner, chat_id, title)
+ })
+ .await
}
/// Refresh only the denormalized `properties` on an existing parent chat doc.
@@ -54,10 +65,14 @@ impl OpensearchClient {
chat_id: &str,
properties: &[IndexedProperty],
) -> Result<()> {
- upsert::chat_message::update_chat_properties(&self.inner, chat_id, properties).await
+ resend_on_lost_connection(|| {
+ upsert::chat_message::update_chat_properties(&self.inner, chat_id, properties)
+ })
+ .await
}
pub async fn delete_chat_by_user_id(&self, user_id: &str) -> Result<()> {
- delete::chat::delete_chat_by_user_id(&self.inner, user_id).await
+ resend_on_lost_connection(|| delete::chat::delete_chat_by_user_id(&self.inner, user_id))
+ .await
}
}
diff --git a/crates/opensearch_client/src/delete/calendar_event.rs b/crates/opensearch_client/src/delete/calendar_event.rs
index 2928f79b3c..a000da5d66 100644
--- a/crates/opensearch_client/src/delete/calendar_event.rs
+++ b/crates/opensearch_client/src/delete/calendar_event.rs
@@ -9,7 +9,7 @@ use crate::{
/// `_id` = event id, so this is a direct delete. A missing-doc 404 is a no-op
/// — the event was never indexed or is already gone. A missing-index 404
/// (`index_not_found_exception`) is a real outage and must propagate.
-#[tracing::instrument(skip(client), err)]
+#[tracing::instrument(skip(client))]
pub async fn delete_calendar_event_by_id(
client: &opensearch::OpenSearch,
event_id: &str,
diff --git a/crates/opensearch_client/src/delete/project.rs b/crates/opensearch_client/src/delete/project.rs
index c53c2eab8a..f56a8c1a32 100644
--- a/crates/opensearch_client/src/delete/project.rs
+++ b/crates/opensearch_client/src/delete/project.rs
@@ -9,7 +9,7 @@ use crate::{
/// project id, so this is a direct delete. A missing-doc 404 is a no-op —
/// the project was never indexed or is already gone. A missing-index 404
/// (`index_not_found_exception`) is a real outage and must propagate.
-#[tracing::instrument(skip(client), err)]
+#[tracing::instrument(skip(client))]
pub async fn delete_project_by_id(
client: &opensearch::OpenSearch,
project_id: &str,
diff --git a/crates/opensearch_client/src/document.rs b/crates/opensearch_client/src/document.rs
index f05a82678d..fd0d2467ba 100644
--- a/crates/opensearch_client/src/document.rs
+++ b/crates/opensearch_client/src/document.rs
@@ -1,5 +1,6 @@
use crate::{
OpensearchClient, Result, delete,
+ retry::resend_on_lost_connection,
upsert::{self, document::UpsertDocumentArgs, properties::IndexedProperty},
};
@@ -7,7 +8,10 @@ impl OpensearchClient {
/// Inserts a document into the opensearch index
#[tracing::instrument(skip(self))]
pub async fn upsert_document(&self, upsert_document_args: &UpsertDocumentArgs) -> Result<()> {
- upsert::document::upsert_document(&self.inner, upsert_document_args, None).await
+ resend_on_lost_connection(|| {
+ upsert::document::upsert_document(&self.inner, upsert_document_args, None)
+ })
+ .await
}
/// Upserts only the parent doc (no content chunks) into the opensearch index
@@ -17,8 +21,14 @@ impl OpensearchClient {
upsert_document_args: &UpsertDocumentArgs,
index_override: Option<&str>,
) -> Result<()> {
- upsert::document::upsert_parent_document(&self.inner, upsert_document_args, index_override)
- .await
+ resend_on_lost_connection(|| {
+ upsert::document::upsert_parent_document(
+ &self.inner,
+ upsert_document_args,
+ index_override,
+ )
+ })
+ .await
}
/// Bulk upserts documents into the opensearch index
@@ -28,7 +38,10 @@ impl OpensearchClient {
documents: &[UpsertDocumentArgs],
index_override: Option<&str>,
) -> Result<upsert::BulkUpsertResult> {
- upsert::document::bulk_upsert_documents(&self.inner, documents, index_override).await
+ resend_on_lost_connection(|| {
+ upsert::document::bulk_upsert_documents(&self.inner, documents, index_override)
+ })
+ .await
}
/// Deletes a document from the opensearch document index
@@ -38,7 +51,10 @@ impl OpensearchClient {
document_id: &str,
index_override: Option<&str>,
) -> Result<()> {
- delete::document::delete_document_by_id(&self.inner, document_id, index_override).await
+ resend_on_lost_connection(|| {
+ delete::document::delete_document_by_id(&self.inner, document_id, index_override)
+ })
+ .await
}
#[tracing::instrument(skip(self))]
@@ -47,7 +63,10 @@ impl OpensearchClient {
document_id: &str,
document_name: &str,
) -> Result<()> {
- upsert::document::update_document_metadata(&self.inner, document_id, document_name).await
+ resend_on_lost_connection(|| {
+ upsert::document::update_document_metadata(&self.inner, document_id, document_name)
+ })
+ .await
}
/// Refresh only the denormalized `properties` on an existing parent doc.
@@ -57,12 +76,17 @@ impl OpensearchClient {
document_id: &str,
properties: &[IndexedProperty],
) -> Result<()> {
- upsert::document::update_document_properties(&self.inner, document_id, properties, None)
- .await
+ resend_on_lost_connection(|| {
+ upsert::document::update_document_properties(&self.inner, document_id, properties, None)
+ })
+ .await
}
#[tracing::instrument(skip(self))]
pub async fn delete_documents_by_owner_id(&self, owner_id: &str) -> Result<()> {
- delete::document::delete_document_by_owner_id(&self.inner, owner_id).await
+ resend_on_lost_connection(|| {
+ delete::document::delete_document_by_owner_id(&self.inner, owner_id)
+ })
+ .await
}
}
diff --git a/crates/opensearch_client/src/email.rs b/crates/opensearch_client/src/email.rs
index 8f17ee22de..5633540020 100644
--- a/crates/opensearch_client/src/email.rs
+++ b/crates/opensearch_client/src/email.rs
@@ -1,5 +1,6 @@
use crate::{
OpensearchClient, Result, delete,
+ retry::resend_on_lost_connection,
upsert::{self, BulkUpsertResult, email::UpsertEmailArgs, properties::IndexedProperty},
};
@@ -7,17 +8,23 @@ impl OpensearchClient {
/// Upserts an email message into the opensearch index
#[tracing::instrument(skip(self))]
pub async fn upsert_email_message(&self, upsert_email_args: &UpsertEmailArgs) -> Result<()> {
- upsert::email::upsert_email_message(&self.inner, upsert_email_args).await
+ resend_on_lost_connection(|| {
+ upsert::email::upsert_email_message(&self.inner, upsert_email_args)
+ })
+ .await
}
/// Bulk upserts email messages into the opensearch index
- #[tracing::instrument(skip(self, messages))]
+ #[tracing::instrument(skip(self, messages), err)]
pub async fn bulk_upsert_email_messages(
&self,
messages: &[UpsertEmailArgs],
index_override: Option<&str>,
) -> Result<BulkUpsertResult> {
- upsert::email::bulk_upsert_email_messages(&self.inner, messages, index_override).await
+ resend_on_lost_connection(|| {
+ upsert::email::bulk_upsert_email_messages(&self.inner, messages, index_override)
+ })
+ .await
}
/// Refresh only the denormalized `properties` on every message doc of a
@@ -28,7 +35,10 @@ impl OpensearchClient {
thread_id: &str,
properties: &[IndexedProperty],
) -> Result<()> {
- upsert::email::update_email_thread_properties(&self.inner, thread_id, properties).await
+ resend_on_lost_connection(|| {
+ upsert::email::update_email_thread_properties(&self.inner, thread_id, properties)
+ })
+ .await
}
/// Deletes a particular email message
@@ -37,15 +47,20 @@ impl OpensearchClient {
message_id: &str,
index_override: Option<&str>,
) -> Result<()> {
- delete::email::delete_email_message_by_id(&self.inner, message_id, index_override).await
+ resend_on_lost_connection(|| {
+ delete::email::delete_email_message_by_id(&self.inner, message_id, index_override)
+ })
+ .await
}
/// Deletes all email messages with the specified link_id
pub async fn delete_email_messages_by_link_id(&self, link_id: &str) -> Result<()> {
- delete::email::delete_email_by_link_id(&self.inner, link_id).await
+ resend_on_lost_connection(|| delete::email::delete_email_by_link_id(&self.inner, link_id))
+ .await
}
pub async fn delete_email_messages_by_user_id(&self, user_id: &str) -> Result<()> {
- delete::email::delete_email_by_user_id(&self.inner, user_id).await
+ resend_on_lost_connection(|| delete::email::delete_email_by_user_id(&self.inner, user_id))
+ .await
}
}
diff --git a/crates/opensearch_client/src/error.rs b/crates/opensearch_client/src/error.rs
index 49a38d34b5..127068b151 100644
--- a/crates/opensearch_client/src/error.rs
+++ b/crates/opensearch_client/src/error.rs
@@ -113,6 +113,13 @@ impl TransportFailure {
return Some(Self::ConnectionClosed);
}
}
+ // Over HTTP/2 a recycled connection arrives as GOAWAY or RST_STREAM, and
+ // `h2::Error` exposes no source for the I/O failure it wraps.
+ if let Some(error) = cause.downcast_ref::<h2::Error>()
+ && (error.is_go_away() || error.is_reset() || error.is_io())
+ {
+ return Some(Self::ConnectionClosed);
+ }
if let Some(error) = cause.downcast_ref::<std::io::Error>() {
use std::io::ErrorKind;
return match error.kind() {
@@ -153,8 +160,8 @@ pub enum OpensearchClientError {
operation: OpensearchOperation,
},
- /// The request produced no response: connect, TLS, timeout, or a
- /// connection closed or reset before the response arrived.
+ /// The request produced no complete response: connect, TLS, timeout, or a
+ /// connection closed or reset before the response finished.
#[error(
"request to opensearch failed. operation: {operation} cause: {cause} details: {details}"
)]
@@ -211,14 +218,33 @@ impl OpensearchClientError {
}
}
- /// A response body that could not be read or parsed, including one cut
- /// off by a dropped connection.
+ /// A response body that could not be read or parsed. A body cut off by a
+ /// dropped connection or deadline is a request failure with that cause.
pub(crate) fn deserialization_failed(operation: OpensearchOperation, error: &Error) -> Self {
- Self::DeserializationFailed {
- details: error_chain(error),
- operation,
+ match TransportFailure::classify(error) {
+ TransportFailure::Other => Self::DeserializationFailed {
+ details: error_chain(error),
+ operation,
+ },
+ cause => Self::RequestFailed {
+ details: error_chain(error),
+ cause,
+ operation,
+ },
}
}
+
+ /// The connection closed or refused before a complete response, so the same
+ /// request can succeed on a new connection.
+ pub(crate) fn lost_connection(&self) -> bool {
+ matches!(
+ self,
+ Self::RequestFailed {
+ cause: TransportFailure::ConnectionClosed | TransportFailure::Connect,
+ ..
+ }
+ )
+ }
}
/// reqwest's `Display` omits the cause (timeout, reset, refused connection),
diff --git a/crates/opensearch_client/src/lib.rs b/crates/opensearch_client/src/lib.rs
index 1eee949c79..95e1819a85 100644
--- a/crates/opensearch_client/src/lib.rs
+++ b/crates/opensearch_client/src/lib.rs
@@ -14,8 +14,12 @@ pub mod document;
pub mod email;
pub mod project;
+mod retry;
+
pub type Result<T> = std::result::Result<T, error::OpensearchClientError>;
+use std::time::Duration;
+
use opensearch::{
OpenSearch,
auth::Credentials,
@@ -27,6 +31,11 @@ use opensearch::{
indices::{IndicesCreateParts, IndicesExistsParts},
};
+/// Bounds a request on a connection that stopped answering. Delete-by-query
+/// across a large mailbox legitimately runs for minutes, so this sits well
+/// above the slowest such request.
+const REQUEST_TIMEOUT: Duration = Duration::from_secs(300);
+
#[derive(Clone, Debug)]
pub struct OpensearchClient {
/// The opensearch client used to interact with opensearch api
@@ -52,6 +61,7 @@ impl OpensearchClient {
.auth(credentials)
.disable_proxy()
.cert_validation(cert_validation)
+ .timeout(REQUEST_TIMEOUT)
.build()?;
let client = OpenSearch::new(transport);
Ok(Self { inner: client })
diff --git a/crates/opensearch_client/src/project.rs b/crates/opensearch_client/src/project.rs
index e157455b4b..0bbbf7d78c 100644
--- a/crates/opensearch_client/src/project.rs
+++ b/crates/opensearch_client/src/project.rs
@@ -1,5 +1,6 @@
use crate::{
OpensearchClient, Result, delete,
+ retry::resend_on_lost_connection,
upsert::{self, project::UpsertProjectArgs, properties::IndexedProperty},
};
@@ -10,7 +11,10 @@ impl OpensearchClient {
upsert_project_args: &UpsertProjectArgs,
index_override: Option<&str>,
) -> Result<()> {
- upsert::project::upsert_project(&self.inner, upsert_project_args, index_override).await
+ resend_on_lost_connection(|| {
+ upsert::project::upsert_project(&self.inner, upsert_project_args, index_override)
+ })
+ .await
}
/// Deletes a project from the opensearch projects index
@@ -20,7 +24,10 @@ impl OpensearchClient {
project_id: &str,
index_override: Option<&str>,
) -> Result<()> {
- delete::project::delete_project_by_id(&self.inner, project_id, index_override).await
+ resend_on_lost_connection(|| {
+ delete::project::delete_project_by_id(&self.inner, project_id, index_override)
+ })
+ .await
}
/// Updates only the denormalized `properties` on an indexed project
@@ -30,6 +37,9 @@ impl OpensearchClient {
project_id: &str,
properties: &[IndexedProperty],
) -> Result<()> {
- upsert::project::update_project_properties(&self.inner, project_id, properties, None).await
+ resend_on_lost_connection(|| {
+ upsert::project::update_project_properties(&self.inner, project_id, properties, None)
+ })
+ .await
}
}
diff --git a/crates/opensearch_client/src/retry.rs b/crates/opensearch_client/src/retry.rs
new file mode 100644
index 0000000000..1beba32717
--- /dev/null
+++ b/crates/opensearch_client/src/retry.rs
@@ -0,0 +1,34 @@
+//! Resend requests that lost their connection before a complete response.
+
+use std::time::Duration;
+
+use tokio_retry::{RetryIf, strategy::ExponentialBackoff};
+
+use crate::{Result, error::OpensearchClientError};
+
+#[cfg(test)]
+mod test;
+
+/// Delays before the second and third attempts: 100 ms, then 400 ms.
+fn resend_delays() -> impl Iterator<Item = Duration> {
+ ExponentialBackoff::from_millis(4).factor(25).take(2)
+}
+
+/// Run `request` (send and read the body), resending it while its connection
+/// closes or refuses before a complete response. Every request this crate sends writes
+/// by explicit id or by query, so a resend cannot apply twice. A timeout is
+/// not resent: OpenSearch may still be working on the first attempt.
+pub(crate) async fn resend_on_lost_connection<T, R, F>(request: R) -> Result<T>
+where
+ R: FnMut() -> F,
+ F: Future<Output = Result<T>>,
+{
+ RetryIf::start(resend_delays(), request, |error: &OpensearchClientError| {
+ let lost = error.lost_connection();
+ if lost {
+ tracing::warn!(%error, "OpenSearch connection lost before a complete response; resending");
+ }
+ lost
+ })
+ .await
+}
diff --git a/crates/opensearch_client/src/retry/test.rs b/crates/opensearch_client/src/retry/test.rs
new file mode 100644
index 0000000000..42b79275be
--- /dev/null
+++ b/crates/opensearch_client/src/retry/test.rs
@@ -0,0 +1,327 @@
+use std::{
+ convert::Infallible,
+ pin::Pin,
+ sync::{
+ Arc,
+ atomic::{AtomicBool, AtomicUsize, Ordering},
+ },
+ task::{Context, Poll},
+ time::Duration,
+};
+
+use bytes::Bytes;
+use hyper::{
+ Request, Response, StatusCode,
+ body::{Frame, Incoming},
+ service::service_fn,
+};
+use hyper_util::rt::{TokioExecutor, TokioIo};
+use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject};
+use tokio::sync::watch;
+
+use crate::{
+ OpensearchClient,
+ agent_session::{AgentSessionMessageAuthor, AgentSessionMessageDocument},
+ date_format::EpochMillis,
+ error::{OpensearchClientError, OpensearchOperation, TransportFailure},
+ upsert::{agent_session::ReconcileAgentSessionArgs, email::UpsertEmailArgs},
+};
+
+const OK_BODY: &str = r#"{"errors":false,"items":[],"failures":[],"timed_out":false}"#;
+
+/// When the server closes a connection with requests still waiting on it.
+#[derive(Clone, Copy)]
+enum Close {
+ /// Close every connection opened so far once this many requests wait on
+ /// them, then answer on new connections: a front end recycling the one
+ /// HTTP/2 connection every request multiplexes over.
+ OnceWhenWaiting(usize),
+ /// Close each connection as soon as a request arrives on it.
+ Always,
+ /// Reset the first response's stream partway through its body, as a
+ /// recycled connection does to a response still in flight.
+ CutFirstBody,
+ /// Answer every request.
+ Never,
+}
+
+/// A response body, reset after its bytes when `cut` is set.
+struct Reply {
+ data: Option<Bytes>,
+ cut: bool,
+}
+
+impl hyper::body::Body for Reply {
+ type Data = Bytes;
+ type Error = std::io::Error;
+
+ fn poll_frame(
+ mut self: Pin<&mut Self>,
+ _: &mut Context<'_>,
+ ) -> Poll<Option<Result<Frame<Bytes>, Self::Error>>> {
+ if let Some(data) = self.data.take() {
+ return Poll::Ready(Some(Ok(Frame::data(data))));
+ }
+ if std::mem::take(&mut self.cut) {
+ return Poll::Ready(Some(Err(std::io::ErrorKind::ConnectionReset.into())));
+ }
+ Poll::Ready(None)
+ }
+}
+
+struct Server {
+ url: String,
+ requests: Arc<AtomicUsize>,
+ connections: Arc<AtomicUsize>,
+}
+
+/// HTTP/2 over TLS, which reqwest negotiates with the OpenSearch domain.
+async fn server(close: Close, status: StatusCode) -> Server {
+ let certs = include_bytes!("../../../../infra/local/certs/localhost.pem");
+ let key = include_bytes!("../../../../infra/local/certs/localhost-key.pem");
+ let mut config = rustls::ServerConfig::builder_with_provider(Arc::new(
+ rustls::crypto::aws_lc_rs::default_provider(),
+ ))
+ .with_safe_default_protocol_versions()
+ .unwrap()
+ .with_no_client_auth()
+ .with_single_cert(
+ CertificateDer::pem_slice_iter(certs)
+ .collect::<Result<_, _>>()
+ .unwrap(),
+ PrivateKeyDer::from_pem_slice(key).unwrap(),
+ )
+ .unwrap();
+ config.alpn_protocols = vec![b"h2".to_vec()];
+ let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config));
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let url = format!(
+ "https://localhost:{}",
+ listener.local_addr().unwrap().port()
+ );
+
+ let requests = Arc::new(AtomicUsize::new(0));
+ let connections = Arc::new(AtomicUsize::new(0));
+ let waiting = Arc::new(AtomicUsize::new(0));
+ let cut_pending = Arc::new(AtomicBool::new(true));
+ let (closed_tx, closed_rx) = watch::channel(false);
+ let closed_tx = Arc::new(closed_tx);
+ let (server_requests, server_connections) = (requests.clone(), connections.clone());
+ tokio::spawn(async move {
+ loop {
+ let (tcp, _) = listener.accept().await.unwrap();
+ let tls = acceptor.accept(tcp).await.unwrap();
+ server_connections.fetch_add(1, Ordering::SeqCst);
+ let holds = match close {
+ Close::OnceWhenWaiting(_) => !*closed_rx.borrow(),
+ Close::Always => true,
+ Close::CutFirstBody | Close::Never => false,
+ };
+ let (requests, waiting, closed_tx, cut_pending) = (
+ server_requests.clone(),
+ waiting.clone(),
+ closed_tx.clone(),
+ cut_pending.clone(),
+ );
+ let (arrived_tx, mut arrived_rx) = watch::channel(false);
+ let mut closed_rx = closed_rx.clone();
+ tokio::spawn(async move {
+ let service = service_fn(move |_: Request<Incoming>| {
+ requests.fetch_add(1, Ordering::SeqCst);
+ let cut = matches!(close, Close::CutFirstBody)
+ && cut_pending.swap(false, Ordering::SeqCst);
+ let (waiting, closed_tx, arrived_tx) =
+ (waiting.clone(), closed_tx.clone(), arrived_tx.clone());
+ async move {
+ if holds {
+ arrived_tx.send_replace(true);
+ if let Close::OnceWhenWaiting(count) = close
+ && waiting.fetch_add(1, Ordering::SeqCst) + 1 == count
+ {
+ closed_tx.send_replace(true);
+ }
+ std::future::pending::<()>().await;
+ }
+ let body = if cut {
+ &OK_BODY[..OK_BODY.len() / 2]
+ } else {
+ OK_BODY
+ };
+ let mut response = Response::new(Reply {
+ data: Some(Bytes::from_static(body.as_bytes())),
+ cut,
+ });
+ *response.status_mut() = status;
+ Ok::<_, Infallible>(response)
+ }
+ });
+ let connection = hyper::server::conn::http2::Builder::new(TokioExecutor::new())
+ .serve_connection(TokioIo::new(tls), service);
+ // Returning drops the connection, closing it under every waiting stream.
+ let close_signal = async {
+ match close {
+ Close::OnceWhenWaiting(_) if holds => {
+ let _ = closed_rx.wait_for(|closed| *closed).await;
+ }
+ Close::Always => {
+ let _ = arrived_rx.wait_for(|arrived| *arrived).await;
+ }
+ _ => std::future::pending().await,
+ }
+ };
+ tokio::select! {
+ _ = connection => {}
+ () = close_signal => {}
+ }
+ });
+ }
+ });
+ Server {
+ url,
+ requests,
+ connections,
+ }
+}
+
+fn client(server: &Server) -> OpensearchClient {
+ OpensearchClient::new(server.url.clone(), "".into(), "".into()).unwrap()
+}
+
+fn session(id: &str) -> ReconcileAgentSessionArgs {
+ ReconcileAgentSessionArgs {
+ agent_session_id: id.to_owned(),
+ name: "Investigate indexing".to_owned(),
+ owner_id: "user-1".to_owned(),
+ bot_id: "bot-1".to_owned(),
+ thread_id: None,
+ originating_message_id: None,
+ created_at_millis: EpochMillis::new(10).unwrap(),
+ updated_at_millis: EpochMillis::new(20).unwrap(),
+ projection_generation: "generation-1".to_owned(),
+ messages: vec![AgentSessionMessageDocument {
+ turn: 1,
+ author: AgentSessionMessageAuthor::Agent,
+ author_user_id: None,
+ content: "Folded output".to_owned(),
+ }],
+ }
+}
+
+fn email(message_id: &str) -> UpsertEmailArgs {
+ UpsertEmailArgs {
+ thread_id: "thread-1".to_owned(),
+ message_id: message_id.to_owned(),
+ sender: "[email protected]".to_owned(),
+ sender_name: None,
+ reply_to: None,
+ recipients: vec![],
+ recipient_names: vec![],
+ cc: vec![],
+ cc_names: vec![],
+ bcc: vec![],
+ bcc_names: vec![],
+ labels: vec![],
+ link_id: "link-1".to_owned(),
+ user_id: "user-1".to_owned(),
+ updated_at_millis: EpochMillis::new(20).unwrap(),
+ subject: None,
+ sent_at_millis: None,
+ content: "Body".to_owned(),
+ properties: vec![],
+ }
+}
+
+/// The production burst: unrelated agent-session and email writes share one
+/// connection, it closes under all of them at once, and each must still land.
+#[tokio::test]
+async fn writes_sharing_a_closed_connection_are_resent_on_a_new_one() {
+ let server = server(Close::OnceWhenWaiting(4), StatusCode::OK).await;
+ let client = client(&server);
+ let (first_session, second_session) = (session("session-1"), session("session-2"));
+ let (first_email, second_email) = ([email("message-1")], [email("message-2")]);
+
+ let results = tokio::time::timeout(Duration::from_secs(20), async {
+ tokio::join!(
+ client.reconcile_agent_session(&first_session, None),
+ client.reconcile_agent_session(&second_session, None),
+ client.bulk_upsert_email_messages(&first_email, None),
+ client.bulk_upsert_email_messages(&second_email, None),
+ )
+ })
+ .await
+ .unwrap();
+
+ results.0.unwrap();
+ results.1.unwrap();
+ results.2.unwrap();
+ results.3.unwrap();
+ // Four first attempts lost, then each reconcile's parent, bulk, and prune
+ // plus each email bulk on a fresh connection.
+ assert_eq!(server.requests.load(Ordering::SeqCst), 4 + 3 * 2 + 2);
+ assert!(server.connections.load(Ordering::SeqCst) >= 2);
+}
+
+/// A response cut off partway through its body is resent, not reported as
+/// an undecodable body.
+#[tokio::test]
+async fn a_response_cut_off_mid_body_is_resent() {
+ let server = server(Close::CutFirstBody, StatusCode::OK).await;
+ let messages = [email("message-1")];
+
+ let result = tokio::time::timeout(
+ Duration::from_secs(20),
+ client(&server).bulk_upsert_email_messages(&messages, None),
+ )
+ .await
+ .unwrap();
+
+ result.unwrap();
+ assert_eq!(server.requests.load(Ordering::SeqCst), 2);
+}
+
+/// A domain that keeps dropping connections still surfaces as a failure.
+#[tokio::test]
+async fn a_connection_lost_on_every_attempt_fails_after_three_attempts() {
+ let server = server(Close::Always, StatusCode::OK).await;
+ let messages = [email("message-1")];
+
+ let result = tokio::time::timeout(
+ Duration::from_secs(20),
+ client(&server).bulk_upsert_email_messages(&messages, None),
+ )
+ .await
+ .unwrap();
+
+ let Err(OpensearchClientError::RequestFailed {
+ cause, operation, ..
+ }) = result
+ else {
+ panic!("expected a lost connection, got {result:?}");
+ };
+ assert_eq!(cause, TransportFailure::ConnectionClosed);
+ assert_eq!(operation, OpensearchOperation::BulkUpsertEmailMessages);
+ assert_eq!(server.requests.load(Ordering::SeqCst), 3);
+}
+
+/// An answered request is never resent, so a server error stays visible.
+#[tokio::test]
+async fn an_error_response_is_not_resent() {
+ let server = server(Close::Never, StatusCode::SERVICE_UNAVAILABLE).await;
+ let messages = [email("message-1")];
+
+ let result = client(&server)
+ .bulk_upsert_email_messages(&messages, None)
+ .await;
+
+ assert!(
+ matches!(
+ result,
+ Err(OpensearchClientError::Unknown {
+ operation: OpensearchOperation::BulkUpsertEmailMessages,
+ ..
+ })
+ ),
+ "{result:?}"
+ );
+ assert_eq!(server.requests.load(Ordering::SeqCst), 1);
+}
diff --git a/crates/opensearch_client/src/search/channels.rs b/crates/opensearch_client/src/search/channels.rs
index f269ad7a3d..98ddcde3ac 100644
--- a/crates/opensearch_client/src/search/channels.rs
+++ b/crates/opensearch_client/src/search/channels.rs
@@ -221,7 +221,7 @@ pub struct ChannelSearchResults {
pub total: i64,
}
-#[tracing::instrument(skip(client, args), err)]
+#[tracing::instrument(skip(client, args))]
pub(crate) async fn search_channel(
client: &opensearch::OpenSearch,
args: ChannelSearchArgs,
diff --git a/crates/opensearch_client/src/search/unified.rs b/crates/opensearch_client/src/search/unified.rs
index a4bbef7a6a..9250068e24 100644
--- a/crates/opensearch_client/src/search/unified.rs
+++ b/crates/opensearch_client/src/search/unified.rs
@@ -960,7 +960,7 @@ fn paginate_unified_hits(
(results, cursor)
}
-#[tracing::instrument(skip(client, args), err)]
+#[tracing::instrument(skip(client, args))]
pub(crate) async fn search_unified(
client: &opensearch::OpenSearch,
args: UnifiedSearchArgs,
diff --git a/crates/opensearch_client/src/unified.rs b/crates/opensearch_client/src/unified.rs
index 8a134cf325..6a25f2493a 100644
--- a/crates/opensearch_client/src/unified.rs
+++ b/crates/opensearch_client/src/unified.rs
@@ -2,16 +2,18 @@ use models_search_cursor::SearchCursorOption;
use crate::{
OpensearchClient, Result,
+ retry::resend_on_lost_connection,
search::{self, model::SearchHit, unified::UnifiedSearchArgs},
};
impl OpensearchClient {
/// Performs a unified search
- #[tracing::instrument(skip(self, args))]
+ #[tracing::instrument(skip(self, args), err)]
pub async fn search_unified(
&self,
args: UnifiedSearchArgs,
) -> Result<(Vec<SearchHit>, SearchCursorOption)> {
- search::unified::search_unified(&self.inner, args).await
+ resend_on_lost_connection(|| search::unified::search_unified(&self.inner, args.clone()))
+ .await
}
}
diff --git a/crates/opensearch_client/src/upsert/agent_session/test.rs b/crates/opensearch_client/src/upsert/agent_session/test.rs
index 7819f49e25..59325107e8 100644
--- a/crates/opensearch_client/src/upsert/agent_session/test.rs
+++ b/crates/opensearch_client/src/upsert/agent_session/test.rs
@@ -192,22 +192,25 @@ async fn a_connection_closed_without_a_response_reports_the_transport_cause() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
+ // One connection per attempt, since a lost connection is resent twice.
let server = std::thread::spawn(move || {
- let (mut stream, _) = listener.accept().unwrap();
- let mut reader = std::io::BufReader::new(&mut stream);
- let mut content_length = 0;
- loop {
- let mut line = String::new();
- reader.read_line(&mut line).unwrap();
- if line == "\r\n" {
- break;
- }
- if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") {
- content_length = value.trim().parse::<usize>().unwrap();
+ for _ in 0..3 {
+ let (mut stream, _) = listener.accept().unwrap();
+ let mut reader = std::io::BufReader::new(&mut stream);
+ let mut content_length = 0;
+ loop {
+ let mut line = String::new();
+ reader.read_line(&mut line).unwrap();
+ if line == "\r\n" {
+ break;
+ }
+ if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") {
+ content_length = value.trim().parse::<usize>().unwrap();
+ }
}
+ reader.read_exact(&mut vec![0; content_length]).unwrap();
+ // Dropping the stream closes the connection before any response.
}
- reader.read_exact(&mut vec![0; content_length]).unwrap();
- // Dropping the stream closes the connection before any response.
});
let client =
crate::OpensearchClient::new(format!("http://{address}"), "".into(), "".into()).unwrap();
diff --git a/crates/opensearch_client/src/upsert/calendar_event.rs b/crates/opensearch_client/src/upsert/calendar_event.rs
index d97bea2854..ab8e4bc796 100644
--- a/crates/opensearch_client/src/upsert/calendar_event.rs
+++ b/crates/opensearch_client/src/upsert/calendar_event.rs
@@ -75,7 +75,7 @@ fn resolve_destination(index_override: Option<&str>) -> &str {
/// Upsert a single calendar event doc. Full-overwrite `index` semantics so
/// omitted optional fields (e.g. `organizer_email`) get cleared on
/// Some→None transitions.
-#[tracing::instrument(skip(client, args), fields(event_id=%args.event_id), err)]
+#[tracing::instrument(skip(client, args), fields(event_id=%args.event_id))]
pub(crate) async fn upsert_calendar_event(
client: &opensearch::OpenSearch,
args: &UpsertCalendarEventArgs,
diff --git a/crates/opensearch_client/src/upsert/call_record.rs b/crates/opensearch_client/src/upsert/call_record.rs
index 8eab77e38d..0ae60ce4dc 100644
--- a/crates/opensearch_client/src/upsert/call_record.rs
+++ b/crates/opensearch_client/src/upsert/call_record.rs
@@ -48,7 +48,7 @@ fn resolve_destination(index_override: Option<&str>) -> &str {
index_override.unwrap_or(SearchIndex::CallRecords.as_ref())
}
-#[tracing::instrument(skip(client), err)]
+#[tracing::instrument(skip(client))]
pub(crate) async fn upsert_call_record_segment(
client: &opensearch::OpenSearch,
args: &UpsertCallRecordSegmentArgs,
@@ -60,7 +60,7 @@ pub(crate) async fn upsert_call_record_segment(
.map(|_| ())
}
-#[tracing::instrument(skip(client, segments), err)]
+#[tracing::instrument(skip(client, segments))]
pub(crate) async fn bulk_upsert_call_record_segments(
client: &opensearch::OpenSearch,
segments: &[UpsertCallRecordSegmentArgs],
diff --git a/crates/opensearch_client/src/upsert/document.rs b/crates/opensearch_client/src/upsert/document.rs
index 5b9f91a25b..4b543fbf86 100644
--- a/crates/opensearch_client/src/upsert/document.rs
+++ b/crates/opensearch_client/src/upsert/document.rs
@@ -335,7 +335,7 @@ fn parent_only_bulk_body(args: &UpsertDocumentArgs) -> Vec<String> {
/// (e.g. archives, media). Writes the same full-overwrite parent body as
/// the chunked paths — `node_id`/`content` on `args` are ignored — and
/// touches no children.
-#[tracing::instrument(skip(client), err)]
+#[tracing::instrument(skip(client))]
pub(crate) async fn upsert_parent_document(
client: &opensearch::OpenSearch,
args: &UpsertDocumentArgs,
diff --git a/crates/opensearch_client/src/upsert/email.rs b/crates/opensearch_client/src/upsert/email.rs
index be5323c979..5f5dbacaa6 100644
--- a/crates/opensearch_client/src/upsert/email.rs
+++ b/crates/opensearch_client/src/upsert/email.rs
@@ -99,7 +99,7 @@ pub(crate) async fn upsert_email_message(
Ok(())
}
-#[tracing::instrument(skip(client, messages), err)]
+#[tracing::instrument(skip(client, messages))]
pub(crate) async fn bulk_upsert_email_messages(
client: &opensearch::OpenSearch,
messages: &[UpsertEmailArgs],
diff --git a/crates/opensearch_client/src/upsert/project.rs b/crates/opensearch_client/src/upsert/project.rs
index 08b57cf0a8..19927a51c8 100644
--- a/crates/opensearch_client/src/upsert/project.rs
+++ b/crates/opensearch_client/src/upsert/project.rs
@@ -38,7 +38,7 @@ fn resolve_destination(index_override: Option<&str>) -> &str {
/// Upsert a single project doc. Full-overwrite `index` semantics so omitted
/// optional fields (e.g. `parent_project_id`) get cleared on Some→None
/// transitions.
-#[tracing::instrument(skip(client, args), fields(project_id=%args.project_id), err)]
+#[tracing::instrument(skip(client, args), fields(project_id=%args.project_id))]
pub(crate) async fn upsert_project(
client: &opensearch::OpenSearch,
args: &UpsertProjectArgs,
diff --git a/crates/workspace-hack/Cargo.toml b/crates/workspace-hack/Cargo.toml
index 2f93a6e500..90f974b7fb 100644
--- a/crates/workspace-hack/Cargo.toml
+++ b/crates/workspace-hack/Cargo.toml
@@ -155,7 +155,7 @@ sync_wrapper = { version = "1", default-features = false, features = ["futures"]
thiserror = { version = "2" }
time = { version = "0.3", features = ["formatting", "macros", "parsing"] }
tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "parking_lot", "process", "rt-multi-thread", "signal", "test-util"] }
-tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] }
+tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs", "logging", "tls12"] }
tokio-stream = { version = "0.1", features = ["fs"] }
tokio-util = { version = "0.7", features = ["codec", "compat", "io", "rt"] }
tower = { version = "0.5", default-features = false, features = ["balance", "buffer", "limit", "load-shed", "log"] }
@@ -326,7 +326,7 @@ sync_wrapper = { version = "1", default-features = false, features = ["futures"]
thiserror = { version = "2" }
time = { version = "0.3", features = ["formatting", "macros", "parsing"] }
tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "parking_lot", "process", "rt-multi-thread", "signal", "test-util"] }
-tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] }
+tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs", "logging", "tls12"] }
tokio-stream = { version = "0.1", features = ["fs"] }
tokio-util = { version = "0.7", features = ["codec", "compat", "io", "rt"] }
tower = { version = "0.5", default-features = false, features = ["balance", "buffer", "limit", "load-shed", "log"] }
diff --git a/services/search_processing_service/README.md b/services/search_processing_service/README.md
index 334c2055e2..a1569ce5e7 100644
--- a/services/search_processing_service/README.md
+++ b/services/search_processing_service/README.md
@@ -108,7 +108,7 @@ Upserts read the current call state from Postgres and overwrite the indexed repr
- The poll loop shards decoded events by entity ordering key across ten sequential workers, each with a bounded 16-message channel. Sending waits when the selected channel is full, stopping polling and committing until that worker catches up. Events for the same entity stay ordered within one consumer, but not end to end: producers can publish them out of order, and a rebalance does not wait for the previous owner's workers, which can still be processing older events while the new owner processes later ones. Handlers must not assume an entity's newest event arrives last.
- An offset is committed asynchronously immediately after a successful in-memory handoff, not after OpenSearch processing. This commit-after-handoff design creates a loss window: a process or host crash can lose committed events still in those buffers. If the worker channel closes before handoff, the offset is left uncommitted for redelivery.
- Malformed, keyless, and unsupported-schema records are logged and committed without handoff so a poison record cannot wedge a partition. Commit failures are logged and may cause duplicate delivery, which is safe because full upserts and per-call deletes are idempotent.
-- Processing gets three total attempts, with retries after 1 and 2 seconds. Exhausted events are logged and dropped so later events can continue.
+- Processing gets three total attempts, with retries after 1 and 2 seconds. Failed attempts that will be retried log warnings; exhausted events are logged as errors and dropped so later events can continue. Within each attempt, `opensearch_client` resends a request whose connection drops before a complete response.
## Running Locally
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/agent_session.rs b/services/search_processing_service/src/inbound/kafka_consumer/agent_session.rs
index 3d668b7453..2391173d28 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer/agent_session.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer/agent_session.rs
@@ -3,7 +3,7 @@
use ::agent_session::domain::events::AgentSessionLifecycleMacroEvent;
use macro_event_broker::MacroEvent as _;
-use super::{EventOutcome, KafkaProcessingContext, retry_processing};
+use super::{EventOutcome, KafkaProcessingContext, MAX_PROCESSING_ATTEMPTS, retry_processing};
pub(super) async fn process_agent_session_event(
context: &KafkaProcessingContext,
@@ -12,7 +12,10 @@ pub(super) async fn process_agent_session_event(
let id = event.event().event.session_id();
match retry_processing(|attempt| async move {
context.agent_session_indexer.reconcile(id, None).await.inspect_err(|error| {
- tracing::error!(error=?error, session_id=%id, attempt, "agent session indexing failed");
+ // An ERROR event marks the event's span failed even when a retry succeeds.
+ if attempt < MAX_PROCESSING_ATTEMPTS {
+ tracing::warn!(error=?error, session_id=%id, attempt, "agent session indexing failed, retrying");
+ }
})
})
.await