Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(search): report OpenSearch transport failures with their cause (#9532)
teo17056f47088a
diff --git a/Cargo.lock b/Cargo.lockindex 0de4526948..ebd499fb0e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -14528,11 +14528,13 @@ version = "0.1.0"dependencies = ["anyhow","chrono",+ "hyper 1.9.0","macro_entrypoint","models_opensearch","models_search_cursor","opensearch","opensearch_query_builder",+ "reqwest 0.13.4","serde","serde_json","strum 0.27.2",diff --git a/crates/opensearch_client/AGENTS.md b/crates/opensearch_client/AGENTS.mdindex 84dc35e0f0..b688491ffa 100644--- a/crates/opensearch_client/AGENTS.md+++ b/crates/opensearch_client/AGENTS.md@@ -44,6 +44,15 @@ projects.- `cargo test -p opensearch_client` (`SQLX_OFFLINE` unset; needs a runningOpenSearch — see `crates/opensearch_query_builder/docker-compose.yml`).+## Gotchas++- Map a failed `.send()` with `OpensearchClientError::request_failed` and a+ failed body read with `deserialization_failed`. Both take an+ `OpensearchOperation`: add a variant for a new request rather than passing a+ name. `request_failed` classifies the cause as a `TransportFailure` (timeout,+ connect, connection closed, other) and keeps the full source chain in+ `details`; reqwest's `Display` alone drops the cause.+## Related- Docs: `README.md`, `examples/`diff --git a/crates/opensearch_client/Cargo.toml b/crates/opensearch_client/Cargo.tomlindex 596365176a..35316ab448 100644--- a/crates/opensearch_client/Cargo.toml+++ b/crates/opensearch_client/Cargo.toml@@ -14,10 +14,12 @@ uuid = { workspace = true }[dependencies]anyhow = { workspace = true }chrono = { workspace = true }+hyper = { workspace = true }models_opensearch = { path = "../models_opensearch" }models_search_cursor = { path = "../models_search_cursor" }opensearch = { version = "2.3.0", default-features = false, features = ["rustls-tls"] }opensearch_query_builder = { path = "../opensearch_query_builder", default-features = false }+reqwest = { workspace = true }serde = { workspace = true }serde_json = { workspace = true }strum = { workspace = true }diff --git a/crates/opensearch_client/src/agent_session.rs b/crates/opensearch_client/src/agent_session.rsindex 672d61dc6c..628315d096 100644--- a/crates/opensearch_client/src/agent_session.rs+++ b/crates/opensearch_client/src/agent_session.rs@@ -1,25 +1,49 @@-use crate::{OpensearchClient, Result, delete, upsert};+use crate::{+ OpensearchClient, Result, delete,+ error::{OpensearchClientError, OpensearchOperation},+ upsert,+};pub use upsert::agent_session::{AgentSessionMessageAuthor, AgentSessionMessageDocument, ReconcileAgentSessionArgs,};+/// Report a failed status with its body as sent; a gateway error page is not JSON.+pub(crate) async fn response_error(+ response: opensearch::http::response::Response,+ operation: OpensearchOperation,+) -> Result<()> {+ let status_code = response.status_code();+ if status_code.is_success() {+ return Ok(());+ }+ let body = response+ .text()+ .await+ .map_err(|error| OpensearchClientError::deserialization_failed(operation, &error))?;+ tracing::error!(%status_code, %body, %operation, "agent-session OpenSearch operation failed");+ Err(OpensearchClientError::Unknown {+ details: body,+ operation,+ })+}+/// Delete-by-query may report partial failure in an otherwise successful HTTP response.pub(crate) async fn check_delete_response(response: opensearch::http::response::Response,- method: &str,+ operation: OpensearchOperation,) -> Result<()> {- let status = response.status_code();- let body: serde_json::Value = response.json().await.map_err(|error| {- crate::error::OpensearchClientError::DeserializationFailed {- details: error.to_string(),- method: Some(method.to_owned()),- }- })?;- if !status.is_success() || delete_failed(&body) {- return Err(crate::error::OpensearchClientError::Unknown {+ if !response.status_code().is_success() {+ return response_error(response, operation).await;+ }+ let body: serde_json::Value = response+ .json()+ .await+ .map_err(|error| OpensearchClientError::deserialization_failed(operation, &error))?;+ if delete_failed(&body) {+ return Err(OpensearchClientError::Unknown {details: body.to_string(),- method: Some(method.to_owned()),+ operation,});}Ok(())diff --git a/crates/opensearch_client/src/delete/agent_session.rs b/crates/opensearch_client/src/delete/agent_session.rsindex bdf1e858ed..41a7f6182d 100644--- a/crates/opensearch_client/src/delete/agent_session.rs+++ b/crates/opensearch_client/src/delete/agent_session.rs@@ -1,6 +1,9 @@use models_opensearch::SearchIndex;-use crate::{Result, error::OpensearchClientError};+use crate::{+ Result,+ error::{OpensearchClientError, OpensearchOperation},+};/// Remove a session parent and all of its message children.pub(crate) async fn delete_agent_session(@@ -19,10 +22,10 @@ pub(crate) async fn delete_agent_session(.refresh(true).send().await- .map_err(|error| OpensearchClientError::Unknown {- details: error.to_string(),- method: Some("delete_agent_session".to_owned()),+ .map_err(|error| {+ OpensearchClientError::request_failed(OpensearchOperation::DeleteAgentSession, &error)})?;- crate::agent_session::check_delete_response(response, "delete_agent_session").await+ crate::agent_session::check_delete_response(response, OpensearchOperation::DeleteAgentSession)+ .await}diff --git a/crates/opensearch_client/src/delete/calendar_event.rs b/crates/opensearch_client/src/delete/calendar_event.rsindex a0bf20cfd8..2928f79b3c 100644--- a/crates/opensearch_client/src/delete/calendar_event.rs+++ b/crates/opensearch_client/src/delete/calendar_event.rs@@ -1,6 +1,9 @@use models_opensearch::SearchIndex;-use crate::{Result, error::OpensearchClientError};+use crate::{+ Result,+ error::{OpensearchClientError, OpensearchOperation},+};/// Deletes a calendar event doc by id. The calendar events index is flat with/// `_id` = event id, so this is a direct delete. A missing-doc 404 is a no-op@@ -17,9 +20,11 @@ pub async fn delete_calendar_event_by_id(.delete(opensearch::DeleteParts::IndexId(index, event_id)).send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("delete_calendar_event_by_id".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(+ OpensearchOperation::DeleteCalendarEventById,+ &err,+ )})?;let status_code = response.status_code();@@ -27,14 +32,12 @@ pub async fn delete_calendar_event_by_id(return Ok(());}- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("delete_calendar_event_by_id".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::DeleteCalendarEventById,+ &err,+ )+ })?;if status_code.as_u16() == 404 {let result = serde_json::from_str::<serde_json::Value>(&body)@@ -54,6 +57,6 @@ pub async fn delete_calendar_event_by_id(Err(OpensearchClientError::Unknown {details: body,- method: Some("delete_calendar_event_by_id".to_string()),+ operation: OpensearchOperation::DeleteCalendarEventById,})}diff --git a/crates/opensearch_client/src/delete/call_record.rs b/crates/opensearch_client/src/delete/call_record.rsindex 9d1ce87186..43c0137527 100644--- a/crates/opensearch_client/src/delete/call_record.rs+++ b/crates/opensearch_client/src/delete/call_record.rs@@ -1,6 +1,9 @@use models_opensearch::SearchIndex;-use crate::{Result, error::OpensearchClientError};+use crate::{+ Result,+ error::{OpensearchClientError, OpensearchOperation},+};#[tracing::instrument(skip(client))]pub async fn delete_call_record_by_id(@@ -12,7 +15,7 @@ pub async fn delete_call_record_by_id(client,"entity_id",call_id,- "delete_call_record_by_id",+ OpensearchOperation::DeleteCallRecordById,index_override,).await@@ -28,7 +31,7 @@ pub async fn delete_call_records_by_channel_id(client,"channel_id",channel_id,- "delete_call_records_by_channel_id",+ OpensearchOperation::DeleteCallRecordsByChannelId,index_override,).await@@ -38,7 +41,7 @@ async fn delete_by_term(client: &opensearch::OpenSearch,field: &str,value: &str,- method: &str,+ operation: OpensearchOperation,index_override: Option<&str>,) -> Result<()> {let query = serde_json::json!({ "query": { "term": { field: value } } });@@ -50,25 +53,18 @@ async fn delete_by_term(.refresh(true).send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some(method.to_string()),- })?;+ .map_err(|err| OpensearchClientError::request_failed(operation, &err))?;let status = response.status_code();if !status.is_success() {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some(method.to_string()),- })?;+ let body = response+ .text()+ .await+ .map_err(|err| OpensearchClientError::deserialization_failed(operation, &err))?;tracing::error!(status_code = ?status, body = ?body, "error deleting call records");return Err(OpensearchClientError::Unknown {details: body,- method: Some(method.to_string()),+ operation,});}diff --git a/crates/opensearch_client/src/delete/channel.rs b/crates/opensearch_client/src/delete/channel.rsindex 4b08e32956..15d37dd537 100644--- a/crates/opensearch_client/src/delete/channel.rs+++ b/crates/opensearch_client/src/delete/channel.rs@@ -1,6 +1,9 @@use models_opensearch::SearchIndex;-use crate::{Result, error::OpensearchClientError};+use crate::{+ Result,+ error::{OpensearchClientError, OpensearchOperation},+};/// Deletes all channel messages with the specified channel_id#[tracing::instrument(skip(client))]@@ -24,22 +27,19 @@ pub async fn delete_channel_by_id(.refresh(true) // Ensure the index reflects changes immediately.send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("delete_channel_by_id".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::DeleteChannelById, &err)})?;let status_code = response.status_code();if !status_code.is_success() {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("delete_channel_by_id".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::DeleteChannelById,+ &err,+ )+ })?;tracing::error!(status_code = ?status_code,@@ -49,7 +49,7 @@ pub async fn delete_channel_by_id(return Err(OpensearchClientError::Unknown {details: body,- method: Some("delete_channel_by_id".to_string()),+ operation: OpensearchOperation::DeleteChannelById,});}@@ -90,22 +90,22 @@ pub async fn delete_channel_message_by_id(.refresh(true) // Ensure the index reflects changes immediately.send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("delete_channel_message_by_id".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(+ OpensearchOperation::DeleteChannelMessageById,+ &err,+ )})?;let status_code = response.status_code();if !status_code.is_success() {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("delete_channel_message_by_id".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::DeleteChannelMessageById,+ &err,+ )+ })?;tracing::error!(status_code = ?status_code,@@ -115,7 +115,7 @@ pub async fn delete_channel_message_by_id(return Err(OpensearchClientError::Unknown {details: body,- method: Some("delete_channel_message_by_id".to_string()),+ operation: OpensearchOperation::DeleteChannelMessageById,});}diff --git a/crates/opensearch_client/src/delete/chat.rs b/crates/opensearch_client/src/delete/chat.rsindex 3b98fb7efd..aec5665ee2 100644--- a/crates/opensearch_client/src/delete/chat.rs+++ b/crates/opensearch_client/src/delete/chat.rs@@ -1,6 +1,9 @@use models_opensearch::SearchIndex;-use crate::{Result, error::OpensearchClientError};+use crate::{+ Result,+ error::{OpensearchClientError, OpensearchOperation},+};/// Deletes all chat messages with the specified chat_id#[tracing::instrument(skip(client))]@@ -24,22 +27,16 @@ pub async fn delete_chat_by_id(.refresh(true) // Ensure the index reflects changes immediately.send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("delete_chat_by_id".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::DeleteChatById, &err)})?;let status_code = response.status_code();if !status_code.is_success() {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("delete_chat_by_id".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(OpensearchOperation::DeleteChatById, &err)+ })?;tracing::error!(status_code = ?status_code,@@ -49,7 +46,7 @@ pub async fn delete_chat_by_id(return Err(OpensearchClientError::Unknown {details: body,- method: Some("delete_chat_by_id".to_string()),+ operation: OpensearchOperation::DeleteChatById,});}@@ -90,22 +87,19 @@ pub async fn delete_chat_message_by_id(.refresh(true) // Ensure the index reflects changes immediately.send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("delete_chat_message_by_id".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::DeleteChatMessageById, &err)})?;let status_code = response.status_code();if !status_code.is_success() {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("delete_chat_message_by_id".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::DeleteChatMessageById,+ &err,+ )+ })?;tracing::error!(status_code = ?status_code,@@ -115,7 +109,7 @@ pub async fn delete_chat_message_by_id(return Err(OpensearchClientError::Unknown {details: body,- method: Some("delete_chat_message_by_id".to_string()),+ operation: OpensearchOperation::DeleteChatMessageById,});}@@ -141,22 +135,19 @@ pub async fn delete_chat_by_user_id(client: &opensearch::OpenSearch, user_id: &s.refresh(true) // Ensure the index reflects changes immediately.send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("delete_chat_by_user_id".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::DeleteChatByUserId, &err)})?;let status_code = response.status_code();if !status_code.is_success() {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("delete_chat_by_user_id".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::DeleteChatByUserId,+ &err,+ )+ })?;tracing::error!(status_code = ?status_code,@@ -166,7 +157,7 @@ pub async fn delete_chat_by_user_id(client: &opensearch::OpenSearch, user_id: &sreturn Err(OpensearchClientError::Unknown {details: body,- method: Some("delete_chat_by_user_id".to_string()),+ operation: OpensearchOperation::DeleteChatByUserId,});}diff --git a/crates/opensearch_client/src/delete/document.rs b/crates/opensearch_client/src/delete/document.rsindex 7d15609b14..6bfdf4814f 100644--- a/crates/opensearch_client/src/delete/document.rs+++ b/crates/opensearch_client/src/delete/document.rs@@ -1,6 +1,9 @@use models_opensearch::SearchIndex;-use crate::{Result, error::OpensearchClientError};+use crate::{+ Result,+ error::{OpensearchClientError, OpensearchOperation},+};/// Deletes all document nodes with the specified document_id#[tracing::instrument(skip(client))]@@ -25,22 +28,19 @@ pub async fn delete_document_by_id(.refresh(true) // Ensure the index reflects changes immediately.send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("delete_document_by_id".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::DeleteDocumentById, &err)})?;let status_code = response.status_code();if !status_code.is_success() {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("delete_document_by_id".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::DeleteDocumentById,+ &err,+ )+ })?;tracing::error!(status_code = ?status_code,@@ -51,7 +51,7 @@ pub async fn delete_document_by_id(return Err(OpensearchClientError::Unknown {details: body,- method: Some("delete_document_by_id".to_string()),+ operation: OpensearchOperation::DeleteDocumentById,});}@@ -92,22 +92,22 @@ pub async fn delete_document_by_owner_id(.refresh(true) // Ensure the index reflects changes immediately.send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("delete_document_by_owner_id".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(+ OpensearchOperation::DeleteDocumentByOwnerId,+ &err,+ )})?;let status_code = response.status_code();if !status_code.is_success() {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("delete_document_by_owner_id".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::DeleteDocumentByOwnerId,+ &err,+ )+ })?;tracing::error!(status_code = ?status_code,@@ -118,7 +118,7 @@ pub async fn delete_document_by_owner_id(return Err(OpensearchClientError::Unknown {details: body,- method: Some("delete_document_by_owner_id".to_string()),+ operation: OpensearchOperation::DeleteDocumentByOwnerId,});}diff --git a/crates/opensearch_client/src/delete/email.rs b/crates/opensearch_client/src/delete/email.rsindex d697a31c88..771a1cccd7 100644--- a/crates/opensearch_client/src/delete/email.rs+++ b/crates/opensearch_client/src/delete/email.rs@@ -1,6 +1,9 @@use models_opensearch::SearchIndex;-use crate::{Result, error::OpensearchClientError};+use crate::{+ Result,+ error::{OpensearchClientError, OpensearchOperation},+};#[cfg(test)]mod test;@@ -28,22 +31,19 @@ pub async fn delete_email_by_link_id(client: &opensearch::OpenSearch, link_id: &.refresh(true) // Ensure the index reflects changes immediately.send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("delete_email_by_link_id".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::DeleteEmailByLinkId, &err)})?;let status_code = response.status_code();if !status_code.is_success() {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("delete_email_by_link_id".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::DeleteEmailByLinkId,+ &err,+ )+ })?;tracing::error!(status_code = ?status_code,@@ -53,7 +53,7 @@ pub async fn delete_email_by_link_id(client: &opensearch::OpenSearch, link_id: &return Err(OpensearchClientError::Unknown {details: body,- method: Some("delete_email_by_link_id".to_string()),+ operation: OpensearchOperation::DeleteEmailByLinkId,});}@@ -82,22 +82,19 @@ pub async fn delete_email_message_by_id(.refresh(true) // Ensure the index reflects changes immediately.send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("delete_email_message_by_id".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::DeleteEmailMessageById, &err)})?;let status_code = response.status_code();if !status_code.is_success() {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("delete_email_message_by_id".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::DeleteEmailMessageById,+ &err,+ )+ })?;tracing::error!(status_code = ?status_code,@@ -107,7 +104,7 @@ pub async fn delete_email_message_by_id(return Err(OpensearchClientError::Unknown {details: body,- method: Some("delete_email_message_by_id".to_string()),+ operation: OpensearchOperation::DeleteEmailMessageById,});}@@ -133,22 +130,19 @@ pub async fn delete_email_by_user_id(client: &opensearch::OpenSearch, user_id: &.refresh(true) // Ensure the index reflects changes immediately.send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("delete_email_by_user_id".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::DeleteEmailByUserId, &err)})?;let status_code = response.status_code();if !status_code.is_success() {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("delete_email_by_user_id".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::DeleteEmailByUserId,+ &err,+ )+ })?;tracing::error!(status_code = ?status_code,@@ -158,7 +152,7 @@ pub async fn delete_email_by_user_id(client: &opensearch::OpenSearch, user_id: &return Err(OpensearchClientError::Unknown {details: body,- method: Some("delete_email_by_user_id".to_string()),+ operation: OpensearchOperation::DeleteEmailByUserId,});}diff --git a/crates/opensearch_client/src/delete/project.rs b/crates/opensearch_client/src/delete/project.rsindex 9c3a3f18bc..c53c2eab8a 100644--- a/crates/opensearch_client/src/delete/project.rs+++ b/crates/opensearch_client/src/delete/project.rs@@ -1,6 +1,9 @@use models_opensearch::SearchIndex;-use crate::{Result, error::OpensearchClientError};+use crate::{+ Result,+ error::{OpensearchClientError, OpensearchOperation},+};/// Deletes a project doc by id. The projects index is flat with `_id` =/// project id, so this is a direct delete. A missing-doc 404 is a no-op —@@ -17,9 +20,8 @@ pub async fn delete_project_by_id(.delete(opensearch::DeleteParts::IndexId(index, project_id)).send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("delete_project_by_id".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::DeleteProjectById, &err)})?;let status_code = response.status_code();@@ -27,14 +29,9 @@ pub async fn delete_project_by_id(return Ok(());}- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("delete_project_by_id".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(OpensearchOperation::DeleteProjectById, &err)+ })?;if status_code.as_u16() == 404 {let result = serde_json::from_str::<serde_json::Value>(&body)@@ -54,6 +51,6 @@ pub async fn delete_project_by_id(Err(OpensearchClientError::Unknown {details: body,- method: Some("delete_project_by_id".to_string()),+ operation: OpensearchOperation::DeleteProjectById,})}diff --git a/crates/opensearch_client/src/error.rs b/crates/opensearch_client/src/error.rsindex f1123bef36..49a38d34b5 100644--- a/crates/opensearch_client/src/error.rs+++ b/crates/opensearch_client/src/error.rs@@ -1,13 +1,144 @@use models_opensearch::OpenSearchEntityType;use opensearch::{Error, http::response::Response};+/// The OpenSearch operation an error came from, one per request the crate sends.+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ serde::Serialize,+ strum::Display,+ strum::AsRefStr,+ strum::IntoStaticStr,+ strum::EnumIter,+)]+#[serde(rename_all = "snake_case")]+#[strum(serialize_all = "snake_case")]+pub enum OpensearchOperation {+ SearchChannel,+ SearchUnified,+ UpsertDocument,+ UpsertParentDocument,+ BulkUpsertSingleChunk,+ UpdateDocumentMetadata,+ UpdateDocumentProperties,+ UpsertChannelMessage,+ UpsertChatMessageParent,+ UpsertChatMessageChild,+ UpdateChatMetadata,+ UpdateChatProperties,+ UpsertEmailMessage,+ BulkUpsertEmailMessages,+ UpdateEmailThreadProperties,+ UpsertProject,+ UpdateProjectProperties,+ UpsertCalendarEvent,+ UpdateCalendarEventProperties,+ BulkUpsertCallRecordSegments,+ UpdateCallRecordProperties,+ ReconcileAgentSessionParent,+ ReconcileAgentSessionBulk,+ ReconcileAgentSessionPrune,+ DeleteDocumentById,+ DeleteDocumentByOwnerId,+ DeleteChannelById,+ DeleteChannelMessageById,+ DeleteChatById,+ DeleteChatMessageById,+ DeleteChatByUserId,+ DeleteEmailByLinkId,+ DeleteEmailMessageById,+ DeleteEmailByUserId,+ DeleteProjectById,+ DeleteCalendarEventById,+ DeleteCallRecordById,+ DeleteCallRecordsByChannelId,+ DeleteAgentSession,+}++/// Why a request produced no response.+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ serde::Serialize,+ strum::Display,+ strum::AsRefStr,+ strum::IntoStaticStr,+)]+#[serde(rename_all = "snake_case")]+#[strum(serialize_all = "snake_case")]+pub enum TransportFailure {+ /// The request or connection attempt exceeded its deadline.+ Timeout,+ /// No connection could be established (refused, DNS, unreachable).+ Connect,+ /// The connection was closed or reset before the response completed.+ ConnectionClosed,+ /// Any other transport failure; `details` carries the cause.+ Other,+}++impl TransportFailure {+ /// Classify by the first cause in the chain that names a known failure.+ pub(crate) fn classify(error: &(dyn std::error::Error + 'static)) -> Self {+ let mut source: Option<&(dyn std::error::Error + 'static)> = Some(error);+ while let Some(cause) = source {+ if let Some(failure) = Self::from_cause(cause) {+ return failure;+ }+ source = cause.source();+ }+ Self::Other+ }++ fn from_cause(cause: &(dyn std::error::Error + 'static)) -> Option<Self> {+ if let Some(error) = cause.downcast_ref::<reqwest::Error>() {+ if error.is_timeout() {+ return Some(Self::Timeout);+ }+ if error.is_connect() {+ return Some(Self::Connect);+ }+ }+ if let Some(error) = cause.downcast_ref::<hyper::Error>() {+ if error.is_timeout() {+ return Some(Self::Timeout);+ }+ if error.is_incomplete_message() || error.is_closed() || error.is_canceled() {+ return Some(Self::ConnectionClosed);+ }+ }+ if let Some(error) = cause.downcast_ref::<std::io::Error>() {+ use std::io::ErrorKind;+ return match error.kind() {+ ErrorKind::TimedOut => Some(Self::Timeout),+ ErrorKind::ConnectionRefused+ | ErrorKind::HostUnreachable+ | ErrorKind::NetworkUnreachable+ | ErrorKind::AddrNotAvailable => Some(Self::Connect),+ ErrorKind::ConnectionReset+ | ErrorKind::ConnectionAborted+ | ErrorKind::BrokenPipe+ | ErrorKind::UnexpectedEof => Some(Self::ConnectionClosed),+ _ => None,+ };+ }+ None+ }+}+#[derive(thiserror::Error, Debug, serde::Serialize, PartialEq)]#[serde(tag = "type")]pub enum OpensearchClientError {- #[error("error deserializing response body. method: {method:?} details: {details}")]+ #[error("error deserializing response body. operation: {operation} details: {details}")]DeserializationFailed {details: String,- method: Option<String>,+ operation: OpensearchOperation,},#[error("error deserializing search response. error: {details}, raw_body: {raw_body}")]@@ -16,10 +147,21 @@ pub enum OpensearchClientError {#[error("unable to grab bytes from http response: {details}")]HttpBytesError { details: String },- #[error("unable to serialize into json. method: {method:?} details: {details}")]+ #[error("unable to serialize into json. operation: {operation} details: {details}")]SerializationFailed {details: String,- method: Option<String>,+ operation: OpensearchOperation,+ },++ /// The request produced no response: connect, TLS, timeout, or a+ /// connection closed or reset before the response arrived.+ #[error(+ "request to opensearch failed. operation: {operation} cause: {cause} details: {details}"+ )]+ RequestFailed {+ details: String,+ cause: TransportFailure,+ operation: OpensearchOperation,},#[error("a network error occurred. status_code: {status_code} message: {message}")]@@ -37,10 +179,10 @@ pub enum OpensearchClientError {#[error("empty search indices provided")]EmptySearchIndices,- #[error("an unknown error occurred. method: {method:?} details: {details}")]+ #[error("an unknown error occurred. operation: {operation} details: {details}")]Unknown {details: String,- method: Option<String>,+ operation: OpensearchOperation,},#[error("no title key is available for name search")]@@ -59,41 +201,68 @@ pub enum OpensearchClientError {SearchWithExhaustedCursor,}-impl From<anyhow::Error> for OpensearchClientError {- fn from(err: anyhow::Error) -> Self {- OpensearchClientError::Unknown {- details: err.to_string(),- method: None,+impl OpensearchClientError {+ /// A request that never produced a response.+ pub(crate) fn request_failed(operation: OpensearchOperation, error: &Error) -> Self {+ Self::RequestFailed {+ details: error_chain(error),+ cause: TransportFailure::classify(error),+ operation,+ }+ }++ /// A response body that could not be read or parsed, including one cut+ /// off by a dropped connection.+ pub(crate) fn deserialization_failed(operation: OpensearchOperation, error: &Error) -> Self {+ Self::DeserializationFailed {+ details: error_chain(error),+ operation,+ }+ }+}++/// reqwest's `Display` omits the cause (timeout, reset, refused connection),+/// which only its source chain carries.+pub(crate) fn error_chain(error: &(dyn std::error::Error + 'static)) -> String {+ let mut details = error.to_string();+ let mut source = error.source();+ while let Some(cause) = source {+ let cause_text = cause.to_string();+ // `opensearch::Error` displays exactly like the reqwest error it wraps.+ if !details.ends_with(&cause_text) {+ details.push_str(": ");+ details.push_str(&cause_text);}+ source = cause.source();}+ details}pub trait ResponseExt {#[allow(async_fn_in_trait)]- async fn map_client_error(self) -> Result<Response, OpensearchClientError>;+ async fn map_client_error(+ self,+ operation: OpensearchOperation,+ ) -> Result<Response, OpensearchClientError>;}-impl ResponseExt for Response {- async fn map_client_error(self) -> Result<Response, OpensearchClientError> {- if self.status_code().is_success() {- Ok(self)+impl ResponseExt for Result<Response, Error> {+ async fn map_client_error(+ self,+ operation: OpensearchOperation,+ ) -> Result<Response, OpensearchClientError> {+ let response =+ self.map_err(|error| OpensearchClientError::request_failed(operation, &error))?;+ if response.status_code().is_success() {+ Ok(response)} else {Err(OpensearchClientError::NetworkError {- status_code: self.status_code().as_u16(),- message: self.text().await.unwrap_or_default(),+ status_code: response.status_code().as_u16(),+ message: response.text().await.unwrap_or_default(),})}}}-impl ResponseExt for Result<Response, Error> {- async fn map_client_error(self) -> Result<Response, OpensearchClientError> {- match self {- Ok(response) => response.map_client_error().await,- Err(e) => Err(OpensearchClientError::Unknown {- details: e.to_string(),- method: None,- }),- }- }-}+#[cfg(test)]+mod test;diff --git a/crates/opensearch_client/src/error/test.rs b/crates/opensearch_client/src/error/test.rsnew file mode 100644index 0000000000..0a0eeaa71c--- /dev/null+++ b/crates/opensearch_client/src/error/test.rs@@ -0,0 +1,125 @@+use std::io::ErrorKind;+use std::time::Duration;++use serde_json::json;+use strum::IntoEnumIterator as _;++use super::*;++#[test]+fn operation_string_forms_agree_with_serde() {+ for operation in OpensearchOperation::iter() {+ let name: &'static str = operation.into();+ assert_eq!(operation.to_string(), name);+ assert_eq!(operation.as_ref(), name);+ assert_eq!(serde_json::to_value(operation).unwrap(), json!(name));+ }+ assert_eq!(+ OpensearchOperation::ReconcileAgentSessionParent.to_string(),+ "reconcile_agent_session_parent"+ );+ assert_eq!(+ OpensearchOperation::DeleteCallRecordsByChannelId.as_ref(),+ "delete_call_records_by_channel_id"+ );+}++#[test]+fn request_failure_names_its_operation_and_cause_in_logs_and_json() {+ let error = OpensearchClientError::RequestFailed {+ details: "error sending request: connection closed before message completed".to_owned(),+ cause: TransportFailure::ConnectionClosed,+ operation: OpensearchOperation::SearchUnified,+ };++ assert_eq!(+ error.to_string(),+ "request to opensearch failed. operation: search_unified cause: connection_closed \+ details: error sending request: connection closed before message completed"+ );+ assert_eq!(+ serde_json::to_value(&error).unwrap(),+ json!({+ "type": "RequestFailed",+ "details": "error sending request: connection closed before message completed",+ "cause": "connection_closed",+ "operation": "search_unified",+ })+ );+}++#[test]+fn io_error_kinds_classify_by_transport_cause() {+ let cases = [+ (ErrorKind::TimedOut, TransportFailure::Timeout),+ (ErrorKind::ConnectionRefused, TransportFailure::Connect),+ (+ ErrorKind::ConnectionReset,+ TransportFailure::ConnectionClosed,+ ),+ (ErrorKind::BrokenPipe, TransportFailure::ConnectionClosed),+ (ErrorKind::UnexpectedEof, TransportFailure::ConnectionClosed),+ (ErrorKind::PermissionDenied, TransportFailure::Other),+ ];+ for (kind, expected) in cases {+ let error = Error::from(std::io::Error::from(kind));+ assert_eq!(TransportFailure::classify(&error), expected, "{kind:?}");+ }+}++#[test]+fn a_non_transport_error_classifies_as_other() {+ let json_error = serde_json::from_str::<serde_json::Value>("{").unwrap_err();+ assert_eq!(+ TransportFailure::classify(&Error::from(json_error)),+ TransportFailure::Other+ );+}++#[tokio::test]+async fn a_refused_connection_is_reported_as_a_connect_failure() {+ let address = std::net::TcpListener::bind("127.0.0.1:0")+ .unwrap()+ .local_addr()+ .unwrap();+ // The listener is dropped, so nothing accepts on this port.+ let client =+ crate::OpensearchClient::new(format!("http://{address}"), "".into(), "".into()).unwrap();++ let result = client.delete_agent_session("session-1", None).await;++ let Err(OpensearchClientError::RequestFailed {+ details,+ cause,+ operation,+ }) = result+ else {+ panic!("expected a request failure, got {result:?}");+ };+ assert_eq!(operation, OpensearchOperation::DeleteAgentSession);+ assert_eq!(cause, TransportFailure::Connect);+ assert!(details.contains("error sending request"), "{details}");+}++#[tokio::test]+async fn a_request_past_its_deadline_is_reported_as_a_timeout() {+ // Never accepted: the kernel completes the handshake and nothing answers.+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let http = reqwest::Client::builder()+ .timeout(Duration::from_millis(100))+ .build()+ .unwrap();++ let reqwest_error = http+ .get(format!("http://{address}"))+ .send()+ .await+ .unwrap_err();+ drop(listener);++ assert_eq!(+ TransportFailure::classify(&Error::from(reqwest_error)),+ TransportFailure::Timeout+ );+}diff --git a/crates/opensearch_client/src/search/channels.rs b/crates/opensearch_client/src/search/channels.rsindex 9632a27b95..f269ad7a3d 100644--- a/crates/opensearch_client/src/search/channels.rs+++ b/crates/opensearch_client/src/search/channels.rs@@ -2,7 +2,7 @@ pub use crate::search::builder::ChannelSortMode;use crate::{Result, delegate_methods,- error::{OpensearchClientError, ResponseExt},+ error::{OpensearchClientError, OpensearchOperation, ResponseExt, error_chain},search::{builder::{SearchQueryBuilder, SearchQueryConfig, thread_sort, updated_at_sort},model::{@@ -236,14 +236,14 @@ pub(crate) async fn search_channel(.body(search_request).send().await- .map_client_error()+ .map_client_error(OpensearchOperation::SearchChannel).await?;let bytes = response.bytes().await.map_err(|e| OpensearchClientError::HttpBytesError {- details: e.to_string(),+ details: error_chain(&e),})?;let result: DefaultSearchResponse<ChannelMessageIndex> = serde_json::from_slice(&bytes)@@ -251,7 +251,7 @@ pub(crate) async fn search_channel(details: e.to_string(),raw_body: String::from_utf8_lossy(&bytes).to_string(),})?;- result.warn_on_shard_failures("search_channel");+ result.warn_on_shard_failures(OpensearchOperation::SearchChannel);let total = result.hits.total.value;diff --git a/crates/opensearch_client/src/search/model.rs b/crates/opensearch_client/src/search/model.rsindex c44582a028..e7a35e0a8e 100644--- a/crates/opensearch_client/src/search/model.rs+++ b/crates/opensearch_client/src/search/model.rs@@ -367,10 +367,10 @@ impl<T> SearchResponse<T> {/// OpenSearch returns HTTP 200 with partial results when individual/// shards fail their query or fetch phase, so every hit on a failed/// shard is silently absent unless callers check `_shards`.- pub(crate) fn warn_on_shard_failures(&self, operation: &str) {+ pub(crate) fn warn_on_shard_failures(&self, operation: crate::error::OpensearchOperation) {if self._shards.failed > 0 {tracing::warn!(- operation,+ %operation,failed = self._shards.failed,total = self._shards.total,failures = ?self._shards.failures,diff --git a/crates/opensearch_client/src/search/unified.rs b/crates/opensearch_client/src/search/unified.rsindex 3c0e46fad0..a4bbef7a6a 100644--- a/crates/opensearch_client/src/search/unified.rs+++ b/crates/opensearch_client/src/search/unified.rs@@ -2,7 +2,7 @@ use std::collections::HashSet;use crate::{Result,- error::{OpensearchClientError, ResponseExt},+ error::{OpensearchClientError, OpensearchOperation, ResponseExt, error_chain},search::{builder::{SearchQueryConfig, updated_at_sort},calendar_events::{@@ -342,7 +342,7 @@ impl UnifiedSearchIndex {details: format!("hit matched no single known entity clause: {matched_queries:?}"),- method: Some("UnifiedSearchIndex::from_matched".to_string()),+ operation: OpensearchOperation::SearchUnified,})?;fn parse<T: serde::de::DeserializeOwned>(@@ -355,7 +355,7 @@ impl UnifiedSearchIndex {"`{}` hit did not match its own shape: {error}",entity.query_name()),- method: Some("UnifiedSearchIndex::from_matched".to_string()),+ operation: OpensearchOperation::SearchUnified,}})}@@ -985,14 +985,14 @@ pub(crate) async fn search_unified(.body(search_request).send().await- .map_client_error()+ .map_client_error(OpensearchOperation::SearchUnified).await?;let bytes = response.bytes().await.map_err(|e| OpensearchClientError::HttpBytesError {- details: e.to_string(),+ details: error_chain(&e),})?;let result: DefaultSearchResponse<serde_json::Value> =@@ -1003,7 +1003,7 @@ pub(crate) async fn search_unified(}})?;- result.warn_on_shard_failures("search_unified");+ result.warn_on_shard_failures(OpensearchOperation::SearchUnified);tracing::info!(response_body_bytes = bytes.len(),diff --git a/crates/opensearch_client/src/upsert.rs b/crates/opensearch_client/src/upsert.rsindex 274be235c1..3d4757913f 100644--- a/crates/opensearch_client/src/upsert.rs+++ b/crates/opensearch_client/src/upsert.rs@@ -8,7 +8,7 @@ pub mod email;pub mod project;pub mod properties;-use crate::error::OpensearchClientError;+use crate::error::{OpensearchClientError, OpensearchOperation};/// Result of a bulk upsert operation#[derive(Debug, Default)]@@ -62,28 +62,21 @@ pub(crate) async fn bulk_upsert_to_index(client: &opensearch::OpenSearch,index: &str,bulk_body: Vec<String>,- method_name: &str,+ operation: OpensearchOperation,) -> crate::Result<BulkUpsertResult> {let response = client.bulk(opensearch::BulkParts::Index(index)).body(bulk_body).send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some(method_name.to_string()),- })?;+ .map_err(|err| OpensearchClientError::request_failed(operation, &err))?;let status_code = response.status_code();if !status_code.is_success() {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some(method_name.to_string()),- })?;+ let body = response+ .text()+ .await+ .map_err(|err| OpensearchClientError::deserialization_failed(operation, &err))?;tracing::error!(status_code = ?status_code,@@ -93,18 +86,14 @@ pub(crate) async fn bulk_upsert_to_index(return Err(OpensearchClientError::Unknown {details: body,- method: Some(method_name.to_string()),+ operation,});}- let response_body: serde_json::Value =- response- .json()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some(method_name.to_string()),- })?;+ let response_body: serde_json::Value = response+ .json()+ .await+ .map_err(|err| OpensearchClientError::deserialization_failed(operation, &err))?;Ok(parse_bulk_response(&response_body))}diff --git a/crates/opensearch_client/src/upsert/agent_session.rs b/crates/opensearch_client/src/upsert/agent_session.rsindex 06bd805879..f6a4598b16 100644--- a/crates/opensearch_client/src/upsert/agent_session.rs+++ b/crates/opensearch_client/src/upsert/agent_session.rs@@ -2,7 +2,12 @@ use models_opensearch::SearchIndex;use opensearch::BulkParts;use serde::Serialize;-use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};+use crate::{+ Result,+ agent_session::response_error,+ date_format::EpochMillis,+ error::{OpensearchClientError, OpensearchOperation},+};#[cfg(test)]mod test;@@ -132,32 +137,16 @@ fn child_document<'a>(}}-fn serialization_error(method: &str, error: impl std::fmt::Display) -> OpensearchClientError {- OpensearchClientError::DeserializationFailed {+fn serialization_error(+ operation: OpensearchOperation,+ error: serde_json::Error,+) -> OpensearchClientError {+ OpensearchClientError::SerializationFailed {details: error.to_string(),- method: Some(method.to_owned()),+ operation,}}-async fn response_error(- response: opensearch::http::response::Response,- method: &str,-) -> Result<()> {- let status_code = response.status_code();- if status_code.is_success() {- return Ok(());- }- let body = response- .text()- .await- .map_err(|error| serialization_error(method, error))?;- tracing::error!(%status_code, %body, "agent-session OpenSearch operation failed");- Err(OpensearchClientError::Unknown {- details: body,- method: Some(method.to_owned()),- })-}-/// Write the current parent and children, then remove documents from older/// projections. Callers serialize reconciles by session id.pub(crate) async fn reconcile_agent_session(@@ -181,8 +170,13 @@ pub(crate) async fn reconcile_agent_session(.refresh(opensearch::params::Refresh::WaitFor).send().await- .map_err(|error| serialization_error("reconcile_agent_session_parent", error))?;- response_error(parent, "reconcile_agent_session_parent").await?;+ .map_err(|error| {+ OpensearchClientError::request_failed(+ OpensearchOperation::ReconcileAgentSessionParent,+ &error,+ )+ })?;+ response_error(parent, OpensearchOperation::ReconcileAgentSessionParent).await?;if !args.messages.is_empty() {let mut body = Vec::with_capacity(args.messages.len() * 2);@@ -194,11 +188,14 @@ pub(crate) async fn reconcile_agent_session("routing": routing,}}))- .map_err(|error| serialization_error("reconcile_agent_session_bulk", error))?,+ .map_err(|error| {+ serialization_error(OpensearchOperation::ReconcileAgentSessionBulk, error)+ })?,);body.push(- serde_json::to_string(&child_document(args, message))- .map_err(|error| serialization_error("reconcile_agent_session_bulk", error))?,+ serde_json::to_string(&child_document(args, message)).map_err(|error| {+ serialization_error(OpensearchOperation::ReconcileAgentSessionBulk, error)+ })?,);}@@ -209,20 +206,26 @@ pub(crate) async fn reconcile_agent_session(.refresh(opensearch::params::Refresh::WaitFor).send().await- .map_err(|error| serialization_error("reconcile_agent_session_bulk", error))?;- let status_code = response.status_code();- if !status_code.is_success() {- return response_error(response, "reconcile_agent_session_bulk").await;+ .map_err(|error| {+ OpensearchClientError::request_failed(+ OpensearchOperation::ReconcileAgentSessionBulk,+ &error,+ )+ })?;+ if !response.status_code().is_success() {+ return response_error(response, OpensearchOperation::ReconcileAgentSessionBulk).await;}- let response_body: serde_json::Value = response- .json()- .await- .map_err(|error| serialization_error("reconcile_agent_session_bulk", error))?;+ let response_body: serde_json::Value = response.json().await.map_err(|error| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::ReconcileAgentSessionBulk,+ &error,+ )+ })?;if response_body["errors"].as_bool() == Some(true) {tracing::error!(body=?response_body, "agent-session bulk reconcile had failed items");return Err(OpensearchClientError::Unknown {details: response_body.to_string(),- method: Some("reconcile_agent_session_bulk".to_owned()),+ operation: OpensearchOperation::ReconcileAgentSessionBulk,});}}@@ -245,6 +248,15 @@ pub(crate) async fn reconcile_agent_session(.refresh(true).send().await- .map_err(|error| serialization_error("reconcile_agent_session_prune", error))?;- crate::agent_session::check_delete_response(response, "reconcile_agent_session_prune").await+ .map_err(|error| {+ OpensearchClientError::request_failed(+ OpensearchOperation::ReconcileAgentSessionPrune,+ &error,+ )+ })?;+ crate::agent_session::check_delete_response(+ response,+ OpensearchOperation::ReconcileAgentSessionPrune,+ )+ .await}diff --git a/crates/opensearch_client/src/upsert/agent_session/test.rs b/crates/opensearch_client/src/upsert/agent_session/test.rsindex 6ee43fdc44..7819f49e25 100644--- a/crates/opensearch_client/src/upsert/agent_session/test.rs+++ b/crates/opensearch_client/src/upsert/agent_session/test.rs@@ -1,6 +1,7 @@use serde_json::json;use super::*;+use crate::error::TransportFailure;fn args() -> ReconcileAgentSessionArgs {ReconcileAgentSessionArgs {@@ -71,8 +72,12 @@ fn index_override_selects_a_physical_backfill_index() {);}-/// Capture actual HTTP requests so both index and bulk serialization are covered.-async fn capture_reconcile_requests(index_override: Option<&str>) -> Vec<String> {+/// Answer the parent, bulk, and prune requests in turn with each status and+/// body, capturing the actual HTTP requests so their serialization is covered.+async fn serve_reconcile(+ responses: [(&'static str, &'static str); 3],+ index_override: Option<&str>,+) -> (Vec<String>, Result<()>) {use std::io::{BufRead, Read, Write};use std::net::TcpListener;use std::time::Duration;@@ -81,11 +86,7 @@ async fn capture_reconcile_requests(index_override: Option<&str>) -> Vec<String>let address = listener.local_addr().unwrap();let server = std::thread::spawn(move || {let mut requests = Vec::new();- for body in [- r#"{}"#,- r#"{"errors":false}"#,- r#"{"failures":[],"timed_out":false}"#,- ] {+ for (status, body) in responses {let (mut stream, _) = listener.accept().unwrap();stream.set_read_timeout(Some(Duration::from_secs(5)))@@ -106,7 +107,7 @@ async fn capture_reconcile_requests(index_override: Option<&str>) -> Vec<String>}}reader.read_exact(&mut vec![0; content_length]).unwrap();- write!(stream, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap();+ write!(stream, "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap();}requests});@@ -115,13 +116,21 @@ async fn capture_reconcile_requests(index_override: Option<&str>) -> Vec<String>let result = client.reconcile_agent_session(&args(), index_override).await;- result.unwrap();- server.join().unwrap()+ (server.join().unwrap(), result)}#[tokio::test]async fn normal_parent_and_bulk_writes_require_an_alias() {- let requests = capture_reconcile_requests(None).await;+ let (requests, result) = serve_reconcile(+ [+ ("200 OK", r#"{}"#),+ ("200 OK", r#"{"errors":false}"#),+ ("200 OK", r#"{"failures":[],"timed_out":false}"#),+ ],+ None,+ )+ .await;+ result.unwrap();assert!(requests[0].starts_with("POST /agent_sessions/_doc/session-1?"),"{requests:?}"@@ -134,7 +143,16 @@ async fn normal_parent_and_bulk_writes_require_an_alias() {#[tokio::test]async fn explicit_backfill_overrides_allow_a_physical_index() {- let requests = capture_reconcile_requests(Some("scratch-index")).await;+ let (requests, result) = serve_reconcile(+ [+ ("200 OK", r#"{}"#),+ ("200 OK", r#"{"errors":false}"#),+ ("200 OK", r#"{"failures":[],"timed_out":false}"#),+ ],+ Some("scratch-index"),+ )+ .await;+ result.unwrap();assert!(requests[0].starts_with("POST /scratch-index/_doc/session-1?"),"{requests:?}"@@ -145,6 +163,78 @@ async fn explicit_backfill_overrides_allow_a_physical_index() {}}+#[tokio::test]+async fn a_load_balancer_error_page_on_prune_is_reported_with_its_body() {+ let (_, result) = serve_reconcile(+ [+ ("200 OK", r#"{}"#),+ ("200 OK", r#"{"errors":false}"#),+ (+ "502 Bad Gateway",+ "<html><head><title>502 Bad Gateway</title></head></html>",+ ),+ ],+ None,+ )+ .await;++ let Err(OpensearchClientError::Unknown { details, operation }) = result else {+ panic!("expected the 502 page to be reported, got {result:?}");+ };+ assert_eq!(operation, OpensearchOperation::ReconcileAgentSessionPrune);+ assert!(details.contains("502 Bad Gateway"), "{details}");+}++#[tokio::test]+async fn a_connection_closed_without_a_response_reports_the_transport_cause() {+ use std::io::{BufRead, Read};+ use std::net::TcpListener;++ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let server = std::thread::spawn(move || {+ let (mut stream, _) = listener.accept().unwrap();+ let mut reader = std::io::BufReader::new(&mut stream);+ let mut content_length = 0;+ loop {+ let mut line = String::new();+ reader.read_line(&mut line).unwrap();+ if line == "\r\n" {+ break;+ }+ if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") {+ content_length = value.trim().parse::<usize>().unwrap();+ }+ }+ reader.read_exact(&mut vec![0; content_length]).unwrap();+ // Dropping the stream closes the connection before any response.+ });+ let client =+ crate::OpensearchClient::new(format!("http://{address}"), "".into(), "".into()).unwrap();+ let result = client.reconcile_agent_session(&args(), None).await;+ server.join().unwrap();++ let Err(OpensearchClientError::RequestFailed {+ details,+ cause,+ operation,+ }) = result+ else {+ panic!("expected a request failure, got {result:?}");+ };+ assert_eq!(operation, OpensearchOperation::ReconcileAgentSessionParent);+ assert_eq!(cause, TransportFailure::ConnectionClosed);+ assert!(+ details.contains("connection closed before message completed"),+ "{details}"+ );+ assert_eq!(+ details.matches("error sending request").count(),+ 1,+ "{details}"+ );+}+/// Uses only a uniquely named scratch index; never changes application indices.#[tokio::test]#[ignore = "requires local OpenSearch at http://localhost:9200"]diff --git a/crates/opensearch_client/src/upsert/calendar_event.rs b/crates/opensearch_client/src/upsert/calendar_event.rsindex 4b8c1ec27e..d97bea2854 100644--- a/crates/opensearch_client/src/upsert/calendar_event.rs+++ b/crates/opensearch_client/src/upsert/calendar_event.rs@@ -1,7 +1,11 @@use models_opensearch::SearchIndex;use super::properties::IndexedProperty;-use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};+use crate::{+ Result,+ date_format::EpochMillis,+ error::{OpensearchClientError, OpensearchOperation},+};/// The arguments for upserting a calendar event into the opensearch index.///@@ -80,7 +84,7 @@ pub(crate) async fn upsert_calendar_event(let index = resolve_destination(index_override);let body = serde_json::to_value(args).map_err(|err| OpensearchClientError::Unknown {details: err.to_string(),- method: Some("upsert_calendar_event".to_string()),+ operation: OpensearchOperation::UpsertCalendarEvent,})?;let response = client@@ -88,9 +92,8 @@ pub(crate) async fn upsert_calendar_event(.body(body).send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("upsert_calendar_event".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::UpsertCalendarEvent, &err)})?;let status_code = response.status_code();@@ -99,14 +102,12 @@ pub(crate) async fn upsert_calendar_event(return Ok(());}- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("upsert_calendar_event".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpsertCalendarEvent,+ &err,+ )+ })?;tracing::error!(status_code=?status_code,@@ -117,7 +118,7 @@ pub(crate) async fn upsert_calendar_event(Err(OpensearchClientError::Unknown {details: body,- method: Some("upsert_calendar_event".to_string()),+ operation: OpensearchOperation::UpsertCalendarEvent,})}@@ -135,7 +136,7 @@ pub(crate) async fn update_calendar_event_properties(let properties_value =serde_json::to_value(properties).map_err(|err| OpensearchClientError::Unknown {details: err.to_string(),- method: Some("update_calendar_event_properties".to_string()),+ operation: OpensearchOperation::UpdateCalendarEventProperties,})?;let body = json!({ "doc": { "properties": properties_value } });@@ -144,9 +145,11 @@ pub(crate) async fn update_calendar_event_properties(.body(body).send().await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_calendar_event_properties".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(+ OpensearchOperation::UpdateCalendarEventProperties,+ &err,+ )})?;let status_code = response.status_code();@@ -154,14 +157,12 @@ pub(crate) async fn update_calendar_event_properties(tracing::trace!(event_id=%event_id, "calendar event properties updated");return Ok(());}- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_calendar_event_properties".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpdateCalendarEventProperties,+ &err,+ )+ })?;// A *missing document* 404 is a no-op: the doc isn't indexed yet, so the// next full upsert will include its properties. A *missing index* 404@@ -188,6 +189,6 @@ pub(crate) async fn update_calendar_event_properties(Err(OpensearchClientError::Unknown {details: body,- method: Some("update_calendar_event_properties".to_string()),+ operation: OpensearchOperation::UpdateCalendarEventProperties,})}diff --git a/crates/opensearch_client/src/upsert/call_record.rs b/crates/opensearch_client/src/upsert/call_record.rsindex 1812b89774..8eab77e38d 100644--- a/crates/opensearch_client/src/upsert/call_record.rs+++ b/crates/opensearch_client/src/upsert/call_record.rs@@ -4,7 +4,11 @@ use models_opensearch::SearchIndex;use super::BulkUpsertResult;use crate::upsert::properties::IndexedProperty;-use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};+use crate::{+ Result,+ date_format::EpochMillis,+ error::{OpensearchClientError, OpensearchOperation},+};#[cfg(test)]mod test;@@ -149,7 +153,7 @@ async fn bulk_upsert_call_record_segments_inner(client,index,bulk_body,- "bulk_upsert_call_record_segments_inner",+ OpensearchOperation::BulkUpsertCallRecordSegments,).await}@@ -168,7 +172,7 @@ pub(crate) async fn update_call_record_properties(let properties_value =serde_json::to_value(properties).map_err(|err| OpensearchClientError::Unknown {details: err.to_string(),- method: Some("update_call_record_properties".to_string()),+ operation: OpensearchOperation::UpdateCallRecordProperties,})?;let body = json!({ "doc": { "properties": properties_value } });@@ -181,9 +185,11 @@ pub(crate) async fn update_call_record_properties(.body(body).send().await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_call_record_properties".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(+ OpensearchOperation::UpdateCallRecordProperties,+ &err,+ )})?;let status_code = response.status_code();@@ -191,14 +197,12 @@ pub(crate) async fn update_call_record_properties(tracing::trace!(call_id=%call_id, "call record properties updated");return Ok(());}- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_call_record_properties".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpdateCallRecordProperties,+ &err,+ )+ })?;// A *missing document* 404 is a no-op: the call isn't indexed yet, so the// next segment upsert will include its properties. A *missing index* 404@@ -225,6 +229,6 @@ pub(crate) async fn update_call_record_properties(Err(OpensearchClientError::Unknown {details: body,- method: Some("update_call_record_properties".to_string()),+ operation: OpensearchOperation::UpdateCallRecordProperties,})}diff --git a/crates/opensearch_client/src/upsert/channel_message.rs b/crates/opensearch_client/src/upsert/channel_message.rsindex 46611e7ec4..bbbb6963b9 100644--- a/crates/opensearch_client/src/upsert/channel_message.rs+++ b/crates/opensearch_client/src/upsert/channel_message.rs@@ -1,6 +1,10 @@use models_opensearch::SearchIndex;-use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};+use crate::{+ Result,+ date_format::EpochMillis,+ error::{OpensearchClientError, OpensearchOperation},+};/// The arguments for upserting a channel message into the opensearch index./// Threadless messages are indexed with `thread_id == message_id`.@@ -38,23 +42,20 @@ pub(crate) async fn upsert_channel_message(.body(args).send().await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("upsert_channel_message".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::UpsertChannelMessage, &err)})?;let status_code = response.status_code();if status_code.is_success() {tracing::trace!(id=%id, "channel message upserted successfully");} else {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("upsert_channel_message".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpsertChannelMessage,+ &err,+ )+ })?;tracing::error!(status_code=%status_code,@@ -64,7 +65,7 @@ pub(crate) async fn upsert_channel_message(return Err(OpensearchClientError::Unknown {details: body,- method: Some("upsert_channel_message".to_string()),+ operation: OpensearchOperation::UpsertChannelMessage,});}Ok(())diff --git a/crates/opensearch_client/src/upsert/chat_message.rs b/crates/opensearch_client/src/upsert/chat_message.rsindex b960fb5d21..c915c5c00e 100644--- a/crates/opensearch_client/src/upsert/chat_message.rs+++ b/crates/opensearch_client/src/upsert/chat_message.rs@@ -1,7 +1,11 @@use models_opensearch::SearchIndex;use crate::upsert::properties::IndexedProperty;-use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};+use crate::{+ Result,+ date_format::EpochMillis,+ error::{OpensearchClientError, OpensearchOperation},+};#[cfg(test)]mod test;@@ -97,18 +101,20 @@ pub(crate) async fn upsert_chat_message(.body(parent_body).send().await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("upsert_chat_message_parent".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(+ OpensearchOperation::UpsertChatMessageParent,+ &err,+ )})?;if !parent_response.status_code().is_success() {let status_code = parent_response.status_code();let body = parent_response.text().await.map_err(|err| {- OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("upsert_chat_message_parent".to_string()),- }+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpsertChatMessageParent,+ &err,+ )})?;tracing::error!(status_code=%status_code,@@ -117,7 +123,7 @@ pub(crate) async fn upsert_chat_message();return Err(OpensearchClientError::Unknown {details: body,- method: Some("upsert_chat_message_parent".to_string()),+ operation: OpensearchOperation::UpsertChatMessageParent,});}@@ -129,9 +135,8 @@ pub(crate) async fn upsert_chat_message(.body(child_body).send().await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("upsert_chat_message_child".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::UpsertChatMessageChild, &err)})?;let status_code = child_response.status_code();@@ -141,10 +146,10 @@ pub(crate) async fn upsert_chat_message(}let body = child_response.text().await.map_err(|err| {- OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("upsert_chat_message_child".to_string()),- }+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpsertChatMessageChild,+ &err,+ )})?;tracing::error!(@@ -155,7 +160,7 @@ pub(crate) async fn upsert_chat_message(Err(OpensearchClientError::Unknown {details: body,- method: Some("upsert_chat_message_child".to_string()),+ operation: OpensearchOperation::UpsertChatMessageChild,})}@@ -182,9 +187,8 @@ pub(crate) async fn update_chat_metadata(.body(body).send().await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_chat_metadata".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::UpdateChatMetadata, &err)})?;let status_code = response.status_code();@@ -197,14 +201,9 @@ pub(crate) async fn update_chat_metadata(return Ok(());}- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_chat_metadata".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(OpensearchOperation::UpdateChatMetadata, &err)+ })?;tracing::error!(status_code=?status_code,@@ -215,7 +214,7 @@ pub(crate) async fn update_chat_metadata(Err(OpensearchClientError::Unknown {details: body,- method: Some("update_chat_metadata".to_string()),+ operation: OpensearchOperation::UpdateChatMetadata,})}@@ -233,7 +232,7 @@ pub(crate) async fn update_chat_properties(let properties_value =serde_json::to_value(properties).map_err(|err| OpensearchClientError::Unknown {details: err.to_string(),- method: Some("update_chat_properties".to_string()),+ operation: OpensearchOperation::UpdateChatProperties,})?;let body = json!({ "doc": { "properties": properties_value } });@@ -246,9 +245,8 @@ pub(crate) async fn update_chat_properties(.body(body).send().await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_chat_properties".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::UpdateChatProperties, &err)})?;let status_code = response.status_code();@@ -256,14 +254,12 @@ pub(crate) async fn update_chat_properties(tracing::trace!(chat_id=%chat_id, "chat properties updated");return Ok(());}- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_chat_properties".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpdateChatProperties,+ &err,+ )+ })?;// A *missing document* 404 is a no-op: the chat isn't indexed yet, so the// next message upsert will include its properties. A *missing index* 404@@ -290,6 +286,6 @@ pub(crate) async fn update_chat_properties(Err(OpensearchClientError::Unknown {details: body,- method: Some("update_chat_properties".to_string()),+ operation: OpensearchOperation::UpdateChatProperties,})}diff --git a/crates/opensearch_client/src/upsert/document.rs b/crates/opensearch_client/src/upsert/document.rsindex 6aaf12683b..5b9f91a25b 100644--- a/crates/opensearch_client/src/upsert/document.rs+++ b/crates/opensearch_client/src/upsert/document.rs@@ -4,7 +4,11 @@ use models_opensearch::SearchIndex;use super::BulkUpsertResult;use super::properties::IndexedProperty;-use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};+use crate::{+ Result,+ date_format::EpochMillis,+ error::{OpensearchClientError, OpensearchOperation},+};/// Relation name for parent docs in the join field.const PARENT_RELATION: &str = "document";@@ -164,8 +168,13 @@ async fn bulk_upsert_single_chunk(bulk_body.push(child_doc_body(doc).to_string());}- let result =- super::bulk_upsert_to_index(client, index, bulk_body, "bulk_upsert_single_chunk").await?;+ let result = super::bulk_upsert_to_index(+ client,+ index,+ bulk_body,+ OpensearchOperation::BulkUpsertSingleChunk,+ )+ .await?;tracing::trace!(chunk_total = documents.len(),@@ -287,7 +296,13 @@ pub(crate) async fn upsert_document(child_doc_body(args).to_string(),];- let result = super::bulk_upsert_to_index(client, index, bulk_body, "upsert_document").await?;+ let result = super::bulk_upsert_to_index(+ client,+ index,+ bulk_body,+ OpensearchOperation::UpsertDocument,+ )+ .await?;if result.failed > 0 {return Err(OpensearchClientError::Unknown {@@ -295,7 +310,7 @@ pub(crate) async fn upsert_document("upsert_document had {} failures: {:?}",result.failed, result.errors),- method: Some("upsert_document".to_string()),+ operation: OpensearchOperation::UpsertDocument,});}@@ -329,8 +344,13 @@ pub(crate) async fn upsert_parent_document(let index = resolve_destination(index_override);let bulk_body = parent_only_bulk_body(args);- let result =- super::bulk_upsert_to_index(client, index, bulk_body, "upsert_parent_document").await?;+ let result = super::bulk_upsert_to_index(+ client,+ index,+ bulk_body,+ OpensearchOperation::UpsertParentDocument,+ )+ .await?;if result.failed > 0 {return Err(OpensearchClientError::Unknown {@@ -338,7 +358,7 @@ pub(crate) async fn upsert_parent_document("upsert_parent_document had {} failures: {:?}",result.failed, result.errors),- method: Some("upsert_parent_document".to_string()),+ operation: OpensearchOperation::UpsertParentDocument,});}@@ -373,9 +393,8 @@ pub(crate) async fn update_document_metadata(.body(body).send().await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_document_metadata".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::UpdateDocumentMetadata, &err)})?;let status_code = response.status_code();@@ -388,14 +407,12 @@ pub(crate) async fn update_document_metadata(return Ok(());}- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_document_metadata".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpdateDocumentMetadata,+ &err,+ )+ })?;// A *missing document* 404 is a no-op: the doc isn't indexed yet, so the// next full index will carry the current name. A *missing index* 404@@ -423,7 +440,7 @@ pub(crate) async fn update_document_metadata(Err(OpensearchClientError::Unknown {details: body,- method: Some("update_document_metadata".to_string()),+ operation: OpensearchOperation::UpdateDocumentMetadata,})}@@ -443,7 +460,7 @@ pub(crate) async fn update_document_properties(let properties_value =serde_json::to_value(properties).map_err(|err| OpensearchClientError::Unknown {details: err.to_string(),- method: Some("update_document_properties".to_string()),+ operation: OpensearchOperation::UpdateDocumentProperties,})?;let body = json!({ "doc": { "properties": properties_value } });@@ -453,9 +470,11 @@ pub(crate) async fn update_document_properties(.body(body).send().await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_document_properties".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(+ OpensearchOperation::UpdateDocumentProperties,+ &err,+ )})?;let status_code = response.status_code();@@ -463,14 +482,12 @@ pub(crate) async fn update_document_properties(tracing::trace!(document_id=%document_id, "document properties updated");return Ok(());}- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_document_properties".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpdateDocumentProperties,+ &err,+ )+ })?;// A *missing document* 404 is a no-op: the doc isn't indexed yet, so the// next full index will include its properties. A *missing index* 404@@ -499,7 +516,7 @@ pub(crate) async fn update_document_properties(Err(OpensearchClientError::Unknown {details: body,- method: Some("update_document_properties".to_string()),+ operation: OpensearchOperation::UpdateDocumentProperties,})}diff --git a/crates/opensearch_client/src/upsert/email.rs b/crates/opensearch_client/src/upsert/email.rsindex 1e41ac35bd..be5323c979 100644--- a/crates/opensearch_client/src/upsert/email.rs+++ b/crates/opensearch_client/src/upsert/email.rs@@ -2,7 +2,11 @@ use models_opensearch::SearchIndex;use super::BulkUpsertResult;use super::properties::IndexedProperty;-use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};+use crate::{+ Result,+ date_format::EpochMillis,+ error::{OpensearchClientError, OpensearchOperation},+};/// The arguments for upserting an email message into the opensearch index#[derive(Debug, serde::Serialize)]@@ -66,23 +70,20 @@ pub(crate) async fn upsert_email_message(.body(args).send().await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("upsert_email_message".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::UpsertEmailMessage, &err)})?;let status_code = response.status_code();if status_code.is_success() {tracing::trace!(id=%id, "email message upserted successfully");} else {- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("upsert_email_message".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpsertEmailMessage,+ &err,+ )+ })?;tracing::error!(status_code=%status_code,@@ -92,7 +93,7 @@ pub(crate) async fn upsert_email_message(return Err(OpensearchClientError::Unknown {details: body,- method: Some("upsert_email_message".to_string()),+ operation: OpensearchOperation::UpsertEmailMessage,});}Ok(())@@ -121,16 +122,22 @@ pub(crate) async fn bulk_upsert_email_messages(bulk_body.push(action.to_string());bulk_body.push(serde_json::to_string(msg).map_err(|e| {- OpensearchClientError::DeserializationFailed {+ OpensearchClientError::SerializationFailed {details: e.to_string(),- method: Some("bulk_upsert_email_messages".to_string()),+ operation: OpensearchOperation::BulkUpsertEmailMessages,}})?);}let index = index_override.unwrap_or(SearchIndex::Emails.as_ref());- super::bulk_upsert_to_index(client, index, bulk_body, "bulk_upsert_email_messages").await+ super::bulk_upsert_to_index(+ client,+ index,+ bulk_body,+ OpensearchOperation::BulkUpsertEmailMessages,+ )+ .await}/// Update only the denormalized `properties` on every message doc of a@@ -148,7 +155,7 @@ pub(crate) async fn update_email_thread_properties(let properties_value =serde_json::to_value(properties).map_err(|err| OpensearchClientError::Unknown {details: err.to_string(),- method: Some("update_email_thread_properties".to_string()),+ operation: OpensearchOperation::UpdateEmailThreadProperties,})?;let body = serde_json::json!({"query": { "term": { "entity_id": thread_id } },@@ -168,22 +175,22 @@ pub(crate) async fn update_email_thread_properties(.body(body).send().await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_email_thread_properties".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(+ OpensearchOperation::UpdateEmailThreadProperties,+ &err,+ )})?;let status_code = response.status_code();if status_code.is_success() {// A 200 can still carry per-document failures in the body.- let body: serde_json::Value =- response- .json()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_email_thread_properties".to_string()),- })?;+ let body: serde_json::Value = response.json().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpdateEmailThreadProperties,+ &err,+ )+ })?;if let Some(failures) = body.get("failures").and_then(|f| f.as_array())&& !failures.is_empty(){@@ -194,20 +201,18 @@ pub(crate) async fn update_email_thread_properties();return Err(OpensearchClientError::Unknown {details: format!("update_by_query failures: {failures:?}"),- method: Some("update_email_thread_properties".to_string()),+ operation: OpensearchOperation::UpdateEmailThreadProperties,});}tracing::trace!(thread_id=%thread_id, "email thread properties updated");return Ok(());}- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_email_thread_properties".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpdateEmailThreadProperties,+ &err,+ )+ })?;tracing::error!(status_code=?status_code,@@ -218,6 +223,6 @@ pub(crate) async fn update_email_thread_properties(Err(OpensearchClientError::Unknown {details: body,- method: Some("update_email_thread_properties".to_string()),+ operation: OpensearchOperation::UpdateEmailThreadProperties,})}diff --git a/crates/opensearch_client/src/upsert/project.rs b/crates/opensearch_client/src/upsert/project.rsindex 859e465183..08b57cf0a8 100644--- a/crates/opensearch_client/src/upsert/project.rs+++ b/crates/opensearch_client/src/upsert/project.rs@@ -1,7 +1,11 @@use models_opensearch::SearchIndex;use super::properties::IndexedProperty;-use crate::{Result, date_format::EpochMillis, error::OpensearchClientError};+use crate::{+ Result,+ date_format::EpochMillis,+ error::{OpensearchClientError, OpensearchOperation},+};/// The arguments for upserting a project into the opensearch index.///@@ -43,7 +47,7 @@ pub(crate) async fn upsert_project(let index = resolve_destination(index_override);let body = serde_json::to_value(args).map_err(|err| OpensearchClientError::Unknown {details: err.to_string(),- method: Some("upsert_project".to_string()),+ operation: OpensearchOperation::UpsertProject,})?;let response = client@@ -51,9 +55,8 @@ pub(crate) async fn upsert_project(.body(body).send().await- .map_err(|err| OpensearchClientError::Unknown {- details: err.to_string(),- method: Some("upsert_project".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(OpensearchOperation::UpsertProject, &err)})?;let status_code = response.status_code();@@ -62,14 +65,9 @@ pub(crate) async fn upsert_project(return Ok(());}- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("upsert_project".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(OpensearchOperation::UpsertProject, &err)+ })?;tracing::error!(status_code=?status_code,@@ -80,7 +78,7 @@ pub(crate) async fn upsert_project(Err(OpensearchClientError::Unknown {details: body,- method: Some("upsert_project".to_string()),+ operation: OpensearchOperation::UpsertProject,})}@@ -100,7 +98,7 @@ pub(crate) async fn update_project_properties(let properties_value =serde_json::to_value(properties).map_err(|err| OpensearchClientError::Unknown {details: err.to_string(),- method: Some("update_project_properties".to_string()),+ operation: OpensearchOperation::UpdateProjectProperties,})?;let body = json!({ "doc": { "properties": properties_value } });@@ -109,9 +107,11 @@ pub(crate) async fn update_project_properties(.body(body).send().await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_project_properties".to_string()),+ .map_err(|err| {+ OpensearchClientError::request_failed(+ OpensearchOperation::UpdateProjectProperties,+ &err,+ )})?;let status_code = response.status_code();@@ -119,14 +119,12 @@ pub(crate) async fn update_project_properties(tracing::trace!(project_id=%project_id, "project properties updated");return Ok(());}- let body =- response- .text()- .await- .map_err(|err| OpensearchClientError::DeserializationFailed {- details: err.to_string(),- method: Some("update_project_properties".to_string()),- })?;+ let body = response.text().await.map_err(|err| {+ OpensearchClientError::deserialization_failed(+ OpensearchOperation::UpdateProjectProperties,+ &err,+ )+ })?;// A *missing document* 404 is a no-op: the doc isn't indexed yet, so the// next full upsert will include its properties. A *missing index* 404@@ -153,6 +151,6 @@ pub(crate) async fn update_project_properties(Err(OpensearchClientError::Unknown {details: body,- method: Some("update_project_properties".to_string()),+ operation: OpensearchOperation::UpdateProjectProperties,})}