Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(graphql): log and trace graphql errors (#7911)

teof86ecbd15fb1
diff --git a/Cargo.lock b/Cargo.lock
index 78b25fc44b..799e4a3e9d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6611,6 +6611,7 @@ dependencies = [
"notification",
"opensearch_client",
"opentelemetry",
+ "opentelemetry_sdk",
"ordered-float",
"projects",
"properties",
@@ -6656,6 +6657,7 @@ dependencies = [
"tower-http",
"tracing",
"tracing-opentelemetry",
+ "tracing-subscriber",
"tungstenite 0.27.0",
"unicode-segmentation",
"urlencoding",
diff --git a/deny.toml b/deny.toml
index edd1d6ddbd..43980027b9 100644
--- a/deny.toml
+++ b/deny.toml
@@ -231,6 +231,7 @@ deny = [
"coding_agent_worker", # Standalone daemon logging setup
"cursor_cloud_agents", # Standalone ACP binary logging; stdout is the protocol, so macro_entrypoint cannot own it
"dictation", # Metadata tracing and content-redaction tests only
+ "document_storage_service", # GraphQL telemetry span and log tests only (dev-dependency)
"documents", # Dev logging only
"email", # Dev logging only
"genai_telemetry", # GenAI span tests
diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.toml
index 2577994875..b26e474e45 100644
--- a/services/document_storage_service/Cargo.toml
+++ b/services/document_storage_service/Cargo.toml
@@ -310,7 +310,11 @@ cool_asserts = "2"
flate2 = "1"
frecency = { path = "../../crates/frecency", features = ["mock"] }
macro_auth = { path = "../../crates/macro_auth", features = ["testing"] }
+opentelemetry = { workspace = true, features = ["trace"] }
+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }
ordered-float = { workspace = true }
+tracing-opentelemetry = { workspace = true }
+tracing-subscriber = { workspace = true }
[package.metadata.cargo-machete]
ignored = ["connection_gateway_client"]
diff --git a/services/document_storage_service/src/api/graphql_soup.rs b/services/document_storage_service/src/api/graphql_soup.rs
index ac0aab0e51..7cb82c90c4 100644
--- a/services/document_storage_service/src/api/graphql_soup.rs
+++ b/services/document_storage_service/src/api/graphql_soup.rs
@@ -1,3 +1,5 @@
+mod telemetry;
+
use crate::api::context::{ApiContext, AuthorizationService};
use crate::api::folders_query::DssFoldersQueryReader;
use agent_session::outbound::postgres::PgAgentSessionRepo;
@@ -49,7 +51,6 @@ fn graphiql_source(endpoint: &str) -> String {
.finish()
}
-#[tracing::instrument(skip_all, name = "graphql.execute")]
async fn graphql_handler(
State(state): State<ApiContext>,
Cached(auth): Cached<
@@ -62,17 +63,19 @@ async fn graphql_handler(
.authorization
.as_ref()
.and_then(UserOrInternalServiceAuthorization::acting_user);
- let request = graphql_query_context_data(
- request.into_inner(),
- &state,
- acting_user.map(|user| user.macro_user_id.clone()),
- acting_user.and_then(|user| user.user_context.organization_id.map(i64::from)),
- acting_user.and_then(opaque_viewer_id),
- );
- let result = state
- .graphql_soup_schema
- .execute(request.data(GraphqlRequestParts::new(request_parts)))
- .await;
+ let result = telemetry::observe(request.into_inner(), acting_user.is_some(), |request| {
+ let request = graphql_query_context_data(
+ request,
+ &state,
+ acting_user.map(|user| user.macro_user_id.clone()),
+ acting_user.and_then(|user| user.user_context.organization_id.map(i64::from)),
+ acting_user.and_then(opaque_viewer_id),
+ );
+ state
+ .graphql_soup_schema
+ .execute(request.data(GraphqlRequestParts::new(request_parts)))
+ })
+ .await;
graphql_response(result)
}
diff --git a/services/document_storage_service/src/api/graphql_soup/telemetry.rs b/services/document_storage_service/src/api/graphql_soup/telemetry.rs
new file mode 100644
index 0000000000..9b27591e18
--- /dev/null
+++ b/services/document_storage_service/src/api/graphql_soup/telemetry.rs
@@ -0,0 +1,267 @@
+#[cfg(test)]
+mod test;
+
+use async_graphql::{
+ PathSegment, Request, Response, ServerError, Value,
+ parser::types::{DocumentOperations, OperationType},
+};
+use tracing::{Instrument as _, Level, Span};
+
+/// Resource for requests whose operation cannot be resolved (parse errors,
+/// unknown operation names). Keeps attacker-supplied text out of the resource.
+const UNKNOWN_RESOURCE: &str = "graphql.unknown";
+const ANONYMOUS_OPERATION: &str = "anonymous";
+/// Unauthenticated callers choose their own operation names. Grouping them
+/// bounds the resource cardinality that Datadog turns into metric tags.
+const UNAUTHENTICATED_OPERATION: &str = "unauthenticated";
+const MAX_OPERATION_NAME_CHARS: usize = 100;
+/// A response can carry one error per aliased field; cap the log lines per request.
+const MAX_LOGGED_ERRORS: usize = 10;
+/// Paths are field names, client-chosen aliases, and list indexes.
+const MAX_ERROR_PATH_CHARS: usize = 200;
+const MAX_ERROR_CODE_CHARS: usize = 64;
+/// Logged in place of an `extensions.code` that is not a plain identifier.
+const UNRECOGNIZED_ERROR_CODE: &str = "UNRECOGNIZED";
+const SERVER_ERROR_CODE_PREFIX: &str = "INTERNAL";
+const SERVER_ERROR_TEXT: &str = "graphql resolver error";
+const CLIENT_ERROR_TEXT: &str = "graphql client error";
+const DATADOG_MEASURED: i64 = 1;
+
+/// Runs `execute` inside a `graphql.execute` span that records the selected
+/// operation and logs the response's errors.
+pub(super) async fn observe<Fut>(
+ mut request: Request,
+ authenticated: bool,
+ execute: impl FnOnce(Request) -> Fut,
+) -> Response
+where
+ Fut: Future<Output = Response>,
+{
+ let span = execute_span();
+ let operation = GraphqlOperation::from_request(&mut request, authenticated);
+ operation.record(&span);
+ let response = execute(request).instrument(span.clone()).await;
+ operation.record_response(&span, &response);
+ response
+}
+
+fn execute_span() -> Span {
+ tracing::info_span!(
+ "graphql.execute",
+ // The Datadog Agent computes `trace.Internal.*` stats for an internal
+ // child span only when this attribute is the OTLP integer 1; a bool or
+ // string is ignored.
+ "_dd.measured" = DATADOG_MEASURED,
+ "resource.name" = tracing::field::Empty,
+ "graphql.operation.type" = tracing::field::Empty,
+ "graphql.operation.name" = tracing::field::Empty,
+ "graphql.error_count" = tracing::field::Empty,
+ otel.status_code = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ )
+}
+
+/// The operation a GraphQL HTTP request selects, recorded on `graphql.execute`.
+#[derive(Debug)]
+struct GraphqlOperation {
+ kind: Option<OperationType>,
+ name: Option<String>,
+ authenticated: bool,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ErrorSeverity {
+ /// The caller sent something the server rejected; logged at warn.
+ Client,
+ /// A resolver failed for an authenticated caller; logged at error and marks the span.
+ Server,
+}
+
+impl GraphqlOperation {
+ /// Parses the request document so its operation can be recorded before
+ /// execution. The parsed document stays on the request and
+ /// `Schema::execute` reuses it, so valid documents are parsed once.
+ fn from_request(request: &mut Request, authenticated: bool) -> Self {
+ let requested_name = request.operation_name.clone();
+ let selected = request.parsed_query().ok().and_then(|document| {
+ select_operation(&document.operations, requested_name.as_deref())
+ .map(|(name, kind)| (name.map(truncate_operation_name), kind))
+ });
+ let (name, kind) = match selected {
+ Some((name, kind)) => (name, Some(kind)),
+ None => (None, None),
+ };
+ Self {
+ kind,
+ name,
+ authenticated,
+ }
+ }
+
+ /// The Datadog resource name, `graphql.<type>.<operationName>`. Datadog
+ /// derives `trace.Internal.{hits,errors,duration}` per resource from it.
+ fn resource_name(&self) -> String {
+ let Some(kind) = self.kind else {
+ return UNKNOWN_RESOURCE.to_owned();
+ };
+ let name = match (&self.name, self.authenticated) {
+ (_, false) => UNAUTHENTICATED_OPERATION,
+ (Some(name), true) => name.as_str(),
+ (None, true) => ANONYMOUS_OPERATION,
+ };
+ format!("graphql.{kind}.{name}")
+ }
+
+ fn record(&self, span: &Span) {
+ span.record("resource.name", self.resource_name());
+ if let Some(kind) = self.kind {
+ span.record("graphql.operation.type", tracing::field::display(kind));
+ }
+ if let Some(name) = &self.name {
+ span.record("graphql.operation.name", name.as_str());
+ }
+ }
+
+ /// Records the error count on `span`, logs each error with its path and
+ /// code, and marks `span` errored when a resolver failed.
+ ///
+ /// Error messages are never recorded: resolvers and async-graphql build
+ /// them from request input (variable values, arguments, upstream error
+ /// text), so no redaction of free text is reliable.
+ fn record_response(&self, span: &Span, response: &Response) {
+ span.record("graphql.error_count", response.errors.len());
+
+ for error in response.errors.iter().take(MAX_LOGGED_ERRORS) {
+ self.log_error(span, error, response.errors.len());
+ }
+
+ if let Some(error) = response
+ .errors
+ .iter()
+ .find(|error| self.error_severity(error) == ErrorSeverity::Server)
+ {
+ span.record("otel.status_code", "ERROR");
+ span.record("otel.status_description", status_description(error));
+ }
+ }
+
+ fn error_severity(&self, error: &ServerError) -> ErrorSeverity {
+ if !self.authenticated || error.path.is_empty() {
+ return ErrorSeverity::Client;
+ }
+ match error_code(error) {
+ Some(code)
+ if !code
+ .to_ascii_uppercase()
+ .starts_with(SERVER_ERROR_CODE_PREFIX) =>
+ {
+ ErrorSeverity::Client
+ }
+ _ => ErrorSeverity::Server,
+ }
+ }
+
+ fn log_error(&self, span: &Span, error: &ServerError, error_count: usize) {
+ let resource = self.resource_name();
+ let path = error_path(&error.path);
+ let code = loggable_error_code(error);
+ macro_rules! log_graphql_error {
+ ($level:expr, $text:expr) => {
+ tracing::event!(
+ parent: span,
+ $level,
+ "graphql.resource" = resource.as_str(),
+ "graphql.operation.type" = self.kind.map(tracing::field::display),
+ "graphql.operation.name" = self.name.as_deref(),
+ "graphql.error.path" = path.as_deref(),
+ "graphql.error.code" = code,
+ "graphql.error_count" = error_count,
+ "{}",
+ $text
+ )
+ };
+ }
+ match self.error_severity(error) {
+ ErrorSeverity::Server => log_graphql_error!(Level::ERROR, SERVER_ERROR_TEXT),
+ ErrorSeverity::Client => log_graphql_error!(Level::WARN, CLIENT_ERROR_TEXT),
+ }
+ }
+}
+
+/// Mirrors async-graphql's operation selection in `Schema::execute`.
+fn select_operation<'a>(
+ operations: &'a DocumentOperations,
+ requested_name: Option<&'a str>,
+) -> Option<(Option<&'a str>, OperationType)> {
+ match (operations, requested_name) {
+ (DocumentOperations::Single(operation), None) => Some((None, operation.node.ty)),
+ (DocumentOperations::Single(_), Some(_)) => None,
+ (DocumentOperations::Multiple(operations), Some(name)) => operations
+ .get(name)
+ .map(|operation| (Some(name), operation.node.ty)),
+ (DocumentOperations::Multiple(operations), None) if operations.len() == 1 => operations
+ .iter()
+ .next()
+ .map(|(name, operation)| (Some(name.as_str()), operation.node.ty)),
+ (DocumentOperations::Multiple(_), None) => None,
+ }
+}
+
+fn truncate_operation_name(name: &str) -> String {
+ name.chars().take(MAX_OPERATION_NAME_CHARS).collect()
+}
+
+fn error_code(error: &ServerError) -> Option<&str> {
+ match error.extensions.as_ref()?.get("code")? {
+ Value::String(code) => Some(code.as_str()),
+ Value::Enum(code) => Some(code.as_str()),
+ _ => None,
+ }
+}
+
+/// Codes are server-chosen identifiers such as `NOT_FOUND`. Anything else is
+/// replaced so a code built from input cannot carry it into telemetry.
+fn loggable_error_code(error: &ServerError) -> Option<&str> {
+ error_code(error).map(|code| {
+ let is_identifier = !code.is_empty()
+ && code.len() <= MAX_ERROR_CODE_CHARS
+ && code
+ .bytes()
+ .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_');
+ if is_identifier {
+ code
+ } else {
+ UNRECOGNIZED_ERROR_CODE
+ }
+ })
+}
+
+/// Fixed text plus the error code, never the error message.
+fn status_description(error: &ServerError) -> String {
+ match loggable_error_code(error) {
+ Some(code) => format!("{SERVER_ERROR_TEXT}: {code}"),
+ None => SERVER_ERROR_TEXT.to_owned(),
+ }
+}
+
+/// `user.soup.items.3.name`; `None` for request-level errors. GraphQL names
+/// are limited to `[_A-Za-z][_0-9A-Za-z]*`, so segments cannot carry free text.
+fn error_path(path: &[PathSegment]) -> Option<String> {
+ if path.is_empty() {
+ return None;
+ }
+ let segments: Vec<String> = path
+ .iter()
+ .map(|segment| match segment {
+ PathSegment::Field(field) => field.clone(),
+ PathSegment::Index(index) => index.to_string(),
+ })
+ .collect();
+ Some(
+ segments
+ .join(".")
+ .chars()
+ .take(MAX_ERROR_PATH_CHARS)
+ .collect(),
+ )
+}
diff --git a/services/document_storage_service/src/api/graphql_soup/telemetry/test.rs b/services/document_storage_service/src/api/graphql_soup/telemetry/test.rs
new file mode 100644
index 0000000000..af84541719
--- /dev/null
+++ b/services/document_storage_service/src/api/graphql_soup/telemetry/test.rs
@@ -0,0 +1,471 @@
+use super::*;
+
+use std::io;
+use std::sync::{Arc, Mutex};
+
+use async_graphql::{
+ EmptyMutation, EmptySubscription, ErrorExtensionValues, ErrorExtensions as _, Object, Schema,
+ Variables,
+};
+use opentelemetry::trace::{Status, TracerProvider as _};
+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
+use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt as _};
+
+struct Query;
+
+#[Object]
+impl Query {
+ async fn ok(&self) -> i32 {
+ 1
+ }
+
+ async fn broken(&self) -> async_graphql::Result<i32> {
+ Err(async_graphql::Error::new(
+ "upstream rejected [email protected]",
+ ))
+ }
+
+ async fn store(&self, value: String) -> async_graphql::Result<String> {
+ Err(async_graphql::Error::new(format!(
+ "could not store {value}"
+ )))
+ }
+
+ async fn missing(&self, value: String) -> async_graphql::Result<i32> {
+ Err(
+ async_graphql::Error::new(format!("no thread named {value}"))
+ .extend_with(|_, extensions| extensions.set("code", "NOT_FOUND")),
+ )
+ }
+
+ async fn rejected(&self, value: String) -> async_graphql::Result<i32> {
+ Err(async_graphql::Error::new("rejected")
+ .extend_with(|_, extensions| extensions.set("code", format!("BAD {value}"))))
+ }
+
+ async fn unavailable(&self) -> async_graphql::Result<i32> {
+ Err(async_graphql::Error::new("service unavailable")
+ .extend_with(|_, extensions| extensions.set("code", "INTERNAL_SERVER_ERROR")))
+ }
+}
+
+#[derive(Clone, Default)]
+struct SharedWriter(Arc<Mutex<Vec<u8>>>);
+
+impl SharedWriter {
+ fn lines(&self) -> Vec<serde_json::Value> {
+ String::from_utf8(self.0.lock().unwrap().clone())
+ .unwrap()
+ .lines()
+ .map(|line| serde_json::from_str(line).unwrap())
+ .collect()
+ }
+}
+
+impl io::Write for SharedWriter {
+ fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+ self.0.lock().unwrap().extend_from_slice(buf);
+ Ok(buf.len())
+ }
+
+ fn flush(&mut self) -> io::Result<()> {
+ Ok(())
+ }
+}
+
+impl<'a> MakeWriter<'a> for SharedWriter {
+ type Writer = SharedWriter;
+
+ fn make_writer(&'a self) -> Self::Writer {
+ self.clone()
+ }
+}
+
+struct Observed {
+ span: SpanData,
+ logs: Vec<serde_json::Value>,
+ raw_logs: String,
+}
+
+impl Observed {
+ fn attribute(&self, key: &str) -> Option<String> {
+ self.attribute_value(key).map(ToString::to_string)
+ }
+
+ fn attribute_value(&self, key: &str) -> Option<&opentelemetry::Value> {
+ self.span
+ .attributes
+ .iter()
+ .find(|attribute| attribute.key.as_str() == key)
+ .map(|attribute| &attribute.value)
+ }
+}
+
+async fn observe_request(request: Request, authenticated: bool) -> Observed {
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let logs = SharedWriter::default();
+ let subscriber = Registry::default()
+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")))
+ .with(
+ tracing_subscriber::fmt::layer()
+ .json()
+ .flatten_event(true)
+ .with_ansi(false)
+ .with_writer(logs.clone()),
+ );
+ let guard = tracing::subscriber::set_default(subscriber);
+ let schema = Schema::new(Query, EmptyMutation, EmptySubscription);
+ observe(request, authenticated, |request| async move {
+ schema.execute(request).await
+ })
+ .await;
+ drop(guard);
+
+ let mut spans = exporter.get_finished_spans().unwrap();
+ assert_eq!(spans.len(), 1);
+ let raw_logs = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap();
+ Observed {
+ span: spans.remove(0),
+ logs: logs.lines(),
+ raw_logs,
+ }
+}
+
+fn operation(query: &str, operation_name: Option<&str>, authenticated: bool) -> GraphqlOperation {
+ let mut request = Request::new(query);
+ if let Some(name) = operation_name {
+ request = request.operation_name(name);
+ }
+ GraphqlOperation::from_request(&mut request, authenticated)
+}
+
+fn resolver_error(code: Option<&str>) -> ServerError {
+ let mut error = ServerError::new("failed", None);
+ error.path = vec![
+ PathSegment::Field("user".to_owned()),
+ PathSegment::Field("soup".to_owned()),
+ ];
+ if let Some(code) = code {
+ let mut extensions = ErrorExtensionValues::default();
+ extensions.set("code", code);
+ error.extensions = Some(extensions);
+ }
+ error
+}
+
+#[test]
+fn resource_name_follows_the_executed_operation() {
+ let cases = [
+ ("query Soup { ok }", None, "graphql.query.Soup"),
+ ("{ ok }", None, "graphql.query.anonymous"),
+ ("mutation Save { ok }", None, "graphql.mutation.Save"),
+ (
+ "subscription Updates { ok }",
+ None,
+ "graphql.subscription.Updates",
+ ),
+ (
+ "mutation Save { ok } query Soup { ok }",
+ Some("Soup"),
+ "graphql.query.Soup",
+ ),
+ (
+ "mutation Save { ok } query Soup { ok }",
+ None,
+ "graphql.unknown",
+ ),
+ ("query Soup { ok }", Some("Other"), "graphql.unknown"),
+ ("{ ok }", Some("Soup"), "graphql.unknown"),
+ ("query Soup {", None, "graphql.unknown"),
+ ];
+ for (query, operation_name, expected) in cases {
+ assert_eq!(
+ operation(query, operation_name, true).resource_name(),
+ expected,
+ "{query} / {operation_name:?}"
+ );
+ }
+}
+
+#[test]
+fn unauthenticated_operations_share_one_resource() {
+ let operation = operation("query AttackerChosenName { ok }", None, false);
+ assert_eq!(operation.resource_name(), "graphql.query.unauthenticated");
+ assert_eq!(operation.name.as_deref(), Some("AttackerChosenName"));
+}
+
+#[test]
+fn operation_names_are_bounded() {
+ let name = "A".repeat(500);
+ let operation = operation(&format!("query {name} {{ ok }}"), None, true);
+ assert_eq!(
+ operation.name.map(|name| name.len()),
+ Some(MAX_OPERATION_NAME_CHARS)
+ );
+}
+
+#[test]
+fn only_uncoded_or_internal_resolver_errors_are_server_errors() {
+ let authenticated = operation("query Soup { ok }", None, true);
+ let anonymous = operation("query Soup { ok }", None, false);
+
+ assert_eq!(
+ authenticated.error_severity(&resolver_error(None)),
+ ErrorSeverity::Server
+ );
+ assert_eq!(
+ authenticated.error_severity(&resolver_error(Some("INTERNAL"))),
+ ErrorSeverity::Server
+ );
+ assert_eq!(
+ authenticated.error_severity(&resolver_error(Some("INTERNAL_SERVER_ERROR"))),
+ ErrorSeverity::Server
+ );
+ for code in ["NOT_FOUND", "BAD_USER_INPUT", "FORBIDDEN", "invalid_input"] {
+ assert_eq!(
+ authenticated.error_severity(&resolver_error(Some(code))),
+ ErrorSeverity::Client,
+ "{code}"
+ );
+ }
+ assert_eq!(
+ authenticated.error_severity(&ServerError::new("Unknown field \"nope\"", None)),
+ ErrorSeverity::Client
+ );
+ assert_eq!(
+ anonymous.error_severity(&resolver_error(None)),
+ ErrorSeverity::Client
+ );
+}
+
+#[test]
+fn error_paths_join_fields_and_indexes() {
+ let path = [
+ PathSegment::Field("user".to_owned()),
+ PathSegment::Field("items".to_owned()),
+ PathSegment::Index(3),
+ PathSegment::Field("name".to_owned()),
+ ];
+ assert_eq!(error_path(&path).as_deref(), Some("user.items.3.name"));
+ assert_eq!(error_path(&[]), None);
+}
+
+#[test]
+fn error_paths_are_bounded() {
+ let path = vec![PathSegment::Field("a".repeat(500))];
+ assert_eq!(
+ error_path(&path).map(|path| path.len()),
+ Some(MAX_ERROR_PATH_CHARS)
+ );
+}
+
+#[test]
+fn only_identifier_error_codes_are_logged() {
+ for code in ["NOT_FOUND", "INTERNAL_SERVER_ERROR", "invalid_input"] {
+ assert_eq!(loggable_error_code(&resolver_error(Some(code))), Some(code));
+ }
+ for code in [
+ "",
+ "invalid-input",
+ &"A".repeat(MAX_ERROR_CODE_CHARS + 1),
+ ] {
+ assert_eq!(
+ loggable_error_code(&resolver_error(Some(code))),
+ Some(UNRECOGNIZED_ERROR_CODE),
+ "{code}"
+ );
+ }
+ assert_eq!(loggable_error_code(&resolver_error(None)), None);
+}
+
+#[test]
+fn status_description_never_contains_the_message() {
+ let mut error = resolver_error(None);
+ error.message = "could not store Q3 board notes".to_owned();
+ assert_eq!(status_description(&error), "graphql resolver error");
+
+ let mut error = resolver_error(Some("INTERNAL"));
+ error.message = "could not store Q3 board notes".to_owned();
+ assert_eq!(
+ status_description(&error),
+ "graphql resolver error: INTERNAL"
+ );
+}
+
+#[tokio::test]
+async fn successful_request_records_operation_without_logging() {
+ let observed = observe_request(Request::new("query Soup { ok }"), true).await;
+
+ assert_eq!(observed.span.name, "graphql.execute");
+ assert_eq!(
+ observed.attribute("resource.name").as_deref(),
+ Some("graphql.query.Soup")
+ );
+ assert_eq!(
+ observed.attribute("graphql.operation.type").as_deref(),
+ Some("query")
+ );
+ assert_eq!(
+ observed.attribute("graphql.operation.name").as_deref(),
+ Some("Soup")
+ );
+ assert_eq!(
+ observed.attribute("graphql.error_count").as_deref(),
+ Some("0")
+ );
+ assert_eq!(observed.span.status, Status::Unset);
+ assert!(observed.logs.is_empty(), "{}", observed.raw_logs);
+}
+
+#[tokio::test]
+async fn every_execute_span_is_measured_as_an_integer() {
+ let requests = [
+ (Request::new("query Soup { ok }"), true),
+ (Request::new("query Broken { broken }"), true),
+ (Request::new("mutation Anything { ok }"), false),
+ (Request::new("query {"), true),
+ ];
+ for (request, authenticated) in requests {
+ let observed = observe_request(request, authenticated).await;
+ assert_eq!(
+ observed.attribute_value("_dd.measured"),
+ Some(&opentelemetry::Value::I64(1)),
+ "{:?}",
+ observed.attribute("resource.name")
+ );
+ }
+}
+
+#[tokio::test]
+async fn resolver_errors_mark_span_and_log_without_messages() {
+ let request = Request::new(
+ "query Broken($value: String!) { ok broken store(value: $value) missing(value: $value) rejected(value: $value) }",
+ )
+ .variables(Variables::from_json(
+ serde_json::json!({ "value": "Q3 board notes [email protected]" }),
+ ));
+ let observed = observe_request(request, true).await;
+
+ assert_eq!(
+ observed.attribute("resource.name").as_deref(),
+ Some("graphql.query.Broken")
+ );
+ assert_eq!(
+ observed.attribute("graphql.error_count").as_deref(),
+ Some("4")
+ );
+ assert_eq!(
+ observed.span.status,
+ Status::error("graphql resolver error")
+ );
+
+ assert_eq!(observed.logs.len(), 4, "{}", observed.raw_logs);
+ let by_path = |path: &str| {
+ observed
+ .logs
+ .iter()
+ .find(|log| log["graphql.error.path"] == path)
+ .unwrap_or_else(|| panic!("no log for {path}: {}", observed.raw_logs))
+ };
+ let broken = by_path("broken");
+ assert_eq!(broken["level"], "ERROR");
+ assert_eq!(broken["message"], "graphql resolver error");
+ assert_eq!(broken["graphql.resource"], "graphql.query.Broken");
+ assert_eq!(broken["graphql.operation.type"], "query");
+ assert_eq!(broken["graphql.operation.name"], "Broken");
+ assert_eq!(broken["graphql.error_count"], 4);
+
+ let store = by_path("store");
+ assert_eq!(store["level"], "ERROR");
+ assert_eq!(store["message"], "graphql resolver error");
+
+ let missing = by_path("missing");
+ assert_eq!(missing["level"], "WARN");
+ assert_eq!(missing["message"], "graphql client error");
+ assert_eq!(missing["graphql.error.code"], "NOT_FOUND");
+
+ let rejected = by_path("rejected");
+ assert_eq!(rejected["level"], "WARN");
+ assert_eq!(rejected["graphql.error.code"], UNRECOGNIZED_ERROR_CODE);
+
+ let span = format!("{:?}", observed.span);
+ for leaked in [
+ "Q3 board notes",
+ "upstream rejected",
+ "could not store",
+ "no thread named",
+ ] {
+ assert!(
+ !observed.raw_logs.contains(leaked),
+ "{leaked}: {}",
+ observed.raw_logs
+ );
+ assert!(!span.contains(leaked), "{leaked}: {span}");
+ }
+}
+
+#[tokio::test]
+async fn internal_error_code_marks_span() {
+ let observed = observe_request(Request::new("query Down { unavailable }"), true).await;
+
+ assert_eq!(
+ observed.span.status,
+ Status::error("graphql resolver error: INTERNAL_SERVER_ERROR")
+ );
+ assert!(!observed.raw_logs.contains("service unavailable"));
+ assert_eq!(observed.logs.len(), 1, "{}", observed.raw_logs);
+ assert_eq!(observed.logs[0]["level"], "ERROR");
+ assert_eq!(
+ observed.logs[0]["graphql.error.code"],
+ "INTERNAL_SERVER_ERROR"
+ );
+}
+
+#[tokio::test]
+async fn client_errors_log_warnings_without_marking_span() {
+ let invalid = observe_request(Request::new("query Typo { nope }"), true).await;
+ assert_eq!(invalid.span.status, Status::Unset);
+ assert_eq!(
+ invalid.attribute("graphql.error_count").as_deref(),
+ Some("1")
+ );
+ assert_eq!(invalid.logs.len(), 1, "{}", invalid.raw_logs);
+ assert_eq!(invalid.logs[0]["level"], "WARN");
+ assert!(invalid.logs[0].get("graphql.error.path").is_none());
+ assert!(!invalid.raw_logs.contains("nope"), "{}", invalid.raw_logs);
+
+ let unparsable = observe_request(Request::new("query {"), true).await;
+ assert_eq!(
+ unparsable.attribute("resource.name").as_deref(),
+ Some("graphql.unknown")
+ );
+ assert_eq!(unparsable.span.status, Status::Unset);
+ assert_eq!(unparsable.logs[0]["level"], "WARN");
+
+ let anonymous = observe_request(Request::new("query Broken { broken }"), false).await;
+ assert_eq!(
+ anonymous.attribute("resource.name").as_deref(),
+ Some("graphql.query.unauthenticated")
+ );
+ assert_eq!(anonymous.span.status, Status::Unset);
+ assert_eq!(anonymous.logs[0]["level"], "WARN");
+}
+
+#[tokio::test]
+async fn logged_errors_are_capped_per_response() {
+ let fields: Vec<String> = (0..25).map(|index| format!("e{index}: broken")).collect();
+ let query = format!("query Many {{ {} }}", fields.join(" "));
+ let observed = observe_request(Request::new(query), true).await;
+
+ assert_eq!(
+ observed.attribute("graphql.error_count").as_deref(),
+ Some("25")
+ );
+ assert_eq!(observed.logs.len(), MAX_LOGGED_ERRORS);
+ assert!(matches!(observed.span.status, Status::Error { .. }));
+}