Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(graphql): log and trace graphql errors (#7911)
teof86ecbd15fb1
diff --git a/Cargo.lock b/Cargo.lockindex 78b25fc44b..799e4a3e9d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -6611,6 +6611,7 @@ dependencies = ["notification","opensearch_client","opentelemetry",+ "opentelemetry_sdk","ordered-float","projects","properties",@@ -6656,6 +6657,7 @@ dependencies = ["tower-http","tracing","tracing-opentelemetry",+ "tracing-subscriber","tungstenite 0.27.0","unicode-segmentation","urlencoding",diff --git a/deny.toml b/deny.tomlindex edd1d6ddbd..43980027b9 100644--- a/deny.toml+++ b/deny.toml@@ -231,6 +231,7 @@ deny = ["coding_agent_worker", # Standalone daemon logging setup"cursor_cloud_agents", # Standalone ACP binary logging; stdout is the protocol, so macro_entrypoint cannot own it"dictation", # Metadata tracing and content-redaction tests only+ "document_storage_service", # GraphQL telemetry span and log tests only (dev-dependency)"documents", # Dev logging only"email", # Dev logging only"genai_telemetry", # GenAI span testsdiff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.tomlindex 2577994875..b26e474e45 100644--- a/services/document_storage_service/Cargo.toml+++ b/services/document_storage_service/Cargo.toml@@ -310,7 +310,11 @@ cool_asserts = "2"flate2 = "1"frecency = { path = "../../crates/frecency", features = ["mock"] }macro_auth = { path = "../../crates/macro_auth", features = ["testing"] }+opentelemetry = { workspace = true, features = ["trace"] }+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }ordered-float = { workspace = true }+tracing-opentelemetry = { workspace = true }+tracing-subscriber = { workspace = true }[package.metadata.cargo-machete]ignored = ["connection_gateway_client"]diff --git a/services/document_storage_service/src/api/graphql_soup.rs b/services/document_storage_service/src/api/graphql_soup.rsindex ac0aab0e51..7cb82c90c4 100644--- a/services/document_storage_service/src/api/graphql_soup.rs+++ b/services/document_storage_service/src/api/graphql_soup.rs@@ -1,3 +1,5 @@+mod telemetry;+use crate::api::context::{ApiContext, AuthorizationService};use crate::api::folders_query::DssFoldersQueryReader;use agent_session::outbound::postgres::PgAgentSessionRepo;@@ -49,7 +51,6 @@ fn graphiql_source(endpoint: &str) -> String {.finish()}-#[tracing::instrument(skip_all, name = "graphql.execute")]async fn graphql_handler(State(state): State<ApiContext>,Cached(auth): Cached<@@ -62,17 +63,19 @@ async fn graphql_handler(.authorization.as_ref().and_then(UserOrInternalServiceAuthorization::acting_user);- let request = graphql_query_context_data(- request.into_inner(),- &state,- acting_user.map(|user| user.macro_user_id.clone()),- acting_user.and_then(|user| user.user_context.organization_id.map(i64::from)),- acting_user.and_then(opaque_viewer_id),- );- let result = state- .graphql_soup_schema- .execute(request.data(GraphqlRequestParts::new(request_parts)))- .await;+ let result = telemetry::observe(request.into_inner(), acting_user.is_some(), |request| {+ let request = graphql_query_context_data(+ request,+ &state,+ acting_user.map(|user| user.macro_user_id.clone()),+ acting_user.and_then(|user| user.user_context.organization_id.map(i64::from)),+ acting_user.and_then(opaque_viewer_id),+ );+ state+ .graphql_soup_schema+ .execute(request.data(GraphqlRequestParts::new(request_parts)))+ })+ .await;graphql_response(result)}diff --git a/services/document_storage_service/src/api/graphql_soup/telemetry.rs b/services/document_storage_service/src/api/graphql_soup/telemetry.rsnew file mode 100644index 0000000000..9b27591e18--- /dev/null+++ b/services/document_storage_service/src/api/graphql_soup/telemetry.rs@@ -0,0 +1,267 @@+#[cfg(test)]+mod test;++use async_graphql::{+ PathSegment, Request, Response, ServerError, Value,+ parser::types::{DocumentOperations, OperationType},+};+use tracing::{Instrument as _, Level, Span};++/// Resource for requests whose operation cannot be resolved (parse errors,+/// unknown operation names). Keeps attacker-supplied text out of the resource.+const UNKNOWN_RESOURCE: &str = "graphql.unknown";+const ANONYMOUS_OPERATION: &str = "anonymous";+/// Unauthenticated callers choose their own operation names. Grouping them+/// bounds the resource cardinality that Datadog turns into metric tags.+const UNAUTHENTICATED_OPERATION: &str = "unauthenticated";+const MAX_OPERATION_NAME_CHARS: usize = 100;+/// A response can carry one error per aliased field; cap the log lines per request.+const MAX_LOGGED_ERRORS: usize = 10;+/// Paths are field names, client-chosen aliases, and list indexes.+const MAX_ERROR_PATH_CHARS: usize = 200;+const MAX_ERROR_CODE_CHARS: usize = 64;+/// Logged in place of an `extensions.code` that is not a plain identifier.+const UNRECOGNIZED_ERROR_CODE: &str = "UNRECOGNIZED";+const SERVER_ERROR_CODE_PREFIX: &str = "INTERNAL";+const SERVER_ERROR_TEXT: &str = "graphql resolver error";+const CLIENT_ERROR_TEXT: &str = "graphql client error";+const DATADOG_MEASURED: i64 = 1;++/// Runs `execute` inside a `graphql.execute` span that records the selected+/// operation and logs the response's errors.+pub(super) async fn observe<Fut>(+ mut request: Request,+ authenticated: bool,+ execute: impl FnOnce(Request) -> Fut,+) -> Response+where+ Fut: Future<Output = Response>,+{+ let span = execute_span();+ let operation = GraphqlOperation::from_request(&mut request, authenticated);+ operation.record(&span);+ let response = execute(request).instrument(span.clone()).await;+ operation.record_response(&span, &response);+ response+}++fn execute_span() -> Span {+ tracing::info_span!(+ "graphql.execute",+ // The Datadog Agent computes `trace.Internal.*` stats for an internal+ // child span only when this attribute is the OTLP integer 1; a bool or+ // string is ignored.+ "_dd.measured" = DATADOG_MEASURED,+ "resource.name" = tracing::field::Empty,+ "graphql.operation.type" = tracing::field::Empty,+ "graphql.operation.name" = tracing::field::Empty,+ "graphql.error_count" = tracing::field::Empty,+ otel.status_code = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ )+}++/// The operation a GraphQL HTTP request selects, recorded on `graphql.execute`.+#[derive(Debug)]+struct GraphqlOperation {+ kind: Option<OperationType>,+ name: Option<String>,+ authenticated: bool,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum ErrorSeverity {+ /// The caller sent something the server rejected; logged at warn.+ Client,+ /// A resolver failed for an authenticated caller; logged at error and marks the span.+ Server,+}++impl GraphqlOperation {+ /// Parses the request document so its operation can be recorded before+ /// execution. The parsed document stays on the request and+ /// `Schema::execute` reuses it, so valid documents are parsed once.+ fn from_request(request: &mut Request, authenticated: bool) -> Self {+ let requested_name = request.operation_name.clone();+ let selected = request.parsed_query().ok().and_then(|document| {+ select_operation(&document.operations, requested_name.as_deref())+ .map(|(name, kind)| (name.map(truncate_operation_name), kind))+ });+ let (name, kind) = match selected {+ Some((name, kind)) => (name, Some(kind)),+ None => (None, None),+ };+ Self {+ kind,+ name,+ authenticated,+ }+ }++ /// The Datadog resource name, `graphql.<type>.<operationName>`. Datadog+ /// derives `trace.Internal.{hits,errors,duration}` per resource from it.+ fn resource_name(&self) -> String {+ let Some(kind) = self.kind else {+ return UNKNOWN_RESOURCE.to_owned();+ };+ let name = match (&self.name, self.authenticated) {+ (_, false) => UNAUTHENTICATED_OPERATION,+ (Some(name), true) => name.as_str(),+ (None, true) => ANONYMOUS_OPERATION,+ };+ format!("graphql.{kind}.{name}")+ }++ fn record(&self, span: &Span) {+ span.record("resource.name", self.resource_name());+ if let Some(kind) = self.kind {+ span.record("graphql.operation.type", tracing::field::display(kind));+ }+ if let Some(name) = &self.name {+ span.record("graphql.operation.name", name.as_str());+ }+ }++ /// Records the error count on `span`, logs each error with its path and+ /// code, and marks `span` errored when a resolver failed.+ ///+ /// Error messages are never recorded: resolvers and async-graphql build+ /// them from request input (variable values, arguments, upstream error+ /// text), so no redaction of free text is reliable.+ fn record_response(&self, span: &Span, response: &Response) {+ span.record("graphql.error_count", response.errors.len());++ for error in response.errors.iter().take(MAX_LOGGED_ERRORS) {+ self.log_error(span, error, response.errors.len());+ }++ if let Some(error) = response+ .errors+ .iter()+ .find(|error| self.error_severity(error) == ErrorSeverity::Server)+ {+ span.record("otel.status_code", "ERROR");+ span.record("otel.status_description", status_description(error));+ }+ }++ fn error_severity(&self, error: &ServerError) -> ErrorSeverity {+ if !self.authenticated || error.path.is_empty() {+ return ErrorSeverity::Client;+ }+ match error_code(error) {+ Some(code)+ if !code+ .to_ascii_uppercase()+ .starts_with(SERVER_ERROR_CODE_PREFIX) =>+ {+ ErrorSeverity::Client+ }+ _ => ErrorSeverity::Server,+ }+ }++ fn log_error(&self, span: &Span, error: &ServerError, error_count: usize) {+ let resource = self.resource_name();+ let path = error_path(&error.path);+ let code = loggable_error_code(error);+ macro_rules! log_graphql_error {+ ($level:expr, $text:expr) => {+ tracing::event!(+ parent: span,+ $level,+ "graphql.resource" = resource.as_str(),+ "graphql.operation.type" = self.kind.map(tracing::field::display),+ "graphql.operation.name" = self.name.as_deref(),+ "graphql.error.path" = path.as_deref(),+ "graphql.error.code" = code,+ "graphql.error_count" = error_count,+ "{}",+ $text+ )+ };+ }+ match self.error_severity(error) {+ ErrorSeverity::Server => log_graphql_error!(Level::ERROR, SERVER_ERROR_TEXT),+ ErrorSeverity::Client => log_graphql_error!(Level::WARN, CLIENT_ERROR_TEXT),+ }+ }+}++/// Mirrors async-graphql's operation selection in `Schema::execute`.+fn select_operation<'a>(+ operations: &'a DocumentOperations,+ requested_name: Option<&'a str>,+) -> Option<(Option<&'a str>, OperationType)> {+ match (operations, requested_name) {+ (DocumentOperations::Single(operation), None) => Some((None, operation.node.ty)),+ (DocumentOperations::Single(_), Some(_)) => None,+ (DocumentOperations::Multiple(operations), Some(name)) => operations+ .get(name)+ .map(|operation| (Some(name), operation.node.ty)),+ (DocumentOperations::Multiple(operations), None) if operations.len() == 1 => operations+ .iter()+ .next()+ .map(|(name, operation)| (Some(name.as_str()), operation.node.ty)),+ (DocumentOperations::Multiple(_), None) => None,+ }+}++fn truncate_operation_name(name: &str) -> String {+ name.chars().take(MAX_OPERATION_NAME_CHARS).collect()+}++fn error_code(error: &ServerError) -> Option<&str> {+ match error.extensions.as_ref()?.get("code")? {+ Value::String(code) => Some(code.as_str()),+ Value::Enum(code) => Some(code.as_str()),+ _ => None,+ }+}++/// Codes are server-chosen identifiers such as `NOT_FOUND`. Anything else is+/// replaced so a code built from input cannot carry it into telemetry.+fn loggable_error_code(error: &ServerError) -> Option<&str> {+ error_code(error).map(|code| {+ let is_identifier = !code.is_empty()+ && code.len() <= MAX_ERROR_CODE_CHARS+ && code+ .bytes()+ .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_');+ if is_identifier {+ code+ } else {+ UNRECOGNIZED_ERROR_CODE+ }+ })+}++/// Fixed text plus the error code, never the error message.+fn status_description(error: &ServerError) -> String {+ match loggable_error_code(error) {+ Some(code) => format!("{SERVER_ERROR_TEXT}: {code}"),+ None => SERVER_ERROR_TEXT.to_owned(),+ }+}++/// `user.soup.items.3.name`; `None` for request-level errors. GraphQL names+/// are limited to `[_A-Za-z][_0-9A-Za-z]*`, so segments cannot carry free text.+fn error_path(path: &[PathSegment]) -> Option<String> {+ if path.is_empty() {+ return None;+ }+ let segments: Vec<String> = path+ .iter()+ .map(|segment| match segment {+ PathSegment::Field(field) => field.clone(),+ PathSegment::Index(index) => index.to_string(),+ })+ .collect();+ Some(+ segments+ .join(".")+ .chars()+ .take(MAX_ERROR_PATH_CHARS)+ .collect(),+ )+}diff --git a/services/document_storage_service/src/api/graphql_soup/telemetry/test.rs b/services/document_storage_service/src/api/graphql_soup/telemetry/test.rsnew file mode 100644index 0000000000..af84541719--- /dev/null+++ b/services/document_storage_service/src/api/graphql_soup/telemetry/test.rs@@ -0,0 +1,471 @@+use super::*;++use std::io;+use std::sync::{Arc, Mutex};++use async_graphql::{+ EmptyMutation, EmptySubscription, ErrorExtensionValues, ErrorExtensions as _, Object, Schema,+ Variables,+};+use opentelemetry::trace::{Status, TracerProvider as _};+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};+use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt as _};++struct Query;++#[Object]+impl Query {+ async fn ok(&self) -> i32 {+ 1+ }++ async fn broken(&self) -> async_graphql::Result<i32> {+ Err(async_graphql::Error::new(+ "upstream rejected [email protected]",+ ))+ }++ async fn store(&self, value: String) -> async_graphql::Result<String> {+ Err(async_graphql::Error::new(format!(+ "could not store {value}"+ )))+ }++ async fn missing(&self, value: String) -> async_graphql::Result<i32> {+ Err(+ async_graphql::Error::new(format!("no thread named {value}"))+ .extend_with(|_, extensions| extensions.set("code", "NOT_FOUND")),+ )+ }++ async fn rejected(&self, value: String) -> async_graphql::Result<i32> {+ Err(async_graphql::Error::new("rejected")+ .extend_with(|_, extensions| extensions.set("code", format!("BAD {value}"))))+ }++ async fn unavailable(&self) -> async_graphql::Result<i32> {+ Err(async_graphql::Error::new("service unavailable")+ .extend_with(|_, extensions| extensions.set("code", "INTERNAL_SERVER_ERROR")))+ }+}++#[derive(Clone, Default)]+struct SharedWriter(Arc<Mutex<Vec<u8>>>);++impl SharedWriter {+ fn lines(&self) -> Vec<serde_json::Value> {+ String::from_utf8(self.0.lock().unwrap().clone())+ .unwrap()+ .lines()+ .map(|line| serde_json::from_str(line).unwrap())+ .collect()+ }+}++impl io::Write for SharedWriter {+ fn write(&mut self, buf: &[u8]) -> io::Result<usize> {+ self.0.lock().unwrap().extend_from_slice(buf);+ Ok(buf.len())+ }++ fn flush(&mut self) -> io::Result<()> {+ Ok(())+ }+}++impl<'a> MakeWriter<'a> for SharedWriter {+ type Writer = SharedWriter;++ fn make_writer(&'a self) -> Self::Writer {+ self.clone()+ }+}++struct Observed {+ span: SpanData,+ logs: Vec<serde_json::Value>,+ raw_logs: String,+}++impl Observed {+ fn attribute(&self, key: &str) -> Option<String> {+ self.attribute_value(key).map(ToString::to_string)+ }++ fn attribute_value(&self, key: &str) -> Option<&opentelemetry::Value> {+ self.span+ .attributes+ .iter()+ .find(|attribute| attribute.key.as_str() == key)+ .map(|attribute| &attribute.value)+ }+}++async fn observe_request(request: Request, authenticated: bool) -> Observed {+ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let logs = SharedWriter::default();+ let subscriber = Registry::default()+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")))+ .with(+ tracing_subscriber::fmt::layer()+ .json()+ .flatten_event(true)+ .with_ansi(false)+ .with_writer(logs.clone()),+ );+ let guard = tracing::subscriber::set_default(subscriber);+ let schema = Schema::new(Query, EmptyMutation, EmptySubscription);+ observe(request, authenticated, |request| async move {+ schema.execute(request).await+ })+ .await;+ drop(guard);++ let mut spans = exporter.get_finished_spans().unwrap();+ assert_eq!(spans.len(), 1);+ let raw_logs = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap();+ Observed {+ span: spans.remove(0),+ logs: logs.lines(),+ raw_logs,+ }+}++fn operation(query: &str, operation_name: Option<&str>, authenticated: bool) -> GraphqlOperation {+ let mut request = Request::new(query);+ if let Some(name) = operation_name {+ request = request.operation_name(name);+ }+ GraphqlOperation::from_request(&mut request, authenticated)+}++fn resolver_error(code: Option<&str>) -> ServerError {+ let mut error = ServerError::new("failed", None);+ error.path = vec![+ PathSegment::Field("user".to_owned()),+ PathSegment::Field("soup".to_owned()),+ ];+ if let Some(code) = code {+ let mut extensions = ErrorExtensionValues::default();+ extensions.set("code", code);+ error.extensions = Some(extensions);+ }+ error+}++#[test]+fn resource_name_follows_the_executed_operation() {+ let cases = [+ ("query Soup { ok }", None, "graphql.query.Soup"),+ ("{ ok }", None, "graphql.query.anonymous"),+ ("mutation Save { ok }", None, "graphql.mutation.Save"),+ (+ "subscription Updates { ok }",+ None,+ "graphql.subscription.Updates",+ ),+ (+ "mutation Save { ok } query Soup { ok }",+ Some("Soup"),+ "graphql.query.Soup",+ ),+ (+ "mutation Save { ok } query Soup { ok }",+ None,+ "graphql.unknown",+ ),+ ("query Soup { ok }", Some("Other"), "graphql.unknown"),+ ("{ ok }", Some("Soup"), "graphql.unknown"),+ ("query Soup {", None, "graphql.unknown"),+ ];+ for (query, operation_name, expected) in cases {+ assert_eq!(+ operation(query, operation_name, true).resource_name(),+ expected,+ "{query} / {operation_name:?}"+ );+ }+}++#[test]+fn unauthenticated_operations_share_one_resource() {+ let operation = operation("query AttackerChosenName { ok }", None, false);+ assert_eq!(operation.resource_name(), "graphql.query.unauthenticated");+ assert_eq!(operation.name.as_deref(), Some("AttackerChosenName"));+}++#[test]+fn operation_names_are_bounded() {+ let name = "A".repeat(500);+ let operation = operation(&format!("query {name} {{ ok }}"), None, true);+ assert_eq!(+ operation.name.map(|name| name.len()),+ Some(MAX_OPERATION_NAME_CHARS)+ );+}++#[test]+fn only_uncoded_or_internal_resolver_errors_are_server_errors() {+ let authenticated = operation("query Soup { ok }", None, true);+ let anonymous = operation("query Soup { ok }", None, false);++ assert_eq!(+ authenticated.error_severity(&resolver_error(None)),+ ErrorSeverity::Server+ );+ assert_eq!(+ authenticated.error_severity(&resolver_error(Some("INTERNAL"))),+ ErrorSeverity::Server+ );+ assert_eq!(+ authenticated.error_severity(&resolver_error(Some("INTERNAL_SERVER_ERROR"))),+ ErrorSeverity::Server+ );+ for code in ["NOT_FOUND", "BAD_USER_INPUT", "FORBIDDEN", "invalid_input"] {+ assert_eq!(+ authenticated.error_severity(&resolver_error(Some(code))),+ ErrorSeverity::Client,+ "{code}"+ );+ }+ assert_eq!(+ authenticated.error_severity(&ServerError::new("Unknown field \"nope\"", None)),+ ErrorSeverity::Client+ );+ assert_eq!(+ anonymous.error_severity(&resolver_error(None)),+ ErrorSeverity::Client+ );+}++#[test]+fn error_paths_join_fields_and_indexes() {+ let path = [+ PathSegment::Field("user".to_owned()),+ PathSegment::Field("items".to_owned()),+ PathSegment::Index(3),+ PathSegment::Field("name".to_owned()),+ ];+ assert_eq!(error_path(&path).as_deref(), Some("user.items.3.name"));+ assert_eq!(error_path(&[]), None);+}++#[test]+fn error_paths_are_bounded() {+ let path = vec![PathSegment::Field("a".repeat(500))];+ assert_eq!(+ error_path(&path).map(|path| path.len()),+ Some(MAX_ERROR_PATH_CHARS)+ );+}++#[test]+fn only_identifier_error_codes_are_logged() {+ for code in ["NOT_FOUND", "INTERNAL_SERVER_ERROR", "invalid_input"] {+ assert_eq!(loggable_error_code(&resolver_error(Some(code))), Some(code));+ }+ for code in [+ "",+ "BAD [email protected]",+ "invalid-input",+ &"A".repeat(MAX_ERROR_CODE_CHARS + 1),+ ] {+ assert_eq!(+ loggable_error_code(&resolver_error(Some(code))),+ Some(UNRECOGNIZED_ERROR_CODE),+ "{code}"+ );+ }+ assert_eq!(loggable_error_code(&resolver_error(None)), None);+}++#[test]+fn status_description_never_contains_the_message() {+ let mut error = resolver_error(None);+ error.message = "could not store Q3 board notes".to_owned();+ assert_eq!(status_description(&error), "graphql resolver error");++ let mut error = resolver_error(Some("INTERNAL"));+ error.message = "could not store Q3 board notes".to_owned();+ assert_eq!(+ status_description(&error),+ "graphql resolver error: INTERNAL"+ );+}++#[tokio::test]+async fn successful_request_records_operation_without_logging() {+ let observed = observe_request(Request::new("query Soup { ok }"), true).await;++ assert_eq!(observed.span.name, "graphql.execute");+ assert_eq!(+ observed.attribute("resource.name").as_deref(),+ Some("graphql.query.Soup")+ );+ assert_eq!(+ observed.attribute("graphql.operation.type").as_deref(),+ Some("query")+ );+ assert_eq!(+ observed.attribute("graphql.operation.name").as_deref(),+ Some("Soup")+ );+ assert_eq!(+ observed.attribute("graphql.error_count").as_deref(),+ Some("0")+ );+ assert_eq!(observed.span.status, Status::Unset);+ assert!(observed.logs.is_empty(), "{}", observed.raw_logs);+}++#[tokio::test]+async fn every_execute_span_is_measured_as_an_integer() {+ let requests = [+ (Request::new("query Soup { ok }"), true),+ (Request::new("query Broken { broken }"), true),+ (Request::new("mutation Anything { ok }"), false),+ (Request::new("query {"), true),+ ];+ for (request, authenticated) in requests {+ let observed = observe_request(request, authenticated).await;+ assert_eq!(+ observed.attribute_value("_dd.measured"),+ Some(&opentelemetry::Value::I64(1)),+ "{:?}",+ observed.attribute("resource.name")+ );+ }+}++#[tokio::test]+async fn resolver_errors_mark_span_and_log_without_messages() {+ let request = Request::new(+ "query Broken($value: String!) { ok broken store(value: $value) missing(value: $value) rejected(value: $value) }",+ )+ .variables(Variables::from_json(+ serde_json::json!({ "value": "Q3 board notes [email protected]" }),+ ));+ let observed = observe_request(request, true).await;++ assert_eq!(+ observed.attribute("resource.name").as_deref(),+ Some("graphql.query.Broken")+ );+ assert_eq!(+ observed.attribute("graphql.error_count").as_deref(),+ Some("4")+ );+ assert_eq!(+ observed.span.status,+ Status::error("graphql resolver error")+ );++ assert_eq!(observed.logs.len(), 4, "{}", observed.raw_logs);+ let by_path = |path: &str| {+ observed+ .logs+ .iter()+ .find(|log| log["graphql.error.path"] == path)+ .unwrap_or_else(|| panic!("no log for {path}: {}", observed.raw_logs))+ };+ let broken = by_path("broken");+ assert_eq!(broken["level"], "ERROR");+ assert_eq!(broken["message"], "graphql resolver error");+ assert_eq!(broken["graphql.resource"], "graphql.query.Broken");+ assert_eq!(broken["graphql.operation.type"], "query");+ assert_eq!(broken["graphql.operation.name"], "Broken");+ assert_eq!(broken["graphql.error_count"], 4);++ let store = by_path("store");+ assert_eq!(store["level"], "ERROR");+ assert_eq!(store["message"], "graphql resolver error");++ let missing = by_path("missing");+ assert_eq!(missing["level"], "WARN");+ assert_eq!(missing["message"], "graphql client error");+ assert_eq!(missing["graphql.error.code"], "NOT_FOUND");++ let rejected = by_path("rejected");+ assert_eq!(rejected["level"], "WARN");+ assert_eq!(rejected["graphql.error.code"], UNRECOGNIZED_ERROR_CODE);++ let span = format!("{:?}", observed.span);+ for leaked in [+ "Q3 board notes",+ "[email protected]",+ "upstream rejected",+ "could not store",+ "no thread named",+ ] {+ assert!(+ !observed.raw_logs.contains(leaked),+ "{leaked}: {}",+ observed.raw_logs+ );+ assert!(!span.contains(leaked), "{leaked}: {span}");+ }+}++#[tokio::test]+async fn internal_error_code_marks_span() {+ let observed = observe_request(Request::new("query Down { unavailable }"), true).await;++ assert_eq!(+ observed.span.status,+ Status::error("graphql resolver error: INTERNAL_SERVER_ERROR")+ );+ assert!(!observed.raw_logs.contains("service unavailable"));+ assert_eq!(observed.logs.len(), 1, "{}", observed.raw_logs);+ assert_eq!(observed.logs[0]["level"], "ERROR");+ assert_eq!(+ observed.logs[0]["graphql.error.code"],+ "INTERNAL_SERVER_ERROR"+ );+}++#[tokio::test]+async fn client_errors_log_warnings_without_marking_span() {+ let invalid = observe_request(Request::new("query Typo { nope }"), true).await;+ assert_eq!(invalid.span.status, Status::Unset);+ assert_eq!(+ invalid.attribute("graphql.error_count").as_deref(),+ Some("1")+ );+ assert_eq!(invalid.logs.len(), 1, "{}", invalid.raw_logs);+ assert_eq!(invalid.logs[0]["level"], "WARN");+ assert!(invalid.logs[0].get("graphql.error.path").is_none());+ assert!(!invalid.raw_logs.contains("nope"), "{}", invalid.raw_logs);++ let unparsable = observe_request(Request::new("query {"), true).await;+ assert_eq!(+ unparsable.attribute("resource.name").as_deref(),+ Some("graphql.unknown")+ );+ assert_eq!(unparsable.span.status, Status::Unset);+ assert_eq!(unparsable.logs[0]["level"], "WARN");++ let anonymous = observe_request(Request::new("query Broken { broken }"), false).await;+ assert_eq!(+ anonymous.attribute("resource.name").as_deref(),+ Some("graphql.query.unauthenticated")+ );+ assert_eq!(anonymous.span.status, Status::Unset);+ assert_eq!(anonymous.logs[0]["level"], "WARN");+}++#[tokio::test]+async fn logged_errors_are_capped_per_response() {+ let fields: Vec<String> = (0..25).map(|index| format!("e{index}: broken")).collect();+ let query = format!("query Many {{ {} }}", fields.join(" "));+ let observed = observe_request(Request::new(query), true).await;++ assert_eq!(+ observed.attribute("graphql.error_count").as_deref(),+ Some("25")+ );+ assert_eq!(observed.logs.len(), MAX_LOGGED_ERRORS);+ assert!(matches!(observed.span.status, Status::Error { .. }));+}