Multiplayer agent harness for work.
Keep long worker runs inside Sentry's transaction size limit (#2498)
Regan Bellf527f9702b96
diff --git a/plugins/chassis/src/error-reporting.ts b/plugins/chassis/src/error-reporting.tsindex b9d966893..fa8998224 100644--- a/plugins/chassis/src/error-reporting.ts+++ b/plugins/chassis/src/error-reporting.ts@@ -37,7 +37,10 @@ const SQL_VERBS = new Set(" ",),);-const POOL_SPANS = new Set(["pg.connect", "pg-pool.connect"]);+const POOL_CHECKOUT = "pg-pool.connect";+const POOL_SPANS = new Set(["pg.connect", POOL_CHECKOUT]);+const MAX_SENT_SPANS = 500;+const IDLE_CHECKOUT_SECONDS = 0.001;const HTTP_METHODS = new Set("GET HEAD POST PUT PATCH DELETE OPTIONS CONNECT TRACE".split(" "));let client: typeof Sentry | undefined;let tracing = false;@@ -49,6 +52,7 @@ export interface TraceCarrier {type TransactionEvent = Parameters<NonNullable<Sentry.NodeOptions["beforeSendTransaction"]>>[0];type ErrorEvent = Parameters<NonNullable<Sentry.NodeOptions["beforeSend"]>>[0];type SpanData = Record<string, unknown>;+type SpanJSON = NonNullable<TransactionEvent["spans"]>[number];function scrubData(data: SpanData | undefined): { statement?: unknown } {if (!data) return {};@@ -105,6 +109,20 @@ function scrubTraceContext<T extends TransactionEvent | ErrorEvent>(event: T): Treturn event;}+function idleCheckout(span: SpanJSON, parents: Set<string | undefined>): boolean {+ return (+ span.description === POOL_CHECKOUT &&+ span.status === "ok" &&+ (span.timestamp ?? Infinity) - span.start_timestamp < IDLE_CHECKOUT_SECONDS &&+ !parents.has(span.span_id)+ );+}++function orderedSpans(spans: SpanJSON[]): SpanJSON[] {+ const parents = new Set(spans.map((span) => span.parent_span_id));+ return spans.filter((span) => !idleCheckout(span, parents)).sort((a, b) => a.start_timestamp - b.start_timestamp);+}+function scrubTransaction(event: TransactionEvent): TransactionEvent | null {const trace = event.contexts?.trace;if (!SENT_ROOTS.has(trace?.op ?? "") || trace?.data?.["sentry.parent_span_already_sent"]) return null;@@ -112,6 +130,12 @@ function scrubTransaction(event: TransactionEvent): TransactionEvent | null {delete event.user;delete event.breadcrumbs;scrubData(trace?.data as SpanData | undefined);+ if (event.spans) {+ const spans = orderedSpans(event.spans);+ if (spans.length > MAX_SENT_SPANS)+ trace!.data = { ...trace!.data, "qm.truncated_spans": spans.length - MAX_SENT_SPANS };+ event.spans = spans.slice(0, MAX_SENT_SPANS);+ }for (const span of event.spans ?? []) {const { statement } = scrubData(span.data);span.description = scrubDescription(span.op, span.description, statement);diff --git a/test/sentry-tracing-pg.test.ts b/test/sentry-tracing-pg.test.tsindex c98e2a854..c597ae98a 100644--- a/test/sentry-tracing-pg.test.ts+++ b/test/sentry-tracing-pg.test.ts@@ -3,8 +3,14 @@ import assert from "node:assert/strict";import type { AddressInfo } from "node:net";import { createServer } from "node:http";import { once } from "node:events";+import { gunzipSync } from "node:zlib";import * as Sentry from "@sentry/node";-import { flushErrorReporting, initializeErrorReporting } from "../plugins/chassis/src/error-reporting.ts";+import {+ flushErrorReporting,+ initializeErrorReporting,+ markFirstToken,+ traceQueued,+} from "../plugins/chassis/src/error-reporting.ts";const URL = process.env.DATABASE_URL;const skip = URL ? false : "set DATABASE_URL (a Postgres) to run the Postgres tracing test";@@ -13,7 +19,8 @@ const captured: Record<string, any>[] = [];const collector = createServer(async (req, res) => {const chunks = [];for await (const chunk of req) chunks.push(chunk);- const lines = Buffer.concat(chunks).toString().split("\n");+ const body = Buffer.concat(chunks);+ const lines = (req.headers["content-encoding"] === "gzip" ? gunzipSync(body) : body).toString().split("\n");for (let i = 1; i + 1 < lines.length; i += 2) {if (JSON.parse(lines[i]!).type === "transaction") captured.push(JSON.parse(lines[i + 1]!));}@@ -77,3 +84,109 @@ test("Postgres queries become pool and query spans that name the SQL verb and noawait pool.end();}});++test(+ "a long run keeps its queries, waiting checkouts and first-token time within Sentry's event size",+ { skip },+ async () => {+ const pg = (await import("pg")).default;+ const pool = new pg.Pool({ connectionString: URL, max: 1 });+ try {+ await traceQueued(undefined, "long-run", {}, async () => {+ await Promise.all([pool.query("SELECT pg_sleep(0.02)"), pool.query("SELECT 'contended'")]);+ for (let i = 0; i < 700; i++) await pool.query("SELECT $1::int", [i]);+ markFirstToken();+ await pool.query("SELECT pg_sleep(0.02)");+ });+ for (let i = 0; i < 40 && !captured.some((event) => event.transaction === "long-run"); i++) {+ await flushErrorReporting();+ await new Promise((resolve) => setTimeout(resolve, 50));+ }+ const run = captured.find((event) => event.transaction === "long-run");+ assert.ok(run, "the transaction arrived");+ assert.ok(run.measurements.time_to_first_token.value > 0, "first-token time survives a late first token");+ const spans: Record<string, any>[] = run.spans;+ assert.ok(spans.length <= 500, `sent ${spans.length} spans`);+ assert.ok(run.contexts.trace.data["qm.truncated_spans"] > 0, "the cut is recorded");+ assert.ok(spans.some((span) => span.description === "SELECT"));+ const checkouts = spans.filter((span) => span.description === "pg-pool.connect");+ assert.ok(+ checkouts.some((span) => span.timestamp - span.start_timestamp >= 0.01),+ "a waiting checkout is kept",+ );+ } finally {+ await pool.end();+ }+ },+);++test("a truncated run keeps every sent span's parent even when parent and child start together", async () => {+ await traceQueued(undefined, "tied-run", {}, async () => {+ const start = Date.now() / 1000;+ for (let i = 0; i < 499; i++) {+ Sentry.startInactiveSpan({ name: "early", op: "task", startTime: start + i / 1000 }).end(+ start + i / 1000 + 0.0005,+ );+ }+ const tied = start + 1;+ Sentry.startSpan({ name: "parent", op: "task", startTime: tied }, () => {+ Sentry.startSpan({ name: "child", op: "task", startTime: tied }, () => {+ Sentry.startInactiveSpan({ name: "grandchild", op: "task", startTime: tied }).end(tied + 0.001);+ });+ });+ });+ for (let i = 0; i < 40 && !captured.some((event) => event.transaction === "tied-run"); i++) {+ await flushErrorReporting();+ await new Promise((resolve) => setTimeout(resolve, 50));+ }+ const run = captured.find((event) => event.transaction === "tied-run");+ assert.ok(run, "the transaction arrived");+ const spans: Record<string, any>[] = run.spans;+ assert.equal(spans.length, 500);+ const ids = new Set([run.contexts.trace.span_id, ...spans.map((sent) => sent.span_id)]);+ for (const sent of spans) assert.ok(ids.has(sent.parent_span_id), "every sent span's parent is sent");+ assert.equal(run.contexts.trace.data["qm.truncated_spans"], 2);+});++test("only quick, successful, childless pool checkouts are left out", async () => {+ const start = 1_700_000_000;+ const traceId = "0123456789abcdef0123456789abcdef";+ const span = (id: string, parent: string, description: string, seconds: number, status = "ok") => ({+ span_id: id.padStart(16, "0"),+ parent_span_id: parent.padStart(16, "0"),+ trace_id: traceId,+ op: "db",+ description,+ start_timestamp: start,+ timestamp: start + seconds,+ status,+ data: {},+ });+ Sentry.captureEvent({+ type: "transaction",+ transaction: "checkouts",+ start_timestamp: start,+ timestamp: start + 1,+ contexts: { trace: { trace_id: traceId, span_id: "1".padStart(16, "0"), op: "queue.process" } },+ spans: [+ span("a", "1", "pg-pool.connect", 0.0002),+ span("b", "1", "pg-pool.connect", 0.005),+ span("c", "1", "pg-pool.connect", 0.0002, "internal_error"),+ span("d", "1", "pg-pool.connect", 0.0002),+ span("e", "d", "pg.connect", 0.0001),+ ],+ });+ for (let i = 0; i < 40 && !captured.some((event) => event.transaction === "checkouts"); i++) {+ await flushErrorReporting();+ await new Promise((resolve) => setTimeout(resolve, 50));+ }+ const run = captured.find((event) => event.transaction === "checkouts");+ assert.ok(run, "the transaction arrived");+ assert.deepEqual(run.spans.map((sent: { span_id: string }) => sent.span_id.replace(/^0+/, "")).sort(), [+ "b",+ "c",+ "d",+ "e",+ ]);+ assert.equal(run.contexts.trace.data?.["qm.truncated_spans"], undefined);+});