Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Keep long worker runs inside Sentry's transaction size limit (#2498)

Regan Bellf527f9702b96
diff --git a/plugins/chassis/src/error-reporting.ts b/plugins/chassis/src/error-reporting.ts
index b9d966893..fa8998224 100644
--- a/plugins/chassis/src/error-reporting.ts
+++ b/plugins/chassis/src/error-reporting.ts
@@ -37,7 +37,10 @@ const SQL_VERBS = new Set(
" ",
),
);
-const POOL_SPANS = new Set(["pg.connect", "pg-pool.connect"]);
+const POOL_CHECKOUT = "pg-pool.connect";
+const POOL_SPANS = new Set(["pg.connect", POOL_CHECKOUT]);
+const MAX_SENT_SPANS = 500;
+const IDLE_CHECKOUT_SECONDS = 0.001;
const HTTP_METHODS = new Set("GET HEAD POST PUT PATCH DELETE OPTIONS CONNECT TRACE".split(" "));
let client: typeof Sentry | undefined;
let tracing = false;
@@ -49,6 +52,7 @@ export interface TraceCarrier {
type TransactionEvent = Parameters<NonNullable<Sentry.NodeOptions["beforeSendTransaction"]>>[0];
type ErrorEvent = Parameters<NonNullable<Sentry.NodeOptions["beforeSend"]>>[0];
type SpanData = Record<string, unknown>;
+type SpanJSON = NonNullable<TransactionEvent["spans"]>[number];
function scrubData(data: SpanData | undefined): { statement?: unknown } {
if (!data) return {};
@@ -105,6 +109,20 @@ function scrubTraceContext<T extends TransactionEvent | ErrorEvent>(event: T): T
return event;
}
+function idleCheckout(span: SpanJSON, parents: Set<string | undefined>): boolean {
+ return (
+ span.description === POOL_CHECKOUT &&
+ span.status === "ok" &&
+ (span.timestamp ?? Infinity) - span.start_timestamp < IDLE_CHECKOUT_SECONDS &&
+ !parents.has(span.span_id)
+ );
+}
+
+function orderedSpans(spans: SpanJSON[]): SpanJSON[] {
+ const parents = new Set(spans.map((span) => span.parent_span_id));
+ return spans.filter((span) => !idleCheckout(span, parents)).sort((a, b) => a.start_timestamp - b.start_timestamp);
+}
+
function scrubTransaction(event: TransactionEvent): TransactionEvent | null {
const trace = event.contexts?.trace;
if (!SENT_ROOTS.has(trace?.op ?? "") || trace?.data?.["sentry.parent_span_already_sent"]) return null;
@@ -112,6 +130,12 @@ function scrubTransaction(event: TransactionEvent): TransactionEvent | null {
delete event.user;
delete event.breadcrumbs;
scrubData(trace?.data as SpanData | undefined);
+ if (event.spans) {
+ const spans = orderedSpans(event.spans);
+ if (spans.length > MAX_SENT_SPANS)
+ trace!.data = { ...trace!.data, "qm.truncated_spans": spans.length - MAX_SENT_SPANS };
+ event.spans = spans.slice(0, MAX_SENT_SPANS);
+ }
for (const span of event.spans ?? []) {
const { statement } = scrubData(span.data);
span.description = scrubDescription(span.op, span.description, statement);
diff --git a/test/sentry-tracing-pg.test.ts b/test/sentry-tracing-pg.test.ts
index c98e2a854..c597ae98a 100644
--- a/test/sentry-tracing-pg.test.ts
+++ b/test/sentry-tracing-pg.test.ts
@@ -3,8 +3,14 @@ import assert from "node:assert/strict";
import type { AddressInfo } from "node:net";
import { createServer } from "node:http";
import { once } from "node:events";
+import { gunzipSync } from "node:zlib";
import * as Sentry from "@sentry/node";
-import { flushErrorReporting, initializeErrorReporting } from "../plugins/chassis/src/error-reporting.ts";
+import {
+ flushErrorReporting,
+ initializeErrorReporting,
+ markFirstToken,
+ traceQueued,
+} from "../plugins/chassis/src/error-reporting.ts";
const URL = process.env.DATABASE_URL;
const skip = URL ? false : "set DATABASE_URL (a Postgres) to run the Postgres tracing test";
@@ -13,7 +19,8 @@ const captured: Record<string, any>[] = [];
const collector = createServer(async (req, res) => {
const chunks = [];
for await (const chunk of req) chunks.push(chunk);
- const lines = Buffer.concat(chunks).toString().split("\n");
+ const body = Buffer.concat(chunks);
+ const lines = (req.headers["content-encoding"] === "gzip" ? gunzipSync(body) : body).toString().split("\n");
for (let i = 1; i + 1 < lines.length; i += 2) {
if (JSON.parse(lines[i]!).type === "transaction") captured.push(JSON.parse(lines[i + 1]!));
}
@@ -77,3 +84,109 @@ test("Postgres queries become pool and query spans that name the SQL verb and no
await pool.end();
}
});
+
+test(
+ "a long run keeps its queries, waiting checkouts and first-token time within Sentry's event size",
+ { skip },
+ async () => {
+ const pg = (await import("pg")).default;
+ const pool = new pg.Pool({ connectionString: URL, max: 1 });
+ try {
+ await traceQueued(undefined, "long-run", {}, async () => {
+ await Promise.all([pool.query("SELECT pg_sleep(0.02)"), pool.query("SELECT 'contended'")]);
+ for (let i = 0; i < 700; i++) await pool.query("SELECT $1::int", [i]);
+ markFirstToken();
+ await pool.query("SELECT pg_sleep(0.02)");
+ });
+ for (let i = 0; i < 40 && !captured.some((event) => event.transaction === "long-run"); i++) {
+ await flushErrorReporting();
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ }
+ const run = captured.find((event) => event.transaction === "long-run");
+ assert.ok(run, "the transaction arrived");
+ assert.ok(run.measurements.time_to_first_token.value > 0, "first-token time survives a late first token");
+ const spans: Record<string, any>[] = run.spans;
+ assert.ok(spans.length <= 500, `sent ${spans.length} spans`);
+ assert.ok(run.contexts.trace.data["qm.truncated_spans"] > 0, "the cut is recorded");
+ assert.ok(spans.some((span) => span.description === "SELECT"));
+ const checkouts = spans.filter((span) => span.description === "pg-pool.connect");
+ assert.ok(
+ checkouts.some((span) => span.timestamp - span.start_timestamp >= 0.01),
+ "a waiting checkout is kept",
+ );
+ } finally {
+ await pool.end();
+ }
+ },
+);
+
+test("a truncated run keeps every sent span's parent even when parent and child start together", async () => {
+ await traceQueued(undefined, "tied-run", {}, async () => {
+ const start = Date.now() / 1000;
+ for (let i = 0; i < 499; i++) {
+ Sentry.startInactiveSpan({ name: "early", op: "task", startTime: start + i / 1000 }).end(
+ start + i / 1000 + 0.0005,
+ );
+ }
+ const tied = start + 1;
+ Sentry.startSpan({ name: "parent", op: "task", startTime: tied }, () => {
+ Sentry.startSpan({ name: "child", op: "task", startTime: tied }, () => {
+ Sentry.startInactiveSpan({ name: "grandchild", op: "task", startTime: tied }).end(tied + 0.001);
+ });
+ });
+ });
+ for (let i = 0; i < 40 && !captured.some((event) => event.transaction === "tied-run"); i++) {
+ await flushErrorReporting();
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ }
+ const run = captured.find((event) => event.transaction === "tied-run");
+ assert.ok(run, "the transaction arrived");
+ const spans: Record<string, any>[] = run.spans;
+ assert.equal(spans.length, 500);
+ const ids = new Set([run.contexts.trace.span_id, ...spans.map((sent) => sent.span_id)]);
+ for (const sent of spans) assert.ok(ids.has(sent.parent_span_id), "every sent span's parent is sent");
+ assert.equal(run.contexts.trace.data["qm.truncated_spans"], 2);
+});
+
+test("only quick, successful, childless pool checkouts are left out", async () => {
+ const start = 1_700_000_000;
+ const traceId = "0123456789abcdef0123456789abcdef";
+ const span = (id: string, parent: string, description: string, seconds: number, status = "ok") => ({
+ span_id: id.padStart(16, "0"),
+ parent_span_id: parent.padStart(16, "0"),
+ trace_id: traceId,
+ op: "db",
+ description,
+ start_timestamp: start,
+ timestamp: start + seconds,
+ status,
+ data: {},
+ });
+ Sentry.captureEvent({
+ type: "transaction",
+ transaction: "checkouts",
+ start_timestamp: start,
+ timestamp: start + 1,
+ contexts: { trace: { trace_id: traceId, span_id: "1".padStart(16, "0"), op: "queue.process" } },
+ spans: [
+ span("a", "1", "pg-pool.connect", 0.0002),
+ span("b", "1", "pg-pool.connect", 0.005),
+ span("c", "1", "pg-pool.connect", 0.0002, "internal_error"),
+ span("d", "1", "pg-pool.connect", 0.0002),
+ span("e", "d", "pg.connect", 0.0001),
+ ],
+ });
+ for (let i = 0; i < 40 && !captured.some((event) => event.transaction === "checkouts"); i++) {
+ await flushErrorReporting();
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ }
+ const run = captured.find((event) => event.transaction === "checkouts");
+ assert.ok(run, "the transaction arrived");
+ assert.deepEqual(run.spans.map((sent: { span_id: string }) => sent.span_id.replace(/^0+/, "")).sort(), [
+ "b",
+ "c",
+ "d",
+ "e",
+ ]);
+ assert.equal(run.contexts.trace.data?.["qm.truncated_spans"], undefined);
+});