Multiplayer agent harness for work.
Add Server-Timing phase timing to transcript, approvals and session-list requests (#2467)
Regan Bell817c283c884f
diff --git a/package.json b/package.jsonindex 5a1f40ca0..a42a33e80 100644--- a/package.json+++ b/package.json@@ -48,7 +48,7 @@"sandbox:local:build": "bash scripts/local-sandbox-build.sh","sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"","deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",- "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/postgres-items-copy.test.ts test/postgres-items-bulk-catchup.test.ts test/postgres-items-parallel-catchup.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-trigger-recipient-consent-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/run-priority.test.ts test/postgres-run-stream.test.ts test/postgres-items-migration.test.ts test/postgres-items-promotion.test.ts test/postgres-items-contract.test.ts test/pi-harness-replay-fidelity.test.ts test/migrate-main.test.ts",+ "test:pg": "node scripts/run-pg-tests.mjs test/server-timing.test.ts test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/postgres-items-copy.test.ts test/postgres-items-bulk-catchup.test.ts test/postgres-items-parallel-catchup.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-trigger-recipient-consent-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/run-priority.test.ts test/postgres-run-stream.test.ts test/postgres-items-migration.test.ts test/postgres-items-promotion.test.ts test/postgres-items-contract.test.ts test/pi-harness-replay-fidelity.test.ts test/migrate-main.test.ts","livetest": "node scripts/api-livetest.ts","bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts","prepare": "node .husky/install.mjs",diff --git a/plugins/chassis/src/core-client.ts b/plugins/chassis/src/core-client.tsindex 3bf411f68..576714264 100644--- a/plugins/chassis/src/core-client.ts+++ b/plugins/chassis/src/core-client.ts@@ -1,3 +1,5 @@+import { performance } from "node:perf_hooks";+import { addPhase, requestTiming } from "./server-timing.ts";import { signedRequestHeaders } from "./source-auth-sign.ts";export type HttpMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE";@@ -43,6 +45,7 @@ export async function fetchCoreText(input: {let phase = "headers";let requestId: string | null = null;let status: number | undefined;+ const startedAt = performance.now();try {const response = await fetch(`${input.origin}${path}`, {method: input.method,@@ -54,7 +57,14 @@ export async function fetchCoreText(input: {status = response.status;phase = "body";requestId = response.headers.get("x-request-id");- return { status: response.status, text: await response.text() };+ const timing = requestTiming();+ const headersAt = performance.now();+ addPhase("core-headers", headersAt - startedAt, timing);+ const text = await response.text();+ addPhase("core-body", performance.now() - headersAt, timing);+ const upstream = response.headers.get("server-timing");+ if (timing && upstream) timing.upstream = upstream;+ return { status: response.status, text };} catch (error) {const cause = error instanceof Error ? error.cause : undefined;const code = cause && typeof cause === "object" && "code" in cause ? cause.code : undefined;diff --git a/plugins/chassis/src/http.ts b/plugins/chassis/src/http.tsindex ef41293f4..d5006f195 100644--- a/plugins/chassis/src/http.ts+++ b/plugins/chassis/src/http.ts@@ -1,5 +1,7 @@import type { IncomingMessage, ServerResponse } from "node:http";+import { performance } from "node:perf_hooks";import { gzip } from "node:zlib";+import { addPhase, timedSync, withServerTiming } from "./server-timing.ts";const COMPRESS_MIN_BYTES = 1024;@@ -39,19 +41,24 @@ export function withVary(res: ServerResponse, value: string): string {export function sendBuffered(res: ServerResponse, status: number, headers: Record<string, string>, body: string): void {const out = { ...headers, vary: withVary(res, "accept-encoding") };if (Buffer.byteLength(body) < COMPRESS_MIN_BYTES || !gzipAccepted(res.req)) {- res.writeHead(status, out);+ res.writeHead(status, withServerTiming(out));res.end(body);return;}+ const started = performance.now();gzip(body, (err, packed) => {+ addPhase("gzip", performance.now() - started);if (res.headersSent || res.writableEnded || res.destroyed) return;try {if (err) {- res.writeHead(status, out);+ res.writeHead(status, withServerTiming(out));res.end(body);return;}- res.writeHead(status, { ...out, "content-encoding": "gzip", "content-length": String(packed.length) });+ res.writeHead(+ status,+ withServerTiming({ ...out, "content-encoding": "gzip", "content-length": String(packed.length) }),+ );res.end(packed);} catch {res.destroy();@@ -60,7 +67,12 @@ export function sendBuffered(res: ServerResponse, status: number, headers: Recor}export function json(res: ServerResponse, status: number, body: unknown): void {- sendBuffered(res, status, { "content-type": "application/json" }, JSON.stringify(body));+ sendBuffered(+ res,+ status,+ { "content-type": "application/json" },+ timedSync("json", () => JSON.stringify(body)),+ );}export async function readBodyBytes(req: IncomingMessage, maxBytes = Infinity): Promise<Buffer> {diff --git a/plugins/chassis/src/server-timing.ts b/plugins/chassis/src/server-timing.tsnew file mode 100644index 000000000..f76c451d7--- /dev/null+++ b/plugins/chassis/src/server-timing.ts@@ -0,0 +1,99 @@+import { AsyncLocalStorage } from "node:async_hooks";+import type { ServerResponse } from "node:http";+import { performance, type EventLoopUtilization } from "node:perf_hooks";++export interface RequestTiming {+ readonly hop: string;+ readonly startedAt: number;+ readonly loop: EventLoopUtilization;+ readonly phases: Map<string, number>;+ readonly db: { waitMs: number; queryMs: number; queries: number; connects: number };+ route?: string;+ upstream?: string;+}++const store = new AsyncLocalStorage<RequestTiming>();++export function withRequestTiming<T>(hop: string, fn: () => T): T {+ return store.run(+ {+ hop,+ startedAt: performance.now(),+ loop: performance.eventLoopUtilization(),+ phases: new Map(),+ db: { waitMs: 0, queryMs: 0, queries: 0, connects: 0 },+ },+ fn,+ );+}++export function requestTiming(): RequestTiming | undefined {+ return store.getStore();+}++export function timeRoute(route: string): void {+ const timing = store.getStore();+ if (timing) timing.route = route;+}++export function addPhase(phase: string, ms: number, timing = store.getStore()): void {+ if (timing) timing.phases.set(phase, (timing.phases.get(phase) ?? 0) + ms);+}++export async function timed<T>(phase: string, fn: () => Promise<T>): Promise<T> {+ const timing = store.getStore();+ if (!timing) return fn();+ const started = performance.now();+ try {+ return await fn();+ } finally {+ addPhase(phase, performance.now() - started, timing);+ }+}++export function timedSync<T>(phase: string, fn: () => T): T {+ const timing = store.getStore();+ if (!timing) return fn();+ const started = performance.now();+ try {+ return fn();+ } finally {+ addPhase(phase, performance.now() - started, timing);+ }+}++const round = (ms: number): number => Math.round(ms * 10) / 10;++function timingMetrics(timing: RequestTiming): Record<string, number> {+ const metrics: Record<string, number> = Object.fromEntries(timing.phases);+ if (timing.db.queries || timing.db.connects) {+ metrics["db-wait"] = timing.db.waitMs;+ metrics["db-query"] = timing.db.queryMs;+ metrics["db-count"] = timing.db.queries;+ metrics["db-connects"] = timing.db.connects;+ }+ metrics.loop = performance.eventLoopUtilization(timing.loop).active;+ metrics.total = performance.now() - timing.startedAt;+ return Object.fromEntries(Object.entries(metrics).map(([name, value]) => [name, round(value)]));+}++export function withServerTiming<T extends Record<string, string | string[]>>(+ headers: T,+ timing = store.getStore(),+): T {+ if (!timing) return headers;+ const own = Object.entries(timingMetrics(timing)).map(([name, value]) => `${timing.hop}-${name};dur=${value}`);+ return { ...headers, "server-timing": [...own, ...(timing.upstream ? [timing.upstream] : [])].join(", ") };+}++export function logRequestTiming(res: ServerResponse, timing: RequestTiming): void {+ if (!timing.route) return;+ console.info(+ `[request-timing] ${JSON.stringify({+ hop: timing.hop,+ route: timing.route,+ status: res.writableFinished ? res.statusCode : 0,+ ...timingMetrics(timing),+ })}`,+ );+}diff --git a/plugins/portal/src/index.ts b/plugins/portal/src/index.tsindex 186f0f8b3..285281a51 100644--- a/plugins/portal/src/index.ts+++ b/plugins/portal/src/index.ts@@ -1,3 +1,4 @@+import { withRequestTiming } from "../../chassis/src/server-timing.ts";import {DESKTOP_LAUNCH_SCRIPT,DESKTOP_LAUNCH_SCRIPT_HASH,@@ -963,7 +964,7 @@ const server = createServer((req, res) => {Math.round(performance.now() - startedAt),);});- void handle(req, res).catch((err: unknown) => {+ void withRequestTiming("portal", () => handle(req, res)).catch((err: unknown) => {errorLogged = true;reportBackendError(err);console.error("[portal] 500 %s %s: %s", req.method ?? "?", (req.url ?? "?").split("?")[0], String(err));diff --git a/plugins/portal/src/proxy.ts b/plugins/portal/src/proxy.tsindex 9ece144ac..c3fa3f90c 100644--- a/plugins/portal/src/proxy.ts+++ b/plugins/portal/src/proxy.ts@@ -1,6 +1,8 @@+import { performance } from "node:perf_hooks";import { request as httpRequest, type IncomingMessage, type ServerResponse } from "node:http";import { signedHeaders } from "../../chassis/src/core-client.ts";import { withVary } from "../../chassis/src/http.ts";+import { addPhase, requestTiming, withServerTiming } from "../../chassis/src/server-timing.ts";import { mintPortalIdentity, PORTAL_IDENTITY_HEADER } from "../../chassis/src/portal-identity.ts";const IDENTITY_TTL_MS = 60_000;@@ -59,6 +61,8 @@ function relay(forwardCookies?: boolean;},): void {+ const timing = requestTiming();+ const sentAt = performance.now();const up = httpRequest({protocol: target.protocol,@@ -78,7 +82,12 @@ function relay(}if (out.vary !== undefined) out.vary = withVary(res, String(out.vary));if (target.honorFramePolicy && declaresFrameAncestors(out)) res.removeHeader("x-frame-options");- res.writeHead(upRes.statusCode ?? 502, out);+ if (timing) {+ addPhase("pre", sentAt - timing.startedAt, timing);+ addPhase("upstream", performance.now() - sentAt, timing);+ if (out["server-timing"]) timing.upstream = [out["server-timing"]].flat().join(", ");+ }+ res.writeHead(upRes.statusCode ?? 502, withServerTiming(out, timing));upRes.on("error", () => res.destroy());upRes.pipe(res);},diff --git a/plugins/portal/test/proxy-server-timing.test.ts b/plugins/portal/test/proxy-server-timing.test.tsnew file mode 100644index 000000000..74d2d8f26--- /dev/null+++ b/plugins/portal/test/proxy-server-timing.test.ts@@ -0,0 +1,52 @@+import test from "node:test";+import assert from "node:assert/strict";+import { createServer } from "node:http";+import type { AddressInfo } from "node:net";+import { withRequestTiming } from "../../chassis/src/server-timing.ts";+import { proxyToSurface } from "../src/proxy.ts";++const upstream = createServer((req, res) => {+ req.resume();+ setTimeout(() => {+ res.writeHead(200, { "content-type": "application/json", "server-timing": "web-total;dur=12.5" });+ res.end("{}");+ }, 20);+});+await new Promise<void>((resolve) => upstream.listen(0, resolve));++const portal = createServer((req, res) =>+ withRequestTiming("portal", () =>+ setTimeout(+ () =>+ proxyToSurface(req, res, {+ upstreamBase: `http://localhost:${(upstream.address() as AddressInfo).port}`,+ forwardPath: "/api/sessions/s1",+ search: "",+ cookieName: "webuiuser",+ principal: "alice",+ }),+ 10,+ ),+ ),+);+await new Promise<void>((resolve) => portal.listen(0, resolve));++test.after(() => {+ portal.close();+ upstream.close();+});++test("the portal reports its own time and its wait on the surface ahead of the surface's timings", async () => {+ const response = await fetch(`http://localhost:${(portal.address() as AddressInfo).port}/api/sessions/s1`);+ await response.text();+ const timing = new Map(+ (response.headers.get("server-timing") ?? "").split(", ").map((part) => {+ const [name, dur] = part.split(";dur=");+ return [name!, Number(dur)];+ }),+ );+ assert.equal(timing.get("web-total"), 12.5);+ assert.ok(timing.get("portal-pre")! >= 8, `portal pre ${timing.get("portal-pre")}`);+ assert.ok(timing.get("portal-upstream")! >= 15, `portal upstream ${timing.get("portal-upstream")}`);+ assert.ok(timing.get("portal-total")! >= timing.get("portal-pre")! + timing.get("portal-upstream")! - 0.2);+});diff --git a/plugins/web-ui/server/index.ts b/plugins/web-ui/server/index.tsindex 9db6c9e2f..6b6e49c20 100644--- a/plugins/web-ui/server/index.ts+++ b/plugins/web-ui/server/index.ts@@ -1,3 +1,10 @@+import {+ logRequestTiming,+ requestTiming,+ timedSync,+ timeRoute,+ withRequestTiming,+} from "../../chassis/src/server-timing.ts";import "./instrument.ts";import { handleVoiceRoute, sweepDictationSpool } from "./voice-routes.ts";import { browserErrorConfig } from "./browser-error-config.ts";@@ -1492,6 +1499,7 @@ const apiRoutes: readonly WebRoute[] = [path: "/api/sessions",handle: async (c) => {const { res, user } = c;+ timeRoute("sessions");return relayCore(res, "GET", `/v1/sessions?principalId=${encodeURIComponent(user)}`);},},@@ -2005,6 +2013,7 @@ const apiRoutes: readonly WebRoute[] = [handle: async (c) => {const { res, user } = c;const id = c.params.id!;+ timeRoute("approvals");return relayCore(res,"GET",@@ -2060,6 +2069,7 @@ const apiRoutes: readonly WebRoute[] = [handle: async (c) => {const { res, url, user } = c;const id = c.params.id!;+ timeRoute("transcript");const qs = new URLSearchParams({ viewer: user });for (const p of ["tailTurns", "sinceSeq", "beforeSeq"] as const) {const v = url.searchParams.get(p);@@ -3356,7 +3366,7 @@ const routeRequest = async (req: IncomingMessage, res: ServerResponse) => {}if (path === "/me" || path.startsWith("/api/")) {- const user = cookieUser(req);+ const user = timedSync("auth", () => cookieUser(req));if (!user) return unauthorized(res, req);if (["/api/dictation", "/api/voice", "/api/voice/sessions"].includes(path)) {return handleVoiceRoute(req, res, path, resolveIdentity(req)!, {@@ -3442,7 +3452,11 @@ export const handler = async (req: IncomingMessage, res: ServerResponse) => {const raw = req.headers[PORTAL_IDENTITY_HEADER];const token = Array.isArray(raw) ? raw[0] : raw;try {- await portalTokenStore.run(token, () => routeRequest(req, res));+ await withRequestTiming("web", () => {+ const timing = requestTiming()!;+ res.once("close", () => logRequestTiming(res, timing));+ return portalTokenStore.run(token, () => routeRequest(req, res));+ });} catch (err) {if (err instanceof PayloadTooLargeError) {if (!res.headersSent) json(res, 413, { error: "payload_too_large", message: err.message });diff --git a/plugins/web-ui/test/server-timing-route.test.ts b/plugins/web-ui/test/server-timing-route.test.tsnew file mode 100644index 000000000..30edb727d--- /dev/null+++ b/plugins/web-ui/test/server-timing-route.test.ts@@ -0,0 +1,66 @@+import { test, mock } from "node:test";+import assert from "node:assert/strict";+import { createServer } from "node:http";+import type { AddressInfo } from "node:net";+import { mintPortalIdentity, PORTAL_IDENTITY_HEADER } from "../../chassis/src/portal-identity.ts";++const core = createServer((req, res) => {+ req.resume();+ req.on("end", () =>+ setTimeout(() => {+ res.writeHead(200, { "content-type": "application/json", "server-timing": "core-total;dur=42.5" });+ res.end(JSON.stringify({ session: { id: "s1" }, entries: [] }));+ }, 20),+ );+});+await new Promise<void>((resolve) => core.listen(0, resolve));++process.env.CORE_API_URL = `http://localhost:${(core.address() as AddressInfo).port}`;+process.env.CORE_SIGNING_SECRET = "server-timing-route-test";+process.env.WEB_UI_PRINCIPALS = "alice";++const { handler } = await import("../server/index.ts");+const surface = createServer((req, res) => void handler(req, res));+await new Promise<void>((resolve) => surface.listen(0, resolve));++test.after(() => {+ surface.close();+ core.close();+});++test("the transcript relay times its core hop and forwards core's Server-Timing", async () => {+ const info = mock.method(console, "info", () => {});+ try {+ const response = await fetch(`http://localhost:${(surface.address() as AddressInfo).port}/api/sessions/s1`, {+ headers: {+ [PORTAL_IDENTITY_HEADER]: mintPortalIdentity(+ { p: "alice", exp: Date.now() + 60_000 },+ "server-timing-route-test",+ ),+ },+ });+ assert.equal(response.status, 200);+ await response.json();+ const timing = new Map(+ (response.headers.get("server-timing") ?? "").split(", ").map((part) => {+ const [name, dur] = part.split(";dur=");+ return [name!, Number(dur)];+ }),+ );+ assert.equal(timing.get("core-total"), 42.5);+ assert.ok(timing.get("web-core-headers")! >= 15, `core hop ${timing.get("web-core-headers")}`);+ for (const name of ["web-auth", "web-core-body", "web-loop"]) assert.ok(Number.isFinite(timing.get(name)), name);+ assert.ok(timing.get("web-total")! >= timing.get("web-core-headers")!);+ await new Promise((resolve) => setImmediate(resolve));+ const line = info.mock.calls+ .map((call) => String(call.arguments[0]))+ .find((text) => text.startsWith("[request-timing] "));+ const logged = JSON.parse(line!.slice("[request-timing] ".length)) as Record<string, unknown>;+ assert.equal(logged.hop, "web");+ assert.equal(logged.route, "transcript");+ assert.equal(logged.status, 200);+ assert.equal(typeof logged["core-headers"], "number");+ } finally {+ info.mock.restore();+ }+});diff --git a/src/api/app-helpers.ts b/src/api/app-helpers.tsindex d95f6cc98..fc42b70a0 100644--- a/src/api/app-helpers.ts+++ b/src/api/app-helpers.ts@@ -1,3 +1,4 @@+import { timed } from "../../plugins/chassis/src/server-timing.ts";import { isSubagentThreadRef } from "../sessions/session-syscalls.ts";import type {PendingApproval,@@ -306,9 +307,11 @@ export function createAppHelpers(deps: AppDeps, app: App) {}async function sessionForViewer(sessionId: string, principalId: string): Promise<Session | null> {- const session = await deps.sessions.getForParticipant(sessionId, principalId);+ const session = await timed("lookup", () => deps.sessions.getForParticipant(sessionId, principalId));if (!session) return null;- return (await managedProjectMembership(session.scopeId, principalId)) === false ? null : session;+ return (await timed("access", () => managedProjectMembership(session.scopeId, principalId))) === false+ ? null+ : session;}async function contextsFor(principalId: string): Promise<ContextSummary[]> {diff --git a/src/api/app-sessions.ts b/src/api/app-sessions.tsindex f53636265..816d9cacf 100644--- a/src/api/app-sessions.ts+++ b/src/api/app-sessions.ts@@ -1,3 +1,4 @@+import { timed, timedSync } from "../../plugins/chassis/src/server-timing.ts";import { composeSoul } from "../resolution/resolution-service.ts";import { createCurrentScopeMembers } from "../resolution/scope-membership.ts";import { notifyDeploymentShared } from "../deploy/share-notice.ts";@@ -295,17 +296,21 @@ export function createSessionMethods(async getSessionForViewer(sessionId, principalId, window) {const found = await sessionForViewer(sessionId, principalId);if (!found) return null;- const session = { ...found, canContinue: await canContinueOnWeb(principalId, found) };+ const session = { ...found, canContinue: await timed("continue", () => canContinueOnWeb(principalId, found)) };let limit = tailWindowLimit(window);const [initialRead, pinRecords, goals, latestRuns] = await Promise.all([- transcripts.forViewer(sessionId, principalId, {- limit,- beforeSeq: window?.beforeSeq,- sinceSeq: window?.sinceSeq,- }),- deps.sessions.listPins(sessionId),- deps.sessions.goals([sessionId]),- window?.beforeSeq === undefined ? deps.runs.latestForThreads([found.threadRef]) : undefined,+ timed("transcript", () =>+ transcripts.forViewer(sessionId, principalId, {+ limit,+ beforeSeq: window?.beforeSeq,+ sinceSeq: window?.sinceSeq,+ }),+ ),+ timed("pins", () => deps.sessions.listPins(sessionId)),+ timed("goals", () => deps.sessions.goals([sessionId])),+ window?.beforeSeq === undefined+ ? timed("runs", () => deps.runs.latestForThreads([found.threadRef]))+ : undefined,]);const goal = openGoal(goals.get(sessionId));const runtime = latestRuns?.get(found.threadRef)?.result?.runtime;@@ -313,12 +318,16 @@ export function createSessionMethods(let visible = read.entries;while (limit !== undefined && read.earlier > 0 && !coversTailWindow(visible, window!.tailTurns!)) {limit *= 2;- read = await transcripts.forViewer(sessionId, principalId, { limit, beforeSeq: window?.beforeSeq });+ read = await timed("transcript", () =>+ transcripts.forViewer(sessionId, principalId, { limit, beforeSeq: window?.beforeSeq }),+ );visible = read.entries;}- const w = windowedTranscript(visible, window);+ const w = timedSync("window", () => windowedTranscript(visible, window));const earlier = w.earlier + read.earlier;- const pins = await decoratedPins(pinRecords, visible, (seq) => viewerStoredEntryAt(sessionId, principalId, seq));+ const pins = await timed("pins", () =>+ decoratedPins(pinRecords, visible, (seq) => viewerStoredEntryAt(sessionId, principalId, seq)),+ );return {session,entries: w.entries,diff --git a/src/api/http.ts b/src/api/http.tsindex 9f458ed92..b0bbe6660 100644--- a/src/api/http.ts+++ b/src/api/http.ts@@ -1,61 +1,9 @@import type { IncomingMessage, ServerResponse } from "node:http";-import { gzip } from "node:zlib";import { verifySignature, type SourceAuth, SOURCE_AUTH_REPLAY_WINDOW_MS } from "../auth/source-auth.ts";-const COMPRESS_MIN_BYTES = 1024;+import { json as sendJson } from "../../plugins/chassis/src/http.ts";-export function gzipAccepted(req: IncomingMessage | undefined): boolean {- const header = req?.headers["accept-encoding"];- if (header === undefined) return false;- let explicit: boolean | undefined;- let wildcard: boolean | undefined;- for (const part of (Array.isArray(header) ? header.join(",") : header).split(",")) {- const [name, ...params] = part.split(";");- const token = (name ?? "").trim().toLowerCase();- if (token !== "gzip" && token !== "*") continue;- const q = params.map((p) => p.trim().toLowerCase()).find((p) => p.startsWith("q="));- const weight = q === undefined ? 1 : Number.parseFloat(q.slice(2));- const allowed = Number.isFinite(weight) && weight > 0;- if (token === "*") wildcard = allowed;- else explicit = allowed;- }- return explicit ?? wildcard ?? false;-}--export function withVary(res: ServerResponse, value: string): string {- const existing = res.getHeader("vary");- const tokens = [...(existing === undefined ? [] : String(existing).split(",")), ...value.split(",")]- .map((token) => token.trim())- .filter(Boolean);- return [...new Map(tokens.map((token) => [token.toLowerCase(), token])).values()].join(", ");-}--export function sendBuffered(res: ServerResponse, status: number, headers: Record<string, string>, body: string): void {- const out = { ...headers, vary: withVary(res, "accept-encoding") };- if (Buffer.byteLength(body) < COMPRESS_MIN_BYTES || !gzipAccepted(res.req)) {- res.writeHead(status, out);- res.end(body);- return;- }- gzip(body, (err, packed) => {- if (res.headersSent || res.writableEnded || res.destroyed) return;- try {- if (err) {- res.writeHead(status, out);- res.end(body);- return;- }- res.writeHead(status, { ...out, "content-encoding": "gzip", "content-length": String(packed.length) });- res.end(packed);- } catch {- res.destroy();- }- });-}--export function sendJson(res: ServerResponse, status: number, body: unknown): void {- sendBuffered(res, status, { "content-type": "application/json" }, JSON.stringify(body));-}+export { sendJson };const BODY_DEADLINE = Symbol.for("qm.bodyDeadline");type DeadlineCarrier = IncomingMessage & { [BODY_DEADLINE]?: { rearm: (ms: number) => void } };diff --git a/src/api/routes/admin/scope-config.ts b/src/api/routes/admin/scope-config.tsindex 927909e11..23eedf1ff 100644--- a/src/api/routes/admin/scope-config.ts+++ b/src/api/routes/admin/scope-config.ts@@ -1,3 +1,4 @@+import { addPhase } from "../../../../plugins/chassis/src/server-timing.ts";import { parseScopeId } from "../../../types.ts";import { encodeRef, serviceCredRef } from "../../../acl/resource-ref.ts";import { computeRetention } from "../../../admin/retention.ts";@@ -445,6 +446,7 @@ export async function getScopeConfig(ctx: ApiCtx): Promise<void> {return await load();} finally {timings[name] = performance.now() - start;+ addPhase(name, timings[name]);}};const actor = await read("authorize", () => authorizeAdmin(ctx, targetScope));@@ -491,12 +493,6 @@ export async function getScopeConfig(ctx: ApiCtx): Promise<void> {scopeModelOptions(deps, values, view !== null && url.searchParams.get("catalog") !== "refresh"),): undefined;- res.setHeader(- "server-timing",- Object.entries(timings)- .map(([name, duration]) => `${name};dur=${duration.toFixed(1)}`)- .join(", "),- );const elapsed = performance.now() - started;if (view && url.searchParams.get("catalog") !== "refresh" && elapsed > 500)console.warn(diff --git a/src/api/routes/surface.ts b/src/api/routes/surface.tsindex bd4764cf3..189f5e282 100644--- a/src/api/routes/surface.ts+++ b/src/api/routes/surface.ts@@ -1,3 +1,4 @@+import { timed, timeRoute } from "../../../plugins/chassis/src/server-timing.ts";import { memoryForRequest } from "./memory-access.ts";import type { MemoryEdit, MemoryService } from "../../memory/memory-service.ts";import { suggestedActivityRoutes } from "./suggested-activities.ts";@@ -174,6 +175,7 @@ async function getSession(ctx: ApiCtx): Promise<void> {message: "tailTurns and beforeSeq must be positive integers, sinceSeq a non-negative one",});}+ timeRoute("transcript");const found = await app.getSessionForViewer(id, viewer, Object.keys(window).length ? window : undefined);if (!found) return sendJson(res, 404, { error: "not_found" });return sendJson(res, 200, found);@@ -225,8 +227,10 @@ async function listSessionApprovals(ctx: ApiCtx): Promise<void> {const id = ctx.params.id!;const viewer = url.searchParams.get("viewer");if (!viewer) return sendJson(res, 400, { error: "bad_request", message: "viewer required" });- const approvals = await app.listSessionApprovals(id, viewer);- return sendJson(res, 200, { approvals: [...approvals, ...(await credentialApprovals(ctx, id, viewer))] });+ timeRoute("approvals");+ const approvals = await timed("approvals", () => app.listSessionApprovals(id, viewer));+ const credentials = await timed("credentials", () => credentialApprovals(ctx, id, viewer));+ return sendJson(res, 200, { approvals: [...approvals, ...credentials] });}// Pending credential requests from this session or any of its sub-agents, shown inline to their owner.@@ -487,7 +491,8 @@ async function listSessions(ctx: ApiCtx): Promise<void> {const { res, app, url } = ctx;const principalId = url.searchParams.get("principalId");if (!principalId) return sendJson(res, 400, { error: "bad_request", message: "principalId required" });- return sendJson(res, 200, { sessions: await app.listSessions(principalId) });+ timeRoute("sessions");+ return sendJson(res, 200, { sessions: await timed("list", () => app.listSessions(principalId)) });}async function searchResources(ctx: ApiCtx): Promise<void> {diff --git a/src/api/server.ts b/src/api/server.tsindex 6d0eba043..b5c580cba 100644--- a/src/api/server.ts+++ b/src/api/server.ts@@ -2,7 +2,15 @@ import { externalSlackCapabilityAllowed } from "./external-slack-capability.ts";import { MemoryDisclosureDenied } from "../memory/disclosure.ts";import { MemoryFactsMissing } from "../memory/facts.ts";import { reportBackendError, startTiming } from "../../plugins/chassis/src/error-reporting.ts";+import {+ addPhase,+ logRequestTiming,+ requestTiming,+ timed,+ withRequestTiming,+} from "../../plugins/chassis/src/server-timing.ts";import { traceStatus } from "../../plugins/chassis/src/timing.ts";+import { performance } from "node:perf_hooks";import {createServer as createHttpServer,type IncomingMessage,@@ -325,7 +333,7 @@ async function gate(const token = Array.isArray(rawToken) ? rawToken[0] : rawToken;actor = token && psecret ? await verifyPortalIdentity(token, psecret, Date.now()) : null;if (actor && deps.identity) {- await deps.identity.refresh(Boolean(actor.authenticatedAs));+ await timed("identity", () => deps.identity!.refresh(Boolean(actor!.authenticatedAs)));if (deps.identity.classify(actor.p).type !== "internal" ||(actor.authenticatedAs &&@@ -367,6 +375,12 @@ async function gate(return { body, capability, actor };}+function timedGate(run: () => Promise<GateResult | null>): Promise<GateResult | null> {+ const timing = requestTiming();+ if (timing) addPhase("pre", performance.now() - timing.startedAt, timing);+ return timed("auth", run);+}+function assertedActorField(field: string, url: URL, body: unknown): unknown {const [where, name] = field.split(".", 2) as [string, string];if (where === "query") return url.searchParams.get(name);@@ -423,7 +437,7 @@ function buildFastify(wiring: Wiring, server: Server): { fastify: FastifyInstancconst base = baseCtx(req, res, wiring);const raw = rawBodies.get(req) ?? "";const found = findRoute(matchRoutes, base.method, base.pathname);- const g = await gate(req, res, wiring, base.method, base.pathname, base.url, raw, found?.route);+ const g = await timedGate(() => gate(req, res, wiring, base.method, base.pathname, base.url, raw, found?.route));if (!g) return;if (found) {const ctx: ApiCtx = {@@ -470,15 +484,17 @@ function buildFastify(wiring: Wiring, server: Server): { fastify: FastifyInstancconst gateHook = async (request: FastifyRequest, reply: FastifyReply): Promise<void> => {const url = new URL(request.raw.url ?? "/", "http://localhost");const raw = rawBodies.get(request.raw) ?? "";- const result = await gate(- request.raw,- reply.raw,- wiring,- request.raw.method ?? "GET",- url.pathname,- url,- raw,- request.routeOptions.config.route,+ const result = await timedGate(() =>+ gate(+ request.raw,+ reply.raw,+ wiring,+ request.raw.method ?? "GET",+ url.pathname,+ url,+ raw,+ request.routeOptions.config.route,+ ),);if (!result) {reply.hijack();@@ -540,23 +556,26 @@ function buildServer(app: App, deps: ServerOptions, allowUnsignedSourceAuth: booallowUnsignedSourceAuth,};const requestNames = new WeakMap<IncomingMessage, string>();- const server = createHttpServer((req, res) => {- const finishTiming =- req.url === "/healthz" || req.url === "/readyz"- ? undefined- : startTiming("http.server", `${req.method ?? "GET"} /*`);- if (finishTiming)- res.once("close", () =>- finishTiming({+ const server = createHttpServer((req, res) =>+ withRequestTiming("core", () => {+ const timing = requestTiming()!;+ const finishTiming =+ req.url === "/healthz" || req.url === "/readyz"+ ? undefined+ : startTiming("http.server", `${req.method ?? "GET"} /*`);+ res.once("close", () => {+ logRequestTiming(res, timing);+ finishTiming?.({name: `${req.method ?? "GET"} ${requestNames.get(req) ?? "/*"}`,status: res.writableFinished ? traceStatus(res.statusCode) : "cancelled",data: { url: req.url, http_status: res.writableFinished ? String(res.statusCode) : undefined },- }),- );- req.on("error", () => res.destroy());- res.on("error", () => res.destroy());- void front(req, res).catch((err: unknown) => respondError(req, res, err));- });+ });+ });+ req.on("error", () => res.destroy());+ res.on("error", () => res.destroy());+ void front(req, res).catch((err: unknown) => respondError(req, res, err));+ }),+ );const { fastify, routing } = buildFastify(wiring, server);const ready = Promise.resolve(fastify.ready());ready.catch((err: unknown) => console.error("[server] fastify initialization failed:", errMessage(err)));diff --git a/src/persistence/pg-pool.ts b/src/persistence/pg-pool.tsindex 8a7f68603..389a3296c 100644--- a/src/persistence/pg-pool.ts+++ b/src/persistence/pg-pool.ts@@ -1,6 +1,8 @@import { createHash } from "node:crypto";import { readFileSync } from "node:fs";import type { Pool, PoolClient } from "pg";+import { requestTiming } from "../../plugins/chassis/src/server-timing.ts";+import { timeClientQueries, timePoolWait } from "./pg-timing.ts";import { createKeyedQueue, sleep } from "../util/async.ts";import { errMessage, swallowAs } from "../util/errors.ts";@@ -62,7 +64,13 @@ async function retainPool(connectionString: string, kind: "query" | "session" |function guardedPool(pool: Pool): Pool {pool.on("error", () => {});- pool.on("connect", (client) => client.on("error", (error) => console.error("[pg] client error:", errMessage(error))));+ pool.on("connect", (client) => {+ client.on("error", (error) => console.error("[pg] client error:", errMessage(error)));+ const timing = requestTiming();+ if (timing) timing.db.connects++;+ timeClientQueries(client);+ });+ timePoolWait(pool);return pool;}diff --git a/src/persistence/pg-timing.ts b/src/persistence/pg-timing.tsnew file mode 100644index 000000000..28f153417--- /dev/null+++ b/src/persistence/pg-timing.ts@@ -0,0 +1,49 @@+import { AsyncResource } from "node:async_hooks";+import { performance } from "node:perf_hooks";+import type { Pool, PoolClient } from "pg";+import { requestTiming } from "../../plugins/chassis/src/server-timing.ts";++type ConnectCallback = (err: Error | undefined, client: PoolClient | undefined, release: (err?: Error) => void) => void;++export function timePoolWait(pool: Pool): void {+ const connect = pool.connect.bind(pool) as (callback?: ConnectCallback) => Promise<PoolClient> | undefined;+ pool.connect = ((callback?: ConnectCallback) => {+ const timing = requestTiming();+ const started = performance.now();+ const waited = () => {+ if (timing) timing.db.waitMs += performance.now() - started;+ };+ if (callback)+ return connect(+ AsyncResource.bind((err: Error | undefined, client: PoolClient | undefined, release: (err?: Error) => void) => {+ waited();+ callback(err, client, release);+ }),+ );+ return timing ? connect()!.finally(waited) : connect();+ }) as Pool["connect"];+}++export function timeClientQueries(client: PoolClient): void {+ const query = (...args: unknown[]) => Reflect.apply(Object.getPrototypeOf(client).query, client, args) as unknown;+ client.query = ((...args: unknown[]) => {+ const timing = requestTiming();+ if (!timing) return query(...args);+ const started = performance.now();+ const finished = () => {+ timing.db.queries++;+ timing.db.queryMs += performance.now() - started;+ };+ const callback = args.at(-1);+ if (typeof callback === "function") {+ args[args.length - 1] = (...result: unknown[]) => {+ finished();+ (callback as (...result: unknown[]) => void)(...result);+ };+ return query(...args);+ }+ const result = query(...args);+ if (result instanceof Promise) result.then(finished, finished);+ return result;+ }) as PoolClient["query"];+}diff --git a/test/admin-settings-load.test.ts b/test/admin-settings-load.test.tsindex 2facc0cc4..cb0f66d97 100644--- a/test/admin-settings-load.test.ts+++ b/test/admin-settings-load.test.ts@@ -44,7 +44,7 @@ test("settings projections preserve values while excluding unrelated payloads",for (const [key, value] of Object.entries(projected)) assert.deepEqual(value, full[key], view + ":" + key);if (view !== "credentials") assert.equal("serviceCredentials" in projected, false, view);else assert.deepEqual(projected.directoryChannels, [{ channelId: "C-settings", name: "settings-test" }]);- assert.match(response.headers.get("server-timing") ?? "", /authorize;dur=/);+ assert.match(response.headers.get("server-timing") ?? "", /core-authorize;dur=/);if (["customize", "credentials", "connectors"].includes(view))assert.equal("modelsByHarness" in projected, false, view);if (view === "slack-settings") {diff --git a/test/response-compression-twins.test.ts b/test/response-compression-twins.test.tsdeleted file mode 100644index 42d63c7e1..000000000--- a/test/response-compression-twins.test.ts+++ /dev/null@@ -1,27 +0,0 @@-import { test } from "node:test";-import assert from "node:assert/strict";-import { readFileSync } from "node:fs";-import { fileURLToPath } from "node:url";--function extract(source: string, name: string): string {- const start = source.indexOf(`export function ${name}(`);- assert.notEqual(start, -1, `${name} not found`);- const end = source.indexOf("\n}\n", start);- assert.notEqual(end, -1, `${name} body not delimited`);- return source.slice(start, end + 2);-}--const core = readFileSync(fileURLToPath(new URL("../src/api/http.ts", import.meta.url)), "utf8");-const chassis = readFileSync(fileURLToPath(new URL("../plugins/chassis/src/http.ts", import.meta.url)), "utf8");--test("the plugin chassis carries a byte-identical copy of the response compressor", () => {- for (const name of ["gzipAccepted", "withVary", "sendBuffered"]) {- assert.equal(- extract(chassis, name),- extract(core, name),- `${name} has drifted between src/api/http.ts and plugins/chassis/src/http.ts — plugins cannot import core, so the copies must be kept in step by hand`,- );- }- const floor = /const COMPRESS_MIN_BYTES = (\d+);/;- assert.equal(floor.exec(chassis)?.[1], floor.exec(core)?.[1]);-});diff --git a/test/response-compression.test.ts b/test/response-compression.test.tsindex 0e9359279..c59e5c515 100644--- a/test/response-compression.test.ts+++ b/test/response-compression.test.ts@@ -12,7 +12,7 @@ import { gunzipSync } from "node:zlib";import { mkdtempSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";-import { gzipAccepted, sendBuffered, sendJson } from "../src/api/http.ts";+import { gzipAccepted, json as sendJson, sendBuffered } from "../plugins/chassis/src/http.ts";import { createInsecureTestServer } from "../src/api/server.ts";import { buildApp } from "../src/wiring.ts";import { testConfig } from "./support/test-config.ts";diff --git a/test/server-timing.test.ts b/test/server-timing.test.tsnew file mode 100644index 000000000..04806a36b--- /dev/null+++ b/test/server-timing.test.ts@@ -0,0 +1,109 @@+import "./support/auto-fake-sprites.ts";++import { test, mock } from "node:test";+import assert from "node:assert/strict";+import { mkdtempSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import type { AddressInfo } from "node:net";+import { createInsecureTestServer } from "../src/api/server.ts";+import { configurePgPooling, createPgPool } from "../src/persistence/pg-pool.ts";+import { buildApp } from "../src/wiring.ts";+import { requestTiming, withRequestTiming, type RequestTiming } from "../plugins/chassis/src/server-timing.ts";+import { testConfig } from "./support/test-config.ts";++const direct = process.env.DATABASE_URL;++function metrics(header: string | null): Map<string, number> {+ return new Map(+ (header ?? "").split(", ").map((part) => {+ const [name, dur] = part.split(";dur=");+ return [name!, Number(dur)];+ }),+ );+}++test("core transcript responses carry Server-Timing phases and log one request-timing line", async () => {+ const built = buildApp(testConfig({ dataDir: mkdtempSync(join(tmpdir(), "server-timing-")) }));+ const server = createInsecureTestServer(built.app);+ server.listen(0);+ const info = mock.method(console, "info", () => {});+ try {+ const response = await fetch(+ `http://localhost:${(server.address() as AddressInfo).port}/v1/sessions/nope?viewer=someone`,+ );+ assert.equal(response.status, 404);+ await response.text();+ const timing = metrics(response.headers.get("server-timing"));+ for (const name of ["core-pre", "core-auth", "core-lookup", "core-json", "core-loop", "core-total"])+ assert.ok(Number.isFinite(timing.get(name)), `${name} in ${response.headers.get("server-timing")}`);+ assert.ok(timing.get("core-total")! >= timing.get("core-lookup")!);+ await new Promise((resolve) => setImmediate(resolve));+ const lines = info.mock.calls+ .map((call) => String(call.arguments[0]))+ .filter((line) => line.startsWith("[request-timing] "));+ assert.equal(lines.length, 1);+ const logged = JSON.parse(lines[0]!.slice("[request-timing] ".length)) as Record<string, unknown>;+ assert.equal(logged.hop, "core");+ assert.equal(logged.route, "transcript");+ assert.equal(logged.status, 404);+ assert.equal(typeof logged.total, "number");+ assert.equal(typeof logged.lookup, "number");+ } finally {+ info.mock.restore();+ await new Promise<void>((resolve) => server.close(() => resolve()));+ }+});++test(+ "Postgres pool wait and queries are charged to the request that issued them",+ { skip: direct ? false : "set DATABASE_URL to run" },+ async () => {+ configurePgPooling({ databaseUrl: direct!, queryMax: 1 });+ const pg = createPgPool(direct!);+ try {+ await pg.q("SELECT 1");+ const run = (sleep: number) =>+ withRequestTiming("t", async () => {+ await pg.q("SELECT pg_sleep($1)", [sleep]);+ return requestTiming()!;+ });+ const [slow, waiting] = (await Promise.all([run(0.1), run(0)])) as [RequestTiming, RequestTiming];+ assert.equal(slow.db.queries, 1);+ assert.equal(waiting.db.queries, 1);+ assert.ok(slow.db.queryMs >= 90, `slow query ${slow.db.queryMs}`);+ assert.ok(waiting.db.waitMs >= 80, `pool wait ${waiting.db.waitMs}`);+ assert.ok(waiting.db.queryMs < slow.db.queryMs);+ } finally {+ await pg.close();+ configurePgPooling({});+ }+ },+);++test(+ "a background query that waits behind a request's connection is not charged to that request",+ { skip: direct ? false : "set DATABASE_URL to run" },+ async () => {+ configurePgPooling({ databaseUrl: direct!, queryMax: 1 });+ const pg = createPgPool(direct!);+ try {+ let started!: () => void;+ const holding = new Promise<void>((resolve) => (started = resolve));+ const pending = withRequestTiming("t", async () => {+ const own = pg.q("SELECT pg_sleep(0.05)");+ started();+ await own;+ return requestTiming()!;+ });+ await holding;+ const background = pg.q("SELECT 1");+ const request = await pending;+ await background;+ assert.equal(request.db.queries, 1);+ } finally {+ await pg.close();+ configurePgPooling({});+ }+ },+);