🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(worker): page score and audit log batch exports by timestamp keyset (#18135)
Niklas Semmlerd62eeb559bdf
diff --git a/worker/src/__tests__/batchExport.test.ts b/worker/src/__tests__/batchExport.test.tsindex 82a2a2a42a..0ecd2c1cee 100644--- a/worker/src/__tests__/batchExport.test.ts+++ b/worker/src/__tests__/batchExport.test.ts@@ -36,6 +36,64 @@ const maybeDescribe =? describe: describe.skip;+// Seven scores over three timestamps, four tied on one millisecond, so pages of+// two must split a tie group.+const createScoresWithTiedTimestamps = async (projectId: string) => {+ const timestamps = [+ "2024-01-01T00:00:00.000Z",+ "2024-01-02T00:00:00.000Z",+ "2024-01-02T00:00:00.000Z",+ "2024-01-02T00:00:00.000Z",+ "2024-01-02T00:00:00.000Z",+ "2024-01-03T00:00:00.000Z",+ "2024-01-03T00:00:00.000Z",+ ];+ const scores = timestamps.map((timestamp) =>+ createTraceScore({+ project_id: projectId,+ trace_id: randomUUID(),+ name: "accuracy",+ value: 0.5,+ data_type: "NUMERIC",+ timestamp: new Date(timestamp).getTime(),+ }),+ );+ await createScoresCh(scores);+ return scores;+};++const exportScoresInPages = async (+ projectId: string,+ order: "ASC" | "DESC",+ useEventsTable: boolean,+) => {+ const stream = await getDatabaseReadStreamPaginated({+ projectId,+ tableName: BatchExportTableName.Scores,+ cutoffCreatedAt: new Date(Date.now() + 1000 * 60 * 60 * 24),+ filter: [],+ orderBy: { column: "timestamp", order },+ useEventsTable,+ pageSize: 2,+ });+ const rows: any[] = [];+ for await (const chunk of stream) {+ rows.push(chunk);+ }+ return rows;+};++const expectEveryScoreOnceInOrder = (+ rows: any[],+ scores: { id: string }[],+ order: "ASC" | "DESC",+) => {+ expect(rows.map((r) => r.id).sort()).toEqual(scores.map((s) => s.id).sort());+ const times = rows.map((r) => new Date(r.timestamp).getTime());+ const sorted = [...times].sort((a, b) => (order === "ASC" ? a - b : b - a));+ expect(times).toEqual(sorted);+};+describe("batch export test suite", () => {it("should export observations", async () => {const { projectId } = await createOrgProjectAndApiKey();@@ -1702,6 +1760,55 @@ describe("batch export test suite", () => {});});+ it.each(["DESC", "ASC"] as const)(+ "should export every score exactly once across pages sorted by timestamp %s",+ async (order) => {+ const { projectId } = await createOrgProjectAndApiKey();+ const scores = await createScoresWithTiedTimestamps(projectId);++ const rows = await exportScoresInPages(projectId, order, false);++ expectEveryScoreOnceInOrder(rows, scores, order);+ },+ );++ it("should export every audit log exactly once across pages", async () => {+ const { projectId, orgId } = await createOrgProjectAndApiKey();+ const tiedCreatedAt = new Date("2024-01-02T10:00:00Z");+ // Four entries share one createdAt so pages of two split a tie group.+ const entries = Array.from({ length: 5 }, (_, i) => ({+ id: randomUUID(),+ projectId,+ orgId,+ type: "USER" as const,+ userId: randomUUID(),+ resourceType: "trace",+ resourceId: randomUUID(),+ action: "CREATE",+ createdAt: i === 0 ? new Date("2024-01-01T10:00:00Z") : tiedCreatedAt,+ updatedAt: tiedCreatedAt,+ }));+ await prisma.auditLog.createMany({ data: entries });++ const stream = await getDatabaseReadStreamPaginated({+ projectId,+ tableName: BatchExportTableName.AuditLogs,+ cutoffCreatedAt: new Date(Date.now() + 1000 * 60 * 60 * 24),+ filter: [],+ orderBy: { column: "createdAt", order: "DESC" },+ pageSize: 2,+ });+ const rows: any[] = [];+ for await (const chunk of stream) {+ rows.push(chunk);+ }++ expect(rows.map((r) => r.id).sort()).toEqual(+ entries.map((e) => e.id).sort(),+ );+ expect(rows.at(-1).id).toBe(entries[0].id);+ });+it("should export traces with searchQuery and searchType filters applied correctly", async () => {const { projectId } = await createOrgProjectAndApiKey();@@ -2504,6 +2611,18 @@ describe("batch export test suite", () => {// ==================== EVENTS TABLE EXPORT TESTS ====================maybeDescribe("events table export tests", () => {+ it.each(["DESC", "ASC"] as const)(+ "should export every score exactly once across pages from the events table sorted by timestamp %s",+ async (order) => {+ const { projectId } = await createOrgProjectAndApiKey();+ const scores = await createScoresWithTiedTimestamps(projectId);++ const rows = await exportScoresInPages(projectId, order, true);++ expectEveryScoreOnceInOrder(rows, scores, order);+ },+ );+it("should export scores with trace metadata from the events table when useEventsTable is true", async () => {const { projectId } = await createOrgProjectAndApiKey();diff --git a/worker/src/features/database-read-stream/getDatabaseReadStream.ts b/worker/src/features/database-read-stream/getDatabaseReadStream.tsindex 9d4dde4992..5502b287d7 100644--- a/worker/src/features/database-read-stream/getDatabaseReadStream.ts+++ b/worker/src/features/database-read-stream/getDatabaseReadStream.ts@@ -102,6 +102,70 @@ export const getChunkWithFlattenedScores = <});};+/**+ * Keyset pagination over a timestamp sort. Each page reads rows at or beyond+ * the last emitted timestamp and excludes the ids already emitted at exactly+ * that timestamp, so a page costs the same however deep into the export it is+ * (unlike OFFSET, which re-reads every earlier row) and no row is skipped or+ * repeated, whatever order the database returns timestamp ties in.+ */+class TimestampKeyset {+ private boundary: Date | undefined;+ private idsAtBoundary: string[] = [];++ constructor(readonly order: "ASC" | "DESC") {}++ get after(): { timestamp: Date; excludeIds: string[] } | undefined {+ return this.boundary+ ? { timestamp: this.boundary, excludeIds: this.idsAtBoundary }+ : undefined;+ }++ advance(rows: { id: string; timestamp: Date }[]) {+ for (const row of rows) {+ if (this.boundary?.getTime() === row.timestamp.getTime()) {+ this.idsAtBoundary.push(row.id);+ } else {+ this.boundary = row.timestamp;+ this.idsAtBoundary = [row.id];+ }+ }+ }+}++// Exports sorted by anything other than the time column keep OFFSET paging.+const getTimestampKeysetOrder = (+ orderBy: OrderByState,+ timeColumn: string,+): "ASC" | "DESC" | null => {+ if (!orderBy) return "DESC";+ return orderBy.column.toLowerCase() === timeColumn.toLowerCase()+ ? orderBy.order+ : null;+};++const keysetFilter = (+ keyset: TimestampKeyset,+ timeColumn: string,+): FilterCondition[] => {+ const after = keyset.after;+ if (!after) return [];+ return [+ {+ type: "datetime",+ column: timeColumn,+ operator: keyset.order === "DESC" ? "<=" : ">=",+ value: after.timestamp,+ },+ {+ type: "stringOptions",+ column: "id",+ operator: "none of",+ value: after.excludeIds,+ },+ ];+};+// Events-backed replacement for the legacy getScoresUiTable traces JOIN:// loads score rows without trace enrichment, then fills traceName /// traceUserId / traceTags from the events table for the page's trace ids.@@ -156,12 +220,14 @@ export const getDatabaseReadStreamPaginated = async ({useEventsTable,preferredClickhouseService,rowLimit = env.BATCH_EXPORT_ROW_LIMIT,+ pageSize = env.BATCH_EXPORT_PAGE_SIZE,}: {projectId: string;cutoffCreatedAt: Date;searchQuery?: string;searchType?: TracingSearchType[];rowLimit?: number;+ pageSize?: number;preferredClickhouseService?: PreferredClickhouseService;} & BatchExportQueryType): Promise<DatabaseReadStream<unknown>> => {// Set createdAt cutoff to prevent exporting data that was created after the job was queued@@ -191,11 +257,23 @@ export const getDatabaseReadStreamPaginated = async ({switch (tableName) {case "scores": {+ const timeColumn = tableNameToTimeFilterColumn[tableName];+ const keysetOrder = getTimestampKeysetOrder(orderBy, timeColumn);+ const keyset = keysetOrder ? new TimestampKeyset(keysetOrder) : null;+return new DatabaseReadStream<unknown>(async (pageSize: number, offset: number) => {- const scoresFilter = filter- ? [...filter, createdAtCutoffFilter]- : [createdAtCutoffFilter];+ const scoresFilter = [+ ...(filter ?? []),+ createdAtCutoffFilter,+ ...(keyset ? keysetFilter(keyset, timeColumn) : []),+ ];+ const page = keyset+ ? {+ orderBy: { column: timeColumn, order: keyset.order },+ offset: 0,+ }+ : { orderBy, offset };// v4-enabled users (snapshotted as useEventsTable at dispatch) read// scores without the legacy traces JOIN; trace metadata (name,@@ -205,20 +283,19 @@ export const getDatabaseReadStreamPaginated = async ({? await getScoresWithTraceMetadataFromEvents({projectId,filter: scoresFilter,- orderBy,limit: pageSize,- offset,+ ...page,clickhouseConfigs,}): await getScoresUiTable({projectId,filter: scoresFilter,- orderBy,limit: pageSize,- offset,+ ...page,clickhouseConfigs,preferredClickhouseService,});+ keyset?.advance(scores);// Get author user info for scores// Only users that have valid project write access may write scores@@ -261,7 +338,7 @@ export const getDatabaseReadStreamPaginated = async ({};});},- env.BATCH_EXPORT_PAGE_SIZE,+ pageSize,rowLimit,);}@@ -349,7 +426,7 @@ export const getDatabaseReadStreamPaginated = async ({comments: sessionComments.get(row.id) ?? [],}));},- env.BATCH_EXPORT_PAGE_SIZE,+ pageSize,rowLimit,);case "observations": {@@ -423,7 +500,7 @@ export const getDatabaseReadStreamPaginated = async ({comments: observationComments.get(obs.id) ?? [],}));},- env.BATCH_EXPORT_PAGE_SIZE,+ pageSize,rowLimit,);}@@ -546,7 +623,7 @@ export const getDatabaseReadStreamPaginated = async ({emptyScoreColumns,);},- env.BATCH_EXPORT_PAGE_SIZE,+ pageSize,rowLimit,);}@@ -597,7 +674,7 @@ export const getDatabaseReadStreamPaginated = async ({};});},- env.BATCH_EXPORT_PAGE_SIZE,+ pageSize,rowLimit,);}@@ -627,27 +704,34 @@ export const getDatabaseReadStreamPaginated = async ({: "",}));},- env.BATCH_EXPORT_PAGE_SIZE,+ pageSize,rowLimit,);}case "audit_logs": {+ const keyset = new TimestampKeyset("DESC");+return new DatabaseReadStream<unknown>(- async (pageSize: number, offset: number) => {+ async (pageSize: number) => {+ const after = keyset.after;const auditLogs = await prisma.auditLog.findMany({where: {projectId: projectId,createdAt: {lt: cutoffCreatedAt,+ ...(after ? { lte: after.timestamp } : {}),},+ ...(after ? { id: { notIn: after.excludeIds } } : {}),},orderBy: {createdAt: "desc",},- skip: offset,take: pageSize,});+ keyset.advance(+ auditLogs.map((log) => ({ id: log.id, timestamp: log.createdAt })),+ );return auditLogs.map((log) => ({id: log.id,@@ -667,7 +751,7 @@ export const getDatabaseReadStreamPaginated = async ({after: log.after,}));},- env.BATCH_EXPORT_PAGE_SIZE,+ pageSize,rowLimit,);}