🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(events): skip io-lane split when a content filter forces events_full (#17542)
Niklas Semmler463d8a287fcd
diff --git a/packages/shared/src/server/repositories/events.test.ts b/packages/shared/src/server/repositories/events.test.tsnew file mode 100644index 0000000000..0be3b4ca53--- /dev/null+++ b/packages/shared/src/server/repositories/events.test.ts@@ -0,0 +1,73 @@+import { describe, it, expect, beforeEach, vi } from "vitest";++const mockQueryClickhouse = vi.hoisted(() => vi.fn());+let charCounter = 0;++vi.mock("./clickhouse", () => ({+ queryClickhouse: mockQueryClickhouse,+ commandClickhouse: vi.fn(),+ queryClickhouseStream: vi.fn(),+ queryClickhouseStreamRawText: vi.fn(),+ queryClickhouseExecRaw: vi.fn(),+ parseClickhouseUTCDateTimeFormat: vi.fn(),+ BLOB_EXPORT_PARQUET_CLICKHOUSE_SETTINGS: {},+ // Return unique names per call so filter parameter variables don't collide+ clickhouseCompliantRandomCharacters: vi.fn(() => `x${++charCounter}`),+}));++import { getObservationsV2FromEventsTableForPublicApi } from "./events";+import { type EventsTableFilterState } from "../../types";++const inputContainsFilter: EventsTableFilterState = [+ { type: "string", column: "input", operator: "contains", value: "needle" },+];++const captureQuery = () => {+ const call = mockQueryClickhouse.mock.calls.at(0);+ return (call?.[0] as { query: string }).query;+};++const baseOpts = { projectId: "proj-1", page: 1, limit: 50 } as const;++describe("getObservationsV2FromEventsTableForPublicApi query shape", () => {+ beforeEach(() => {+ charCounter = 0;+ vi.clearAllMocks();+ mockQueryClickhouse.mockResolvedValue([]);+ });++ it("skips the io-lane split when a content filter forces base onto events_full", async () => {+ await getObservationsV2FromEventsTableForPublicApi(+ {+ ...baseOpts,+ fields: ["core", "io"],+ advancedFilters: inputContainsFilter,+ },+ { allowUnindexedIoFilters: true },+ );++ const query = captureQuery();+ // Single-pass scan on events_full: no base/io CTE split, io read inline.+ expect(query).not.toContain("FROM base");+ expect(query).not.toContain("_io_start_time");+ expect(query).toContain("e.input");+ expect(query).toContain("e.output");+ // Exactly one events_full read (the split would reference it twice).+ expect((query.match(/events_full/g) ?? []).length).toBe(1);+ expect(query).not.toContain("events_core");+ });++ it("keeps the io-lane split when no content filter forces events_full", async () => {+ await getObservationsV2FromEventsTableForPublicApi({+ ...baseOpts,+ fields: ["core", "io"],+ });++ const query = captureQuery();+ // Base filters on the cheap truncated table; io lane reads full columns+ // only for the matched, page-sized rows.+ expect(query).toContain("FROM events_core");+ expect(query).toContain("FROM base");+ expect(query).toContain("_io_start_time");+ });+});diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 316237b4c7..6026451442 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -1438,6 +1438,7 @@ function buildObservationsQueryComponents(name: string;queryWithParams: { query: string; params: Record<string, any> };}>;+ filtersNeedFullTable: boolean;} {const { projectId, advancedFilters, ...filterParams } = opts;@@ -1493,7 +1494,7 @@ function buildObservationsQueryComponents().where(appliedFilter);- return { queryBuilder, externalCTEs };+ return { queryBuilder, externalCTEs, filtersNeedFullTable };}function buildObservationsQueryBase(@@ -1681,36 +1682,50 @@ export const getObservationsV2FromEventsTableForPublicApi = async (expandMetadataKeys != null &&expandMetadataKeys.length > 0;const needsIOCTE = needsIO || needsExpandedMetadata;- // Metadata goes to io CTE when in CTE mode and metadata is requested- const metadataFromFullTable =- needsIOCTE && requestedFields.includes("metadata");// Shared: build base query with field sets, ordering, pagination- const { queryBuilder: baseBuilder, externalCTEs } =- buildObservationsQueryComponents(- opts,- eventsTableNativeUiColumnDefinitions,- options,- );+ const {+ queryBuilder: baseBuilder,+ externalCTEs,+ filtersNeedFullTable,+ } = buildObservationsQueryComponents(+ opts,+ eventsTableNativeUiColumnDefinitions,+ options,+ );++ // The io lane only pays off when base reads the cheap truncated table+ // (events_core) and the split defers the heavy input/output/metadata columns+ // to a page-size lookup on events_full. When a filter touches input/output,+ // `filtersNeedFullTable` already forces base onto events_full to evaluate that+ // filter, so there is nothing left to defer — the split just re-scans+ // events_full several times (base is referenced from the io lane and the final+ // join, and ClickHouse inlines CTEs). Read io/metadata inline on base instead.+ const useSplit = needsIOCTE && !filtersNeedFullTable;++ // Metadata goes to the io CTE only when we actually split; on the simple path+ // base already reads events_full (forced by the filter) so metadata is+ // untruncated there.+ const metadataFromIoCte = useSplit && requestedFields.includes("metadata");// Shared steps: ordering and pagination apply to every path and are// independent of which columns each path projects.applyOrderByForObservationsQuery(baseBuilder);applyCursorPagination(opts, baseBuilder);- // Pick the query shape. When IO/expanded metadata is requested, use the- // CTE+JOIN split query that fetches those columns from events_full for the- // matched rows only; otherwise everything comes from events_core directly.- const queryPath: "simple" | "cte-join" = needsIOCTE ? "cte-join" : "simple";+ const queryPath: "simple" | "cte-join" = useSplit ? "cte-join" : "simple";- // Field groups projected on the base builder.- // `io` (and `metadata` when it must come untruncated from events_full) is- // fetched by the io CTE instead; selecting it on events_core would return- // truncated values.+ // Field groups projected on the base builder. In the split, `io` (and+ // `metadata`) is fetched by the io CTE instead; selecting it on events_core+ // would return truncated values. On the simple path base is on events_full+ // whenever io/metadata is requested (filter-forced), so it is selected inline.const selectBaseFieldSets = () => {baseBuilder.selectFieldSet("core");- const excludeFromBase = new Set<string>(["core", "io"]);- if (metadataFromFullTable) excludeFromBase.add("metadata");+ const excludeFromBase = new Set<string>(["core"]);+ if (useSplit) {+ excludeFromBase.add("io");+ if (metadataFromIoCte) excludeFromBase.add("metadata");+ }requestedFields.filter((fg) => !excludeFromBase.has(fg)).forEach((fg) => baseBuilder.selectFieldSet(fg));@@ -1733,7 +1748,7 @@ export const getObservationsV2FromEventsTableForPublicApi = async (projectId,baseBuilder,includeIO: needsIO,- includeMetadata: metadataFromFullTable,+ includeMetadata: metadataFromIoCte,externalCTEs,}).orderByColumns(orderByForObservationsQuery("b", "id"));break;