🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(events): mirror base index predicates onto the io lane (#17480)
Niklas Semmler42b6f96b0ecc
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.tsindex 7fe98aebe8..4c8ea2bae6 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts@@ -168,4 +168,28 @@ describe("buildEventsFullTableSplitQuery", () => {expect(query).toContain("i.output as output");expect(query).toContain("i.metadata as metadata");});++ it("bounds the io lane to base's start_time range and keeps the semi-join", () => {+ const { query } = buildEventsFullTableSplitQuery({+ projectId: "test-project",+ baseBuilder: buildBase(),+ includeIO: true,+ includeMetadata: false,+ }).buildWithParams();++ // The bound is derived from base (no re-serialized params) so events_full+ // can prune partitions/primary key; the semi-join stays for join exactness.+ expect(query).toContain(+ "SELECT min(start_time) AS io_min_start_time, max(start_time) AS io_max_start_time FROM base",+ );+ expect(query).toContain(+ "AND e.start_time >= (SELECT io_min_start_time FROM io_bounds)",+ );+ expect(query).toContain(+ "AND e.start_time <= (SELECT io_max_start_time FROM io_bounds)",+ );+ expect(query).toContain(+ 'AND (e.start_time, e.trace_id, e.span_id) IN (SELECT "start_time", "trace_id", id FROM base)',+ );+ });});diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.tsindex 3bcde13662..eadebcf135 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts@@ -2136,12 +2136,21 @@ export function buildEventsFullTableSplitQuery(opts: {"mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) as metadata",);}+ // The tuple semi-join alone cannot prune events_full's primary key (raw+ // start_time/trace_id vs the toStartOfMinute/xxHash32 key expressions), so+ // bound start_time to base's own matched range. Derived from base (not the+ // request filter) so it also tightens lookups that arrive without a time+ // filter, and the values are never re-serialized as params.const ioQuery = [`SELECT ${ioSelectParts.join(", ")}`,"FROM events_full e","WHERE e.project_id = {projectId: String}",+ "AND e.start_time >= (SELECT io_min_start_time FROM io_bounds)",+ "AND e.start_time <= (SELECT io_max_start_time FROM io_bounds)",'AND (e.start_time, e.trace_id, e.span_id) IN (SELECT "start_time", "trace_id", id FROM base)',].join("\n");+ const ioBoundsQuery =+ "SELECT min(start_time) AS io_min_start_time, max(start_time) AS io_max_start_time FROM base";// Compose final query using CTEQueryBuilderlet cteBuilder = new CTEQueryBuilder();@@ -2154,13 +2163,19 @@ export function buildEventsFullTableSplitQuery(opts: {});}- // Register base and io CTEs, set up FROM and JOIN+ // Register base, io_bounds, and io CTEs, set up FROM and JOIN. io_bounds must+ // follow base (it aggregates over it) and precede io (which reads from it).cteBuilder = cteBuilder.withCTE("base", {query: baseQuery,params: baseParams,schema: [] as string[],})+ .withCTE("io_bounds", {+ query: ioBoundsQuery,+ params: {},+ schema: [] as string[],+ }).withCTE("io", {query: ioQuery,params: { projectId: opts.projectId },diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 7a06b21a1a..f54bde9a02 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -1462,7 +1462,6 @@ function buildObservationsQueryComponents();const filtersNeedFullTable = filtersRequireEventsFull(observationsFilter);- // Extract time filter and apply filtersconst startTimeFrom = extractTimeFilter(observationsFilter);const appliedFilter = observationsFilter.apply();