Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

perf(events): mirror base index predicates onto the io lane (#17480)

Niklas Semmler42b6f96b0ecc
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts
index 7fe98aebe8..4c8ea2bae6 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts
@@ -168,4 +168,28 @@ describe("buildEventsFullTableSplitQuery", () => {
expect(query).toContain("i.output as output");
expect(query).toContain("i.metadata as metadata");
});
+
+ it("bounds the io lane to base's start_time range and keeps the semi-join", () => {
+ const { query } = buildEventsFullTableSplitQuery({
+ projectId: "test-project",
+ baseBuilder: buildBase(),
+ includeIO: true,
+ includeMetadata: false,
+ }).buildWithParams();
+
+ // The bound is derived from base (no re-serialized params) so events_full
+ // can prune partitions/primary key; the semi-join stays for join exactness.
+ expect(query).toContain(
+ "SELECT min(start_time) AS io_min_start_time, max(start_time) AS io_max_start_time FROM base",
+ );
+ expect(query).toContain(
+ "AND e.start_time >= (SELECT io_min_start_time FROM io_bounds)",
+ );
+ expect(query).toContain(
+ "AND e.start_time <= (SELECT io_max_start_time FROM io_bounds)",
+ );
+ expect(query).toContain(
+ 'AND (e.start_time, e.trace_id, e.span_id) IN (SELECT "start_time", "trace_id", id FROM base)',
+ );
+ });
});
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
index 3bcde13662..eadebcf135 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
@@ -2136,12 +2136,21 @@ export function buildEventsFullTableSplitQuery(opts: {
"mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) as metadata",
);
}
+ // The tuple semi-join alone cannot prune events_full's primary key (raw
+ // start_time/trace_id vs the toStartOfMinute/xxHash32 key expressions), so
+ // bound start_time to base's own matched range. Derived from base (not the
+ // request filter) so it also tightens lookups that arrive without a time
+ // filter, and the values are never re-serialized as params.
const ioQuery = [
`SELECT ${ioSelectParts.join(", ")}`,
"FROM events_full e",
"WHERE e.project_id = {projectId: String}",
+ "AND e.start_time >= (SELECT io_min_start_time FROM io_bounds)",
+ "AND e.start_time <= (SELECT io_max_start_time FROM io_bounds)",
'AND (e.start_time, e.trace_id, e.span_id) IN (SELECT "start_time", "trace_id", id FROM base)',
].join("\n");
+ const ioBoundsQuery =
+ "SELECT min(start_time) AS io_min_start_time, max(start_time) AS io_max_start_time FROM base";
// Compose final query using CTEQueryBuilder
let cteBuilder = new CTEQueryBuilder();
@@ -2154,13 +2163,19 @@ export function buildEventsFullTableSplitQuery(opts: {
});
}
- // Register base and io CTEs, set up FROM and JOIN
+ // Register base, io_bounds, and io CTEs, set up FROM and JOIN. io_bounds must
+ // follow base (it aggregates over it) and precede io (which reads from it).
cteBuilder = cteBuilder
.withCTE("base", {
query: baseQuery,
params: baseParams,
schema: [] as string[],
})
+ .withCTE("io_bounds", {
+ query: ioBoundsQuery,
+ params: {},
+ schema: [] as string[],
+ })
.withCTE("io", {
query: ioQuery,
params: { projectId: opts.projectId },
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.ts
index 7a06b21a1a..f54bde9a02 100644
--- a/packages/shared/src/server/repositories/events.ts
+++ b/packages/shared/src/server/repositories/events.ts
@@ -1462,7 +1462,6 @@ function buildObservationsQueryComponents(
);
const filtersNeedFullTable = filtersRequireEventsFull(observationsFilter);
- // Extract time filter and apply filters
const startTimeFrom = extractTimeFilter(observationsFilter);
const appliedFilter = observationsFilter.apply();