🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(query-ast): read events through the logical relation (#19029)
Niklas Semmler36a277fb2ec0
diff --git a/packages/shared/src/server/repositories/__snapshots__/environments.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/environments.golden.test.ts.snapindex a6715a551b..7a7e243d82 100644--- a/packages/shared/src/server/repositories/__snapshots__/environments.golden.test.ts.snap+++ b/packages/shared/src/server/repositories/__snapshots__/environments.golden.test.ts.snap@@ -11,9 +11,9 @@ exports[`golden: environments.getEnvironmentsForProject > writeMode=events_only},"route": "environments.getEnvironmentsForProject","sql": "SELECT DISTINCT environment-FROM events_core-WHERE (project_id = {param1:String})- AND (events_core.is_deleted = {param2:UInt8})+FROM events_core AS events+WHERE (events.project_id = {param1:String})+ AND (events.is_deleted = {param2:UInt8})AND (start_time >= {param3:DateTime64(6)})",},{@@ -48,9 +48,9 @@ exports[`golden: environments.getEnvironmentsForProject > writeMode=events_only},"route": "environments.getEnvironmentsForProject","sql": "SELECT DISTINCT environment-FROM events_core-WHERE (project_id = {param1:String})- AND (events_core.is_deleted = {param2:UInt8})",+FROM events_core AS events+WHERE (events.project_id = {param1:String})+ AND (events.is_deleted = {param2:UInt8})",},{"fn": "queryClickhouse",diff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snapindex bae881367b..ee2ad2d9d5 100644--- a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap@@ -513,9 +513,9 @@ WHERE (e.project_id = {param1:String})AND ((e.trace_id, e.span_id) IN ({param5:Array(Tuple(String, String))}))AND (e.trace_id IN (SELECT trace_id- FROM events_core- WHERE (project_id = {param1:String})- AND (events_core.is_deleted = {param2:UInt8})+ FROM events_core AS events+ WHERE (events.project_id = {param1:String})+ AND (events.is_deleted = {param2:UInt8})AND (trace_id IN ({param4:Array(String)}))GROUP BY trace_idHAVING argMaxIf(session_id, event_ts, session_id != '') = {param6:String}@@ -2932,9 +2932,9 @@ exports[`golden: events point-reads & existence family > hasAnySessionFromEvents},"route": "unknown","sql": "SELECT 1 AS one-FROM events_core-WHERE (project_id = {param1:String})- AND (events_core.is_deleted = {param2:UInt8})+FROM events_core AS events+WHERE (events.project_id = {param1:String})+ AND (events.is_deleted = {param2:UInt8})AND (session_id IS NOT NULL)AND (session_id != {param3:String})LIMIT {param4:Int64}",@@ -2953,9 +2953,9 @@ exports[`golden: events point-reads & existence family > hasAnyTraceFromEventsTa},"route": "unknown","sql": "SELECT 1 AS one-FROM events_core-WHERE (project_id = {param1:String})- AND (events_core.is_deleted = {param2:UInt8})+FROM events_core AS events+WHERE (events.project_id = {param1:String})+ AND (events.is_deleted = {param2:UInt8})LIMIT {param3:Int64}",},]@@ -2973,9 +2973,9 @@ exports[`golden: events point-reads & existence family > hasAnyUserFromEventsTab},"route": "unknown","sql": "SELECT 1 AS one-FROM events_core-WHERE (project_id = {param1:String})- AND (events_core.is_deleted = {param2:UInt8})+FROM events_core AS events+WHERE (events.project_id = {param1:String})+ AND (events.is_deleted = {param2:UInt8})AND (user_id IS NOT NULL)AND (user_id != {param3:String})LIMIT {param4:Int64}",@@ -3416,9 +3416,9 @@ exports[`golden: events simple aggregates family > getObservationsWithPromptName"sql": "SELECTuniq(span_id) AS count,prompt_name-FROM events_core-WHERE (project_id = {param1:String})- AND (events_core.is_deleted = {param2:UInt8})+FROM events_core AS events+WHERE (events.project_id = {param1:String})+ AND (events.is_deleted = {param2:UInt8})AND (prompt_name IN ({param3:Array(String)}))AND (prompt_name != {param4:String})GROUP BY prompt_name",@@ -3444,9 +3444,9 @@ exports[`golden: events simple aggregates family > getObservationsWithPromptName"sql": "SELECTuniq(span_id) AS count,prompt_name-FROM events_core-WHERE (project_id = {param1:String})- AND (events_core.is_deleted = {param2:UInt8})+FROM events_core AS events+WHERE (events.project_id = {param1:String})+ AND (events.is_deleted = {param2:UInt8})AND (prompt_name IN ({param3:Array(String)}))AND (prompt_name != {param4:String})AND (start_time >= {param5:DateTime64(6)})@@ -3474,9 +3474,9 @@ exports[`golden: events simple aggregates family > getObservationsWithPromptName"sql": "SELECTuniq(span_id) AS count,prompt_name-FROM events_core-WHERE (project_id = {param1:String})- AND (events_core.is_deleted = {param2:UInt8})+FROM events_core AS events+WHERE (events.project_id = {param1:String})+ AND (events.is_deleted = {param2:UInt8})AND (prompt_name IN ({param3:Array(String)}))AND (prompt_name != {param4:String})AND (start_time >= {param5:DateTime64(6)})@@ -3504,9 +3504,9 @@ exports[`golden: events simple aggregates family > getObservationsWithPromptName"sql": "SELECTuniq(span_id) AS count,prompt_name-FROM events_core-WHERE (project_id = {param1:String})- AND (events_core.is_deleted = {param2:UInt8})+FROM events_core AS events+WHERE (events.project_id = {param1:String})+ AND (events.is_deleted = {param2:UInt8})AND (prompt_name IN ({param3:Array(String)}))AND (prompt_name != {param4:String})AND (start_time <= {param5:DateTime64(6)})@@ -3532,9 +3532,9 @@ exports[`golden: events simple aggregates family > getObservationsWithPromptName"sql": "SELECTuniq(span_id) AS count,prompt_name-FROM events_core-WHERE (project_id = {param1:String})- AND (events_core.is_deleted = {param2:UInt8})+FROM events_core AS events+WHERE (events.project_id = {param1:String})+ AND (events.is_deleted = {param2:UInt8})AND (prompt_name IN ({param3:Array(String)}))AND (prompt_name != {param4:String})GROUP BY prompt_name",diff --git a/packages/shared/src/server/repositories/environments.ts b/packages/shared/src/server/repositories/environments.tsindex cea6d96b04..06e3035fa1 100644--- a/packages/shared/src/server/repositories/environments.ts+++ b/packages/shared/src/server/repositories/environments.ts@@ -41,7 +41,7 @@ export const getEnvironmentsForProject = async (),): db- .selectFrom("events_core")+ .selectFrom("events").select("environment").distinct().$if(fromTimestamp != null, (qb) =>diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 6de758985f..6b086b5b91 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -735,7 +735,7 @@ export const getObservationsWithPromptNameFromEvents = async (}: { fromTimestamp?: Date; toTimestamp?: Date } = {},) => {const builder = getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select((eb) => [uniq(eb.ref("span_id")).as("count"), "prompt_name"]).where("prompt_name", "in", promptNames).where("prompt_name", "!=", "")@@ -1388,8 +1388,8 @@ function compileHasAnyFromEventsTable(opts: {const db = getClickhouseKysely();const query = db- .selectFrom("events_core")- .select(sql<number>`1`.as("one"))+ .selectFrom("events")+ .select(literal(1).as("one")).$if(opts.kind === "user", (qb) =>qb.where("user_id", "is not", null).where("user_id", "!=", ""),)@@ -2856,7 +2856,7 @@ export const getObservationsBatchIOFromEventsTable = async <.$if(opts.sessionId !== undefined, (qb) =>qb.where("e.trace_id", "in", (eb) =>eb- .selectFrom("events_core")+ .selectFrom("events").select("trace_id").where("trace_id", "in", traceIds).groupBy("trace_id")@@ -2964,7 +2964,7 @@ function compileObservationIOFieldRead<A extends string>(select: (field: AliasableExpression<string>) => AliasedExpression<unknown, A>,) {const query = getClickhouseKysely()- .selectFrom("events_full as e")+ .selectFrom("events as e").$call(softDeleteAfterLimit()).select((eb) =>select(@@ -3084,7 +3084,7 @@ export const getObservationFullIOForSessionFromEventsTable = async (opts: {const db = getClickhouseKysely();// A soft-deleted latest version reads as not found.const builder = db- .selectFrom("events_full as e")+ .selectFrom("events as e").$call(softDeleteAfterLimit()).select((eb) => ["e.span_id as id",@@ -3154,7 +3154,7 @@ export const getObservationsTraceIdsFromEventsTable = async (opts: {const db = getClickhouseKysely();const builder = db- .selectFrom("events_core as e")+ .selectFrom("events as e").select(["e.trace_id as trace_id", "e.span_id as span_id"]).where("e.span_id", "in", observationIds);@@ -3633,7 +3633,7 @@ export const getTraceMetadataByIdsFromEvents = async (props: {const db = getClickhouseKysely();const builder = db- .selectFrom("events_core as e")+ .selectFrom("events as e").select(["e.trace_id as id",eventsTraceName.as("name"),@@ -3917,7 +3917,7 @@ export const getSessionMetricsFromEvents = async <const { projectId, sessionIds, fields, queryFromTimestamp } = props;const builder = getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select(fields.map((field) => SESSION_AGGREGATION_SELECTS[field])).where("session_id", "in", sessionIds).$if(queryFromTimestamp != null, (qb) =>@@ -3973,7 +3973,7 @@ export async function getLatestSdkVersionInfoFromEvents(params: {const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);const builder = getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select((eb) => ["e.ingestion_sdk_name as ingestion_sdk_name","e.ingestion_sdk_version as ingestion_sdk_version",diff --git a/packages/shared/src/server/repositories/experiments.ts b/packages/shared/src/server/repositories/experiments.tsindex 839f7df8d2..53676bce9e 100644--- a/packages/shared/src/server/repositories/experiments.ts+++ b/packages/shared/src/server/repositories/experiments.ts@@ -359,7 +359,7 @@ export const getExperimentMetricsFromEvents = async (props: {const { projectId, experimentIds } = props;const builder = getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select((eb) => ["e.experiment_id as experiment_id",eb.fn.sum("e.total_cost").as("total_cost"),