🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(web-callouts): bound observation lookups by trace timestamp (#16980)
Niklas Semmler4f9227d62150
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 3e4c5a5408..fda517cd3e 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -17,6 +17,7 @@ import {} from "../clickhouse/client";import { recordDistribution } from "../instrumentation";import { logger } from "../logger";+import { OBSERVATIONS_TO_TRACE_INTERVAL } from "./constants";import {convertClickhouseToDomain,convertClickhouseTracesListToDomain,@@ -921,6 +922,7 @@ export const getObservationByIdFromEventsTable = async ({projectId,fetchWithInputOutput = false,startTime,+ startTimeLowerBound,type,traceId,renderingProps = DEFAULT_RENDERING_PROPS,@@ -930,6 +932,7 @@ export const getObservationByIdFromEventsTable = async ({projectId: string;fetchWithInputOutput?: boolean;startTime?: Date;+ startTimeLowerBound?: Date;type?: ObservationType;traceId?: string;renderingProps?: RenderingProps;@@ -940,6 +943,7 @@ export const getObservationByIdFromEventsTable = async ({projectId,fetchWithInputOutput,startTime,+ startTimeLowerBound,type,traceId,renderingProps,@@ -988,6 +992,7 @@ async function getObservationByIdFromEventsTableInternal({projectId,fetchWithInputOutput = false,startTime,+ startTimeLowerBound,type,traceId,renderingProps = DEFAULT_RENDERING_PROPS,@@ -997,6 +1002,7 @@ async function getObservationByIdFromEventsTableInternal({projectId: string;fetchWithInputOutput?: boolean;startTime?: Date;+ startTimeLowerBound?: Date;type?: ObservationType;traceId?: string;renderingProps?: RenderingProps;@@ -1016,6 +1022,19 @@ async function getObservationByIdFromEventsTableInternal({startTime: convertDateToClickhouseDateTime(startTime!),}),)+ // Lower-bound start_time on an anchor (e.g. the parent trace's timestamp) so+ // the lookup can prune events_full parts/partitions. Subtract the skew+ // interval because an observation may start slightly before its anchor.+ .when(Boolean(startTimeLowerBound), (b) =>+ b.whereRaw(+ `start_time >= {startTimeLowerBound: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}`,+ {+ startTimeLowerBound: convertDateToClickhouseDateTime(+ startTimeLowerBound!,+ ),+ },+ ),+ ).when(Boolean(type), (b) => b.whereRaw("type = {type: String}", { type })).when(Boolean(traceId), (b) =>b.whereRaw("trace_id = {traceId: String}", { traceId }),diff --git a/packages/shared/src/server/repositories/observations.ts b/packages/shared/src/server/repositories/observations.tsindex 6a255e86c2..0be5d6076f 100644--- a/packages/shared/src/server/repositories/observations.ts+++ b/packages/shared/src/server/repositories/observations.ts@@ -333,6 +333,7 @@ export const getObservationByIdFromObservationsTable = async ({projectId,fetchWithInputOutput = false,startTime,+ startTimeLowerBound,type,traceId,renderingProps = DEFAULT_RENDERING_PROPS,@@ -342,6 +343,7 @@ export const getObservationByIdFromObservationsTable = async ({projectId: string;fetchWithInputOutput?: boolean;startTime?: Date;+ startTimeLowerBound?: Date;type?: ObservationType;traceId?: string;renderingProps?: RenderingProps;@@ -352,6 +354,7 @@ export const getObservationByIdFromObservationsTable = async ({projectId,fetchWithInputOutput,startTime,+ startTimeLowerBound,type,traceId,renderingProps,@@ -442,6 +445,7 @@ const getObservationByIdInternal = async ({projectId,fetchWithInputOutput = false,startTime,+ startTimeLowerBound,type,traceId,renderingProps = DEFAULT_RENDERING_PROPS,@@ -451,6 +455,7 @@ const getObservationByIdInternal = async ({projectId: string;fetchWithInputOutput?: boolean;startTime?: Date;+ startTimeLowerBound?: Date;type?: ObservationType;traceId?: string;renderingProps?: RenderingProps;@@ -496,6 +501,7 @@ const getObservationByIdInternal = async ({WHERE id = {id: String}AND project_id = {projectId: String}${startTime ? `AND toDate(start_time) = toDate({startTime: DateTime64(3)})` : ""}+ ${startTimeLowerBound ? `AND start_time >= {startTimeLowerBound: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}${type ? `AND type = {type: String}` : ""}${traceId ? `AND trace_id = {traceId: String}` : ""}ORDER BY event_ts desc@@ -508,6 +514,12 @@ const getObservationByIdInternal = async ({...(startTime? { startTime: convertDateToClickhouseDateTime(startTime) }: {}),+ ...(startTimeLowerBound+ ? {+ startTimeLowerBound:+ convertDateToClickhouseDateTime(startTimeLowerBound),+ }+ : {}),...(traceId ? { traceId } : {}),},tags: { projectId },diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.tsindex 6211e11165..9018ff6991 100644--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts@@ -867,6 +867,56 @@ describe("Clickhouse Events Repository Test", () => {});});+ maybe("getObservationByIdFromEventsTable startTimeLowerBound", () => {+ const DAY_MS = 24 * 60 * 60 * 1000;++ it("prunes on start_time while absorbing trace-to-observation skew", async () => {+ const traceId = randomUUID();+ const observationId = randomUUID();+ const observationStart = Date.now() - 5 * DAY_MS;++ await createEventsCh([+ createEvent({+ id: observationId,+ span_id: observationId,+ project_id: projectId,+ trace_id: traceId,+ type: "GENERATION",+ start_time: observationStart,+ }),+ ]);++ const atStart = await getObservationByIdFromEventsTable({+ id: observationId,+ projectId,+ traceId,+ startTimeLowerBound: new Date(observationStart),+ });+ expect(atStart?.id).toBe(observationId);++ // Anchor one day after start (e.g. a trace whose timestamp trails the+ // observation): still returned because the 2-day skew lookback covers it.+ const withinSkew = await getObservationByIdFromEventsTable({+ id: observationId,+ projectId,+ traceId,+ startTimeLowerBound: new Date(observationStart + DAY_MS),+ });+ expect(withinSkew?.id).toBe(observationId);++ // Anchor three days after start: the lower bound (anchor - 2 days) now+ // excludes the observation, proving the bound actually prunes.+ await expect(+ getObservationByIdFromEventsTable({+ id: observationId,+ projectId,+ traceId,+ startTimeLowerBound: new Date(observationStart + 3 * DAY_MS),+ }),+ ).rejects.toThrow();+ });+ });+maybe("getObservationsCountFromEventsTable", () => {it("should return 0 for non-existent project", async () => {const nonExistentProjectId = randomUUID();diff --git a/web/src/__tests__/server/web-callouts-trpc.servertest.ts b/web/src/__tests__/server/web-callouts-trpc.servertest.tsindex ef8d45b5a1..236198dafc 100644--- a/web/src/__tests__/server/web-callouts-trpc.servertest.ts+++ b/web/src/__tests__/server/web-callouts-trpc.servertest.ts@@ -60,10 +60,12 @@ const buildSession = ({orgId,projectId,projectRole = "ADMIN",+ v4BetaEnabled = false,}: {orgId: string;projectId: string;projectRole?: "ADMIN" | "MEMBER" | "NONE" | "OWNER" | "VIEWER";+ v4BetaEnabled?: boolean;}): Session => ({expires: "1",user: {@@ -103,7 +105,7 @@ const buildSession = ({experimentsV4Enabled: false,},admin: false,- v4BetaEnabled: false,+ v4BetaEnabled,},environment: {} as any,});@@ -268,15 +270,17 @@ const createPrismaStub = () => {const prepare = async ({projectRole = "ADMIN",+ v4BetaEnabled = false,}: {projectRole?: "ADMIN" | "MEMBER" | "NONE" | "OWNER" | "VIEWER";+ v4BetaEnabled?: boolean;} = {}) => {const orgId = "org-1";const projectId = "project-1";const prismaStub = createPrismaStub();const ctx = createInnerTRPCContext({- session: buildSession({ orgId, projectId, projectRole }),+ session: buildSession({ orgId, projectId, projectRole, v4BetaEnabled }),headers: {},});@@ -679,6 +683,149 @@ describe("webCallouts router", () => {);});+ it("widens the observation lookup bound when the trace timestamp misses it", async () => {+ const { caller, projectId } = await prepare();+ const traceTimestamp = new Date("2099-01-01T00:00:00.000Z");+ (getTraceById as Mock).mockResolvedValue({+ id: "trace-1",+ sessionId: null,+ timestamp: traceTimestamp,+ });+ // A trace whose timestamp sits far after the observation's start_time: the+ // first lookup, bounded by that timestamp, misses; the widened retry finds+ // it, so validation must succeed rather than 404.+ (getObservationById as Mock)+ .mockResolvedValueOnce(undefined)+ .mockResolvedValue({+ id: "observation-1",+ traceId: "trace-1",+ sessionId: null,+ });+ await createEndpoint(caller, projectId);++ await expect(+ caller.webCallouts.invoke({+ projectId,+ traceId: "trace-1",+ observationId: "observation-1",+ sessionId: null,+ }),+ ).resolves.toEqual({ success: true, status: 204 });++ expect(getObservationById).toHaveBeenCalledTimes(2);+ const calls = (getObservationById as Mock).mock.calls;+ expect(calls[0][0].startTimeLowerBound).toEqual(traceTimestamp);+ // No safe lower bound exists on a miss (legitimate backfills predate any+ // project-derived floor), so the retry is unbounded.+ expect(calls[1][0].startTimeLowerBound).toBeUndefined();+ });++ it("does not widen the observation lookup bound when the bounded lookup errors", async () => {+ const { caller, projectId } = await prepare();+ const traceTimestamp = new Date("2099-01-01T00:00:00.000Z");+ (getTraceById as Mock).mockResolvedValue({+ id: "trace-1",+ sessionId: null,+ timestamp: traceTimestamp,+ });+ // A transient failure (e.g. a ClickHouse timeout) must not be mistaken for a+ // genuine miss: retrying unbounded would issue a second expensive scan while+ // the backend is already struggling. Validation fails fast to a 404 instead.+ (getObservationById as Mock).mockRejectedValue(+ new Error("ClickHouse timeout"),+ );+ await createEndpoint(caller, projectId);++ await expect(+ caller.webCallouts.invoke({+ projectId,+ traceId: "trace-1",+ observationId: "observation-1",+ sessionId: null,+ }),+ ).rejects.toMatchObject({ code: "NOT_FOUND" });++ expect(getObservationById).toHaveBeenCalledTimes(1);+ expect(+ (getObservationById as Mock).mock.calls[0][0].startTimeLowerBound,+ ).toEqual(traceTimestamp);+ });++ it("still widens the bound when an earlier source in the chain transient-errors but a later one cleanly misses", async () => {+ const { caller, projectId } = await prepare({ v4BetaEnabled: true });+ const traceTimestamp = new Date("2099-01-01T00:00:00.000Z");+ (getTraceByIdFromEventsTable as Mock).mockResolvedValue({+ id: "trace-1",+ sessionId: null,+ timestamp: traceTimestamp,+ });+ // Bounded pass: the events-table lookup transient-errors, then the legacy+ // lookup cleanly misses. The transient error on the earlier source must not+ // suppress the unbounded retry, which finds the backfilled observation.+ (getObservationByIdFromEventsTable as Mock)+ .mockRejectedValueOnce(new Error("ClickHouse timeout"))+ .mockResolvedValue({+ id: "observation-1",+ traceId: "trace-1",+ sessionId: null,+ });+ (getObservationById as Mock).mockResolvedValue(undefined);+ await createEndpoint(caller, projectId);++ await expect(+ caller.webCallouts.invoke({+ projectId,+ traceId: "trace-1",+ observationId: "observation-1",+ sessionId: null,+ }),+ ).resolves.toEqual({ success: true, status: 204 });++ expect(getObservationByIdFromEventsTable).toHaveBeenCalledTimes(2);+ const eventsCalls = (getObservationByIdFromEventsTable as Mock).mock.calls;+ expect(eventsCalls[0][0].startTimeLowerBound).toEqual(traceTimestamp);+ expect(eventsCalls[1][0].startTimeLowerBound).toBeUndefined();+ });++ it("still widens the bound when an earlier source cleanly misses but a later one transient-errors", async () => {+ const { caller, projectId } = await prepare({ v4BetaEnabled: true });+ const traceTimestamp = new Date("2099-01-01T00:00:00.000Z");+ (getTraceByIdFromEventsTable as Mock).mockResolvedValue({+ id: "trace-1",+ sessionId: null,+ timestamp: traceTimestamp,+ });+ // Bounded pass: the events-table lookup cleanly misses, then the legacy+ // lookup transient-errors. The later transient error must not discard the+ // earlier clean-miss signal — the unbounded retry still runs and finds the+ // backfilled observation via the events table.+ (getObservationByIdFromEventsTable as Mock)+ .mockResolvedValueOnce(undefined)+ .mockResolvedValue({+ id: "observation-1",+ traceId: "trace-1",+ sessionId: null,+ });+ (getObservationById as Mock).mockRejectedValue(+ new Error("ClickHouse timeout"),+ );+ await createEndpoint(caller, projectId);++ await expect(+ caller.webCallouts.invoke({+ projectId,+ traceId: "trace-1",+ observationId: "observation-1",+ sessionId: null,+ }),+ ).resolves.toEqual({ success: true, status: 204 });++ expect(getObservationByIdFromEventsTable).toHaveBeenCalledTimes(2);+ const eventsCalls = (getObservationByIdFromEventsTable as Mock).mock.calls;+ expect(eventsCalls[0][0].startTimeLowerBound).toEqual(traceTimestamp);+ expect(eventsCalls[1][0].startTimeLowerBound).toBeUndefined();+ });+it("rate limits before target validation and outbound delivery", async () => {const { caller, projectId } = await prepare();await createEndpoint(caller, projectId);diff --git a/web/src/features/web-callouts/server/targetValidation.ts b/web/src/features/web-callouts/server/targetValidation.tsindex d3d8e43fcb..bc0752ceaa 100644--- a/web/src/features/web-callouts/server/targetValidation.ts+++ b/web/src/features/web-callouts/server/targetValidation.ts@@ -2,6 +2,7 @@ import { TRPCError } from "@trpc/server";import { env } from "@/src/env.mjs";import { type WebCalloutInvokeInput } from "@/src/features/web-callouts/types";+import { LangfuseNotFoundError } from "@langfuse/shared";import { type PrismaClient } from "@langfuse/shared/src/db";import {getObservationById,@@ -52,11 +53,18 @@ export const assertTargetBelongsToProject = async ({});}+ // The observation belongs to this trace, so the trace timestamp is a tight+ // lower bound for its start_time and lets the lookup prune events_full. On a+ // genuine miss the lookup retries unbounded internally: trace.timestamp is+ // an independent SDK-supplied field that can post-date a child observation's+ // start_time, and legitimate backfills mean an observation can be older than+ // any project-derived floor, so no safe fallback bound exists.const observation = await getObservationForProject({observationId: input.observationId,traceId: input.traceId,projectId: input.projectId,useEventsTable,+ startTimeLowerBound: trace?.timestamp,});if (!observation) {@@ -115,38 +123,130 @@ const getTraceForProject = async ({return undefined;};+// Validation only needs the observation's existence, so the lookup keeps just+// its identity — this stays assignable from both the observations- and+// events-table getters, whose full return shapes differ.+type WebCalloutObservation = { id: string };++// A single-source lookup either resolves to an observation, a genuine not-found+// (undefined observation, transientError false), or a transient failure+// (transientError true) that the helpers swallow so the caller can decide+// whether to retry.+type ObservationLookupResult = {+ observation?: WebCalloutObservation;+ transientError: boolean;+};++// The aggregate result of running the source chain at one bound. cleanMiss is+// true when at least one source returned a genuine not-found under the bound —+// the signal that an unbounded retry is worthwhile.+type ObservationChainResult = {+ observation?: WebCalloutObservation;+ cleanMiss: boolean;+};+const getObservationForProject = async ({observationId,traceId,projectId,useEventsTable,+ startTimeLowerBound,}: {observationId: string;traceId: string;projectId: string;useEventsTable: boolean;+ startTimeLowerBound?: Date;}) => {- if (useEventsTable) {- const observation = await tryGetObservationFromEvents({+ const bounded = await runObservationLookup({+ observationId,+ traceId,+ projectId,+ useEventsTable,+ startTimeLowerBound,+ });+ if (bounded.observation) return bounded.observation;++ // Retry unbounded only when at least one bounded source produced a genuine+ // clean miss: the observation could legitimately be older than the bound+ // (backfills, or trace.timestamp post-dating a child observation). If every+ // source instead transient-errored (timeout, ClickHouse unavailable) the+ // bounded pass told us nothing, so skip the second expensive unbounded scan+ // and fail fast — matching the pre-bounding behavior.+ if (startTimeLowerBound && bounded.cleanMiss) {+ const unbounded = await runObservationLookup({observationId,traceId,projectId,+ useEventsTable,+ startTimeLowerBound: undefined,});- if (observation) return observation;+ if (unbounded.observation) return unbounded.observation;}- const observation = await tryGetObservation({- observationId,- traceId,- projectId,- });- if (observation) return observation;+ return undefined;+};++const runObservationLookup = async ({+ observationId,+ traceId,+ projectId,+ useEventsTable,+ startTimeLowerBound,+}: {+ observationId: string;+ traceId: string;+ projectId: string;+ useEventsTable: boolean;+ startTimeLowerBound?: Date;+}): Promise<ObservationChainResult> => {+ // Remember a genuine clean miss from any source, even if a later source in the+ // chain transient-errors: the earlier clean miss still means the observation+ // could exist outside the bound, so the unbounded retry must stay eligible.+ let cleanMiss = false;++ const consider = (result: ObservationLookupResult) => {+ if (!result.observation && !result.transientError) {+ cleanMiss = true;+ }+ return result.observation;+ };++ if (useEventsTable) {+ const observation = consider(+ await tryGetObservationFromEvents({+ observationId,+ traceId,+ projectId,+ startTimeLowerBound,+ }),+ );+ if (observation) return { observation, cleanMiss };+ }++ const observation = consider(+ await tryGetObservation({+ observationId,+ traceId,+ projectId,+ startTimeLowerBound,+ }),+ );+ if (observation) return { observation, cleanMiss };if (shouldTryEventsTableFallback(useEventsTable)) {- return tryGetObservationFromEvents({ observationId, traceId, projectId });+ const fallback = consider(+ await tryGetObservationFromEvents({+ observationId,+ traceId,+ projectId,+ startTimeLowerBound,+ }),+ );+ if (fallback) return { observation: fallback, cleanMiss };}- return undefined;+ return { cleanMiss };};const tryGetTrace = async ({@@ -204,25 +304,40 @@ const tryGetObservation = async ({observationId,traceId,projectId,+ startTimeLowerBound,}: {observationId: string;traceId: string;projectId: string;-}) => {+ startTimeLowerBound?: Date;+}): Promise<ObservationLookupResult> => {try {// eslint-disable-next-line @typescript-eslint/no-deprecated -- Per customer requirement, web callouts should also work on v3.- return await getObservationById({+ const observation = await getObservationById({id: observationId,projectId,traceId,+ startTimeLowerBound,fetchWithInputOutput: false,renderingProps: {truncated: true,shouldJsonParse: false,},});- } catch {- return undefined;+ return { observation, transientError: false };+ } catch (error) {+ if (error instanceof LangfuseNotFoundError) {+ return { transientError: false };+ }+ logger.warn(+ "Failed to validate web callout observation via observations table",+ {+ projectId,+ observationId,+ error: error instanceof Error ? error.message : "Unknown error",+ },+ );+ return { transientError: true };}};@@ -230,29 +345,36 @@ const tryGetObservationFromEvents = async ({observationId,traceId,projectId,+ startTimeLowerBound,}: {observationId: string;traceId: string;projectId: string;-}) => {+ startTimeLowerBound?: Date;+}): Promise<ObservationLookupResult> => {try {- return await getObservationByIdFromEventsTable({+ const observation = await getObservationByIdFromEventsTable({id: observationId,projectId,traceId,+ startTimeLowerBound,fetchWithInputOutput: false,renderingProps: {truncated: true,shouldJsonParse: false,},});+ return { observation, transientError: false };} catch (error) {+ if (error instanceof LangfuseNotFoundError) {+ return { transientError: false };+ }logger.warn("Failed to validate web callout observation via events table", {projectId,observationId,error: error instanceof Error ? error.message : "Unknown error",});- return undefined;+ return { transientError: true };}};