Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

perf(web-callouts): bound observation lookups by trace timestamp (#16980)

Niklas Semmler4f9227d62150
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.ts
index 3e4c5a5408..fda517cd3e 100644
--- a/packages/shared/src/server/repositories/events.ts
+++ b/packages/shared/src/server/repositories/events.ts
@@ -17,6 +17,7 @@ import {
} from "../clickhouse/client";
import { recordDistribution } from "../instrumentation";
import { logger } from "../logger";
+import { OBSERVATIONS_TO_TRACE_INTERVAL } from "./constants";
import {
convertClickhouseToDomain,
convertClickhouseTracesListToDomain,
@@ -921,6 +922,7 @@ export const getObservationByIdFromEventsTable = async ({
projectId,
fetchWithInputOutput = false,
startTime,
+ startTimeLowerBound,
type,
traceId,
renderingProps = DEFAULT_RENDERING_PROPS,
@@ -930,6 +932,7 @@ export const getObservationByIdFromEventsTable = async ({
projectId: string;
fetchWithInputOutput?: boolean;
startTime?: Date;
+ startTimeLowerBound?: Date;
type?: ObservationType;
traceId?: string;
renderingProps?: RenderingProps;
@@ -940,6 +943,7 @@ export const getObservationByIdFromEventsTable = async ({
projectId,
fetchWithInputOutput,
startTime,
+ startTimeLowerBound,
type,
traceId,
renderingProps,
@@ -988,6 +992,7 @@ async function getObservationByIdFromEventsTableInternal({
projectId,
fetchWithInputOutput = false,
startTime,
+ startTimeLowerBound,
type,
traceId,
renderingProps = DEFAULT_RENDERING_PROPS,
@@ -997,6 +1002,7 @@ async function getObservationByIdFromEventsTableInternal({
projectId: string;
fetchWithInputOutput?: boolean;
startTime?: Date;
+ startTimeLowerBound?: Date;
type?: ObservationType;
traceId?: string;
renderingProps?: RenderingProps;
@@ -1016,6 +1022,19 @@ async function getObservationByIdFromEventsTableInternal({
startTime: convertDateToClickhouseDateTime(startTime!),
}),
)
+ // Lower-bound start_time on an anchor (e.g. the parent trace's timestamp) so
+ // the lookup can prune events_full parts/partitions. Subtract the skew
+ // interval because an observation may start slightly before its anchor.
+ .when(Boolean(startTimeLowerBound), (b) =>
+ b.whereRaw(
+ `start_time >= {startTimeLowerBound: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}`,
+ {
+ startTimeLowerBound: convertDateToClickhouseDateTime(
+ startTimeLowerBound!,
+ ),
+ },
+ ),
+ )
.when(Boolean(type), (b) => b.whereRaw("type = {type: String}", { type }))
.when(Boolean(traceId), (b) =>
b.whereRaw("trace_id = {traceId: String}", { traceId }),
diff --git a/packages/shared/src/server/repositories/observations.ts b/packages/shared/src/server/repositories/observations.ts
index 6a255e86c2..0be5d6076f 100644
--- a/packages/shared/src/server/repositories/observations.ts
+++ b/packages/shared/src/server/repositories/observations.ts
@@ -333,6 +333,7 @@ export const getObservationByIdFromObservationsTable = async ({
projectId,
fetchWithInputOutput = false,
startTime,
+ startTimeLowerBound,
type,
traceId,
renderingProps = DEFAULT_RENDERING_PROPS,
@@ -342,6 +343,7 @@ export const getObservationByIdFromObservationsTable = async ({
projectId: string;
fetchWithInputOutput?: boolean;
startTime?: Date;
+ startTimeLowerBound?: Date;
type?: ObservationType;
traceId?: string;
renderingProps?: RenderingProps;
@@ -352,6 +354,7 @@ export const getObservationByIdFromObservationsTable = async ({
projectId,
fetchWithInputOutput,
startTime,
+ startTimeLowerBound,
type,
traceId,
renderingProps,
@@ -442,6 +445,7 @@ const getObservationByIdInternal = async ({
projectId,
fetchWithInputOutput = false,
startTime,
+ startTimeLowerBound,
type,
traceId,
renderingProps = DEFAULT_RENDERING_PROPS,
@@ -451,6 +455,7 @@ const getObservationByIdInternal = async ({
projectId: string;
fetchWithInputOutput?: boolean;
startTime?: Date;
+ startTimeLowerBound?: Date;
type?: ObservationType;
traceId?: string;
renderingProps?: RenderingProps;
@@ -496,6 +501,7 @@ const getObservationByIdInternal = async ({
WHERE id = {id: String}
AND project_id = {projectId: String}
${startTime ? `AND toDate(start_time) = toDate({startTime: DateTime64(3)})` : ""}
+ ${startTimeLowerBound ? `AND start_time >= {startTimeLowerBound: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
${type ? `AND type = {type: String}` : ""}
${traceId ? `AND trace_id = {traceId: String}` : ""}
ORDER BY event_ts desc
@@ -508,6 +514,12 @@ const getObservationByIdInternal = async ({
...(startTime
? { startTime: convertDateToClickhouseDateTime(startTime) }
: {}),
+ ...(startTimeLowerBound
+ ? {
+ startTimeLowerBound:
+ convertDateToClickhouseDateTime(startTimeLowerBound),
+ }
+ : {}),
...(traceId ? { traceId } : {}),
},
tags: { projectId },
diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.ts
index 6211e11165..9018ff6991 100644
--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts
@@ -867,6 +867,56 @@ describe("Clickhouse Events Repository Test", () => {
});
});
+ maybe("getObservationByIdFromEventsTable startTimeLowerBound", () => {
+ const DAY_MS = 24 * 60 * 60 * 1000;
+
+ it("prunes on start_time while absorbing trace-to-observation skew", async () => {
+ const traceId = randomUUID();
+ const observationId = randomUUID();
+ const observationStart = Date.now() - 5 * DAY_MS;
+
+ await createEventsCh([
+ createEvent({
+ id: observationId,
+ span_id: observationId,
+ project_id: projectId,
+ trace_id: traceId,
+ type: "GENERATION",
+ start_time: observationStart,
+ }),
+ ]);
+
+ const atStart = await getObservationByIdFromEventsTable({
+ id: observationId,
+ projectId,
+ traceId,
+ startTimeLowerBound: new Date(observationStart),
+ });
+ expect(atStart?.id).toBe(observationId);
+
+ // Anchor one day after start (e.g. a trace whose timestamp trails the
+ // observation): still returned because the 2-day skew lookback covers it.
+ const withinSkew = await getObservationByIdFromEventsTable({
+ id: observationId,
+ projectId,
+ traceId,
+ startTimeLowerBound: new Date(observationStart + DAY_MS),
+ });
+ expect(withinSkew?.id).toBe(observationId);
+
+ // Anchor three days after start: the lower bound (anchor - 2 days) now
+ // excludes the observation, proving the bound actually prunes.
+ await expect(
+ getObservationByIdFromEventsTable({
+ id: observationId,
+ projectId,
+ traceId,
+ startTimeLowerBound: new Date(observationStart + 3 * DAY_MS),
+ }),
+ ).rejects.toThrow();
+ });
+ });
+
maybe("getObservationsCountFromEventsTable", () => {
it("should return 0 for non-existent project", async () => {
const nonExistentProjectId = randomUUID();
diff --git a/web/src/__tests__/server/web-callouts-trpc.servertest.ts b/web/src/__tests__/server/web-callouts-trpc.servertest.ts
index ef8d45b5a1..236198dafc 100644
--- a/web/src/__tests__/server/web-callouts-trpc.servertest.ts
+++ b/web/src/__tests__/server/web-callouts-trpc.servertest.ts
@@ -60,10 +60,12 @@ const buildSession = ({
orgId,
projectId,
projectRole = "ADMIN",
+ v4BetaEnabled = false,
}: {
orgId: string;
projectId: string;
projectRole?: "ADMIN" | "MEMBER" | "NONE" | "OWNER" | "VIEWER";
+ v4BetaEnabled?: boolean;
}): Session => ({
expires: "1",
user: {
@@ -103,7 +105,7 @@ const buildSession = ({
experimentsV4Enabled: false,
},
admin: false,
- v4BetaEnabled: false,
+ v4BetaEnabled,
},
environment: {} as any,
});
@@ -268,15 +270,17 @@ const createPrismaStub = () => {
const prepare = async ({
projectRole = "ADMIN",
+ v4BetaEnabled = false,
}: {
projectRole?: "ADMIN" | "MEMBER" | "NONE" | "OWNER" | "VIEWER";
+ v4BetaEnabled?: boolean;
} = {}) => {
const orgId = "org-1";
const projectId = "project-1";
const prismaStub = createPrismaStub();
const ctx = createInnerTRPCContext({
- session: buildSession({ orgId, projectId, projectRole }),
+ session: buildSession({ orgId, projectId, projectRole, v4BetaEnabled }),
headers: {},
});
@@ -679,6 +683,149 @@ describe("webCallouts router", () => {
);
});
+ it("widens the observation lookup bound when the trace timestamp misses it", async () => {
+ const { caller, projectId } = await prepare();
+ const traceTimestamp = new Date("2099-01-01T00:00:00.000Z");
+ (getTraceById as Mock).mockResolvedValue({
+ id: "trace-1",
+ sessionId: null,
+ timestamp: traceTimestamp,
+ });
+ // A trace whose timestamp sits far after the observation's start_time: the
+ // first lookup, bounded by that timestamp, misses; the widened retry finds
+ // it, so validation must succeed rather than 404.
+ (getObservationById as Mock)
+ .mockResolvedValueOnce(undefined)
+ .mockResolvedValue({
+ id: "observation-1",
+ traceId: "trace-1",
+ sessionId: null,
+ });
+ await createEndpoint(caller, projectId);
+
+ await expect(
+ caller.webCallouts.invoke({
+ projectId,
+ traceId: "trace-1",
+ observationId: "observation-1",
+ sessionId: null,
+ }),
+ ).resolves.toEqual({ success: true, status: 204 });
+
+ expect(getObservationById).toHaveBeenCalledTimes(2);
+ const calls = (getObservationById as Mock).mock.calls;
+ expect(calls[0][0].startTimeLowerBound).toEqual(traceTimestamp);
+ // No safe lower bound exists on a miss (legitimate backfills predate any
+ // project-derived floor), so the retry is unbounded.
+ expect(calls[1][0].startTimeLowerBound).toBeUndefined();
+ });
+
+ it("does not widen the observation lookup bound when the bounded lookup errors", async () => {
+ const { caller, projectId } = await prepare();
+ const traceTimestamp = new Date("2099-01-01T00:00:00.000Z");
+ (getTraceById as Mock).mockResolvedValue({
+ id: "trace-1",
+ sessionId: null,
+ timestamp: traceTimestamp,
+ });
+ // A transient failure (e.g. a ClickHouse timeout) must not be mistaken for a
+ // genuine miss: retrying unbounded would issue a second expensive scan while
+ // the backend is already struggling. Validation fails fast to a 404 instead.
+ (getObservationById as Mock).mockRejectedValue(
+ new Error("ClickHouse timeout"),
+ );
+ await createEndpoint(caller, projectId);
+
+ await expect(
+ caller.webCallouts.invoke({
+ projectId,
+ traceId: "trace-1",
+ observationId: "observation-1",
+ sessionId: null,
+ }),
+ ).rejects.toMatchObject({ code: "NOT_FOUND" });
+
+ expect(getObservationById).toHaveBeenCalledTimes(1);
+ expect(
+ (getObservationById as Mock).mock.calls[0][0].startTimeLowerBound,
+ ).toEqual(traceTimestamp);
+ });
+
+ it("still widens the bound when an earlier source in the chain transient-errors but a later one cleanly misses", async () => {
+ const { caller, projectId } = await prepare({ v4BetaEnabled: true });
+ const traceTimestamp = new Date("2099-01-01T00:00:00.000Z");
+ (getTraceByIdFromEventsTable as Mock).mockResolvedValue({
+ id: "trace-1",
+ sessionId: null,
+ timestamp: traceTimestamp,
+ });
+ // Bounded pass: the events-table lookup transient-errors, then the legacy
+ // lookup cleanly misses. The transient error on the earlier source must not
+ // suppress the unbounded retry, which finds the backfilled observation.
+ (getObservationByIdFromEventsTable as Mock)
+ .mockRejectedValueOnce(new Error("ClickHouse timeout"))
+ .mockResolvedValue({
+ id: "observation-1",
+ traceId: "trace-1",
+ sessionId: null,
+ });
+ (getObservationById as Mock).mockResolvedValue(undefined);
+ await createEndpoint(caller, projectId);
+
+ await expect(
+ caller.webCallouts.invoke({
+ projectId,
+ traceId: "trace-1",
+ observationId: "observation-1",
+ sessionId: null,
+ }),
+ ).resolves.toEqual({ success: true, status: 204 });
+
+ expect(getObservationByIdFromEventsTable).toHaveBeenCalledTimes(2);
+ const eventsCalls = (getObservationByIdFromEventsTable as Mock).mock.calls;
+ expect(eventsCalls[0][0].startTimeLowerBound).toEqual(traceTimestamp);
+ expect(eventsCalls[1][0].startTimeLowerBound).toBeUndefined();
+ });
+
+ it("still widens the bound when an earlier source cleanly misses but a later one transient-errors", async () => {
+ const { caller, projectId } = await prepare({ v4BetaEnabled: true });
+ const traceTimestamp = new Date("2099-01-01T00:00:00.000Z");
+ (getTraceByIdFromEventsTable as Mock).mockResolvedValue({
+ id: "trace-1",
+ sessionId: null,
+ timestamp: traceTimestamp,
+ });
+ // Bounded pass: the events-table lookup cleanly misses, then the legacy
+ // lookup transient-errors. The later transient error must not discard the
+ // earlier clean-miss signal — the unbounded retry still runs and finds the
+ // backfilled observation via the events table.
+ (getObservationByIdFromEventsTable as Mock)
+ .mockResolvedValueOnce(undefined)
+ .mockResolvedValue({
+ id: "observation-1",
+ traceId: "trace-1",
+ sessionId: null,
+ });
+ (getObservationById as Mock).mockRejectedValue(
+ new Error("ClickHouse timeout"),
+ );
+ await createEndpoint(caller, projectId);
+
+ await expect(
+ caller.webCallouts.invoke({
+ projectId,
+ traceId: "trace-1",
+ observationId: "observation-1",
+ sessionId: null,
+ }),
+ ).resolves.toEqual({ success: true, status: 204 });
+
+ expect(getObservationByIdFromEventsTable).toHaveBeenCalledTimes(2);
+ const eventsCalls = (getObservationByIdFromEventsTable as Mock).mock.calls;
+ expect(eventsCalls[0][0].startTimeLowerBound).toEqual(traceTimestamp);
+ expect(eventsCalls[1][0].startTimeLowerBound).toBeUndefined();
+ });
+
it("rate limits before target validation and outbound delivery", async () => {
const { caller, projectId } = await prepare();
await createEndpoint(caller, projectId);
diff --git a/web/src/features/web-callouts/server/targetValidation.ts b/web/src/features/web-callouts/server/targetValidation.ts
index d3d8e43fcb..bc0752ceaa 100644
--- a/web/src/features/web-callouts/server/targetValidation.ts
+++ b/web/src/features/web-callouts/server/targetValidation.ts
@@ -2,6 +2,7 @@ import { TRPCError } from "@trpc/server";
import { env } from "@/src/env.mjs";
import { type WebCalloutInvokeInput } from "@/src/features/web-callouts/types";
+import { LangfuseNotFoundError } from "@langfuse/shared";
import { type PrismaClient } from "@langfuse/shared/src/db";
import {
getObservationById,
@@ -52,11 +53,18 @@ export const assertTargetBelongsToProject = async ({
});
}
+ // The observation belongs to this trace, so the trace timestamp is a tight
+ // lower bound for its start_time and lets the lookup prune events_full. On a
+ // genuine miss the lookup retries unbounded internally: trace.timestamp is
+ // an independent SDK-supplied field that can post-date a child observation's
+ // start_time, and legitimate backfills mean an observation can be older than
+ // any project-derived floor, so no safe fallback bound exists.
const observation = await getObservationForProject({
observationId: input.observationId,
traceId: input.traceId,
projectId: input.projectId,
useEventsTable,
+ startTimeLowerBound: trace?.timestamp,
});
if (!observation) {
@@ -115,38 +123,130 @@ const getTraceForProject = async ({
return undefined;
};
+// Validation only needs the observation's existence, so the lookup keeps just
+// its identity — this stays assignable from both the observations- and
+// events-table getters, whose full return shapes differ.
+type WebCalloutObservation = { id: string };
+
+// A single-source lookup either resolves to an observation, a genuine not-found
+// (undefined observation, transientError false), or a transient failure
+// (transientError true) that the helpers swallow so the caller can decide
+// whether to retry.
+type ObservationLookupResult = {
+ observation?: WebCalloutObservation;
+ transientError: boolean;
+};
+
+// The aggregate result of running the source chain at one bound. cleanMiss is
+// true when at least one source returned a genuine not-found under the bound —
+// the signal that an unbounded retry is worthwhile.
+type ObservationChainResult = {
+ observation?: WebCalloutObservation;
+ cleanMiss: boolean;
+};
+
const getObservationForProject = async ({
observationId,
traceId,
projectId,
useEventsTable,
+ startTimeLowerBound,
}: {
observationId: string;
traceId: string;
projectId: string;
useEventsTable: boolean;
+ startTimeLowerBound?: Date;
}) => {
- if (useEventsTable) {
- const observation = await tryGetObservationFromEvents({
+ const bounded = await runObservationLookup({
+ observationId,
+ traceId,
+ projectId,
+ useEventsTable,
+ startTimeLowerBound,
+ });
+ if (bounded.observation) return bounded.observation;
+
+ // Retry unbounded only when at least one bounded source produced a genuine
+ // clean miss: the observation could legitimately be older than the bound
+ // (backfills, or trace.timestamp post-dating a child observation). If every
+ // source instead transient-errored (timeout, ClickHouse unavailable) the
+ // bounded pass told us nothing, so skip the second expensive unbounded scan
+ // and fail fast — matching the pre-bounding behavior.
+ if (startTimeLowerBound && bounded.cleanMiss) {
+ const unbounded = await runObservationLookup({
observationId,
traceId,
projectId,
+ useEventsTable,
+ startTimeLowerBound: undefined,
});
- if (observation) return observation;
+ if (unbounded.observation) return unbounded.observation;
}
- const observation = await tryGetObservation({
- observationId,
- traceId,
- projectId,
- });
- if (observation) return observation;
+ return undefined;
+};
+
+const runObservationLookup = async ({
+ observationId,
+ traceId,
+ projectId,
+ useEventsTable,
+ startTimeLowerBound,
+}: {
+ observationId: string;
+ traceId: string;
+ projectId: string;
+ useEventsTable: boolean;
+ startTimeLowerBound?: Date;
+}): Promise<ObservationChainResult> => {
+ // Remember a genuine clean miss from any source, even if a later source in the
+ // chain transient-errors: the earlier clean miss still means the observation
+ // could exist outside the bound, so the unbounded retry must stay eligible.
+ let cleanMiss = false;
+
+ const consider = (result: ObservationLookupResult) => {
+ if (!result.observation && !result.transientError) {
+ cleanMiss = true;
+ }
+ return result.observation;
+ };
+
+ if (useEventsTable) {
+ const observation = consider(
+ await tryGetObservationFromEvents({
+ observationId,
+ traceId,
+ projectId,
+ startTimeLowerBound,
+ }),
+ );
+ if (observation) return { observation, cleanMiss };
+ }
+
+ const observation = consider(
+ await tryGetObservation({
+ observationId,
+ traceId,
+ projectId,
+ startTimeLowerBound,
+ }),
+ );
+ if (observation) return { observation, cleanMiss };
if (shouldTryEventsTableFallback(useEventsTable)) {
- return tryGetObservationFromEvents({ observationId, traceId, projectId });
+ const fallback = consider(
+ await tryGetObservationFromEvents({
+ observationId,
+ traceId,
+ projectId,
+ startTimeLowerBound,
+ }),
+ );
+ if (fallback) return { observation: fallback, cleanMiss };
}
- return undefined;
+ return { cleanMiss };
};
const tryGetTrace = async ({
@@ -204,25 +304,40 @@ const tryGetObservation = async ({
observationId,
traceId,
projectId,
+ startTimeLowerBound,
}: {
observationId: string;
traceId: string;
projectId: string;
-}) => {
+ startTimeLowerBound?: Date;
+}): Promise<ObservationLookupResult> => {
try {
// eslint-disable-next-line @typescript-eslint/no-deprecated -- Per customer requirement, web callouts should also work on v3.
- return await getObservationById({
+ const observation = await getObservationById({
id: observationId,
projectId,
traceId,
+ startTimeLowerBound,
fetchWithInputOutput: false,
renderingProps: {
truncated: true,
shouldJsonParse: false,
},
});
- } catch {
- return undefined;
+ return { observation, transientError: false };
+ } catch (error) {
+ if (error instanceof LangfuseNotFoundError) {
+ return { transientError: false };
+ }
+ logger.warn(
+ "Failed to validate web callout observation via observations table",
+ {
+ projectId,
+ observationId,
+ error: error instanceof Error ? error.message : "Unknown error",
+ },
+ );
+ return { transientError: true };
}
};
@@ -230,29 +345,36 @@ const tryGetObservationFromEvents = async ({
observationId,
traceId,
projectId,
+ startTimeLowerBound,
}: {
observationId: string;
traceId: string;
projectId: string;
-}) => {
+ startTimeLowerBound?: Date;
+}): Promise<ObservationLookupResult> => {
try {
- return await getObservationByIdFromEventsTable({
+ const observation = await getObservationByIdFromEventsTable({
id: observationId,
projectId,
traceId,
+ startTimeLowerBound,
fetchWithInputOutput: false,
renderingProps: {
truncated: true,
shouldJsonParse: false,
},
});
+ return { observation, transientError: false };
} catch (error) {
+ if (error instanceof LangfuseNotFoundError) {
+ return { transientError: false };
+ }
logger.warn("Failed to validate web callout observation via events table", {
projectId,
observationId,
error: error instanceof Error ? error.message : "Unknown error",
});
- return undefined;
+ return { transientError: true };
}
};