Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(v4): expose migration observation evidence (#16047)

Hassieb Pakzad2b508078b866
diff --git a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts
index 7c8e47738a..2c08e7a3f4 100644
--- a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts
+++ b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts
@@ -1287,6 +1287,14 @@ describe("v4TransitionRouter", () => {
expect(clickhouseQuery?.query).toContain(
"ingestion_sdk_name NOT IN {internalSdkNames: Array(String)}",
);
+ const [eventsUsageQuery, scoresUsageQuery] =
+ clickhouseQuery?.query.split("UNION ALL") ?? [];
+ expect(eventsUsageQuery).toContain(
+ "AND NOT startsWith(environment, 'langfuse-')",
+ );
+ expect(scoresUsageQuery).toContain(
+ "AND NOT startsWith(environment, 'langfuse-')",
+ );
expect(clickhouseQuery?.query).not.toContain("toDate(start_time)");
expect(clickhouseQuery?.query).not.toContain("toDate(timestamp)");
expect(clickhouseQuery?.query).toContain(
@@ -1763,6 +1771,14 @@ describe("v4TransitionRouter", () => {
expect(usageQuery?.query).toContain(
"ingestion_sdk_name NOT IN {internalSdkNames: Array(String)}",
);
+ const [eventsUsageQuery, scoresUsageQuery] =
+ usageQuery?.query.split("UNION ALL") ?? [];
+ expect(eventsUsageQuery).toContain(
+ "AND NOT startsWith(environment, 'langfuse-')",
+ );
+ expect(scoresUsageQuery).toContain(
+ "AND NOT startsWith(environment, 'langfuse-')",
+ );
expect(usageQuery?.query).not.toContain("toDate(start_time)");
expect(usageQuery?.query).not.toContain("toDate(timestamp)");
expect(usageQuery?.params).toMatchObject({
diff --git a/web/src/features/v4-migration/V4MigrationContent.clienttest.tsx b/web/src/features/v4-migration/V4MigrationContent.clienttest.tsx
index 26e38244bf..3e0db83b3c 100644
--- a/web/src/features/v4-migration/V4MigrationContent.clienttest.tsx
+++ b/web/src/features/v4-migration/V4MigrationContent.clienttest.tsx
@@ -421,12 +421,13 @@ describe("V4MigrationDetailsContent", () => {
expect(screen.getByText("Python 2.60.3")).toBeInTheDocument();
expect(screen.getByText("Python 4.7.1")).toBeInTheDocument();
expect(screen.getByText(/upgrade required/)).toBeInTheDocument();
- // The public key deep-links to the exact evidence: the events table
- // filtered by this key + SDK name/version over the detection lookback
- // window, so two SDK versions on the same key link to distinct results.
- expect(
- screen.getByRole("link", { name: "pk-lf-123…abcdef" }),
- ).toHaveAttribute(
+ // The explicit evidence link targets this key + SDK name/version over the
+ // detection lookback window, so versions on the same key stay distinct.
+ const outdatedRow = screen.getByText("Python 2.60.3").closest("li")!;
+ const evidenceLink = within(outdatedRow).getByRole("link", {
+ name: "View observations",
+ });
+ expect(evidenceLink).toHaveAttribute(
"href",
`/project/project-1/observations?filter=${encodeURIComponent(
"ingestionApiKey;stringOptions;;any of;pk-lf-1234567890abcdef," +
@@ -434,6 +435,13 @@ describe("V4MigrationDetailsContent", () => {
"ingestionSdkVersion;stringOptions;;any of;2.60.3",
)}&dateRange=14d`,
);
+ expect(evidenceLink).toHaveAttribute("target", "_blank");
+ expect(evidenceLink).toHaveAttribute("rel", "noopener noreferrer");
+ expect(
+ within(outdatedRow).queryByRole("link", {
+ name: "pk-lf-123…abcdef",
+ }),
+ ).not.toBeInTheDocument();
expect(
screen.queryByText("Update OTel Instrumentation"),
).not.toBeInTheDocument();
@@ -442,7 +450,7 @@ describe("V4MigrationDetailsContent", () => {
it("renders the key as plain text when an offender has no observation evidence", () => {
// A scores-only offender: detection counts score ingestions, but the
// events table has nothing for this key — a link would open an empty
- // table, so the key must stay plain text (LFE-14859).
+ // table, so the evidence link must stay hidden.
mocks.migrationData.sdk = {
status: "legacy",
sdkUsageSeries: [
@@ -461,7 +469,29 @@ describe("V4MigrationDetailsContent", () => {
expect(screen.getByText("Python 2.60.3")).toBeInTheDocument();
expect(screen.getByText("pk-lf-123…abcdef")).toBeInTheDocument();
expect(
- screen.queryByRole("link", { name: "pk-lf-123…abcdef" }),
+ screen.queryByRole("link", { name: "View observations" }),
+ ).not.toBeInTheDocument();
+ });
+
+ it("does not link observations when an offender has no public key", () => {
+ mocks.migrationData.sdk = {
+ status: "legacy",
+ sdkUsageSeries: [
+ makeSdkUsageSeries({
+ sdkVersion: "2.60.3",
+ v4MigrationStatus: "upgrade_required",
+ publicKey: "",
+ }),
+ ],
+ upgradeRequiredCount: 1,
+ delayedOtelIngestionCount: 0,
+ };
+
+ render(<V4MigrationDetailsContent projectId="project-1" />);
+
+ expect(screen.getByText("No API key")).toBeInTheDocument();
+ expect(
+ screen.queryByRole("link", { name: "View observations" }),
).not.toBeInTheDocument();
});
@@ -523,6 +553,7 @@ describe("V4MigrationDetailsContent", () => {
});
it("captures section_expanded and evidence_link_clicked in the SDK section", () => {
+ const onNavigate = vi.fn();
mocks.migrationData.sdk = {
status: "legacy",
sdkUsageSeries: [
@@ -535,7 +566,12 @@ describe("V4MigrationDetailsContent", () => {
delayedOtelIngestionCount: 0,
};
- render(<V4MigrationDetailsContent projectId="project-1" />);
+ render(
+ <V4MigrationDetailsContent
+ projectId="project-1"
+ onNavigate={onNavigate}
+ />,
+ );
fireEvent.click(screen.getByText("Update SDK").closest("button")!);
expect(mocks.capture).toHaveBeenCalledWith(
@@ -551,7 +587,8 @@ describe("V4MigrationDetailsContent", () => {
),
).toHaveLength(1);
- fireEvent.click(screen.getByRole("link", { name: "pk-lf-123…abcdef" }));
+ fireEvent.click(screen.getByRole("link", { name: "View observations" }));
+ expect(onNavigate).not.toHaveBeenCalled();
// Only bounded values: the canonical SDK enum and status enums — never
// the raw client-supplied sdkName/sdkVersion header strings.
expect(mocks.capture).toHaveBeenCalledWith(
@@ -670,7 +707,7 @@ describe("V4MigrationDetailsContent", () => {
expect(screen.getByText("pk-lf-123…abcdef")).toBeInTheDocument();
expect(
- screen.queryByRole("link", { name: "pk-lf-123…abcdef" }),
+ screen.queryByRole("link", { name: "View observations" }),
).not.toBeInTheDocument();
});
@@ -701,14 +738,20 @@ describe("V4MigrationDetailsContent", () => {
expect(screen.getByText("Custom instrumentation")).toBeInTheDocument();
// Unattributed series ("unknown" SDK name/version) keep a key-only
// evidence link — "unknown" is the fallback bucket, not an exact value.
- expect(
- screen.getByRole("link", { name: "pk-lf-123…abcdef" }),
- ).toHaveAttribute(
+ const customInstrumentationRow = screen
+ .getByText("Custom instrumentation")
+ .closest("li")!;
+ const evidenceLink = within(customInstrumentationRow).getByRole("link", {
+ name: "View observations",
+ });
+ expect(evidenceLink).toHaveAttribute(
"href",
`/project/project-1/observations?filter=${encodeURIComponent(
"ingestionApiKey;stringOptions;;any of;pk-lf-1234567890abcdef",
)}&dateRange=14d`,
);
+ expect(evidenceLink).toHaveAttribute("target", "_blank");
+ expect(evidenceLink).toHaveAttribute("rel", "noopener noreferrer");
expect(
screen.getByRole("link", { name: "Python or JS SDK" }),
).toHaveAttribute(
diff --git a/web/src/features/v4-migration/V4MigrationContent.tsx b/web/src/features/v4-migration/V4MigrationContent.tsx
index 041899d8eb..5be1b1eece 100644
--- a/web/src/features/v4-migration/V4MigrationContent.tsx
+++ b/web/src/features/v4-migration/V4MigrationContent.tsx
@@ -301,7 +301,6 @@ function SdkUsageSeriesRows({
suffix,
unknownSeriesLabel = "OTLP exporter",
projectId,
- onNavigate,
analyticsSection,
}: {
series: V4MigrationSdkUsageSeries[];
@@ -310,9 +309,8 @@ function SdkUsageSeriesRows({
suffix: (series: V4MigrationSdkUsageSeries) => ReactNode;
/** Row label when the series carries no usable SDK name. */
unknownSeriesLabel?: string;
- /** Enables the evidence deep link on the public key. */
+ /** Enables the evidence deep link. */
projectId?: string;
- onNavigate?: () => void;
/** Funnel dimension for the evidence_link_clicked event (snake_case). */
analyticsSection: string;
}) {
@@ -333,6 +331,12 @@ function SdkUsageSeriesRows({
usage.publicKey.length > 18
? `${usage.publicKey.slice(0, 9)}…${usage.publicKey.slice(-6)}`
: usage.publicKey || "No API key";
+ const evidenceHref =
+ projectId && usage.publicKey && usage.eventsCount > 0
+ ? `/project/${projectId}/observations?filter=${encodeURIComponent(
+ buildSdkUsageEvidenceFilter(usage),
+ )}&dateRange=${V4_MIGRATION_LOOKBACK_DAYS}d`
+ : null;
return (
<li
@@ -349,44 +353,43 @@ function SdkUsageSeriesRows({
</div>
{/* Metadata line, indented under the label (emoji + gap). */}
<div className="text-muted-foreground flex flex-wrap items-baseline gap-x-1.5 pl-5">
- {/* Deep link to the exact evidence: the events table filtered by
- this public key (plus SDK name/version when attributed) over
- the same lookback window the detection used; see
- buildSdkUsageEvidenceFilter. Scores-only offenders
- (eventsCount 0) stay plain text — the events table has
- nothing for them and a link would open an empty result. */}
- {projectId && usage.publicKey && usage.eventsCount > 0 ? (
- <Link
- // The events page's `filter` param carries the semicolon
- // filter encoding (column;type;key;operator;value), not the
- // search-bar grammar; the bar re-derives its text from it.
- href={`/project/${projectId}/observations?filter=${encodeURIComponent(
- buildSdkUsageEvidenceFilter(usage),
- )}&dateRange=${V4_MIGRATION_LOOKBACK_DAYS}d`}
- onClick={() => {
- // Bounded values only: raw sdkName/sdkVersion are
- // client-supplied ingestion headers (user-controlled,
- // unbounded cardinality) and must not reach PostHog.
- capture("v4_migration:evidence_link_clicked", {
- section: analyticsSection,
- sdkName: usage.canonicalSdkName ?? "other",
- v4MigrationStatus: usage.v4MigrationStatus,
- attributionStatus: usage.attributionStatus,
- });
- onNavigate?.();
- }}
- className="underline"
- title={usage.publicKey}
- >
- {publicKey}
- </Link>
- ) : (
- <span title={usage.publicKey || undefined}>{publicKey}</span>
- )}
+ <span title={usage.publicKey || undefined}>{publicKey}</span>
<span>
· last seen{" "}
{formatCompactRelativeTime(new Date(usage.lastSeen))}
</span>
+ {/* Deep link to the exact evidence: the events table filtered by
+ this public key, plus SDK name and version when attributed,
+ over the detection lookback. Rows without a public key and
+ scores-only offenders stay unlinked because their target
+ would be overly broad or empty. */}
+ {evidenceHref ? (
+ <>
+ <span aria-hidden="true">·</span>
+ <Link
+ // The events page's `filter` param carries the semicolon
+ // filter encoding (column;type;key;operator;value), not the
+ // search-bar grammar; the bar re-derives its text from it.
+ href={evidenceHref}
+ target="_blank"
+ rel="noopener noreferrer"
+ onClick={() => {
+ // Bounded values only: raw sdkName/sdkVersion are
+ // client-supplied ingestion headers (user-controlled,
+ // unbounded cardinality) and must not reach PostHog.
+ capture("v4_migration:evidence_link_clicked", {
+ section: analyticsSection,
+ sdkName: usage.canonicalSdkName ?? "other",
+ v4MigrationStatus: usage.v4MigrationStatus,
+ attributionStatus: usage.attributionStatus,
+ });
+ }}
+ className="underline"
+ >
+ View observations
+ </Link>
+ </>
+ ) : null}
</div>
</li>
);
@@ -404,12 +407,10 @@ export function V4MigrationSdkSection({
sdk,
defaultOpen,
projectId,
- onNavigate,
}: {
sdk: V4MigrationSdkState;
defaultOpen?: boolean;
projectId?: string;
- onNavigate?: () => void;
}) {
const section = getSdkSectionState(sdk);
if (section.status === "latest" || section.status === "no_data") return null;
@@ -456,7 +457,6 @@ export function V4MigrationSdkSection({
<SdkUsageSeriesRows
series={section.series}
projectId={projectId}
- onNavigate={onNavigate}
analyticsSection="sdk"
needsAction={(usage) =>
(usage.v4MigrationStatus === "upgrade_required" &&
@@ -485,12 +485,10 @@ export function V4MigrationOtelSection({
sdk,
defaultOpen,
projectId,
- onNavigate,
}: {
sdk: V4MigrationSdkState;
defaultOpen?: boolean;
projectId?: string;
- onNavigate?: () => void;
}) {
const section = getOtelSectionState(sdk);
if (
@@ -523,7 +521,6 @@ export function V4MigrationOtelSection({
<SdkUsageSeriesRows
series={section.series}
projectId={projectId}
- onNavigate={onNavigate}
analyticsSection="otel"
needsAction={(usage) => usage.hasDelayedOtelEvents === true}
suffix={(usage) =>
@@ -545,12 +542,10 @@ export function V4MigrationCustomInstrumentationSection({
sdk,
defaultOpen,
projectId,
- onNavigate,
}: {
sdk: V4MigrationSdkState;
defaultOpen?: boolean;
projectId?: string;
- onNavigate?: () => void;
}) {
const section = getCustomInstrumentationSectionState(sdk);
if (
@@ -595,7 +590,6 @@ export function V4MigrationCustomInstrumentationSection({
<SdkUsageSeriesRows
series={section.series}
projectId={projectId}
- onNavigate={onNavigate}
unknownSeriesLabel="Custom instrumentation"
analyticsSection="custom_instrumentation"
needsAction={() => true}
@@ -1314,17 +1308,14 @@ export function V4MigrationDetailsContent({
<V4MigrationSdkSection
sdk={migrationData.sdk}
projectId={evidenceProjectId}
- onNavigate={onNavigate}
/>
<V4MigrationOtelSection
sdk={migrationData.sdk}
projectId={evidenceProjectId}
- onNavigate={onNavigate}
/>
<V4MigrationCustomInstrumentationSection
sdk={migrationData.sdk}
projectId={evidenceProjectId}
- onNavigate={onNavigate}
/>
{!evalsClean && (
diff --git a/web/src/features/v4/server/v4TransitionRouter.ts b/web/src/features/v4/server/v4TransitionRouter.ts
index 0445ce3f01..55fa824e40 100644
--- a/web/src/features/v4/server/v4TransitionRouter.ts
+++ b/web/src/features/v4/server/v4TransitionRouter.ts
@@ -852,6 +852,7 @@ ORDER BY bucket_time ASC, score_name ASC
project_id = {projectId: String}
AND timestamp >= {fromTimestamp: DateTime64(3)}
AND timestamp <= {toTimestamp: DateTime64(3)}
+ AND NOT startsWith(environment, 'langfuse-')
AND ingestion_sdk_name NOT IN {internalSdkNames: Array(String)}
AND is_deleted = 0`;
@@ -870,6 +871,7 @@ WITH selected AS (
project_id = {projectId: String}
AND start_time >= {fromTimestamp: DateTime64(3)}
AND start_time <= {toTimestamp: DateTime64(3)}
+ AND NOT startsWith(environment, 'langfuse-')
AND ingestion_sdk_name NOT IN {internalSdkNames: Array(String)}
AND is_deleted = 0
${scoresUnionSql}
@@ -949,6 +951,7 @@ ORDER BY ${bucketTimeSql} ASC, sdk_name ASC, sdk_version ASC, public_key ASC
project_id IN {projectIds: Array(String)}
AND timestamp >= {fromTimestamp: DateTime64(3)}
AND timestamp <= {toTimestamp: DateTime64(3)}
+ AND NOT startsWith(environment, 'langfuse-')
AND ingestion_sdk_name NOT IN {internalSdkNames: Array(String)}
AND is_deleted = 0`;
@@ -970,6 +973,7 @@ WITH selected AS (
project_id IN {projectIds: Array(String)}
AND start_time >= {fromTimestamp: DateTime64(3)}
AND start_time <= {toTimestamp: DateTime64(3)}
+ AND NOT startsWith(environment, 'langfuse-')
AND ingestion_sdk_name NOT IN {internalSdkNames: Array(String)}
AND is_deleted = 0
${scoresUnionSql}