🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(v4): expose migration observation evidence (#16047)
Hassieb Pakzad2b508078b866
diff --git a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.tsindex 7c8e47738a..2c08e7a3f4 100644--- a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts+++ b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts@@ -1287,6 +1287,14 @@ describe("v4TransitionRouter", () => {expect(clickhouseQuery?.query).toContain("ingestion_sdk_name NOT IN {internalSdkNames: Array(String)}",);+ const [eventsUsageQuery, scoresUsageQuery] =+ clickhouseQuery?.query.split("UNION ALL") ?? [];+ expect(eventsUsageQuery).toContain(+ "AND NOT startsWith(environment, 'langfuse-')",+ );+ expect(scoresUsageQuery).toContain(+ "AND NOT startsWith(environment, 'langfuse-')",+ );expect(clickhouseQuery?.query).not.toContain("toDate(start_time)");expect(clickhouseQuery?.query).not.toContain("toDate(timestamp)");expect(clickhouseQuery?.query).toContain(@@ -1763,6 +1771,14 @@ describe("v4TransitionRouter", () => {expect(usageQuery?.query).toContain("ingestion_sdk_name NOT IN {internalSdkNames: Array(String)}",);+ const [eventsUsageQuery, scoresUsageQuery] =+ usageQuery?.query.split("UNION ALL") ?? [];+ expect(eventsUsageQuery).toContain(+ "AND NOT startsWith(environment, 'langfuse-')",+ );+ expect(scoresUsageQuery).toContain(+ "AND NOT startsWith(environment, 'langfuse-')",+ );expect(usageQuery?.query).not.toContain("toDate(start_time)");expect(usageQuery?.query).not.toContain("toDate(timestamp)");expect(usageQuery?.params).toMatchObject({diff --git a/web/src/features/v4-migration/V4MigrationContent.clienttest.tsx b/web/src/features/v4-migration/V4MigrationContent.clienttest.tsxindex 26e38244bf..3e0db83b3c 100644--- a/web/src/features/v4-migration/V4MigrationContent.clienttest.tsx+++ b/web/src/features/v4-migration/V4MigrationContent.clienttest.tsx@@ -421,12 +421,13 @@ describe("V4MigrationDetailsContent", () => {expect(screen.getByText("Python 2.60.3")).toBeInTheDocument();expect(screen.getByText("Python 4.7.1")).toBeInTheDocument();expect(screen.getByText(/upgrade required/)).toBeInTheDocument();- // The public key deep-links to the exact evidence: the events table- // filtered by this key + SDK name/version over the detection lookback- // window, so two SDK versions on the same key link to distinct results.- expect(- screen.getByRole("link", { name: "pk-lf-123…abcdef" }),- ).toHaveAttribute(+ // The explicit evidence link targets this key + SDK name/version over the+ // detection lookback window, so versions on the same key stay distinct.+ const outdatedRow = screen.getByText("Python 2.60.3").closest("li")!;+ const evidenceLink = within(outdatedRow).getByRole("link", {+ name: "View observations",+ });+ expect(evidenceLink).toHaveAttribute("href",`/project/project-1/observations?filter=${encodeURIComponent("ingestionApiKey;stringOptions;;any of;pk-lf-1234567890abcdef," +@@ -434,6 +435,13 @@ describe("V4MigrationDetailsContent", () => {"ingestionSdkVersion;stringOptions;;any of;2.60.3",)}&dateRange=14d`,);+ expect(evidenceLink).toHaveAttribute("target", "_blank");+ expect(evidenceLink).toHaveAttribute("rel", "noopener noreferrer");+ expect(+ within(outdatedRow).queryByRole("link", {+ name: "pk-lf-123…abcdef",+ }),+ ).not.toBeInTheDocument();expect(screen.queryByText("Update OTel Instrumentation"),).not.toBeInTheDocument();@@ -442,7 +450,7 @@ describe("V4MigrationDetailsContent", () => {it("renders the key as plain text when an offender has no observation evidence", () => {// A scores-only offender: detection counts score ingestions, but the// events table has nothing for this key — a link would open an empty- // table, so the key must stay plain text (LFE-14859).+ // table, so the evidence link must stay hidden.mocks.migrationData.sdk = {status: "legacy",sdkUsageSeries: [@@ -461,7 +469,29 @@ describe("V4MigrationDetailsContent", () => {expect(screen.getByText("Python 2.60.3")).toBeInTheDocument();expect(screen.getByText("pk-lf-123…abcdef")).toBeInTheDocument();expect(- screen.queryByRole("link", { name: "pk-lf-123…abcdef" }),+ screen.queryByRole("link", { name: "View observations" }),+ ).not.toBeInTheDocument();+ });++ it("does not link observations when an offender has no public key", () => {+ mocks.migrationData.sdk = {+ status: "legacy",+ sdkUsageSeries: [+ makeSdkUsageSeries({+ sdkVersion: "2.60.3",+ v4MigrationStatus: "upgrade_required",+ publicKey: "",+ }),+ ],+ upgradeRequiredCount: 1,+ delayedOtelIngestionCount: 0,+ };++ render(<V4MigrationDetailsContent projectId="project-1" />);++ expect(screen.getByText("No API key")).toBeInTheDocument();+ expect(+ screen.queryByRole("link", { name: "View observations" }),).not.toBeInTheDocument();});@@ -523,6 +553,7 @@ describe("V4MigrationDetailsContent", () => {});it("captures section_expanded and evidence_link_clicked in the SDK section", () => {+ const onNavigate = vi.fn();mocks.migrationData.sdk = {status: "legacy",sdkUsageSeries: [@@ -535,7 +566,12 @@ describe("V4MigrationDetailsContent", () => {delayedOtelIngestionCount: 0,};- render(<V4MigrationDetailsContent projectId="project-1" />);+ render(+ <V4MigrationDetailsContent+ projectId="project-1"+ onNavigate={onNavigate}+ />,+ );fireEvent.click(screen.getByText("Update SDK").closest("button")!);expect(mocks.capture).toHaveBeenCalledWith(@@ -551,7 +587,8 @@ describe("V4MigrationDetailsContent", () => {),).toHaveLength(1);- fireEvent.click(screen.getByRole("link", { name: "pk-lf-123…abcdef" }));+ fireEvent.click(screen.getByRole("link", { name: "View observations" }));+ expect(onNavigate).not.toHaveBeenCalled();// Only bounded values: the canonical SDK enum and status enums — never// the raw client-supplied sdkName/sdkVersion header strings.expect(mocks.capture).toHaveBeenCalledWith(@@ -670,7 +707,7 @@ describe("V4MigrationDetailsContent", () => {expect(screen.getByText("pk-lf-123…abcdef")).toBeInTheDocument();expect(- screen.queryByRole("link", { name: "pk-lf-123…abcdef" }),+ screen.queryByRole("link", { name: "View observations" }),).not.toBeInTheDocument();});@@ -701,14 +738,20 @@ describe("V4MigrationDetailsContent", () => {expect(screen.getByText("Custom instrumentation")).toBeInTheDocument();// Unattributed series ("unknown" SDK name/version) keep a key-only// evidence link — "unknown" is the fallback bucket, not an exact value.- expect(- screen.getByRole("link", { name: "pk-lf-123…abcdef" }),- ).toHaveAttribute(+ const customInstrumentationRow = screen+ .getByText("Custom instrumentation")+ .closest("li")!;+ const evidenceLink = within(customInstrumentationRow).getByRole("link", {+ name: "View observations",+ });+ expect(evidenceLink).toHaveAttribute("href",`/project/project-1/observations?filter=${encodeURIComponent("ingestionApiKey;stringOptions;;any of;pk-lf-1234567890abcdef",)}&dateRange=14d`,);+ expect(evidenceLink).toHaveAttribute("target", "_blank");+ expect(evidenceLink).toHaveAttribute("rel", "noopener noreferrer");expect(screen.getByRole("link", { name: "Python or JS SDK" }),).toHaveAttribute(diff --git a/web/src/features/v4-migration/V4MigrationContent.tsx b/web/src/features/v4-migration/V4MigrationContent.tsxindex 041899d8eb..5be1b1eece 100644--- a/web/src/features/v4-migration/V4MigrationContent.tsx+++ b/web/src/features/v4-migration/V4MigrationContent.tsx@@ -301,7 +301,6 @@ function SdkUsageSeriesRows({suffix,unknownSeriesLabel = "OTLP exporter",projectId,- onNavigate,analyticsSection,}: {series: V4MigrationSdkUsageSeries[];@@ -310,9 +309,8 @@ function SdkUsageSeriesRows({suffix: (series: V4MigrationSdkUsageSeries) => ReactNode;/** Row label when the series carries no usable SDK name. */unknownSeriesLabel?: string;- /** Enables the evidence deep link on the public key. */+ /** Enables the evidence deep link. */projectId?: string;- onNavigate?: () => void;/** Funnel dimension for the evidence_link_clicked event (snake_case). */analyticsSection: string;}) {@@ -333,6 +331,12 @@ function SdkUsageSeriesRows({usage.publicKey.length > 18? `${usage.publicKey.slice(0, 9)}…${usage.publicKey.slice(-6)}`: usage.publicKey || "No API key";+ const evidenceHref =+ projectId && usage.publicKey && usage.eventsCount > 0+ ? `/project/${projectId}/observations?filter=${encodeURIComponent(+ buildSdkUsageEvidenceFilter(usage),+ )}&dateRange=${V4_MIGRATION_LOOKBACK_DAYS}d`+ : null;return (<li@@ -349,44 +353,43 @@ function SdkUsageSeriesRows({</div>{/* Metadata line, indented under the label (emoji + gap). */}<div className="text-muted-foreground flex flex-wrap items-baseline gap-x-1.5 pl-5">- {/* Deep link to the exact evidence: the events table filtered by- this public key (plus SDK name/version when attributed) over- the same lookback window the detection used; see- buildSdkUsageEvidenceFilter. Scores-only offenders- (eventsCount 0) stay plain text — the events table has- nothing for them and a link would open an empty result. */}- {projectId && usage.publicKey && usage.eventsCount > 0 ? (- <Link- // The events page's `filter` param carries the semicolon- // filter encoding (column;type;key;operator;value), not the- // search-bar grammar; the bar re-derives its text from it.- href={`/project/${projectId}/observations?filter=${encodeURIComponent(- buildSdkUsageEvidenceFilter(usage),- )}&dateRange=${V4_MIGRATION_LOOKBACK_DAYS}d`}- onClick={() => {- // Bounded values only: raw sdkName/sdkVersion are- // client-supplied ingestion headers (user-controlled,- // unbounded cardinality) and must not reach PostHog.- capture("v4_migration:evidence_link_clicked", {- section: analyticsSection,- sdkName: usage.canonicalSdkName ?? "other",- v4MigrationStatus: usage.v4MigrationStatus,- attributionStatus: usage.attributionStatus,- });- onNavigate?.();- }}- className="underline"- title={usage.publicKey}- >- {publicKey}- </Link>- ) : (- <span title={usage.publicKey || undefined}>{publicKey}</span>- )}+ <span title={usage.publicKey || undefined}>{publicKey}</span><span>· last seen{" "}{formatCompactRelativeTime(new Date(usage.lastSeen))}</span>+ {/* Deep link to the exact evidence: the events table filtered by+ this public key, plus SDK name and version when attributed,+ over the detection lookback. Rows without a public key and+ scores-only offenders stay unlinked because their target+ would be overly broad or empty. */}+ {evidenceHref ? (+ <>+ <span aria-hidden="true">·</span>+ <Link+ // The events page's `filter` param carries the semicolon+ // filter encoding (column;type;key;operator;value), not the+ // search-bar grammar; the bar re-derives its text from it.+ href={evidenceHref}+ target="_blank"+ rel="noopener noreferrer"+ onClick={() => {+ // Bounded values only: raw sdkName/sdkVersion are+ // client-supplied ingestion headers (user-controlled,+ // unbounded cardinality) and must not reach PostHog.+ capture("v4_migration:evidence_link_clicked", {+ section: analyticsSection,+ sdkName: usage.canonicalSdkName ?? "other",+ v4MigrationStatus: usage.v4MigrationStatus,+ attributionStatus: usage.attributionStatus,+ });+ }}+ className="underline"+ >+ View observations+ </Link>+ </>+ ) : null}</div></li>);@@ -404,12 +407,10 @@ export function V4MigrationSdkSection({sdk,defaultOpen,projectId,- onNavigate,}: {sdk: V4MigrationSdkState;defaultOpen?: boolean;projectId?: string;- onNavigate?: () => void;}) {const section = getSdkSectionState(sdk);if (section.status === "latest" || section.status === "no_data") return null;@@ -456,7 +457,6 @@ export function V4MigrationSdkSection({<SdkUsageSeriesRowsseries={section.series}projectId={projectId}- onNavigate={onNavigate}analyticsSection="sdk"needsAction={(usage) =>(usage.v4MigrationStatus === "upgrade_required" &&@@ -485,12 +485,10 @@ export function V4MigrationOtelSection({sdk,defaultOpen,projectId,- onNavigate,}: {sdk: V4MigrationSdkState;defaultOpen?: boolean;projectId?: string;- onNavigate?: () => void;}) {const section = getOtelSectionState(sdk);if (@@ -523,7 +521,6 @@ export function V4MigrationOtelSection({<SdkUsageSeriesRowsseries={section.series}projectId={projectId}- onNavigate={onNavigate}analyticsSection="otel"needsAction={(usage) => usage.hasDelayedOtelEvents === true}suffix={(usage) =>@@ -545,12 +542,10 @@ export function V4MigrationCustomInstrumentationSection({sdk,defaultOpen,projectId,- onNavigate,}: {sdk: V4MigrationSdkState;defaultOpen?: boolean;projectId?: string;- onNavigate?: () => void;}) {const section = getCustomInstrumentationSectionState(sdk);if (@@ -595,7 +590,6 @@ export function V4MigrationCustomInstrumentationSection({<SdkUsageSeriesRowsseries={section.series}projectId={projectId}- onNavigate={onNavigate}unknownSeriesLabel="Custom instrumentation"analyticsSection="custom_instrumentation"needsAction={() => true}@@ -1314,17 +1308,14 @@ export function V4MigrationDetailsContent({<V4MigrationSdkSectionsdk={migrationData.sdk}projectId={evidenceProjectId}- onNavigate={onNavigate}/><V4MigrationOtelSectionsdk={migrationData.sdk}projectId={evidenceProjectId}- onNavigate={onNavigate}/><V4MigrationCustomInstrumentationSectionsdk={migrationData.sdk}projectId={evidenceProjectId}- onNavigate={onNavigate}/>{!evalsClean && (diff --git a/web/src/features/v4/server/v4TransitionRouter.ts b/web/src/features/v4/server/v4TransitionRouter.tsindex 0445ce3f01..55fa824e40 100644--- a/web/src/features/v4/server/v4TransitionRouter.ts+++ b/web/src/features/v4/server/v4TransitionRouter.ts@@ -852,6 +852,7 @@ ORDER BY bucket_time ASC, score_name ASCproject_id = {projectId: String}AND timestamp >= {fromTimestamp: DateTime64(3)}AND timestamp <= {toTimestamp: DateTime64(3)}+ AND NOT startsWith(environment, 'langfuse-')AND ingestion_sdk_name NOT IN {internalSdkNames: Array(String)}AND is_deleted = 0`;@@ -870,6 +871,7 @@ WITH selected AS (project_id = {projectId: String}AND start_time >= {fromTimestamp: DateTime64(3)}AND start_time <= {toTimestamp: DateTime64(3)}+ AND NOT startsWith(environment, 'langfuse-')AND ingestion_sdk_name NOT IN {internalSdkNames: Array(String)}AND is_deleted = 0${scoresUnionSql}@@ -949,6 +951,7 @@ ORDER BY ${bucketTimeSql} ASC, sdk_name ASC, sdk_version ASC, public_key ASCproject_id IN {projectIds: Array(String)}AND timestamp >= {fromTimestamp: DateTime64(3)}AND timestamp <= {toTimestamp: DateTime64(3)}+ AND NOT startsWith(environment, 'langfuse-')AND ingestion_sdk_name NOT IN {internalSdkNames: Array(String)}AND is_deleted = 0`;@@ -970,6 +973,7 @@ WITH selected AS (project_id IN {projectIds: Array(String)}AND start_time >= {fromTimestamp: DateTime64(3)}AND start_time <= {toTimestamp: DateTime64(3)}+ AND NOT startsWith(environment, 'langfuse-')AND ingestion_sdk_name NOT IN {internalSdkNames: Array(String)}AND is_deleted = 0${scoresUnionSql}