🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(blob-export): add opt-in trace field groups for the legacy traces export (#19244)
Niklas Semmlerbd1ad4f71f67
diff --git a/packages/shared/src/domain/export-field-groups.ts b/packages/shared/src/domain/export-field-groups.tsindex 02f500dc44..2b3d51fc6d 100644--- a/packages/shared/src/domain/export-field-groups.ts+++ b/packages/shared/src/domain/export-field-groups.ts@@ -59,15 +59,59 @@ export const SCORE_EXPORT_GROUP_FIELDS = {],} as const satisfies Record<ScoreExportFieldGroup, readonly string[]>;+/**+ * Field groups of the legacy traces blob export, prefixed with `trace_` like+ * the score groups. `trace_core` is always exported; it is stored to mark the+ * trace selection as configured.+ *+ * A selection without any trace group exports every trace column, so+ * integrations saved before trace groups existed keep their export unchanged.+ */+export const TRACE_EXPORT_FIELD_GROUPS = [+ "trace_core",+ "trace_io",+ "trace_metadata",+] as const;++export type TraceExportFieldGroup = (typeof TRACE_EXPORT_FIELD_GROUPS)[number];++/**+ * Columns of the traces export and the group each belongs to, in the column+ * order of the full export (kept stable so a full selection is unchanged).+ */+export const TRACE_EXPORT_FIELDS = [+ { field: "id", group: "trace_core" },+ { field: "timestamp", group: "trace_core" },+ { field: "name", group: "trace_core" },+ { field: "environment", group: "trace_core" },+ { field: "project_id", group: "trace_core" },+ { field: "metadata", group: "trace_metadata" },+ { field: "user_id", group: "trace_core" },+ { field: "session_id", group: "trace_core" },+ { field: "release", group: "trace_core" },+ { field: "version", group: "trace_core" },+ { field: "public", group: "trace_core" },+ { field: "bookmarked", group: "trace_core" },+ { field: "tags", group: "trace_core" },+ { field: "input", group: "trace_io" },+ { field: "output", group: "trace_io" },+ { field: "created_at", group: "trace_core" },+ { field: "updated_at", group: "trace_core" },+] as const satisfies readonly {+ field: string;+ group: TraceExportFieldGroup;+}[];+/*** Every value that can be stored in an integration's `exportFieldGroups`.* Each group belongs to exactly one exported table; split a selection with- * `getObservationFieldGroups` / `getScoreFieldGroups` before handing it to a- * table's query.+ * `getObservationFieldGroups` / `getScoreFieldGroups` /+ * `getTraceFieldGroups` before handing it to a table's query.*/export const EXPORT_FIELD_GROUPS = [...OBSERVATION_FIELD_GROUPS_FULL,...SCORE_EXPORT_FIELD_GROUPS,+ ...TRACE_EXPORT_FIELD_GROUPS,] as const;export type ExportFieldGroup = (typeof EXPORT_FIELD_GROUPS)[number];@@ -85,3 +129,22 @@ export const getScoreFieldGroups = (groups.filter((group): group is ScoreExportFieldGroup =>(SCORE_EXPORT_FIELD_GROUPS as readonly string[]).includes(group),);++/** True when the selection configures the traces export (any trace group stored). */+export const hasTraceFieldGroups = (groups: readonly string[]): boolean =>+ groups.some((group) =>+ (TRACE_EXPORT_FIELD_GROUPS as readonly string[]).includes(group),+ );++/**+ * Trace groups to export: all of them when the selection holds no trace group,+ * otherwise the selected ones plus the always-exported `trace_core`.+ */+export const getTraceFieldGroups = (+ groups: readonly string[],+): TraceExportFieldGroup[] =>+ hasTraceFieldGroups(groups)+ ? TRACE_EXPORT_FIELD_GROUPS.filter(+ (group) => group === "trace_core" || groups.includes(group),+ )+ : [...TRACE_EXPORT_FIELD_GROUPS];diff --git a/packages/shared/src/features/analytics-integrations/index.ts b/packages/shared/src/features/analytics-integrations/index.tsindex cd27ff1b0f..cfbb612146 100644--- a/packages/shared/src/features/analytics-integrations/index.ts+++ b/packages/shared/src/features/analytics-integrations/index.ts@@ -12,6 +12,9 @@ import {SCORE_EXPORT_FIELD_GROUPS,SCORE_EXPORT_GROUP_FIELDS,type ScoreExportFieldGroup,+ TRACE_EXPORT_FIELDS,+ TRACE_EXPORT_FIELD_GROUPS,+ type TraceExportFieldGroup,} from "../../domain/export-field-groups";export * from "./export-source-policy";@@ -175,3 +178,20 @@ export const SCORE_EXPORT_FIELD_GROUP_OPTIONS = SCORE_EXPORT_FIELD_GROUPS.map(description: SCORE_EXPORT_GROUP_FIELDS[value].join(", "),}),);++/** Trace group labels, in the order of the matching observation groups. */+const TRACE_EXPORT_FIELD_GROUP_LABELS = {+ trace_core: { label: "Core" },+ trace_io: { label: "Input / Output" },+ trace_metadata: { label: "Metadata" },+} satisfies Record<TraceExportFieldGroup, { label: string }>;++export const TRACE_EXPORT_FIELD_GROUP_OPTIONS = TRACE_EXPORT_FIELD_GROUPS.map(+ (value) => ({+ value,+ ...TRACE_EXPORT_FIELD_GROUP_LABELS[value],+ description: TRACE_EXPORT_FIELDS.filter((f) => f.group === value)+ .map((f) => f.field)+ .join(", "),+ }),+);diff --git a/packages/shared/src/server/repositories/traces.ts b/packages/shared/src/server/repositories/traces.tsindex d4193aef99..dea4070b28 100644--- a/packages/shared/src/server/repositories/traces.ts+++ b/packages/shared/src/server/repositories/traces.ts@@ -22,6 +22,11 @@ import {resolveScoreDataRequirement,} from "../queries/clickhouse-sql/score-filter-pushdown";import { LISTABLE_SCORE_TYPES } from "../../domain/scores";+import {+ TRACE_EXPORT_FIELDS,+ TRACE_EXPORT_FIELD_GROUPS,+ type TraceExportFieldGroup,+} from "../../domain/export-field-groups";import { OrderByState } from "../../interfaces/orderBy";import snakeCase from "lodash/snakeCase";import { FilterState } from "../../types";@@ -1207,33 +1212,22 @@ export const getUserMetrics = async (};// Shared query for both the standard (parsed) and Parquet (`exec` binary)-// trace export paths, so the SELECT/WHERE stays in one place.+// trace export paths, so the SELECT/WHERE stays in one place. Deselected+// field group columns are dropped; the remaining columns keep their order.const buildTracesForBlobStorageExportQuery = (projectId: string,minTimestamp: Date,maxTimestamp: Date,+ fieldGroups: TraceExportFieldGroup[],) => {const traceTable = "traces";+ const columns = TRACE_EXPORT_FIELDS.filter(({ group }) =>+ fieldGroups.includes(group),+ ).map(({ field }) => field);const query = `SELECT- id,- timestamp,- name,- environment,- project_id,- metadata,- user_id,- session_id,- release,- version,- public as public,- bookmarked as bookmarked,- tags,- input as input,- output as output,- created_at,- updated_at+ ${columns.join(",\n ")}FROM ${traceTable} FINALWHERE project_id = {projectId: String}AND timestamp >= {minTimestamp: DateTime64(3)}@@ -1260,9 +1254,15 @@ export const getTracesForBlobStorageExport = function (projectId: string,minTimestamp: Date,maxTimestamp: Date,+ fieldGroups: TraceExportFieldGroup[] = [...TRACE_EXPORT_FIELD_GROUPS],) {return queryClickhouseStream<Record<string, unknown>>(- buildTracesForBlobStorageExportQuery(projectId, minTimestamp, maxTimestamp),+ buildTracesForBlobStorageExportQuery(+ projectId,+ minTimestamp,+ maxTimestamp,+ fieldGroups,+ ),);};@@ -1272,12 +1272,14 @@ export const getTracesForBlobStorageExportParquet = function (projectId: string,minTimestamp: Date,maxTimestamp: Date,+ fieldGroups: TraceExportFieldGroup[] = [...TRACE_EXPORT_FIELD_GROUPS],) {return queryClickhouseExecRaw({...buildTracesForBlobStorageExportQuery(projectId,minTimestamp,maxTimestamp,+ fieldGroups,),format: "Parquet",clickhouseSettings: BLOB_EXPORT_PARQUET_CLICKHOUSE_SETTINGS,diff --git a/web/src/__tests__/blob-storage-form-field-groups.clienttest.ts b/web/src/__tests__/blob-storage-form-field-groups.clienttest.tsindex 53623027de..ee45c80b71 100644--- a/web/src/__tests__/blob-storage-form-field-groups.clienttest.ts+++ b/web/src/__tests__/blob-storage-form-field-groups.clienttest.ts@@ -12,7 +12,12 @@ import {BlobStorageIntegrationFileType,BlobStorageIntegrationType,EXPORT_FIELD_GROUP_OPTIONS,+ type ExportSourceContext,} from "@langfuse/shared";+import {+ buildBlobStorageFormValues,+ toStoredFieldGroups,+} from "@/src/features/blobstorage-integration/components/formValues";const VALID_BASE: BlobStorageIntegrationFormSchema = {type: BlobStorageIntegrationType.S3,@@ -162,3 +167,64 @@ describe("EXPORT_FIELD_GROUP_OPTIONS — parquet description", () => {}});});++describe("buildBlobStorageFormValues — trace field groups", () => {+ const ctx: ExportSourceContext = {+ isCloud: false,+ enrichedAvailable: true,+ legacyWritesActive: true,+ };++ it.each([+ // no stored trace group = full traces export, so every trace group is shown+ [+ ["core", "io"],+ ["core", "io", "trace_core", "trace_io", "trace_metadata"],+ ],+ [+ ["core", "trace_core"],+ ["core", "trace_core"],+ ],+ // trace_core is locked in the form and kept once any trace group is stored+ [+ ["core", "trace_io"],+ ["core", "trace_io", "trace_core"],+ ],+ ])("maps stored groups %j to form groups %j", (stored, expected) => {+ const values = buildBlobStorageFormValues(+ { exportFieldGroups: stored },+ ctx,+ );+ expect(values.exportFieldGroups).toEqual(expected);+ });+});++describe("toStoredFieldGroups", () => {+ const ctx: ExportSourceContext = {+ isCloud: false,+ enrichedAvailable: true,+ legacyWritesActive: true,+ };++ it.each([+ // an untouched save stores the same list it loaded (no trace groups)+ [+ ["core", "io"],+ ["core", "io"],+ ],+ [+ ["core", "trace_core"],+ ["core", "trace_core"],+ ],+ [+ ["core", "trace_core", "trace_io"],+ ["core", "trace_core", "trace_io"],+ ],+ ])("round-trips stored groups %j as %j", (stored, expected) => {+ const formGroups = buildBlobStorageFormValues(+ { exportFieldGroups: stored },+ ctx,+ ).exportFieldGroups;+ expect(toStoredFieldGroups(formGroups ?? [])).toEqual(expected);+ });+});diff --git a/web/src/__tests__/server/blob-storage-integration-api.servertest.ts b/web/src/__tests__/server/blob-storage-integration-api.servertest.tsindex 115eb1d908..fad878a760 100644--- a/web/src/__tests__/server/blob-storage-integration-api.servertest.ts+++ b/web/src/__tests__/server/blob-storage-integration-api.servertest.ts@@ -1023,6 +1023,39 @@ describe("Blob Storage Integrations API", () => {expect(saved?.exportFieldGroups).toStrictEqual(["core", "io"]);});+ it("PUT LEGACY_TRACES_OBSERVATIONS hides UI-set trace groups and keeps them on update", async () => {+ await seedGrandfatheredLegacyRow({+ exportFieldGroups: ["core", "io", "trace_core", "trace_metadata"],+ });++ const putResponse = await makeZodVerifiedAPICall(+ BlobStorageIntegrationResponseSchema,+ "PUT",+ "/api/public/integrations/blob-storage",+ {+ ...validBlobStorageConfig,+ projectId: testProject1Id,+ exportSource: "LEGACY_TRACES_OBSERVATIONS" as const,+ exportFieldGroups: ["core", "metadata"],+ },+ createBasicAuthHeader(testApiKey, testApiSecretKey),+ );+ expect(putResponse.body.exportFieldGroups).toStrictEqual([+ "core",+ "metadata",+ ]);++ const saved = await prisma.blobStorageIntegration.findUnique({+ where: { projectId: testProject1Id },+ });+ expect(saved?.exportFieldGroups).toStrictEqual([+ "core",+ "metadata",+ "trace_core",+ "trace_metadata",+ ]);+ });+// ---- Response shape ----it("PUT response includes exportSource and exportFieldGroups for LEGACY_TRACES_OBSERVATIONS", async () => {diff --git a/web/src/features/blobstorage-integration/components/BlobStorageIntegrationContainer.tsx b/web/src/features/blobstorage-integration/components/BlobStorageIntegrationContainer.tsxindex 718697b429..342b8558e4 100644--- a/web/src/features/blobstorage-integration/components/BlobStorageIntegrationContainer.tsx+++ b/web/src/features/blobstorage-integration/components/BlobStorageIntegrationContainer.tsx@@ -16,7 +16,10 @@ import {import { type BlobStorageIntegrationFormSchema } from "@/src/features/blobstorage-integration/types";import { useLangfuseCloudRegion } from "@/src/features/organizations";import { useQueryProject } from "@/src/features/projects";-import { buildBlobStorageFormValues } from "@/src/features/blobstorage-integration/components/formValues";+import {+ buildBlobStorageFormValues,+ toStoredFieldGroups,+} from "@/src/features/blobstorage-integration/components/formValues";import { BlobStorageIntegrationForm } from "@/src/features/blobstorage-integration/components/BlobStorageIntegrationForm";// State layer. Owns everything async and entity-scoped: availability@@ -112,7 +115,14 @@ export const BlobStorageIntegrationContainer = ({onSaved: () => void,) => {capture("integrations:blob_storage_form_submitted");- mut.mutate({ projectId, ...values }, { onSuccess: onSaved });+ mut.mutate(+ {+ projectId,+ ...values,+ exportFieldGroups: toStoredFieldGroups(values.exportFieldGroups),+ },+ { onSuccess: onSaved },+ );};return (diff --git a/web/src/features/blobstorage-integration/components/ExportFieldGroupsField.tsx b/web/src/features/blobstorage-integration/components/ExportFieldGroupsField.tsxindex 6f02905c1d..b6f881c24c 100644--- a/web/src/features/blobstorage-integration/components/ExportFieldGroupsField.tsx+++ b/web/src/features/blobstorage-integration/components/ExportFieldGroupsField.tsx@@ -1,5 +1,7 @@/* eslint-disable no-nested-ternary */import { useFormState, useWatch } from "react-hook-form";+import { AlertTriangle } from "@langfuse/icons";+import { Alert } from "@/src/components/design-system/Alert/Alert";import {FormDescription,FormField,@@ -13,6 +15,7 @@ import {EXPORT_FIELD_GROUP_OPTIONS,SCORE_EXPORT_CORE_FIELD_GROUP_OPTION,SCORE_EXPORT_FIELD_GROUP_OPTIONS,+ TRACE_EXPORT_FIELD_GROUP_OPTIONS,} from "@langfuse/shared";import { type BlobStorageFormControl } from "@/src/features/blobstorage-integration/components/formValues";import { FieldGroupCheckbox } from "@/src/features/blobstorage-integration/components/FieldGroupCheckbox";@@ -63,16 +66,49 @@ export const ExportFieldGroupsField = ({<FormItem><FormLabel>Export Field Groups</FormLabel><FormDescription>- Choose which field groups to include in the observation and score- exports. Deselect large groups (e.g. Input / Output) to reduce- export size, or privacy-sensitive groups (e.g. Metadata) to avoid- storing user data.+ Choose which field groups to include in the exports. Deselect large+ groups (e.g. Input / Output) to reduce export size, or+ privacy-sensitive groups (e.g. Metadata) to avoid storing user data.{includesLegacyExport? isLegacyOnlyExport- ? " Traces are always exported in full. Field groups that only exist on the enriched observations (e.g. Trace Context) are not available for this export source."- : " Traces are always exported in full. Fields that only exist on the enriched observations (e.g. Trace Context) are omitted from the legacy observations export."+ ? " Field groups that only exist on the enriched observations (e.g. Trace Context) are not available for this export source."+ : " Fields that only exist on the enriched observations (e.g. Trace Context) are omitted from the legacy observations export.": ""}</FormDescription>+ {includesLegacyExport && (+ <>+ <div className="text-sm font-bold">Traces</div>+ <div className="mt-2 space-y-2">+ {TRACE_EXPORT_FIELD_GROUP_OPTIONS.map((option) =>+ option.value === "trace_core" ? (+ <FieldGroupCheckbox+ key={option.value}+ id="field-group-trace-core"+ required+ label={option.label}+ description={option.description}+ />+ ) : (+ <FieldGroupCheckbox+ key={option.value}+ value={option.value}+ label={option.label}+ description={option.description}+ selected={field.value ?? []}+ onChange={field.onChange}+ />+ ),+ )}+ </div>+ <Alert variant="warning" size="sm" icon={AlertTriangle}>+ <Alert.Description>+ Trace field groups can only be set here. The public API+ doesn't show them, and updating field groups through the+ API keeps this selection.+ </Alert.Description>+ </Alert>+ </>+ )}<div className="text-sm font-bold">Observations</div><div className="mt-2 space-y-2">{EXPORT_FIELD_GROUP_OPTIONS.filter(diff --git a/web/src/features/blobstorage-integration/components/formValues.ts b/web/src/features/blobstorage-integration/components/formValues.tsindex 2b47ddff3b..162f40e67d 100644--- a/web/src/features/blobstorage-integration/components/formValues.ts+++ b/web/src/features/blobstorage-integration/components/formValues.ts@@ -5,6 +5,8 @@ import {BlobStorageIntegrationFileType,BlobStorageExportMode,OBSERVATION_FIELD_GROUPS_FULL,+ TRACE_EXPORT_FIELD_GROUPS,+ hasTraceFieldGroups,type BlobStorageIntegration,type ExportFieldGroup,type ExportSourceContext,@@ -28,6 +30,33 @@ export type BlobStorageFormControl = Control<BlobStorageIntegrationFormSchema>;+// Without a stored trace group the traces export is full, so show every trace+// group as selected; saving then stores the same full selection explicitly.+// trace_core is locked in the form and always kept in the selection.+function withTraceFieldGroups(groups: ExportFieldGroup[]): ExportFieldGroup[] {+ if (!hasTraceFieldGroups(groups)) {+ return [...groups, ...TRACE_EXPORT_FIELD_GROUPS];+ }+ return groups.includes("trace_core") ? groups : [...groups, "trace_core"];+}++// A full trace selection exports the same as no trace group, so it is stored+// without trace groups; the stored list then only names trace groups when the+// user deselected one.+export function toStoredFieldGroups(+ groups: ExportFieldGroup[],+): ExportFieldGroup[] {+ const isFullTraceSelection = TRACE_EXPORT_FIELD_GROUPS.every((group) =>+ groups.includes(group),+ );+ return isFullTraceSelection+ ? groups.filter(+ (group) =>+ !(TRACE_EXPORT_FIELD_GROUPS as readonly string[]).includes(group),+ )+ : groups;+}+export function buildBlobStorageFormValues(state:| (Partial<BlobStorageIntegration> & { hasSecretAccessKey?: boolean })@@ -62,9 +91,11 @@ export function buildBlobStorageFormValues(),// Empty array in the DB means "export everything" (the worker falls back// to all groups), so surface it as the full selection in the form.- exportFieldGroups: state?.exportFieldGroups?.length- ? (state.exportFieldGroups as ExportFieldGroup[])- : [...OBSERVATION_FIELD_GROUPS_FULL],+ exportFieldGroups: withTraceFieldGroups(+ state?.exportFieldGroups?.length+ ? (state.exportFieldGroups as ExportFieldGroup[])+ : [...OBSERVATION_FIELD_GROUPS_FULL],+ ),compressed: state?.compressed ?? true,};}diff --git a/web/src/features/public-api/types/blob-storage-integrations.ts b/web/src/features/public-api/types/blob-storage-integrations.tsindex 2a36a8409f..3d2c57e579 100644--- a/web/src/features/public-api/types/blob-storage-integrations.ts+++ b/web/src/features/public-api/types/blob-storage-integrations.ts@@ -1,7 +1,8 @@import { z } from "zod";import {AnalyticsIntegrationExportSource,- EXPORT_FIELD_GROUPS,+ OBSERVATION_FIELD_GROUPS_FULL,+ SCORE_EXPORT_FIELD_GROUPS,BLOB_STORAGE_REGION_INVALID_MESSAGE,BLOB_STORAGE_REGION_REGEX,} from "@langfuse/shared";@@ -89,7 +90,23 @@ export const toPublicExportSource = (): z.infer<typeof BlobStorageExportSource> =>INTERNAL_TO_PUBLIC_EXPORT_SOURCE[internalValue];-const BlobStorageExportFieldGroup = z.enum(EXPORT_FIELD_GROUPS);+// Trace groups are configurable in the UI only and never cross the public API,+// so SDKs never receive a value outside this enum.+const PUBLIC_EXPORT_FIELD_GROUPS = [+ ...OBSERVATION_FIELD_GROUPS_FULL,+ ...SCORE_EXPORT_FIELD_GROUPS,+] as const;++const BlobStorageExportFieldGroup = z.enum(PUBLIC_EXPORT_FIELD_GROUPS);++type PublicExportFieldGroup = z.infer<typeof BlobStorageExportFieldGroup>;++export const toPublicExportFieldGroups = (+ groups: readonly string[],+): PublicExportFieldGroup[] =>+ groups.filter((group): group is PublicExportFieldGroup =>+ (PUBLIC_EXPORT_FIELD_GROUPS as readonly string[]).includes(group),+ );/*** Request/Response Typesdiff --git a/web/src/pages/api/public/integrations/blob-storage/index.ts b/web/src/pages/api/public/integrations/blob-storage/index.tsindex 640fba8f4a..44836648f4 100644--- a/web/src/pages/api/public/integrations/blob-storage/index.ts+++ b/web/src/pages/api/public/integrations/blob-storage/index.ts@@ -4,10 +4,15 @@ import { type NextApiRequest, type NextApiResponse } from "next";import {CreateBlobStorageIntegrationRequest,toInternalExportSource,+ toPublicExportFieldGroups,toPublicExportSource,type BlobStorageIntegrationResponseType,} from "@/src/features/public-api/types/blob-storage-integrations";-import { type ExportFieldGroup, LangfuseNotFoundError } from "@langfuse/shared";+import {+ LangfuseNotFoundError,+ TRACE_EXPORT_FIELD_GROUPS,+ type ExportFieldGroup,+} from "@langfuse/shared";import { upsertBlobStorageIntegration } from "@/src/features/blobstorage-integration/service";import { resolveExportSource } from "@/src/features/analytics-integrations/server";import { auditLog } from "@/src/features/audit-logs/server";@@ -56,7 +61,9 @@ async function handleGetBlobStorageIntegrations(exportStartDate: integration.exportStartDate,compressed: integration.compressed,exportSource: toPublicExportSource(integration.exportSource),- exportFieldGroups: integration.exportFieldGroups as ExportFieldGroup[],+ exportFieldGroups: toPublicExportFieldGroups(+ integration.exportFieldGroups,+ ),nextSyncAt: integration.nextSyncAt,lastSyncAt: integration.lastSyncAt,lastError: integration.lastError,@@ -100,7 +107,7 @@ async function handleUpsertBlobStorageIntegration(// enriched value is rejected.const existingIntegration = await prisma.blobStorageIntegration.findUnique({where: { projectId: validatedData.projectId },- select: { createdAt: true, exportSource: true },+ select: { createdAt: true, exportSource: true, exportFieldGroups: true },});// Explicit sources must pass every check; an omitted source keeps the@@ -117,6 +124,14 @@ async function handleUpsertBlobStorageIntegration(existingIntegration,});+ // Trace groups are set in the UI only; keep them when the API replaces the+ // public groups.+ const storedTraceFieldGroups = (+ existingIntegration?.exportFieldGroups ?? []+ ).filter((group): group is ExportFieldGroup =>+ (TRACE_EXPORT_FIELD_GROUPS as readonly string[]).includes(group),+ );+await auditLog({action: "update",resourceType: "blobStorageIntegration",@@ -145,7 +160,9 @@ async function handleUpsertBlobStorageIntegration(exportStartDate: validatedData.exportStartDate ?? null,compressed: validatedData.compressed,exportSource: internalExportSource,- exportFieldGroups: validatedData.exportFieldGroups ?? undefined,+ exportFieldGroups: validatedData.exportFieldGroups+ ? [...validatedData.exportFieldGroups, ...storedTraceFieldGroups]+ : undefined,},});@@ -167,7 +184,7 @@ async function handleUpsertBlobStorageIntegration(exportStartDate: integration.exportStartDate,compressed: integration.compressed,exportSource: toPublicExportSource(integration.exportSource),- exportFieldGroups: integration.exportFieldGroups as ExportFieldGroup[],+ exportFieldGroups: toPublicExportFieldGroups(integration.exportFieldGroups),nextSyncAt: integration.nextSyncAt,lastSyncAt: integration.lastSyncAt,lastError: integration.lastError,diff --git a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts b/worker/src/__tests__/blobStorageIntegrationProcessing.test.tsindex b7ea55d8e4..71e03e6027 100644--- a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts+++ b/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts@@ -1313,6 +1313,116 @@ describe("BlobStorageIntegrationProcessingJob", () => {},);+ const TRACE_CORE_COLUMNS = [+ "id",+ "timestamp",+ "name",+ "environment",+ "project_id",+ "user_id",+ "session_id",+ "release",+ "version",+ "public",+ "bookmarked",+ "tags",+ "created_at",+ "updated_at",+ ];++ it.each([+ // no trace group stored: every trace column, in the original order+ [+ ["core", "io", "metadata"],+ [+ "id",+ "timestamp",+ "name",+ "environment",+ "project_id",+ "metadata",+ "user_id",+ "session_id",+ "release",+ "version",+ "public",+ "bookmarked",+ "tags",+ "input",+ "output",+ "created_at",+ "updated_at",+ ],+ ],+ [["core", "io", "metadata", "trace_core"], TRACE_CORE_COLUMNS],+ [+ ["core", "trace_core", "trace_io"],+ [...TRACE_CORE_COLUMNS.slice(0, 12), "input", "output"].concat(+ TRACE_CORE_COLUMNS.slice(12),+ ),+ ],+ // trace_core is implied once any trace group is stored+ [+ ["core", "trace_metadata"],+ [...TRACE_CORE_COLUMNS.slice(0, 5), "metadata"].concat(+ TRACE_CORE_COLUMNS.slice(5),+ ),+ ],+ ])(+ "should export trace columns for field groups %j",+ async (exportFieldGroups, expectedColumns) => {+ const { projectId } = await createOrgProjectAndApiKey();+ s3Prefix = projectId;+ const now = new Date();+ const twoHoursAgo = new Date(now.getTime() - 2 * 60 * 60 * 1000);++ await prisma.blobStorageIntegration.create({+ data: {+ projectId,+ type: BlobStorageIntegrationType.S3,+ bucketName,+ prefix: s3Prefix,+ accessKeyId: minioAccessKeyId,+ secretAccessKey: encrypt(minioAccessKeySecret),+ region: region ? region : "auto",+ endpoint: minioEndpoint,+ forcePathStyle:+ env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",+ enabled: true,+ exportFrequency: "hourly",+ exportSource: "TRACES_OBSERVATIONS",+ exportFieldGroups,+ nextSyncAt: twoHoursAgo,+ lastSyncAt: twoHoursAgo,+ compressed: false,+ fileType: BlobStorageIntegrationFileType.JSONL,+ },+ });++ await createTracesCh([+ createTrace({+ project_id: projectId,+ timestamp: now.getTime() - 90 * 60 * 1000,+ input: JSON.stringify({ q: "trace input" }),+ output: JSON.stringify({ a: "trace output" }),+ metadata: { source: "trace" },+ }),+ ]);++ await handleBlobStorageIntegrationProjectJob({+ data: { payload: { projectId } },+ } as Job);++ const files = await s3StorageService.listFiles(s3Prefix);+ const traceFile = files.find((f) => f.file.includes("/traces/"));+ expect(traceFile).toBeDefined();+ const content = await s3StorageService.download(traceFile!.file);+ const traceRow = JSON.parse(content.trim().split("\n")[0]);++ expect(Object.keys(traceRow)).toEqual(expectedColumns);+ },+ );+it("should respect export frequency when setting nextSyncAt", async () => {const { projectId } = await createOrgProjectAndApiKey();diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.tsindex 744e3b3f9a..69274f71c8 100644--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts@@ -66,6 +66,7 @@ import {type ExportFieldGroup,getObservationFieldGroups,getScoreFieldGroups,+ getTraceFieldGroups,isLegacyExportSource,isLegacyExporter,resolveBlobExportTuning,@@ -557,7 +558,8 @@ const processBlobStorageExport = async (config: {})();// Each table's query only receives its own groups. Score groups are- // opt-in, so the empty-selection fallback covers observation groups only.+ // opt-in, so the empty-selection fallback covers observation groups only;+ // traces export in full unless a trace group is stored.const selectedFieldGroups: ExportFieldGroup[] =config.exportFieldGroups && config.exportFieldGroups.length > 0? config.exportFieldGroups@@ -565,6 +567,7 @@ const processBlobStorageExport = async (config: {const observationFieldGroups =getObservationFieldGroups(selectedFieldGroups);const scoreFieldGroups = getScoreFieldGroups(selectedFieldGroups);+ const traceFieldGroups = getTraceFieldGroups(selectedFieldGroups);// blob.path already encodes parquet (no per-table fallback), so no// separate blob.config.parquet attribute is needed.@@ -623,6 +626,7 @@ const processBlobStorageExport = async (config: {config.projectId,config.minTimestamp,config.maxTimestamp,+ traceFieldGroups,)).stream;break;@@ -717,6 +721,7 @@ const processBlobStorageExport = async (config: {config.projectId,config.minTimestamp,config.maxTimestamp,+ traceFieldGroups,);break;case "observations":