Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(blob-export): add opt-in trace field groups for the legacy traces export (#19244)

Niklas Semmlerbd1ad4f71f67
diff --git a/packages/shared/src/domain/export-field-groups.ts b/packages/shared/src/domain/export-field-groups.ts
index 02f500dc44..2b3d51fc6d 100644
--- a/packages/shared/src/domain/export-field-groups.ts
+++ b/packages/shared/src/domain/export-field-groups.ts
@@ -59,15 +59,59 @@ export const SCORE_EXPORT_GROUP_FIELDS = {
],
} as const satisfies Record<ScoreExportFieldGroup, readonly string[]>;
+/**
+ * Field groups of the legacy traces blob export, prefixed with `trace_` like
+ * the score groups. `trace_core` is always exported; it is stored to mark the
+ * trace selection as configured.
+ *
+ * A selection without any trace group exports every trace column, so
+ * integrations saved before trace groups existed keep their export unchanged.
+ */
+export const TRACE_EXPORT_FIELD_GROUPS = [
+ "trace_core",
+ "trace_io",
+ "trace_metadata",
+] as const;
+
+export type TraceExportFieldGroup = (typeof TRACE_EXPORT_FIELD_GROUPS)[number];
+
+/**
+ * Columns of the traces export and the group each belongs to, in the column
+ * order of the full export (kept stable so a full selection is unchanged).
+ */
+export const TRACE_EXPORT_FIELDS = [
+ { field: "id", group: "trace_core" },
+ { field: "timestamp", group: "trace_core" },
+ { field: "name", group: "trace_core" },
+ { field: "environment", group: "trace_core" },
+ { field: "project_id", group: "trace_core" },
+ { field: "metadata", group: "trace_metadata" },
+ { field: "user_id", group: "trace_core" },
+ { field: "session_id", group: "trace_core" },
+ { field: "release", group: "trace_core" },
+ { field: "version", group: "trace_core" },
+ { field: "public", group: "trace_core" },
+ { field: "bookmarked", group: "trace_core" },
+ { field: "tags", group: "trace_core" },
+ { field: "input", group: "trace_io" },
+ { field: "output", group: "trace_io" },
+ { field: "created_at", group: "trace_core" },
+ { field: "updated_at", group: "trace_core" },
+] as const satisfies readonly {
+ field: string;
+ group: TraceExportFieldGroup;
+}[];
+
/**
* Every value that can be stored in an integration's `exportFieldGroups`.
* Each group belongs to exactly one exported table; split a selection with
- * `getObservationFieldGroups` / `getScoreFieldGroups` before handing it to a
- * table's query.
+ * `getObservationFieldGroups` / `getScoreFieldGroups` /
+ * `getTraceFieldGroups` before handing it to a table's query.
*/
export const EXPORT_FIELD_GROUPS = [
...OBSERVATION_FIELD_GROUPS_FULL,
...SCORE_EXPORT_FIELD_GROUPS,
+ ...TRACE_EXPORT_FIELD_GROUPS,
] as const;
export type ExportFieldGroup = (typeof EXPORT_FIELD_GROUPS)[number];
@@ -85,3 +129,22 @@ export const getScoreFieldGroups = (
groups.filter((group): group is ScoreExportFieldGroup =>
(SCORE_EXPORT_FIELD_GROUPS as readonly string[]).includes(group),
);
+
+/** True when the selection configures the traces export (any trace group stored). */
+export const hasTraceFieldGroups = (groups: readonly string[]): boolean =>
+ groups.some((group) =>
+ (TRACE_EXPORT_FIELD_GROUPS as readonly string[]).includes(group),
+ );
+
+/**
+ * Trace groups to export: all of them when the selection holds no trace group,
+ * otherwise the selected ones plus the always-exported `trace_core`.
+ */
+export const getTraceFieldGroups = (
+ groups: readonly string[],
+): TraceExportFieldGroup[] =>
+ hasTraceFieldGroups(groups)
+ ? TRACE_EXPORT_FIELD_GROUPS.filter(
+ (group) => group === "trace_core" || groups.includes(group),
+ )
+ : [...TRACE_EXPORT_FIELD_GROUPS];
diff --git a/packages/shared/src/features/analytics-integrations/index.ts b/packages/shared/src/features/analytics-integrations/index.ts
index cd27ff1b0f..cfbb612146 100644
--- a/packages/shared/src/features/analytics-integrations/index.ts
+++ b/packages/shared/src/features/analytics-integrations/index.ts
@@ -12,6 +12,9 @@ import {
SCORE_EXPORT_FIELD_GROUPS,
SCORE_EXPORT_GROUP_FIELDS,
type ScoreExportFieldGroup,
+ TRACE_EXPORT_FIELDS,
+ TRACE_EXPORT_FIELD_GROUPS,
+ type TraceExportFieldGroup,
} from "../../domain/export-field-groups";
export * from "./export-source-policy";
@@ -175,3 +178,20 @@ export const SCORE_EXPORT_FIELD_GROUP_OPTIONS = SCORE_EXPORT_FIELD_GROUPS.map(
description: SCORE_EXPORT_GROUP_FIELDS[value].join(", "),
}),
);
+
+/** Trace group labels, in the order of the matching observation groups. */
+const TRACE_EXPORT_FIELD_GROUP_LABELS = {
+ trace_core: { label: "Core" },
+ trace_io: { label: "Input / Output" },
+ trace_metadata: { label: "Metadata" },
+} satisfies Record<TraceExportFieldGroup, { label: string }>;
+
+export const TRACE_EXPORT_FIELD_GROUP_OPTIONS = TRACE_EXPORT_FIELD_GROUPS.map(
+ (value) => ({
+ value,
+ ...TRACE_EXPORT_FIELD_GROUP_LABELS[value],
+ description: TRACE_EXPORT_FIELDS.filter((f) => f.group === value)
+ .map((f) => f.field)
+ .join(", "),
+ }),
+);
diff --git a/packages/shared/src/server/repositories/traces.ts b/packages/shared/src/server/repositories/traces.ts
index d4193aef99..dea4070b28 100644
--- a/packages/shared/src/server/repositories/traces.ts
+++ b/packages/shared/src/server/repositories/traces.ts
@@ -22,6 +22,11 @@ import {
resolveScoreDataRequirement,
} from "../queries/clickhouse-sql/score-filter-pushdown";
import { LISTABLE_SCORE_TYPES } from "../../domain/scores";
+import {
+ TRACE_EXPORT_FIELDS,
+ TRACE_EXPORT_FIELD_GROUPS,
+ type TraceExportFieldGroup,
+} from "../../domain/export-field-groups";
import { OrderByState } from "../../interfaces/orderBy";
import snakeCase from "lodash/snakeCase";
import { FilterState } from "../../types";
@@ -1207,33 +1212,22 @@ export const getUserMetrics = async (
};
// Shared query for both the standard (parsed) and Parquet (`exec` binary)
-// trace export paths, so the SELECT/WHERE stays in one place.
+// trace export paths, so the SELECT/WHERE stays in one place. Deselected
+// field group columns are dropped; the remaining columns keep their order.
const buildTracesForBlobStorageExportQuery = (
projectId: string,
minTimestamp: Date,
maxTimestamp: Date,
+ fieldGroups: TraceExportFieldGroup[],
) => {
const traceTable = "traces";
+ const columns = TRACE_EXPORT_FIELDS.filter(({ group }) =>
+ fieldGroups.includes(group),
+ ).map(({ field }) => field);
const query = `
SELECT
- id,
- timestamp,
- name,
- environment,
- project_id,
- metadata,
- user_id,
- session_id,
- release,
- version,
- public as public,
- bookmarked as bookmarked,
- tags,
- input as input,
- output as output,
- created_at,
- updated_at
+ ${columns.join(",\n ")}
FROM ${traceTable} FINAL
WHERE project_id = {projectId: String}
AND timestamp >= {minTimestamp: DateTime64(3)}
@@ -1260,9 +1254,15 @@ export const getTracesForBlobStorageExport = function (
projectId: string,
minTimestamp: Date,
maxTimestamp: Date,
+ fieldGroups: TraceExportFieldGroup[] = [...TRACE_EXPORT_FIELD_GROUPS],
) {
return queryClickhouseStream<Record<string, unknown>>(
- buildTracesForBlobStorageExportQuery(projectId, minTimestamp, maxTimestamp),
+ buildTracesForBlobStorageExportQuery(
+ projectId,
+ minTimestamp,
+ maxTimestamp,
+ fieldGroups,
+ ),
);
};
@@ -1272,12 +1272,14 @@ export const getTracesForBlobStorageExportParquet = function (
projectId: string,
minTimestamp: Date,
maxTimestamp: Date,
+ fieldGroups: TraceExportFieldGroup[] = [...TRACE_EXPORT_FIELD_GROUPS],
) {
return queryClickhouseExecRaw({
...buildTracesForBlobStorageExportQuery(
projectId,
minTimestamp,
maxTimestamp,
+ fieldGroups,
),
format: "Parquet",
clickhouseSettings: BLOB_EXPORT_PARQUET_CLICKHOUSE_SETTINGS,
diff --git a/web/src/__tests__/blob-storage-form-field-groups.clienttest.ts b/web/src/__tests__/blob-storage-form-field-groups.clienttest.ts
index 53623027de..ee45c80b71 100644
--- a/web/src/__tests__/blob-storage-form-field-groups.clienttest.ts
+++ b/web/src/__tests__/blob-storage-form-field-groups.clienttest.ts
@@ -12,7 +12,12 @@ import {
BlobStorageIntegrationFileType,
BlobStorageIntegrationType,
EXPORT_FIELD_GROUP_OPTIONS,
+ type ExportSourceContext,
} from "@langfuse/shared";
+import {
+ buildBlobStorageFormValues,
+ toStoredFieldGroups,
+} from "@/src/features/blobstorage-integration/components/formValues";
const VALID_BASE: BlobStorageIntegrationFormSchema = {
type: BlobStorageIntegrationType.S3,
@@ -162,3 +167,64 @@ describe("EXPORT_FIELD_GROUP_OPTIONS — parquet description", () => {
}
});
});
+
+describe("buildBlobStorageFormValues — trace field groups", () => {
+ const ctx: ExportSourceContext = {
+ isCloud: false,
+ enrichedAvailable: true,
+ legacyWritesActive: true,
+ };
+
+ it.each([
+ // no stored trace group = full traces export, so every trace group is shown
+ [
+ ["core", "io"],
+ ["core", "io", "trace_core", "trace_io", "trace_metadata"],
+ ],
+ [
+ ["core", "trace_core"],
+ ["core", "trace_core"],
+ ],
+ // trace_core is locked in the form and kept once any trace group is stored
+ [
+ ["core", "trace_io"],
+ ["core", "trace_io", "trace_core"],
+ ],
+ ])("maps stored groups %j to form groups %j", (stored, expected) => {
+ const values = buildBlobStorageFormValues(
+ { exportFieldGroups: stored },
+ ctx,
+ );
+ expect(values.exportFieldGroups).toEqual(expected);
+ });
+});
+
+describe("toStoredFieldGroups", () => {
+ const ctx: ExportSourceContext = {
+ isCloud: false,
+ enrichedAvailable: true,
+ legacyWritesActive: true,
+ };
+
+ it.each([
+ // an untouched save stores the same list it loaded (no trace groups)
+ [
+ ["core", "io"],
+ ["core", "io"],
+ ],
+ [
+ ["core", "trace_core"],
+ ["core", "trace_core"],
+ ],
+ [
+ ["core", "trace_core", "trace_io"],
+ ["core", "trace_core", "trace_io"],
+ ],
+ ])("round-trips stored groups %j as %j", (stored, expected) => {
+ const formGroups = buildBlobStorageFormValues(
+ { exportFieldGroups: stored },
+ ctx,
+ ).exportFieldGroups;
+ expect(toStoredFieldGroups(formGroups ?? [])).toEqual(expected);
+ });
+});
diff --git a/web/src/__tests__/server/blob-storage-integration-api.servertest.ts b/web/src/__tests__/server/blob-storage-integration-api.servertest.ts
index 115eb1d908..fad878a760 100644
--- a/web/src/__tests__/server/blob-storage-integration-api.servertest.ts
+++ b/web/src/__tests__/server/blob-storage-integration-api.servertest.ts
@@ -1023,6 +1023,39 @@ describe("Blob Storage Integrations API", () => {
expect(saved?.exportFieldGroups).toStrictEqual(["core", "io"]);
});
+ it("PUT LEGACY_TRACES_OBSERVATIONS hides UI-set trace groups and keeps them on update", async () => {
+ await seedGrandfatheredLegacyRow({
+ exportFieldGroups: ["core", "io", "trace_core", "trace_metadata"],
+ });
+
+ const putResponse = await makeZodVerifiedAPICall(
+ BlobStorageIntegrationResponseSchema,
+ "PUT",
+ "/api/public/integrations/blob-storage",
+ {
+ ...validBlobStorageConfig,
+ projectId: testProject1Id,
+ exportSource: "LEGACY_TRACES_OBSERVATIONS" as const,
+ exportFieldGroups: ["core", "metadata"],
+ },
+ createBasicAuthHeader(testApiKey, testApiSecretKey),
+ );
+ expect(putResponse.body.exportFieldGroups).toStrictEqual([
+ "core",
+ "metadata",
+ ]);
+
+ const saved = await prisma.blobStorageIntegration.findUnique({
+ where: { projectId: testProject1Id },
+ });
+ expect(saved?.exportFieldGroups).toStrictEqual([
+ "core",
+ "metadata",
+ "trace_core",
+ "trace_metadata",
+ ]);
+ });
+
// ---- Response shape ----
it("PUT response includes exportSource and exportFieldGroups for LEGACY_TRACES_OBSERVATIONS", async () => {
diff --git a/web/src/features/blobstorage-integration/components/BlobStorageIntegrationContainer.tsx b/web/src/features/blobstorage-integration/components/BlobStorageIntegrationContainer.tsx
index 718697b429..342b8558e4 100644
--- a/web/src/features/blobstorage-integration/components/BlobStorageIntegrationContainer.tsx
+++ b/web/src/features/blobstorage-integration/components/BlobStorageIntegrationContainer.tsx
@@ -16,7 +16,10 @@ import {
import { type BlobStorageIntegrationFormSchema } from "@/src/features/blobstorage-integration/types";
import { useLangfuseCloudRegion } from "@/src/features/organizations";
import { useQueryProject } from "@/src/features/projects";
-import { buildBlobStorageFormValues } from "@/src/features/blobstorage-integration/components/formValues";
+import {
+ buildBlobStorageFormValues,
+ toStoredFieldGroups,
+} from "@/src/features/blobstorage-integration/components/formValues";
import { BlobStorageIntegrationForm } from "@/src/features/blobstorage-integration/components/BlobStorageIntegrationForm";
// State layer. Owns everything async and entity-scoped: availability
@@ -112,7 +115,14 @@ export const BlobStorageIntegrationContainer = ({
onSaved: () => void,
) => {
capture("integrations:blob_storage_form_submitted");
- mut.mutate({ projectId, ...values }, { onSuccess: onSaved });
+ mut.mutate(
+ {
+ projectId,
+ ...values,
+ exportFieldGroups: toStoredFieldGroups(values.exportFieldGroups),
+ },
+ { onSuccess: onSaved },
+ );
};
return (
diff --git a/web/src/features/blobstorage-integration/components/ExportFieldGroupsField.tsx b/web/src/features/blobstorage-integration/components/ExportFieldGroupsField.tsx
index 6f02905c1d..b6f881c24c 100644
--- a/web/src/features/blobstorage-integration/components/ExportFieldGroupsField.tsx
+++ b/web/src/features/blobstorage-integration/components/ExportFieldGroupsField.tsx
@@ -1,5 +1,7 @@
/* eslint-disable no-nested-ternary */
import { useFormState, useWatch } from "react-hook-form";
+import { AlertTriangle } from "@langfuse/icons";
+import { Alert } from "@/src/components/design-system/Alert/Alert";
import {
FormDescription,
FormField,
@@ -13,6 +15,7 @@ import {
EXPORT_FIELD_GROUP_OPTIONS,
SCORE_EXPORT_CORE_FIELD_GROUP_OPTION,
SCORE_EXPORT_FIELD_GROUP_OPTIONS,
+ TRACE_EXPORT_FIELD_GROUP_OPTIONS,
} from "@langfuse/shared";
import { type BlobStorageFormControl } from "@/src/features/blobstorage-integration/components/formValues";
import { FieldGroupCheckbox } from "@/src/features/blobstorage-integration/components/FieldGroupCheckbox";
@@ -63,16 +66,49 @@ export const ExportFieldGroupsField = ({
<FormItem>
<FormLabel>Export Field Groups</FormLabel>
<FormDescription>
- Choose which field groups to include in the observation and score
- exports. Deselect large groups (e.g. Input / Output) to reduce
- export size, or privacy-sensitive groups (e.g. Metadata) to avoid
- storing user data.
+ Choose which field groups to include in the exports. Deselect large
+ groups (e.g. Input / Output) to reduce export size, or
+ privacy-sensitive groups (e.g. Metadata) to avoid storing user data.
{includesLegacyExport
? isLegacyOnlyExport
- ? " Traces are always exported in full. Field groups that only exist on the enriched observations (e.g. Trace Context) are not available for this export source."
- : " Traces are always exported in full. Fields that only exist on the enriched observations (e.g. Trace Context) are omitted from the legacy observations export."
+ ? " Field groups that only exist on the enriched observations (e.g. Trace Context) are not available for this export source."
+ : " Fields that only exist on the enriched observations (e.g. Trace Context) are omitted from the legacy observations export."
: ""}
</FormDescription>
+ {includesLegacyExport && (
+ <>
+ <div className="text-sm font-bold">Traces</div>
+ <div className="mt-2 space-y-2">
+ {TRACE_EXPORT_FIELD_GROUP_OPTIONS.map((option) =>
+ option.value === "trace_core" ? (
+ <FieldGroupCheckbox
+ key={option.value}
+ id="field-group-trace-core"
+ required
+ label={option.label}
+ description={option.description}
+ />
+ ) : (
+ <FieldGroupCheckbox
+ key={option.value}
+ value={option.value}
+ label={option.label}
+ description={option.description}
+ selected={field.value ?? []}
+ onChange={field.onChange}
+ />
+ ),
+ )}
+ </div>
+ <Alert variant="warning" size="sm" icon={AlertTriangle}>
+ <Alert.Description>
+ Trace field groups can only be set here. The public API
+ doesn&apos;t show them, and updating field groups through the
+ API keeps this selection.
+ </Alert.Description>
+ </Alert>
+ </>
+ )}
<div className="text-sm font-bold">Observations</div>
<div className="mt-2 space-y-2">
{EXPORT_FIELD_GROUP_OPTIONS.filter(
diff --git a/web/src/features/blobstorage-integration/components/formValues.ts b/web/src/features/blobstorage-integration/components/formValues.ts
index 2b47ddff3b..162f40e67d 100644
--- a/web/src/features/blobstorage-integration/components/formValues.ts
+++ b/web/src/features/blobstorage-integration/components/formValues.ts
@@ -5,6 +5,8 @@ import {
BlobStorageIntegrationFileType,
BlobStorageExportMode,
OBSERVATION_FIELD_GROUPS_FULL,
+ TRACE_EXPORT_FIELD_GROUPS,
+ hasTraceFieldGroups,
type BlobStorageIntegration,
type ExportFieldGroup,
type ExportSourceContext,
@@ -28,6 +30,33 @@ export type BlobStorageFormControl = Control<
BlobStorageIntegrationFormSchema
>;
+// Without a stored trace group the traces export is full, so show every trace
+// group as selected; saving then stores the same full selection explicitly.
+// trace_core is locked in the form and always kept in the selection.
+function withTraceFieldGroups(groups: ExportFieldGroup[]): ExportFieldGroup[] {
+ if (!hasTraceFieldGroups(groups)) {
+ return [...groups, ...TRACE_EXPORT_FIELD_GROUPS];
+ }
+ return groups.includes("trace_core") ? groups : [...groups, "trace_core"];
+}
+
+// A full trace selection exports the same as no trace group, so it is stored
+// without trace groups; the stored list then only names trace groups when the
+// user deselected one.
+export function toStoredFieldGroups(
+ groups: ExportFieldGroup[],
+): ExportFieldGroup[] {
+ const isFullTraceSelection = TRACE_EXPORT_FIELD_GROUPS.every((group) =>
+ groups.includes(group),
+ );
+ return isFullTraceSelection
+ ? groups.filter(
+ (group) =>
+ !(TRACE_EXPORT_FIELD_GROUPS as readonly string[]).includes(group),
+ )
+ : groups;
+}
+
export function buildBlobStorageFormValues(
state:
| (Partial<BlobStorageIntegration> & { hasSecretAccessKey?: boolean })
@@ -62,9 +91,11 @@ export function buildBlobStorageFormValues(
),
// Empty array in the DB means "export everything" (the worker falls back
// to all groups), so surface it as the full selection in the form.
- exportFieldGroups: state?.exportFieldGroups?.length
- ? (state.exportFieldGroups as ExportFieldGroup[])
- : [...OBSERVATION_FIELD_GROUPS_FULL],
+ exportFieldGroups: withTraceFieldGroups(
+ state?.exportFieldGroups?.length
+ ? (state.exportFieldGroups as ExportFieldGroup[])
+ : [...OBSERVATION_FIELD_GROUPS_FULL],
+ ),
compressed: state?.compressed ?? true,
};
}
diff --git a/web/src/features/public-api/types/blob-storage-integrations.ts b/web/src/features/public-api/types/blob-storage-integrations.ts
index 2a36a8409f..3d2c57e579 100644
--- a/web/src/features/public-api/types/blob-storage-integrations.ts
+++ b/web/src/features/public-api/types/blob-storage-integrations.ts
@@ -1,7 +1,8 @@
import { z } from "zod";
import {
AnalyticsIntegrationExportSource,
- EXPORT_FIELD_GROUPS,
+ OBSERVATION_FIELD_GROUPS_FULL,
+ SCORE_EXPORT_FIELD_GROUPS,
BLOB_STORAGE_REGION_INVALID_MESSAGE,
BLOB_STORAGE_REGION_REGEX,
} from "@langfuse/shared";
@@ -89,7 +90,23 @@ export const toPublicExportSource = (
): z.infer<typeof BlobStorageExportSource> =>
INTERNAL_TO_PUBLIC_EXPORT_SOURCE[internalValue];
-const BlobStorageExportFieldGroup = z.enum(EXPORT_FIELD_GROUPS);
+// Trace groups are configurable in the UI only and never cross the public API,
+// so SDKs never receive a value outside this enum.
+const PUBLIC_EXPORT_FIELD_GROUPS = [
+ ...OBSERVATION_FIELD_GROUPS_FULL,
+ ...SCORE_EXPORT_FIELD_GROUPS,
+] as const;
+
+const BlobStorageExportFieldGroup = z.enum(PUBLIC_EXPORT_FIELD_GROUPS);
+
+type PublicExportFieldGroup = z.infer<typeof BlobStorageExportFieldGroup>;
+
+export const toPublicExportFieldGroups = (
+ groups: readonly string[],
+): PublicExportFieldGroup[] =>
+ groups.filter((group): group is PublicExportFieldGroup =>
+ (PUBLIC_EXPORT_FIELD_GROUPS as readonly string[]).includes(group),
+ );
/**
* Request/Response Types
diff --git a/web/src/pages/api/public/integrations/blob-storage/index.ts b/web/src/pages/api/public/integrations/blob-storage/index.ts
index 640fba8f4a..44836648f4 100644
--- a/web/src/pages/api/public/integrations/blob-storage/index.ts
+++ b/web/src/pages/api/public/integrations/blob-storage/index.ts
@@ -4,10 +4,15 @@ import { type NextApiRequest, type NextApiResponse } from "next";
import {
CreateBlobStorageIntegrationRequest,
toInternalExportSource,
+ toPublicExportFieldGroups,
toPublicExportSource,
type BlobStorageIntegrationResponseType,
} from "@/src/features/public-api/types/blob-storage-integrations";
-import { type ExportFieldGroup, LangfuseNotFoundError } from "@langfuse/shared";
+import {
+ LangfuseNotFoundError,
+ TRACE_EXPORT_FIELD_GROUPS,
+ type ExportFieldGroup,
+} from "@langfuse/shared";
import { upsertBlobStorageIntegration } from "@/src/features/blobstorage-integration/service";
import { resolveExportSource } from "@/src/features/analytics-integrations/server";
import { auditLog } from "@/src/features/audit-logs/server";
@@ -56,7 +61,9 @@ async function handleGetBlobStorageIntegrations(
exportStartDate: integration.exportStartDate,
compressed: integration.compressed,
exportSource: toPublicExportSource(integration.exportSource),
- exportFieldGroups: integration.exportFieldGroups as ExportFieldGroup[],
+ exportFieldGroups: toPublicExportFieldGroups(
+ integration.exportFieldGroups,
+ ),
nextSyncAt: integration.nextSyncAt,
lastSyncAt: integration.lastSyncAt,
lastError: integration.lastError,
@@ -100,7 +107,7 @@ async function handleUpsertBlobStorageIntegration(
// enriched value is rejected.
const existingIntegration = await prisma.blobStorageIntegration.findUnique({
where: { projectId: validatedData.projectId },
- select: { createdAt: true, exportSource: true },
+ select: { createdAt: true, exportSource: true, exportFieldGroups: true },
});
// Explicit sources must pass every check; an omitted source keeps the
@@ -117,6 +124,14 @@ async function handleUpsertBlobStorageIntegration(
existingIntegration,
});
+ // Trace groups are set in the UI only; keep them when the API replaces the
+ // public groups.
+ const storedTraceFieldGroups = (
+ existingIntegration?.exportFieldGroups ?? []
+ ).filter((group): group is ExportFieldGroup =>
+ (TRACE_EXPORT_FIELD_GROUPS as readonly string[]).includes(group),
+ );
+
await auditLog({
action: "update",
resourceType: "blobStorageIntegration",
@@ -145,7 +160,9 @@ async function handleUpsertBlobStorageIntegration(
exportStartDate: validatedData.exportStartDate ?? null,
compressed: validatedData.compressed,
exportSource: internalExportSource,
- exportFieldGroups: validatedData.exportFieldGroups ?? undefined,
+ exportFieldGroups: validatedData.exportFieldGroups
+ ? [...validatedData.exportFieldGroups, ...storedTraceFieldGroups]
+ : undefined,
},
});
@@ -167,7 +184,7 @@ async function handleUpsertBlobStorageIntegration(
exportStartDate: integration.exportStartDate,
compressed: integration.compressed,
exportSource: toPublicExportSource(integration.exportSource),
- exportFieldGroups: integration.exportFieldGroups as ExportFieldGroup[],
+ exportFieldGroups: toPublicExportFieldGroups(integration.exportFieldGroups),
nextSyncAt: integration.nextSyncAt,
lastSyncAt: integration.lastSyncAt,
lastError: integration.lastError,
diff --git a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts b/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts
index b7ea55d8e4..71e03e6027 100644
--- a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts
+++ b/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts
@@ -1313,6 +1313,116 @@ describe("BlobStorageIntegrationProcessingJob", () => {
},
);
+ const TRACE_CORE_COLUMNS = [
+ "id",
+ "timestamp",
+ "name",
+ "environment",
+ "project_id",
+ "user_id",
+ "session_id",
+ "release",
+ "version",
+ "public",
+ "bookmarked",
+ "tags",
+ "created_at",
+ "updated_at",
+ ];
+
+ it.each([
+ // no trace group stored: every trace column, in the original order
+ [
+ ["core", "io", "metadata"],
+ [
+ "id",
+ "timestamp",
+ "name",
+ "environment",
+ "project_id",
+ "metadata",
+ "user_id",
+ "session_id",
+ "release",
+ "version",
+ "public",
+ "bookmarked",
+ "tags",
+ "input",
+ "output",
+ "created_at",
+ "updated_at",
+ ],
+ ],
+ [["core", "io", "metadata", "trace_core"], TRACE_CORE_COLUMNS],
+ [
+ ["core", "trace_core", "trace_io"],
+ [...TRACE_CORE_COLUMNS.slice(0, 12), "input", "output"].concat(
+ TRACE_CORE_COLUMNS.slice(12),
+ ),
+ ],
+ // trace_core is implied once any trace group is stored
+ [
+ ["core", "trace_metadata"],
+ [...TRACE_CORE_COLUMNS.slice(0, 5), "metadata"].concat(
+ TRACE_CORE_COLUMNS.slice(5),
+ ),
+ ],
+ ])(
+ "should export trace columns for field groups %j",
+ async (exportFieldGroups, expectedColumns) => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ s3Prefix = projectId;
+ const now = new Date();
+ const twoHoursAgo = new Date(now.getTime() - 2 * 60 * 60 * 1000);
+
+ await prisma.blobStorageIntegration.create({
+ data: {
+ projectId,
+ type: BlobStorageIntegrationType.S3,
+ bucketName,
+ prefix: s3Prefix,
+ accessKeyId: minioAccessKeyId,
+ secretAccessKey: encrypt(minioAccessKeySecret),
+ region: region ? region : "auto",
+ endpoint: minioEndpoint,
+ forcePathStyle:
+ env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
+ enabled: true,
+ exportFrequency: "hourly",
+ exportSource: "TRACES_OBSERVATIONS",
+ exportFieldGroups,
+ nextSyncAt: twoHoursAgo,
+ lastSyncAt: twoHoursAgo,
+ compressed: false,
+ fileType: BlobStorageIntegrationFileType.JSONL,
+ },
+ });
+
+ await createTracesCh([
+ createTrace({
+ project_id: projectId,
+ timestamp: now.getTime() - 90 * 60 * 1000,
+ input: JSON.stringify({ q: "trace input" }),
+ output: JSON.stringify({ a: "trace output" }),
+ metadata: { source: "trace" },
+ }),
+ ]);
+
+ await handleBlobStorageIntegrationProjectJob({
+ data: { payload: { projectId } },
+ } as Job);
+
+ const files = await s3StorageService.listFiles(s3Prefix);
+ const traceFile = files.find((f) => f.file.includes("/traces/"));
+ expect(traceFile).toBeDefined();
+ const content = await s3StorageService.download(traceFile!.file);
+ const traceRow = JSON.parse(content.trim().split("\n")[0]);
+
+ expect(Object.keys(traceRow)).toEqual(expectedColumns);
+ },
+ );
+
it("should respect export frequency when setting nextSyncAt", async () => {
const { projectId } = await createOrgProjectAndApiKey();
diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
index 744e3b3f9a..69274f71c8 100644
--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
@@ -66,6 +66,7 @@ import {
type ExportFieldGroup,
getObservationFieldGroups,
getScoreFieldGroups,
+ getTraceFieldGroups,
isLegacyExportSource,
isLegacyExporter,
resolveBlobExportTuning,
@@ -557,7 +558,8 @@ const processBlobStorageExport = async (config: {
})();
// Each table's query only receives its own groups. Score groups are
- // opt-in, so the empty-selection fallback covers observation groups only.
+ // opt-in, so the empty-selection fallback covers observation groups only;
+ // traces export in full unless a trace group is stored.
const selectedFieldGroups: ExportFieldGroup[] =
config.exportFieldGroups && config.exportFieldGroups.length > 0
? config.exportFieldGroups
@@ -565,6 +567,7 @@ const processBlobStorageExport = async (config: {
const observationFieldGroups =
getObservationFieldGroups(selectedFieldGroups);
const scoreFieldGroups = getScoreFieldGroups(selectedFieldGroups);
+ const traceFieldGroups = getTraceFieldGroups(selectedFieldGroups);
// blob.path already encodes parquet (no per-table fallback), so no
// separate blob.config.parquet attribute is needed.
@@ -623,6 +626,7 @@ const processBlobStorageExport = async (config: {
config.projectId,
config.minTimestamp,
config.maxTimestamp,
+ traceFieldGroups,
)
).stream;
break;
@@ -717,6 +721,7 @@ const processBlobStorageExport = async (config: {
config.projectId,
config.minTimestamp,
config.maxTimestamp,
+ traceFieldGroups,
);
break;
case "observations":