Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

25.0 KB
Raw Blame
import { Processor } from "bullmq";
import { Readable } from "node:stream";
import { DashboardWidgetChartType } from "@prisma/client";
import pLimit from "p-limit";
import {
  logger,
  StorageServiceFactory,
  type StorageService,
} from "@langfuse/shared/src/server";
import { metricAggregations, viewDeclarations } from "@langfuse/shared/query";
import { prisma } from "@langfuse/shared/src/db";
import { env } from "../env";

const CORE_DATA_EXPORT_PAGE_SIZE = 1_000;
const CORE_DATA_EXPORT_PART_SIZE_BYTES = 100 * 1024 * 1024;
// Each table export holds at most one postgres connection at a time (short
// keyset page queries). Keep this below the worker's Prisma pool size
// (default: 5 connections) so other queue jobs on the same worker never
// starve on connection acquisition.
const CORE_DATA_EXPORT_TABLE_CONCURRENCY = 3;

let s3StorageServiceClient: StorageService;

const getS3StorageServiceClient = (bucketName: string): StorageService => {
  if (!s3StorageServiceClient) {
    s3StorageServiceClient = StorageServiceFactory.getInstance({
      bucketName,
      accessKeyId: env.LANGFUSE_S3_CORE_DATA_UPLOAD_ACCESS_KEY_ID,
      secretAccessKey: env.LANGFUSE_S3_CORE_DATA_UPLOAD_SECRET_ACCESS_KEY,
      endpoint: env.LANGFUSE_S3_CORE_DATA_UPLOAD_ENDPOINT,
      region: env.LANGFUSE_S3_CORE_DATA_UPLOAD_REGION,
      forcePathStyle:
        env.LANGFUSE_S3_CORE_DATA_UPLOAD_FORCE_PATH_STYLE === "true",
      awsSse: env.LANGFUSE_S3_CORE_DATA_UPLOAD_SSE,
      awsSseKmsKeyId: env.LANGFUSE_S3_CORE_DATA_UPLOAD_SSE_KMS_KEY_ID,
    });
  }
  return s3StorageServiceClient;
};

type UserCoreDataInput = {
  password: string | null;
  accounts: { provider: string }[];
} & Record<string, unknown>;

// Derives the auth methods per user from the linked next-auth accounts and the
// presence of a password hash ("credentials"). The hash itself must never
// reach the export.
export const mapUserToCoreDataRow = ({
  password,
  accounts,
  ...user
}: UserCoreDataInput) => ({
  ...user,
  authMethods: [
    ...(password ? ["credentials"] : []),
    ...Array.from(new Set(accounts.map((account) => account.provider))),
  ],
});

type JobConfigurationCoreDataInput = {
  evalTemplate: { name: string } | null;
  sampling: { toNumber: () => number };
  variableMapping?: unknown;
} & Record<string, unknown>;

// Airbyte's JSONL schema inference cannot merge null with array for one column.
// Empty mappings are exported as [] so every row has an array.
const variableMappingForExport = (variableMapping: unknown) =>
  variableMapping == null ? [] : variableMapping;

export const mapJobConfigurationToCoreDataRow = ({
  evalTemplate,
  sampling,
  variableMapping,
  ...jobConfiguration
}: JobConfigurationCoreDataInput) => ({
  ...jobConfiguration,
  variableMapping: variableMappingForExport(variableMapping),
  sampling: sampling.toNumber(),
  evalTemplateName: evalTemplate?.name ?? null,
});

type EvaluationRuleCoreDataInput = {
  sampling: { toNumber: () => number };
} & Record<string, unknown>;

export const mapEvaluationRuleToCoreDataRow = ({
  sampling,
  ...evaluationRule
}: EvaluationRuleCoreDataInput) => ({
  ...evaluationRule,
  sampling: sampling.toNumber(),
});

type EvaluatorVersionCoreDataInput = {
  variableMapping: unknown;
} & Record<string, unknown>;

export const mapEvaluatorVersionToCoreDataRow = ({
  variableMapping,
  ...version
}: EvaluatorVersionCoreDataInput) => ({
  ...version,
  variableMapping: variableMappingForExport(variableMapping),
});

type EvaluationRuleEvaluatorAssignmentCoreDataInput = {
  variableMapping: unknown;
} & Record<string, unknown>;

export const mapEvaluationRuleEvaluatorAssignmentToCoreDataRow = ({
  variableMapping,
  ...assignment
}: EvaluationRuleEvaluatorAssignmentCoreDataInput) => ({
  ...assignment,
  variableMapping: variableMappingForExport(variableMapping),
});

// Widget dimensions/metrics/chart config are persisted as free-form strings
// (DimensionSchema/MetricSchema accept any z.string()), so an API client can
// smuggle arbitrary text into them. The export therefore allowlists every
// string against the query-model declarations and replaces unknown values
// with a sentinel instead of forwarding them.
const KNOWN_WIDGET_MEASURES = new Set(
  Object.values(viewDeclarations).flatMap((versionViews) =>
    Object.values(versionViews).flatMap((view) => Object.keys(view.measures)),
  ),
);
const KNOWN_WIDGET_DIMENSIONS = new Set(
  Object.values(viewDeclarations).flatMap((versionViews) =>
    Object.values(versionViews).flatMap((view) => Object.keys(view.dimensions)),
  ),
);
const KNOWN_WIDGET_AGGREGATIONS = new Set<string>(metricAggregations.options);
const KNOWN_WIDGET_CHART_TYPES = new Set<string>(
  Object.values(DashboardWidgetChartType),
);
const INVALID_SENTINEL = "__invalid__";

const allowlisted = (value: unknown, allowlist: Set<string>): string | null => {
  if (value == null) {
    return null;
  }
  if (typeof value === "string" && allowlist.has(value)) {
    return value;
  }
  return INVALID_SENTINEL;
};

type DashboardWidgetCoreDataInput = {
  dimensions: unknown;
  metrics: unknown;
  chartConfig: unknown;
  /** Drops a supplied filters column so it cannot leak through the rest spread. */
  filters?: unknown;
} & Record<string, unknown>;

/** Allowlists widget dimensions, metrics, and chart type, and keeps numeric chartConfig limits. */
export const mapDashboardWidgetToCoreDataRow = ({
  dimensions,
  metrics,
  filters: _filters,
  chartConfig,
  ...widget
}: DashboardWidgetCoreDataInput) => {
  const config = (chartConfig ?? {}) as Record<string, unknown>;
  return {
    ...widget,
    dimensions: Array.isArray(dimensions)
      ? dimensions.map((dimension: Record<string, unknown>) => ({
          field: allowlisted(dimension.field, KNOWN_WIDGET_DIMENSIONS),
        }))
      : [],
    metrics: Array.isArray(metrics)
      ? metrics.map((metric: Record<string, unknown>) => ({
          measure: allowlisted(metric.measure, KNOWN_WIDGET_MEASURES),
          agg: allowlisted(metric.agg, KNOWN_WIDGET_AGGREGATIONS),
        }))
      : [],
    chartConfig: {
      type: allowlisted(config.type, KNOWN_WIDGET_CHART_TYPES),
      row_limit: typeof config.row_limit === "number" ? config.row_limit : null,
      bins: typeof config.bins === "number" ? config.bins : null,
    },
  };
};

type DashboardCoreDataInput = {
  definition: unknown;
} & Record<string, unknown>;

// Dashboard definitions hold widget placements; explicit field picks keep any
// unexpected persisted keys out of the export.
export const mapDashboardToCoreDataRow = ({
  definition,
  ...dashboard
}: DashboardCoreDataInput) => {
  const widgets = (definition as { widgets?: unknown })?.widgets;
  return {
    ...dashboard,
    definition: {
      widgets: Array.isArray(widgets)
        ? widgets.map((placement: Record<string, unknown>) => ({
            type: placement.type ?? null,
            widgetId: placement.widgetId ?? null,
            x: placement.x ?? null,
            y: placement.y ?? null,
            x_size: placement.x_size ?? null,
            y_size: placement.y_size ?? null,
          }))
        : [],
    },
  };
};

type TablePageArgs<TCursor> = {
  lastRow: TCursor | null;
  take: number;
};

type FetchTablePage<TRow> = (args: {
  lastRow: TRow | null;
  take: number;
}) => Promise<TRow[]>;

async function* createTableJsonlStream<TRow>({
  fetchPage,
  mapRow,
  pageSize,
  onPage,
}: {
  fetchPage: FetchTablePage<TRow>;
  mapRow?: (row: TRow) => unknown;
  pageSize: number;
  onPage?: (pageRowCount: number) => void;
}): AsyncGenerator<string> {
  let lastRow: TRow | null = null;
  let isFirstRow = true;

  while (true) {
    const rows = await fetchPage({ lastRow, take: pageSize });

    if (rows.length === 0) {
      break;
    }

    for (const row of rows) {
      yield `${isFirstRow ? "" : "\n"}${JSON.stringify(mapRow ? mapRow(row) : row)}`;
      isFirstRow = false;
    }

    onPage?.(rows.length);

    if (rows.length < pageSize) {
      break;
    }

    lastRow = rows[rows.length - 1];
  }
}

// Streams a table to S3 as JSONL via keyset pagination so memory usage stays
// O(page size) regardless of table size and no postgres connection is held
// across pages. `fetchPage` must return rows in a stable unique order,
// resuming strictly after `lastRow`.
export const uploadTableCoreDataJsonl = async <TRow>({
  s3Client,
  uploadPrefix,
  tableName,
  fetchPage,
  mapRow,
  pageSize = CORE_DATA_EXPORT_PAGE_SIZE,
}: {
  s3Client: StorageService;
  uploadPrefix: string;
  tableName: string;
  fetchPage: FetchTablePage<TRow>;
  mapRow?: (row: TRow) => unknown;
  pageSize?: number;
}): Promise<void> => {
  logger.info(`[CORE DATA] Exporting table ${tableName}`);

  let rowCount = 0;

  try {
    await s3Client.uploadFileBuffered({
      fileName: `${uploadPrefix}${tableName}.jsonl`,
      fileType: "application/x-ndjson",
      data: Readable.from(
        createTableJsonlStream({
          fetchPage,
          mapRow,
          pageSize,
          onPage: (pageRowCount) => {
            rowCount += pageRowCount;
          },
        }),
      ),
      partSizeBytes: CORE_DATA_EXPORT_PART_SIZE_BYTES,
    });
  } catch (error) {
    logger.error(
      `[CORE DATA] Export of table ${tableName} failed after ${rowCount} rows`,
      error,
    );
    throw error;
  }

  logger.info(`[CORE DATA] Finished table ${tableName} (${rowCount} rows)`);
};

// One entry per exported table. The tableName doubles as the S3 object base
// name — keep names stable, downstream DWH consumers depend on them.
export const coreDataTableExports: Array<
  (args: { s3Client: StorageService; uploadPrefix: string }) => Promise<void>
> = [
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "projects",
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.project.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            name: true,
            orgId: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "users",
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.user.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            name: true,
            admin: true,
            email: true,
            featureFlags: true,
            v4BetaEnabled: true,
            createdAt: true,
            updatedAt: true,
            // password and accounts are mapped to authMethods below and must
            // not be exported as-is
            password: true,
            accounts: {
              select: {
                provider: true,
              },
            },
          },
        }),
      mapRow: mapUserToCoreDataRow,
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "organizations",
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.organization.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            name: true,
            cloudConfig: true,
            sfdcOrgId: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "orgMemberships",
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.organizationMembership.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            role: true,
            orgId: true,
            userId: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "projectMemberships",
      fetchPage: ({
        lastRow,
        take,
      }: TablePageArgs<{ projectId: string; userId: string }>) =>
        prisma.projectMembership.findMany({
          take,
          ...(lastRow
            ? {
                cursor: {
                  projectId_userId: {
                    projectId: lastRow.projectId,
                    userId: lastRow.userId,
                  },
                },
                skip: 1,
              }
            : {}),
          orderBy: [{ projectId: "asc" }, { userId: "asc" }],
          select: {
            role: true,
            projectId: true,
            userId: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "billingMeterBackup",
      fetchPage: ({
        lastRow,
        take,
      }: TablePageArgs<{
        stripeCustomerId: string;
        meterId: string;
        startTime: Date;
        endTime: Date;
      }>) =>
        prisma.billingMeterBackup.findMany({
          take,
          ...(lastRow
            ? {
                cursor: {
                  stripeCustomerId_meterId_startTime_endTime: {
                    stripeCustomerId: lastRow.stripeCustomerId,
                    meterId: lastRow.meterId,
                    startTime: lastRow.startTime,
                    endTime: lastRow.endTime,
                  },
                },
                skip: 1,
              }
            : {}),
          orderBy: [
            { stripeCustomerId: "asc" },
            { meterId: "asc" },
            { startTime: "asc" },
            { endTime: "asc" },
          ],
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "surveys",
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.survey.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            surveyName: true,
            response: true,
            userId: true,
            userEmail: true,
            orgId: true,
            createdAt: true,
          },
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "blobStorageIntegrations",
      fetchPage: ({ lastRow, take }: TablePageArgs<{ projectId: string }>) =>
        prisma.blobStorageIntegration.findMany({
          take,
          ...(lastRow
            ? { cursor: { projectId: lastRow.projectId }, skip: 1 }
            : {}),
          orderBy: { projectId: "asc" },
          select: {
            projectId: true,
            type: true,
            bucketName: true,
            prefix: true,
            region: true,
            endpoint: true,
            forcePathStyle: true,
            nextSyncAt: true,
            lastSyncAt: true,
            enabled: true,
            exportFrequency: true,
            fileType: true,
            exportMode: true,
            exportStartDate: true,
            exportSource: true,
            exportFieldGroups: true,
            compressed: true,
            lastError: true,
            lastErrorAt: true,
            lastFailureNotificationSentAt: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "posthogIntegrations",
      // encryptedPosthogApiKey is excluded as it holds the project's API key
      fetchPage: ({ lastRow, take }: TablePageArgs<{ projectId: string }>) =>
        prisma.posthogIntegration.findMany({
          take,
          ...(lastRow
            ? { cursor: { projectId: lastRow.projectId }, skip: 1 }
            : {}),
          orderBy: { projectId: "asc" },
          select: {
            projectId: true,
            posthogHostName: true,
            lastSyncAt: true,
            enabled: true,
            exportSource: true,
            createdAt: true,
          },
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "mixpanelIntegrations",
      // encryptedMixpanelProjectToken is excluded as it holds the project token
      fetchPage: ({ lastRow, take }: TablePageArgs<{ projectId: string }>) =>
        prisma.mixpanelIntegration.findMany({
          take,
          ...(lastRow
            ? { cursor: { projectId: lastRow.projectId }, skip: 1 }
            : {}),
          orderBy: { projectId: "asc" },
          select: {
            projectId: true,
            mixpanelRegion: true,
            lastSyncAt: true,
            enabled: true,
            exportSource: true,
            createdAt: true,
          },
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "ssoConfigs",
      // authConfig is excluded as it may contain client secrets
      fetchPage: ({ lastRow, take }: TablePageArgs<{ domain: string }>) =>
        prisma.ssoConfig.findMany({
          take,
          ...(lastRow ? { cursor: { domain: lastRow.domain }, skip: 1 } : {}),
          orderBy: { domain: "asc" },
          select: {
            domain: true,
            authProvider: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "verifiedDomains",
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.verifiedDomain.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            organizationId: true,
            domain: true,
            verifiedAt: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "prompts",
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.prompt.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            name: true,
            projectId: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "jobConfigurations",
      /** Omits the customer-authored job-configuration filter. */
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.jobConfiguration.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            projectId: true,
            jobType: true,
            status: true,
            blockedAt: true,
            blockReason: true,
            evalTemplateId: true,
            scoreName: true,
            targetObject: true,
            variableMapping: true,
            sampling: true,
            delay: true,
            timeScope: true,
            createdAt: true,
            updatedAt: true,
            evalTemplate: { select: { name: true } },
          },
        }),
      mapRow: mapJobConfigurationToCoreDataRow,
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "evaluators",
      // Customer-authored descriptions and block messages are free text and
      // intentionally excluded from the core-data analytics export.
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.evaluator.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            projectId: true,
            name: true,
            type: true,
            createdByUserId: true,
            blockedAt: true,
            blockReason: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "evaluatorVersions",
      /** Exports version metadata, vars, and variableMapping, and omits prompt, code, and model params. */
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.evaluatorVersion.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            evaluatorId: true,
            version: true,
            createdByUserId: true,
            partner: true,
            model: true,
            provider: true,
            vars: true,
            variableMapping: true,
            sourceCodeLanguage: true,
            createdAt: true,
          },
        }),
      mapRow: mapEvaluatorVersionToCoreDataRow,
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "evaluationRules",
      /** Omits the customer-authored evaluation-rule filter. */
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.evaluationRule.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            projectId: true,
            createdByUserId: true,
            name: true,
            status: true,
            targetObject: true,
            sampling: true,
            delay: true,
            timeScope: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
      mapRow: mapEvaluationRuleToCoreDataRow,
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "evaluationRuleEvaluatorAssignments",
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.evaluationRuleEvaluatorAssignment.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            projectId: true,
            evaluationRuleId: true,
            evaluatorId: true,
            variableMapping: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
      mapRow: mapEvaluationRuleEvaluatorAssignmentToCoreDataRow,
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "dashboards",
      // Customer-authored name/description are free text and intentionally
      // excluded; the definition JSON holds only widget placements (ids,
      // positions, sizes). projectId NULL marks Langfuse-owned templates.
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.dashboard.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            projectId: true,
            createdBy: true,
            definition: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
      mapRow: mapDashboardToCoreDataRow,
    }),
  (args) =>
    uploadTableCoreDataJsonl({
      ...args,
      tableName: "dashboardWidgets",
      /** Allowlists widget dimensions, metrics, and chart type, and omits customer-authored filters. */
      fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
        prisma.dashboardWidget.findMany({
          take,
          ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
          orderBy: { id: "asc" },
          select: {
            id: true,
            projectId: true,
            createdBy: true,
            view: true,
            dimensions: true,
            metrics: true,
            chartType: true,
            chartConfig: true,
            minVersion: true,
            createdAt: true,
            updatedAt: true,
          },
        }),
      mapRow: mapDashboardWidgetToCoreDataRow,
    }),
];

export const coreDataS3ExportProcessor: Processor = async (): Promise<void> => {
  if (!env.LANGFUSE_S3_CORE_DATA_UPLOAD_BUCKET) {
    logger.error("[CORE DATA] No bucket name provided for core data S3 export");
    throw new Error(
      "Must provide LANGFUSE_S3_CORE_DATA_UPLOAD_BUCKET to use core data S3 exports",
    );
  }

  logger.info("[CORE DATA] Starting core data S3 export");

  const s3Client = getS3StorageServiceClient(
    env.LANGFUSE_S3_CORE_DATA_UPLOAD_BUCKET,
  );
  const uploadPrefix = env.LANGFUSE_S3_CORE_DATA_UPLOAD_PREFIX;

  const limit = pLimit(CORE_DATA_EXPORT_TABLE_CONCURRENCY);
  const results = await Promise.allSettled(
    coreDataTableExports.map((exportTable) =>
      limit(() => exportTable({ s3Client, uploadPrefix })),
    ),
  );

  const failures = results.filter(
    (result): result is PromiseRejectedResult => result.status === "rejected",
  );
  if (failures.length > 0) {
    logger.error(
      `[CORE DATA] Core data S3 export failed for ${failures.length} of ${results.length} tables`,
    );
    throw failures[0].reason;
  }

  logger.info("[CORE DATA] Finished core data S3 export");
};