🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { type PrismaClient } from "@langfuse/shared/src/db";
import {
BlobStorageExportMode,
BlobStorageIntegrationType,
InvalidRequestError,
type AnalyticsIntegrationExportSource,
BlobStorageIntegrationFileType,
type ExportFieldGroup,
BLOB_STORAGE_REGION_INVALID_MESSAGE,
normalizeBlobStorageRegion,
GCPServiceAccountKeySchema,
GCS_USE_DEFAULT_CREDENTIALS,
} from "@langfuse/shared";
import { assertPersistedExportSourceAllowed } from "@/src/features/analytics-integrations/server/exportSource";
import { isSameStorageDestination } from "@/src/features/automations/destinationUrl";
import { encrypt } from "@langfuse/shared/encryption";
import { env } from "@/src/env.mjs";
import {
assertGcsBlobStorageBucketAllowed,
validateBlobStorageBucketHost,
validateBlobStorageEndpoint,
} from "@langfuse/shared/src/server";
type UpsertBlobStorageIntegrationInput = {
type: BlobStorageIntegrationType;
bucketName: string;
endpoint: string | null;
region: string;
accessKeyId: string | null;
secretAccessKey: string | null; // plain text — encrypted by this service
prefix: string;
exportFrequency: string;
enabled: boolean;
forcePathStyle: boolean;
// Optional: undefined preserves the persisted value on UPDATE (Prisma omits
// the column) and falls back to PARQUET on CREATE.
fileType?: BlobStorageIntegrationFileType;
exportMode: BlobStorageExportMode;
exportStartDate: Date | null;
exportSource?: AnalyticsIntegrationExportSource;
exportFieldGroups?: ExportFieldGroup[];
compressed?: boolean;
};
// Same shape check the Vertex AI LLM connection applies to its JSON key.
function assertValidGcsServiceAccountKey(secret: string): void {
let parsed: unknown;
try {
parsed = JSON.parse(secret);
} catch {
throw new InvalidRequestError(
"GCS credentials must be a GCP service account JSON key",
);
}
if (!GCPServiceAccountKeySchema.safeParse(parsed).success) {
throw new InvalidRequestError("Invalid GCP service account JSON key");
}
}
// Which providers can read each other's stored secret. S3 and S3-compatible
// share the access key shape; an Azure account key and a GCS JSON key do not.
function secretFamily(type: BlobStorageIntegrationType) {
switch (type) {
case BlobStorageIntegrationType.S3:
case BlobStorageIntegrationType.S3_COMPATIBLE:
return "s3";
case BlobStorageIntegrationType.AZURE_BLOB_STORAGE:
return "azure";
case BlobStorageIntegrationType.GOOGLE_CLOUD_STORAGE:
return "gcs";
default: {
const _exhaustive: never = type;
_exhaustive;
return null;
}
}
}
function resolveExportStartDate(params: {
exportMode: BlobStorageExportMode;
exportStartDate: Date | null;
}): Date | null {
switch (params.exportMode) {
case BlobStorageExportMode.FROM_TODAY:
return new Date();
case BlobStorageExportMode.FROM_CUSTOM_DATE:
return params.exportStartDate || new Date();
case BlobStorageExportMode.FULL_HISTORY:
return null;
default: {
const _exhaustive: never = params.exportMode;
_exhaustive;
return null;
}
}
}
export async function upsertBlobStorageIntegration(params: {
prisma: PrismaClient;
projectId: string;
data: UpsertBlobStorageIntegrationInput;
// The source a CREATE lands, already validated and resolved by the caller via
// resolveExportSource. Always concrete, so the CREATE branch never falls
// through to the Prisma column default (TRACES_OBSERVATIONS). An UPDATE keeps
// using data.exportSource, where undefined preserves the persisted value.
createExportSource: AnalyticsIntegrationExportSource;
}) {
const { prisma, projectId, data } = params;
const isSelfHosted = !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
const isGcs = data.type === BlobStorageIntegrationType.GOOGLE_CLOUD_STORAGE;
const canUseHostCredentials =
isSelfHosted && data.type === BlobStorageIntegrationType.S3;
// GCS, like the Vertex AI LLM connection, takes either a service account JSON
// key (stored encrypted, in secretAccessKey) or the default-credentials
// sentinel (ADC; self-hosted only, bucket must be allowlisted). An omitted
// secret on UPDATE keeps whatever is stored. GCS never uses accessKeyId.
// The sentinel is never a real secret: the form keeps it when the provider is
// switched away from GCS, so treat it as "no secret" for every type.
const rawSecret = data.secretAccessKey?.trim() || null;
const isSentinel = rawSecret === GCS_USE_DEFAULT_CREDENTIALS;
const isKeylessGcs = isGcs && isSentinel;
const secretAccessKey = isSentinel ? null : rawSecret;
if (isGcs && secretAccessKey) {
assertValidGcsServiceAccountKey(secretAccessKey);
}
const accessKeyId = isGcs ? null : data.accessKeyId?.trim() || null;
// The GCS client always talks to Google, so an endpoint (e.g. one the form
// kept from another provider) is neither validated nor stored.
const endpoint = isGcs ? null : data.endpoint;
let region: string;
try {
region = normalizeBlobStorageRegion(data.region);
} catch {
throw new InvalidRequestError(BLOB_STORAGE_REGION_INVALID_MESSAGE);
}
if (endpoint) {
try {
await validateBlobStorageEndpoint(endpoint);
await validateBlobStorageBucketHost({
type: data.type,
endpoint,
bucketName: data.bucketName,
forcePathStyle: data.forcePathStyle,
});
} catch (error) {
throw new InvalidRequestError(
`Invalid blob storage endpoint: ${error instanceof Error ? error.message : "Endpoint validation failed"}`,
);
}
}
if (!isGcs && !canUseHostCredentials && !accessKeyId) {
throw new InvalidRequestError(
"Access Key ID and Secret Access Key are required",
);
}
const resolvedExportStartDate = resolveExportStartDate({
exportMode: data.exportMode,
exportStartDate: data.exportStartDate,
});
const writeData = {
type: data.type,
bucketName: data.bucketName,
endpoint,
region,
accessKeyId,
prefix: data.prefix,
exportFrequency: data.exportFrequency,
enabled: data.enabled,
forcePathStyle: data.forcePathStyle,
fileType: data.fileType,
exportMode: data.exportMode,
exportStartDate: resolvedExportStartDate,
exportSource: data.exportSource,
exportFieldGroups: data.exportFieldGroups,
compressed: data.compressed ?? true,
};
return prisma.$transaction(async (tx) => {
const existing = await tx.blobStorageIntegration.findUnique({
where: { projectId },
// createdAt/exportSource feed the post-upsert backstop below; the
// destination fields and secretAccessKey decide whether an update keeps
// a stored secret.
select: {
enabled: true,
exportMode: true,
lastError: true,
runStartedAt: true,
createdAt: true,
exportSource: true,
type: true,
bucketName: true,
endpoint: true,
accessKeyId: true,
secretAccessKey: true,
},
});
// A stored secret is only reusable within the same credential family, and
// only for the destination it was entered for.
const hasStoredSecretOfFamily =
!!existing?.secretAccessKey &&
secretFamily(existing.type) === secretFamily(data.type);
const canKeepStoredSecret =
hasStoredSecretOfFamily &&
isSameStorageDestination(existing, {
type: data.type,
bucketName: data.bucketName,
endpoint,
accessKeyId,
});
// Switching a self-hosted S3 integration to host credentials drops the
// stored secret instead of asking for it again.
const isSwitchingToHostCredentials = canUseHostCredentials && !accessKeyId;
if (
hasStoredSecretOfFamily &&
!canKeepStoredSecret &&
!secretAccessKey &&
!isKeylessGcs &&
!isSwitchingToHostCredentials
) {
throw new InvalidRequestError(
"Secret access key is required when changing the storage provider, bucket, endpoint, or access key ID",
);
}
// A GCS integration that will run keyless: explicitly requested, or an
// update that sends no secret and has no stored GCS key to keep. Keyless
// runs as the deployment identity, so it is gated by the bucket allowlist.
const willRunKeyless =
isGcs && (isKeylessGcs || (!secretAccessKey && !canKeepStoredSecret));
if (willRunKeyless) {
try {
assertGcsBlobStorageBucketAllowed(data.bucketName);
} catch (error) {
throw new InvalidRequestError(
error instanceof Error ? error.message : "GCS bucket not allowed",
);
}
}
// New GCS integrations must pick a mode explicitly.
if (!existing && isGcs && !isKeylessGcs && !secretAccessKey) {
throw new InvalidRequestError(
"A GCP service account JSON key or default credentials is required",
);
}
// S3/Azure need a secret (new, or a reusable stored one) unless using host
// credentials.
const isUsingHostCredentials =
canUseHostCredentials && (!accessKeyId || !secretAccessKey);
if (
!isGcs &&
!isUsingHostCredentials &&
!secretAccessKey &&
!canKeepStoredSecret
) {
throw new InvalidRequestError("Secret access key is required");
}
const modeChanged = existing && existing.exportMode !== data.exportMode;
const justEnabled = data.enabled && !existing?.enabled;
const encryptedSecret = secretAccessKey ? encrypt(secretAccessKey) : null;
// An update without a new secret writes back the stored secret checked
// above, never omits the column: a concurrent save that changed the secret
// in between must not end up paired with this request's destination.
// Keyless GCS, or a destination the stored secret was not entered for,
// clears it.
const nextSecretAccessKey =
encryptedSecret ??
(canKeepStoredSecret && !willRunKeyless
? existing.secretAccessKey
: null);
// The CREATE payload always carries a concrete source, resolved by the
// caller through resolveExportSource. Applying it unconditionally (rather
// than behind a `!existing` guard) closes a TOCTOU: under READ COMMITTED,
// tx.findUnique and tx.upsert take independent snapshots, so a concurrent
// DELETE between the two could otherwise leave this undefined and let
// Postgres apply the @default(TRACES_OBSERVATIONS) column default on INSERT.
// ON CONFLICT decides CREATE vs UPDATE atomically regardless of what
// findUnique saw, and UPDATE uses writeData.exportSource (undefined → Prisma
// omits the column → preserves the existing value), so caller intent is
// honored on both paths.
const result = await tx.blobStorageIntegration.upsert({
where: { projectId },
create: {
...writeData,
exportSource: params.createExportSource,
// Parquet is the default export format; apply it when the caller omits
// fileType on CREATE. This app-level fallback (not the Prisma column
// default) is the source of truth for the default across every write path.
fileType: data.fileType ?? BlobStorageIntegrationFileType.PARQUET,
projectId,
secretAccessKey: encryptedSecret,
},
update: {
...writeData,
secretAccessKey: nextSecretAccessKey,
// Schedule an immediate retry when saving an errored integration
// so the scheduler picks it up via the nextSyncAt clause.
...(existing?.lastError && data.enabled && !modeChanged
? { nextSyncAt: new Date() }
: {}),
// Reset sync state when export mode changes so the new mode's
// start-date logic takes effect instead of continuing from the
// previous mode's lastSyncAt.
...(modeChanged ? { lastSyncAt: null, nextSyncAt: new Date() } : {}),
// Both restart the export from history; the worker clears the flag
// once it reaches the live tail. CREATE gets it from the column default.
...(modeChanged || justEnabled ? { backfill: true } : {}),
// Saving enabled resets the failure-notification cooldown: the
// customer just acted, so a fresh failure should email promptly.
...(data.enabled ? { lastFailureNotificationSentAt: null } : {}),
runStartedAt: null,
},
});
// Race-free backstop over the row that actually landed, shared with the
// PostHog and Mixpanel routers. The pre-flight `existing` snapshot (and the
// router's pre-flight gate) are racy under READ COMMITTED: a concurrent
// DELETE can flip this upsert to a CREATE after those reads. Throwing here
// rolls the transaction back. See export-source-policy.ts.
assertPersistedExportSourceAllowed({
existingIntegration: existing,
result,
});
return result;
});
}