🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable @typescript-eslint/no-use-before-define */
import { createHash, getFips } from "crypto";
import { createServer } from "http";
import { PassThrough, Readable } from "stream";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { S3Client } from "@aws-sdk/client-s3";
import type { NodeHttpHandler } from "@smithy/node-http-handler";
import { BLOB_STORAGE_REGION_INVALID_MESSAGE } from "../../utils/stringChecks";
import { env } from "../../env";
import { logger } from "../logger";
import { resolveMediaStorageEndpoints } from "../s3";
import { StorageServiceFactory } from "./StorageService";
// Capture the options handed to lib-storage's Upload so the non-buffered
// fallback's part size can be asserted without real network I/O.
const s3UploadCtorOptions = vi.hoisted(
() => [] as Array<{ partSize?: number; queueSize?: number }>,
);
vi.mock("@aws-sdk/lib-storage", () => ({
Upload: class {
constructor(options: { partSize?: number; queueSize?: number }) {
s3UploadCtorOptions.push(options);
}
done() {
return Promise.resolve(undefined);
}
},
}));
describe("S3StorageService region normalization", () => {
it("configures a failing request timeout when requested", async () => {
const server = createServer(() => {});
await new Promise<void>((resolve) =>
server.listen(0, "127.0.0.1", resolve),
);
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("Expected a local HTTP port");
}
try {
const service = StorageServiceFactory.getInstance({
accessKeyId: "test-access-key",
secretAccessKey: "test-secret-key",
bucketName: "test-bucket",
endpoint: undefined,
region: "us-west-2",
forcePathStyle: false,
useAzureBlob: false,
useGoogleCloudStorage: false,
useOCIObjectStorage: false,
awsSse: undefined,
awsSseKmsKeyId: undefined,
s3RequestTimeoutMs: 50,
});
const client = (service as unknown as { client: S3Client }).client;
const handler = client.config.requestHandler as NodeHttpHandler;
await expect(
handler.handle({
protocol: "http:",
hostname: "127.0.0.1",
port: address.port,
method: "GET",
path: "/",
} as Parameters<NodeHttpHandler["handle"]>[0]),
).rejects.toMatchObject({ name: "TimeoutError" });
expect(handler.httpHandlerConfigs()).toMatchObject({
requestTimeout: 50,
throwOnRequestTimeout: true,
});
} finally {
server.closeAllConnections();
server.close();
}
});
it("trims a persisted region before configuring the AWS client", async () => {
const service = StorageServiceFactory.getInstance({
accessKeyId: "test-access-key",
secretAccessKey: "test-secret-key",
bucketName: "test-bucket",
endpoint: undefined,
region: " us-west-2",
forcePathStyle: false,
useAzureBlob: false,
useGoogleCloudStorage: false,
useOCIObjectStorage: false,
awsSse: undefined,
awsSseKmsKeyId: undefined,
});
const client = (service as unknown as { client: S3Client }).client;
await expect(client.config.region()).resolves.toBe("us-west-2");
});
it("rejects malformed persisted regions before creating an AWS client", () => {
expect(() =>
StorageServiceFactory.getInstance({
accessKeyId: "test-access-key",
secretAccessKey: "test-secret-key",
bucketName: "test-bucket",
endpoint: undefined,
region: "us west-2",
forcePathStyle: false,
useAzureBlob: false,
useGoogleCloudStorage: false,
useOCIObjectStorage: false,
awsSse: undefined,
awsSseKmsKeyId: undefined,
}),
).toThrow(BLOB_STORAGE_REGION_INVALID_MESSAGE);
});
it.each(["europe-west1", "eastus", "auto"])(
"configures an S3-compatible client with region %s",
async (region) => {
const service = StorageServiceFactory.getInstance({
accessKeyId: "test-access-key",
secretAccessKey: "test-secret-key",
bucketName: "test-bucket",
endpoint: "https://storage.googleapis.com",
region,
forcePathStyle: true,
useAzureBlob: false,
useGoogleCloudStorage: false,
useOCIObjectStorage: false,
awsSse: undefined,
awsSseKmsKeyId: undefined,
});
const client = (service as unknown as { client: S3Client }).client;
await expect(client.config.region()).resolves.toBe(region);
},
);
it("follows S3's region redirect when region is auto on AWS", async () => {
const service = StorageServiceFactory.getInstance({
accessKeyId: "test-access-key",
secretAccessKey: "test-secret-key",
bucketName: "test-bucket",
endpoint: undefined,
region: "auto",
forcePathStyle: false,
useAzureBlob: false,
useGoogleCloudStorage: false,
useOCIObjectStorage: false,
awsSse: undefined,
awsSseKmsKeyId: undefined,
});
const client = (service as unknown as { client: S3Client }).client;
const hostnames: string[] = [];
// Answer the first request like S3 does for a bucket in another region,
// then succeed. No network I/O happens.
const fakeS3 = () => async (args: { request: unknown }) => {
hostnames.push((args.request as { hostname: string }).hostname);
const redirect = hostnames.length === 1;
return {
response: {
statusCode: redirect ? 301 : 200,
headers: {
"content-type": "application/xml",
...(redirect ? { "x-amz-bucket-region": "eu-west-1" } : {}),
},
body: Readable.from([
Buffer.from(
redirect
? "<Error><Code>PermanentRedirect</Code></Error>"
: "<ListBucketResult></ListBucketResult>",
),
]),
},
};
};
client.middlewareStack.add(
fakeS3 as unknown as Parameters<typeof client.middlewareStack.add>[0],
{ step: "deserialize", priority: "low", name: "fakeS3", override: true },
);
await service.listFiles("exports/");
expect(hostnames).toEqual([
"test-bucket.s3.us-east-1.amazonaws.com",
"test-bucket.s3.eu-west-1.amazonaws.com",
]);
});
});
describe("resolveMediaStorageEndpoints", () => {
it("keeps the existing endpoint for both server access and signed URLs by default", () => {
expect(
resolveMediaStorageEndpoints({
endpoint: "http://localhost:9090",
internalEndpoint: undefined,
}),
).toEqual({
endpoint: "http://localhost:9090",
externalEndpoint: undefined,
});
});
it("uses the internal override for server access and the existing endpoint for signed URLs", () => {
expect(
resolveMediaStorageEndpoints({
endpoint: "http://localhost:9090",
internalEndpoint: "http://minio:9000",
}),
).toEqual({
endpoint: "http://minio:9000",
externalEndpoint: "http://localhost:9090",
});
});
});
/**
* Regression tests for the Azure Blob download path.
*
* `AzureBlobStorageService.streamToString` used to decode each download
* stream chunk individually with `data.toString()` and concatenate the
* resulting strings. Node.js stream chunk boundaries can fall in the middle
* of a multibyte UTF-8 sequence, so any character straddling a boundary was
* split and decoded as invalid bytes, producing U+FFFD. The fix aggregates
* the raw Buffers and decodes once at the end.
*/
describe("AzureBlobStorageService.streamToString", () => {
// streamToString is private; reach it through the Azure instance for a
// focused unit test without standing up a real Azure backend (the
// constructor does not perform any network calls).
const getStreamToString = () => {
const service = StorageServiceFactory.getInstance({
accessKeyId: "test-account",
secretAccessKey: Buffer.from("test-secret-key").toString("base64"),
bucketName: "test-container",
endpoint: "https://test.blob.core.windows.net",
region: undefined,
forcePathStyle: false,
useAzureBlob: true,
awsSse: undefined,
awsSseKmsKeyId: undefined,
});
return (chunks: Buffer[]): Promise<string> =>
(
service as unknown as {
streamToString(stream: NodeJS.ReadableStream): Promise<string>;
}
).streamToString(streamFromChunks(chunks));
};
// Emits each provided buffer as its own discrete "data" chunk, so we can
// reproduce exact chunk boundaries.
const streamFromChunks = (chunks: Buffer[]): Readable => {
let index = 0;
return new Readable({
read() {
if (index < chunks.length) {
this.push(chunks[index++]);
} else {
this.push(null);
}
},
});
};
const splitBufferAt = (buffer: Buffer, offset: number): Buffer[] => [
buffer.subarray(0, offset),
buffer.subarray(offset),
];
it("reassembles a 3-byte character split across a chunk boundary", async () => {
const streamToString = getStreamToString();
const text = "について の ご相談"; // の = E3 81 AE
const buffer = Buffer.from(text, "utf-8");
const splitOffset = buffer.indexOf(Buffer.from("の", "utf-8")) + 1; // inside の
const result = await streamToString(splitBufferAt(buffer, splitOffset));
expect(result).toBe(text);
expect(result).not.toContain("�");
});
it("handles a boundary at every byte (worst case)", async () => {
const streamToString = getStreamToString();
const text = "支払いは退院後になります。🙂 café";
const buffer = Buffer.from(text, "utf-8");
const oneBytePerChunk = Array.from(buffer, (byte) => Buffer.from([byte]));
const result = await streamToString(oneBytePerChunk);
expect(result).toBe(text);
});
it("leaves ASCII-only content unchanged", async () => {
const streamToString = getStreamToString();
const text = "plain ascii payload";
const buffer = Buffer.from(text, "utf-8");
const result = await streamToString(splitBufferAt(buffer, 4));
expect(result).toBe(text);
});
it("returns an empty string for an empty stream", async () => {
const streamToString = getStreamToString();
const result = await streamToString([]);
expect(result).toBe("");
});
});
/**
* Regression tests for Google Cloud Storage signed-URL generation.
*
* Generating a v4 signed URL calls Google's `iamcredentials:signBlob` endpoint
* through `@google-cloud/storage` -> `google-auth-library` -> `gaxios`. That
* call can fail transiently with "Premature close" (the socket to Google's IAM
* API is dropped mid-response), which previously surfaced as a hard failure for
* the whole export/media flow (issue #14460). `deleteFiles` already wraps its
* GCS call in `backOff`; these tests cover the same hardening now applied to
* `getSignedUrl` / `getSignedUploadUrl`: a single transient failure is retried,
* a persistent failure still propagates (never masked), and retries are bounded.
*/
describe("GoogleCloudStorageService signed-URL retry", () => {
// A "Premature close" from the underlying signBlob HTTP call, as seen in #14460.
const prematureClose = () =>
Object.assign(
new Error(
"Invalid response body while trying to fetch " +
"https://iamcredentials.googleapis.com/...:signBlob: Premature close",
),
{ name: "SigningError" },
);
// Build a real GoogleCloudStorageService (its constructor performs no network
// I/O without credentials) and replace its private `bucket` with a stub whose
// `file().getSignedUrl` we control, so we exercise the retry wrapper without a
// live GCS backend.
const makeService = (getSignedUrl: ReturnType<typeof vi.fn>) => {
const service = StorageServiceFactory.getInstance({
accessKeyId: undefined,
secretAccessKey: undefined,
bucketName: "test-bucket",
endpoint: undefined,
region: undefined,
forcePathStyle: false,
useGoogleCloudStorage: true,
awsSse: undefined,
awsSseKmsKeyId: undefined,
});
(
service as unknown as { bucket: { file: (name: string) => unknown } }
).bucket = {
file: () => ({ getSignedUrl }),
};
return service as unknown as {
getSignedUrl(fileName: string, ttlSeconds: number): Promise<string>;
getSignedUrlNonRetrying(
fileName: string,
ttlSeconds: number,
): Promise<string>;
getSignedUploadUrl(params: {
path: string;
ttlSeconds: number;
sha256Hash: string;
contentType: string;
contentLength: number;
}): Promise<string>;
};
};
const uploadParams = {
path: "media/p.png",
ttlSeconds: 3600,
sha256Hash: "hash",
contentType: "image/png",
contentLength: 1,
};
// Reproduction: the un-retried path (old behavior) fails on a single
// transient "Premature close" -- exactly what issue #14460 reports.
it("getSignedUrlNonRetrying rejects on a single transient failure (repro)", async () => {
const getSignedUrl = vi.fn().mockRejectedValueOnce(prematureClose());
const service = makeService(getSignedUrl);
await expect(
service.getSignedUrlNonRetrying("f.png", 3600),
).rejects.toThrow();
expect(getSignedUrl).toHaveBeenCalledTimes(1);
});
// Fix: the public method retries the transient failure and succeeds.
it("getSignedUrl retries a transient failure and returns the URL", async () => {
const getSignedUrl = vi
.fn()
.mockRejectedValueOnce(prematureClose())
.mockResolvedValue(["https://signed-read-url"]);
const service = makeService(getSignedUrl);
await expect(service.getSignedUrl("f.png", 3600)).resolves.toBe(
"https://signed-read-url",
);
expect(getSignedUrl).toHaveBeenCalledTimes(2);
});
it("getSignedUploadUrl retries a transient failure and returns the URL", async () => {
const getSignedUrl = vi
.fn()
.mockRejectedValueOnce(prematureClose())
.mockResolvedValue(["https://signed-write-url"]);
const service = makeService(getSignedUrl);
await expect(service.getSignedUploadUrl(uploadParams)).resolves.toBe(
"https://signed-write-url",
);
expect(getSignedUrl).toHaveBeenCalledTimes(2);
});
// Guard: a persistent failure is not masked -- it still throws, after a
// bounded number of attempts (3), not indefinitely.
it("getSignedUrl gives up after 3 attempts and still throws", async () => {
const getSignedUrl = vi.fn().mockRejectedValue(prematureClose());
const service = makeService(getSignedUrl);
await expect(service.getSignedUrl("f.png", 3600)).rejects.toThrow();
expect(getSignedUrl).toHaveBeenCalledTimes(3);
});
// No retry overhead on the happy path.
it("getSignedUrl calls the backend once when it succeeds immediately", async () => {
const getSignedUrl = vi.fn().mockResolvedValue(["https://signed-read-url"]);
const service = makeService(getSignedUrl);
await expect(service.getSignedUrl("f.png", 3600)).resolves.toBe(
"https://signed-read-url",
);
expect(getSignedUrl).toHaveBeenCalledTimes(1);
});
});
describe("StorageServiceFactory GCS credentials", () => {
const original = env.LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS;
afterEach(() => {
env.LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS = original;
});
const keyFilenameOf = (service: unknown) =>
(service as { storage: { authClient: { keyFilename?: string } } }).storage
.authClient.keyFilename;
it("uses ADC, not the deployment storage key, when GCS is selected explicitly without credentials", () => {
env.LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS = "/etc/deployment-key.json";
const service = StorageServiceFactory.getInstance({
accessKeyId: undefined,
secretAccessKey: undefined,
bucketName: "test-bucket",
endpoint: undefined,
region: undefined,
forcePathStyle: false,
useGoogleCloudStorage: true,
awsSse: undefined,
awsSseKmsKeyId: undefined,
});
expect(keyFilenameOf(service)).toBeUndefined();
});
it("uses the deployment storage key when GCS is selected by env", () => {
env.LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS = "/etc/deployment-key.json";
const originalUseGcs = env.LANGFUSE_USE_GOOGLE_CLOUD_STORAGE;
env.LANGFUSE_USE_GOOGLE_CLOUD_STORAGE = "true";
try {
const service = StorageServiceFactory.getInstance({
accessKeyId: undefined,
secretAccessKey: undefined,
bucketName: "test-bucket",
endpoint: undefined,
region: undefined,
forcePathStyle: false,
awsSse: undefined,
awsSseKmsKeyId: undefined,
});
expect(keyFilenameOf(service)).toBe("/etc/deployment-key.json");
} finally {
env.LANGFUSE_USE_GOOGLE_CLOUD_STORAGE = originalUseGcs;
}
});
});
/**
* Source-stream errors on GCS uploads must reject the awaited upload instead
* of escaping as an uncaught 'error' event. `.pipe().on("error")` only
* listens on the destination; `pipeline()` forwards source errors too.
*/
describe("GoogleCloudStorageService.uploadFile source stream errors", () => {
it("rejects when the source stream fails without leaking process events", async () => {
const service = StorageServiceFactory.getInstance({
accessKeyId: undefined,
secretAccessKey: undefined,
bucketName: "test-bucket",
endpoint: undefined,
region: undefined,
forcePathStyle: false,
useGoogleCloudStorage: true,
awsSse: undefined,
awsSseKmsKeyId: undefined,
});
(
service as unknown as { bucket: { file: (name: string) => unknown } }
).bucket = {
file: () => ({
createWriteStream: () => new PassThrough(),
}),
};
const unhandled: unknown[] = [];
const uncaught: unknown[] = [];
const onUnhandled = (reason: unknown) => {
unhandled.push(reason);
};
const onUncaught = (err: unknown) => {
uncaught.push(err);
};
process.on("unhandledRejection", onUnhandled);
process.on("uncaughtException", onUncaught);
try {
const source = new Readable({
read() {
this.destroy(new Error("source stream failed"));
},
});
await expect(
(
service as unknown as {
uploadFile(params: {
fileName: string;
fileType: string;
data: Readable;
}): Promise<void>;
}
).uploadFile({
fileName: "export.json",
fileType: "application/json",
data: source,
}),
).rejects.toThrow();
await new Promise<void>((resolve) => {
setImmediate(resolve);
});
expect(unhandled).toHaveLength(0);
expect(uncaught).toHaveLength(0);
} finally {
process.off("unhandledRejection", onUnhandled);
process.off("uncaughtException", onUncaught);
}
});
});
/**
* Regression tests for the DeleteObjects checksum sent to S3-compatible
* stores.
*
* Since AWS SDK v3.729 the S3 client attaches a CRC32 flexible checksum to
* DeleteObjects even under `requestChecksumCalculation: "WHEN_REQUIRED"`,
* because the S3 model marks the operation as checksum-required. Older
* S3-compatible stores (e.g. the MinIO bundled with langfuse-k8s) only accept
* the legacy Content-MD5 header for multi-object deletes and reject the CRC32
* variant with 400 MissingContentMD5, silently breaking data retention and
* deletion jobs (https://github.com/langfuse/langfuse-k8s/issues/356).
* LANGFUSE_S3_DELETE_OBJECTS_CHECKSUM_ALGORITHM lets deployments pick the
* algorithm the store accepts ("MD5" maps to the Content-MD5 header); it is
* unset by default because MD5 is unavailable on FIPS runtimes.
*/
describe("S3StorageService DeleteObjects checksum", () => {
const EMPTY_DELETE_RESULT_XML = `<?xml version="1.0" encoding="UTF-8"?><DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></DeleteResult>`;
const setChecksumAlgorithm = (value: string | undefined) => {
(
env as { LANGFUSE_S3_DELETE_OBJECTS_CHECKSUM_ALGORITHM?: string }
).LANGFUSE_S3_DELETE_OBJECTS_CHECKSUM_ALGORITHM = value;
};
afterEach(() => {
setChecksumAlgorithm(undefined);
});
type CapturedRequest = {
method: string;
headers: Record<string, string>;
body?: unknown;
};
// Build a real S3StorageService whose client runs the full middleware stack
// (serialization, flexible checksums, signing) but short-circuits right
// before the HTTP handler, capturing the final outgoing request and
// returning a canned success response. No network I/O happens.
const makeServiceWithCapture = (responseXml: string) => {
const service = StorageServiceFactory.getInstance({
accessKeyId: "test-access-key",
secretAccessKey: "test-secret-key",
bucketName: "test-bucket",
endpoint: "http://127.0.0.1:9000",
region: "us-east-1",
forcePathStyle: true,
useAzureBlob: false,
useGoogleCloudStorage: false,
useOCIObjectStorage: false,
awsSse: undefined,
awsSseKmsKeyId: undefined,
});
const client = (service as unknown as { client: S3Client }).client;
const captured: CapturedRequest[] = [];
const captureMiddleware = () => async (args: { request: unknown }) => {
captured.push(args.request as CapturedRequest);
return {
response: {
statusCode: 200,
reason: "OK",
headers: { "content-type": "application/xml" },
body: Readable.from([Buffer.from(responseXml)]),
},
};
};
client.middlewareStack.add(
captureMiddleware as unknown as Parameters<
typeof client.middlewareStack.add
>[0],
{
step: "deserialize",
priority: "low",
name: "testCaptureRequest",
override: true,
},
);
return { service, captured };
};
const findHeader = (
request: CapturedRequest,
name: string,
): string | undefined => {
const key = Object.keys(request.headers).find(
(header) => header.toLowerCase() === name,
);
return key === undefined ? undefined : request.headers[key];
};
it("keeps the SDK's CRC32 checksum on DeleteObjects by default", async () => {
const { service, captured } = makeServiceWithCapture(
EMPTY_DELETE_RESULT_XML,
);
await service.deleteFiles([
"events/project-1/file-1.json",
"media/project-1/file-2.png",
]);
expect(captured).toHaveLength(1);
const request = captured[0];
expect(findHeader(request, "x-amz-checksum-crc32")).toBeDefined();
expect(findHeader(request, "content-md5")).toBeUndefined();
});
// MD5 checksums are unavailable when OpenSSL runs in FIPS mode.
it.skipIf(getFips() === 1)(
"sends Content-MD5 on DeleteObjects when the algorithm is set to MD5",
async () => {
setChecksumAlgorithm("MD5");
const { service, captured } = makeServiceWithCapture(
EMPTY_DELETE_RESULT_XML,
);
await service.deleteFiles([
"events/project-1/file-1.json",
"media/project-1/file-2.png",
]);
expect(captured).toHaveLength(1);
const request = captured[0];
expect(typeof request.body).toBe("string");
const expectedMd5 = createHash("md5")
.update(request.body as string)
.digest("base64");
expect(findHeader(request, "content-md5")).toBe(expectedMd5);
expect(findHeader(request, "x-amz-checksum-crc32")).toBeUndefined();
},
);
});
describe("S3StorageService non-buffered upload part size", () => {
const bufferedKey = "LANGFUSE_S3_UPLOAD_ENABLE_BUFFERED" as const;
const originalBuffered = env[bufferedKey];
afterEach(() => {
(env as Record<string, unknown>)[bufferedKey] = originalBuffered;
s3UploadCtorOptions.length = 0;
});
const makeService = () =>
StorageServiceFactory.getInstance({
accessKeyId: "test-access-key",
secretAccessKey: "test-secret-key",
bucketName: "test-bucket",
endpoint: "http://127.0.0.1:9000",
region: "us-east-1",
forcePathStyle: true,
useAzureBlob: false,
useGoogleCloudStorage: false,
useOCIObjectStorage: false,
awsSse: undefined,
awsSseKmsKeyId: undefined,
});
it("forwards the caller's part size so the 10k-part cap does not truncate large exports", async () => {
(env as Record<string, unknown>)[bufferedKey] = "false";
await makeService().uploadFileBuffered({
fileName: "export.csv",
fileType: "text/csv",
data: Readable.from(["a,b,c\n"]),
partSizeBytes: 100 * 1024 * 1024,
});
expect(s3UploadCtorOptions).toHaveLength(1);
const { partSize } = s3UploadCtorOptions[0];
expect(partSize).toBe(100 * 1024 * 1024);
// Above lib-storage's 5 MiB default, which caps a single object at ~48.83 GiB.
expect(partSize).toBeGreaterThan(5 * 1024 * 1024);
});
it("caps concurrency so parallel callers do not multiply peak buffer memory", async () => {
(env as Record<string, unknown>)[bufferedKey] = "false";
await makeService().uploadFileBuffered({
fileName: "export.csv",
fileType: "text/csv",
data: Readable.from(["a,b,c\n"]),
partSizeBytes: 100 * 1024 * 1024,
maxConcurrentParts: 2,
});
expect(s3UploadCtorOptions).toHaveLength(1);
// Caller's concurrency is honored instead of lib-storage's default of 4,
// which would otherwise buffer partSize x 4 per concurrent upload.
expect(s3UploadCtorOptions[0].queueSize).toBe(2);
});
});
describe("S3StorageService SlowDown logging", () => {
const slowDown = Object.assign(
new Error("Please reduce your request rate."),
{ name: "SlowDown" },
);
function makeService(logSlowDownAsWarning = false) {
return StorageServiceFactory.getInstance({
accessKeyId: "test-access-key",
secretAccessKey: "test-secret-key",
bucketName: "test-bucket",
endpoint: "http://127.0.0.1:9000",
region: "us-east-1",
forcePathStyle: true,
useAzureBlob: false,
useGoogleCloudStorage: false,
useOCIObjectStorage: false,
awsSse: undefined,
awsSseKmsKeyId: undefined,
logSlowDownAsWarning,
});
}
async function rejectUpload(
service: ReturnType<typeof makeService>,
err: unknown,
) {
vi.spyOn(
(service as unknown as { client: { send: () => Promise<unknown> } })
.client,
"send",
).mockRejectedValue(err);
await expect(service.uploadJson("events/key.json", [])).rejects.toThrow(
/Failed to upload JSON/,
);
}
afterEach(() => {
vi.restoreAllMocks();
});
it("logs SlowDown as an error unless the caller owns retries", async () => {
const errorSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
const warnSpy = vi.spyOn(logger, "warn").mockImplementation(() => logger);
await rejectUpload(makeService(), slowDown);
expect(errorSpy).toHaveBeenCalled();
expect(warnSpy).not.toHaveBeenCalled();
});
it("logs SlowDown as a warning when the caller owns retries", async () => {
const errorSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
const warnSpy = vi.spyOn(logger, "warn").mockImplementation(() => logger);
await rejectUpload(makeService(true), slowDown);
expect(warnSpy).toHaveBeenCalled();
expect(errorSpy).not.toHaveBeenCalled();
});
it("still logs other failures as errors when SlowDown is a warning", async () => {
const errorSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
const warnSpy = vi.spyOn(logger, "warn").mockImplementation(() => logger);
await rejectUpload(
makeService(true),
Object.assign(new Error("missing"), { name: "NoSuchKey" }),
);
expect(errorSpy).toHaveBeenCalled();
expect(warnSpy).not.toHaveBeenCalled();
});
});
describe("S3StorageService request handler socket limit", () => {
const originalConcurrentWrites = env.LANGFUSE_S3_CONCURRENT_WRITES;
const setConcurrentWrites = (value: number) => {
(
env as { LANGFUSE_S3_CONCURRENT_WRITES: number }
).LANGFUSE_S3_CONCURRENT_WRITES = value;
};
afterEach(() => {
setConcurrentWrites(originalConcurrentWrites);
});
type ResolvedHandlerConfig = {
httpAgentProvider: () => Promise<{ maxSockets: number }>;
httpsAgent: { maxSockets: number };
};
const resolveHandlerConfig = (endpoint: string) => {
const service = StorageServiceFactory.getInstance({
accessKeyId: "test-access-key",
secretAccessKey: "test-secret-key",
bucketName: "test-bucket",
endpoint,
region: "us-east-1",
forcePathStyle: true,
useAzureBlob: false,
useGoogleCloudStorage: false,
useOCIObjectStorage: false,
awsSse: undefined,
awsSseKmsKeyId: undefined,
});
const client = (service as unknown as { client: S3Client }).client;
const handler = client.config.requestHandler as unknown as {
configProvider: Promise<ResolvedHandlerConfig>;
};
return handler.configProvider;
};
it("applies LANGFUSE_S3_CONCURRENT_WRITES to plain-HTTP endpoints such as MinIO", async () => {
setConcurrentWrites(123);
const config = await resolveHandlerConfig("http://127.0.0.1:9000");
// The HTTP agent is created lazily on the first non-TLS request.
const httpAgent = await config.httpAgentProvider();
expect(httpAgent.maxSockets).toBe(123);
expect(config.httpsAgent.maxSockets).toBe(123);
});
});