🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(ingestion): log S3 SlowDown upload rejections as warn only (#18547)
Niklas Semmler1306b7a20404
diff --git a/packages/shared/src/server/ingestion/processEventBatch.ts b/packages/shared/src/server/ingestion/processEventBatch.tsindex 9e2c212517..d3beff713a 100644--- a/packages/shared/src/server/ingestion/processEventBatch.ts+++ b/packages/shared/src/server/ingestion/processEventBatch.ts@@ -297,11 +297,10 @@ export const processEventBatch = async (source: "process_event_batch",reason: "s3_upload_error",});+ logger.error("Failed to upload event to S3", {+ error: result.reason,+ });}-- logger.error("Failed to upload event to S3", {- error: result.reason,- });}});});diff --git a/worker/src/__tests__/processEventBatch.test.ts b/worker/src/__tests__/processEventBatch.test.tsindex 12d5c56644..9e6c0b98eb 100644--- a/worker/src/__tests__/processEventBatch.test.ts+++ b/worker/src/__tests__/processEventBatch.test.ts@@ -2,6 +2,7 @@ import { beforeEach, describe, it, expect, assert, vi } from "vitest";import { eventTypes } from "../../../packages/shared/src/server/ingestion/types";import { processEventBatch } from "../../../packages/shared/src/server/ingestion/processEventBatch";import { ServiceUnavailableError } from "../../../packages/shared/src/errors";+import { logger } from "../../../packages/shared/src/server/logger";import {createUnknownSdkIngestionAttribution,UNKNOWN_INGESTION_SDK_VALUE,@@ -62,6 +63,8 @@ const createTraceCreateEvent = () => {};describe("processEventBatch", () => {+ const loggerErrorSpy = vi.spyOn(logger, "error");+beforeEach(() => {vi.clearAllMocks();getQueueInstanceMock.mockReturnValue({ add: queueAddMock });@@ -177,6 +180,10 @@ describe("processEventBatch", () => {expect(error).toBeInstanceOf(ServiceUnavailableError);expect(error).toMatchObject({ httpCode: 503, retryAfterSeconds: 2 });expect(queueAddMock).not.toHaveBeenCalled();+ expect(loggerErrorSpy).not.toHaveBeenCalledWith(+ "Failed to upload event to S3",+ expect.anything(),+ );});it("does not report a retryable 503 when another upload failed for a different reason", async () => {@@ -208,5 +215,14 @@ describe("processEventBatch", () => {expect(error).toBeInstanceOf(Error);expect(error).not.toBeInstanceOf(ServiceUnavailableError);+ const uploadErrorLogs = loggerErrorSpy.mock.calls.filter(+ ([message]) => message === "Failed to upload event to S3",+ );+ expect(uploadErrorLogs).toEqual([+ [+ "Failed to upload event to S3",+ { error: expect.objectContaining({ message: "Access Denied" }) },+ ],+ ]);});});