AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
fix(observability): logs sent to Loki are retried instead of lost during brief Loki outages (#15913)
Bartosz Majewskid036c1bf0ae6
diff --git a/packages/server/utils/src/evlog-drains.ts b/packages/server/utils/src/evlog-drains.tsindex 523ac8e707c..762fbdda253 100644--- a/packages/server/utils/src/evlog-drains.ts+++ b/packages/server/utils/src/evlog-drains.ts@@ -1,5 +1,6 @@import { existsSync } from 'node:fs'import { dirname, join } from 'node:path'+import { isNil, tryCatch } from '@activepieces/core-utils'import { DrainContext } from 'evlog'import { createAxiomDrain } from 'evlog/axiom'import { createBetterStackDrain } from 'evlog/better-stack'@@ -7,7 +8,7 @@ import { createFsDrain } from 'evlog/fs'import { createHyperDXDrain } from 'evlog/hyperdx'import { createOTLPDrain } from 'evlog/otlp'import { createDrainPipeline, PipelineDrainFn } from 'evlog/pipeline'-import { AxiosInstance } from 'axios'+import { AxiosInstance, isAxiosError } from 'axios'import { safeHttp } from './safe-http'function normalizeBetterstackEndpoint(host: string): string {@@ -42,12 +43,15 @@ function createLokiDrain({ url, username, password, serviceName, httpClient }: Cvalues,}))- try {- await client.post(`${url}/loki/api/v1/push`, { streams })+ const { error } = await tryCatch(() => client.post(`${url}/loki/api/v1/push`, { streams }))+ if (isNil(error)) {+ return}- catch (err) {- console.error('[evlog-loki] failed to push batch:', err instanceof Error ? err.message : String(err))+ const status = isAxiosError(error) ? error.response?.status : undefined+ if (isNil(status) || status === 408 || status === 429 || status >= 500) {+ throw error}+ console.error(`[evlog-loki] dropped ${batch.length} events, Loki rejected the batch with status ${status}:`, error.message)}}diff --git a/packages/server/utils/test/evlog-loki-drain.test.ts b/packages/server/utils/test/evlog-loki-drain.test.tsindex f6d75347709..959b94c6fdc 100644--- a/packages/server/utils/test/evlog-loki-drain.test.ts+++ b/packages/server/utils/test/evlog-loki-drain.test.ts@@ -1,7 +1,9 @@-import { describe, it, expect, vi, beforeEach } from 'vitest'+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'import type { DrainContext, WideEvent } from 'evlog'import { evlogDrains } from '../src/evlog-drains'-import type { AxiosInstance } from 'axios'+import { safeHttp } from '../src/safe-http'+import { AxiosError } from 'axios'+import type { AxiosInstance, AxiosResponse } from 'axios'function makeEvent(overrides: Partial<WideEvent> = {}): WideEvent {return {@@ -17,6 +19,10 @@ function makeContext(event: WideEvent): DrainContext {return { event }}+function makeHttpError(status: number): AxiosError {+ return new AxiosError(`Request failed with status code ${status}`, 'ERR_BAD_RESPONSE', undefined, undefined, { status } as AxiosResponse)+}+function makeHttpClient(postFn = vi.fn().mockResolvedValue({ status: 204 })): AxiosInstance {return { post: postFn } as unknown as AxiosInstance}@@ -94,10 +100,9 @@ describe('evlogDrains.createLokiDrain', () => {expect(postFn).not.toHaveBeenCalled()})- it('logs to console.error on HTTP failure (no rethrow)', async () => {+ it('rejects on HTTP failure so the pipeline can retry the batch', async () => {const postFn = vi.fn().mockRejectedValue(new Error('network error'))const client = makeHttpClient(postFn)- const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)const drain = evlogDrains.createLokiDrain({url: 'http://loki:3100',@@ -105,9 +110,34 @@ describe('evlogDrains.createLokiDrain', () => {httpClient: client,})+ await expect(drain([makeContext(makeEvent())])).rejects.toThrow('network error')+ })++ it.each([408, 429, 503])('rejects on a retryable %i response so the pipeline can retry the batch', async (status) => {+ const postFn = vi.fn().mockRejectedValue(makeHttpError(status))++ const drain = evlogDrains.createLokiDrain({+ url: 'http://loki:3100',+ serviceName: 'svc',+ httpClient: makeHttpClient(postFn),+ })++ await expect(drain([makeContext(makeEvent())])).rejects.toThrow(`status code ${status}`)+ })++ it('logs and resolves when Loki rejects the batch with a 4xx response', async () => {+ const postFn = vi.fn().mockRejectedValue(makeHttpError(400))+ const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)++ const drain = evlogDrains.createLokiDrain({+ url: 'http://loki:3100',+ serviceName: 'svc',+ httpClient: makeHttpClient(postFn),+ })+await expect(drain([makeContext(makeEvent())])).resolves.toBeUndefined()expect(consoleSpy).toHaveBeenCalledOnce()- expect(consoleSpy.mock.calls[0][0]).toContain('[evlog-loki]')+ expect(consoleSpy.mock.calls[0][0]).toContain('[evlog-loki] dropped 1 events')consoleSpy.mockRestore()})@@ -169,3 +199,58 @@ describe('evlogDrains.resolve', () => {expect(result.drain).toBeUndefined()})})++describe('evlogDrains.resolve with lokiUrl', () => {+ beforeEach(() => {+ vi.useFakeTimers()+ })++ afterEach(() => {+ vi.useRealTimers()+ vi.restoreAllMocks()+ })++ async function pushAndFlush(postFn: ReturnType<typeof vi.fn>): Promise<void> {+ vi.spyOn(safeHttp, 'createAxios').mockReturnValue(makeHttpClient(postFn))+ const result = evlogDrains.resolve({+ config: { serviceName: 'svc', lokiUrl: 'http://loki:3100' },+ })+ await result.drain?.(makeContext(makeEvent()))+ const flushed = result.flush()+ await vi.runAllTimersAsync()+ await flushed+ }++ it('retries a batch after a transient push failure', async () => {+ const postFn = vi.fn()+ .mockRejectedValueOnce(makeHttpError(503))+ .mockResolvedValueOnce({ status: 204 })+ const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)++ await pushAndFlush(postFn)++ expect(postFn).toHaveBeenCalledTimes(2)+ expect(consoleSpy).not.toHaveBeenCalled()+ })++ it('reports dropped events after exhausting retries', async () => {+ const postFn = vi.fn().mockRejectedValue(makeHttpError(503))+ const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)++ await pushAndFlush(postFn)++ expect(postFn).toHaveBeenCalledTimes(3)+ expect(consoleSpy).toHaveBeenCalledWith('[evlog-pipeline] dropped 1 events', 'Request failed with status code 503')+ })++ it('does not retry a batch Loki rejects with a 4xx response', async () => {+ const postFn = vi.fn().mockRejectedValue(makeHttpError(400))+ const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)++ await pushAndFlush(postFn)++ expect(postFn).toHaveBeenCalledOnce()+ expect(consoleSpy).toHaveBeenCalledOnce()+ expect(consoleSpy.mock.calls[0][0]).toContain('[evlog-loki] dropped 1 events')+ })+})