Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(observability): logs sent to Loki are retried instead of lost during brief Loki outages (#15913)

Bartosz Majewskid036c1bf0ae6
diff --git a/packages/server/utils/src/evlog-drains.ts b/packages/server/utils/src/evlog-drains.ts
index 523ac8e707c..762fbdda253 100644
--- a/packages/server/utils/src/evlog-drains.ts
+++ b/packages/server/utils/src/evlog-drains.ts
@@ -1,5 +1,6 @@
import { existsSync } from 'node:fs'
import { dirname, join } from 'node:path'
+import { isNil, tryCatch } from '@activepieces/core-utils'
import { DrainContext } from 'evlog'
import { createAxiomDrain } from 'evlog/axiom'
import { createBetterStackDrain } from 'evlog/better-stack'
@@ -7,7 +8,7 @@ import { createFsDrain } from 'evlog/fs'
import { createHyperDXDrain } from 'evlog/hyperdx'
import { createOTLPDrain } from 'evlog/otlp'
import { createDrainPipeline, PipelineDrainFn } from 'evlog/pipeline'
-import { AxiosInstance } from 'axios'
+import { AxiosInstance, isAxiosError } from 'axios'
import { safeHttp } from './safe-http'
function normalizeBetterstackEndpoint(host: string): string {
@@ -42,12 +43,15 @@ function createLokiDrain({ url, username, password, serviceName, httpClient }: C
values,
}))
- try {
- await client.post(`${url}/loki/api/v1/push`, { streams })
+ const { error } = await tryCatch(() => client.post(`${url}/loki/api/v1/push`, { streams }))
+ if (isNil(error)) {
+ return
}
- catch (err) {
- console.error('[evlog-loki] failed to push batch:', err instanceof Error ? err.message : String(err))
+ const status = isAxiosError(error) ? error.response?.status : undefined
+ if (isNil(status) || status === 408 || status === 429 || status >= 500) {
+ throw error
}
+ console.error(`[evlog-loki] dropped ${batch.length} events, Loki rejected the batch with status ${status}:`, error.message)
}
}
diff --git a/packages/server/utils/test/evlog-loki-drain.test.ts b/packages/server/utils/test/evlog-loki-drain.test.ts
index f6d75347709..959b94c6fdc 100644
--- a/packages/server/utils/test/evlog-loki-drain.test.ts
+++ b/packages/server/utils/test/evlog-loki-drain.test.ts
@@ -1,7 +1,9 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest'
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
import type { DrainContext, WideEvent } from 'evlog'
import { evlogDrains } from '../src/evlog-drains'
-import type { AxiosInstance } from 'axios'
+import { safeHttp } from '../src/safe-http'
+import { AxiosError } from 'axios'
+import type { AxiosInstance, AxiosResponse } from 'axios'
function makeEvent(overrides: Partial<WideEvent> = {}): WideEvent {
return {
@@ -17,6 +19,10 @@ function makeContext(event: WideEvent): DrainContext {
return { event }
}
+function makeHttpError(status: number): AxiosError {
+ return new AxiosError(`Request failed with status code ${status}`, 'ERR_BAD_RESPONSE', undefined, undefined, { status } as AxiosResponse)
+}
+
function makeHttpClient(postFn = vi.fn().mockResolvedValue({ status: 204 })): AxiosInstance {
return { post: postFn } as unknown as AxiosInstance
}
@@ -94,10 +100,9 @@ describe('evlogDrains.createLokiDrain', () => {
expect(postFn).not.toHaveBeenCalled()
})
- it('logs to console.error on HTTP failure (no rethrow)', async () => {
+ it('rejects on HTTP failure so the pipeline can retry the batch', async () => {
const postFn = vi.fn().mockRejectedValue(new Error('network error'))
const client = makeHttpClient(postFn)
- const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
const drain = evlogDrains.createLokiDrain({
url: 'http://loki:3100',
@@ -105,9 +110,34 @@ describe('evlogDrains.createLokiDrain', () => {
httpClient: client,
})
+ await expect(drain([makeContext(makeEvent())])).rejects.toThrow('network error')
+ })
+
+ it.each([408, 429, 503])('rejects on a retryable %i response so the pipeline can retry the batch', async (status) => {
+ const postFn = vi.fn().mockRejectedValue(makeHttpError(status))
+
+ const drain = evlogDrains.createLokiDrain({
+ url: 'http://loki:3100',
+ serviceName: 'svc',
+ httpClient: makeHttpClient(postFn),
+ })
+
+ await expect(drain([makeContext(makeEvent())])).rejects.toThrow(`status code ${status}`)
+ })
+
+ it('logs and resolves when Loki rejects the batch with a 4xx response', async () => {
+ const postFn = vi.fn().mockRejectedValue(makeHttpError(400))
+ const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+
+ const drain = evlogDrains.createLokiDrain({
+ url: 'http://loki:3100',
+ serviceName: 'svc',
+ httpClient: makeHttpClient(postFn),
+ })
+
await expect(drain([makeContext(makeEvent())])).resolves.toBeUndefined()
expect(consoleSpy).toHaveBeenCalledOnce()
- expect(consoleSpy.mock.calls[0][0]).toContain('[evlog-loki]')
+ expect(consoleSpy.mock.calls[0][0]).toContain('[evlog-loki] dropped 1 events')
consoleSpy.mockRestore()
})
@@ -169,3 +199,58 @@ describe('evlogDrains.resolve', () => {
expect(result.drain).toBeUndefined()
})
})
+
+describe('evlogDrains.resolve with lokiUrl', () => {
+ beforeEach(() => {
+ vi.useFakeTimers()
+ })
+
+ afterEach(() => {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ })
+
+ async function pushAndFlush(postFn: ReturnType<typeof vi.fn>): Promise<void> {
+ vi.spyOn(safeHttp, 'createAxios').mockReturnValue(makeHttpClient(postFn))
+ const result = evlogDrains.resolve({
+ config: { serviceName: 'svc', lokiUrl: 'http://loki:3100' },
+ })
+ await result.drain?.(makeContext(makeEvent()))
+ const flushed = result.flush()
+ await vi.runAllTimersAsync()
+ await flushed
+ }
+
+ it('retries a batch after a transient push failure', async () => {
+ const postFn = vi.fn()
+ .mockRejectedValueOnce(makeHttpError(503))
+ .mockResolvedValueOnce({ status: 204 })
+ const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+
+ await pushAndFlush(postFn)
+
+ expect(postFn).toHaveBeenCalledTimes(2)
+ expect(consoleSpy).not.toHaveBeenCalled()
+ })
+
+ it('reports dropped events after exhausting retries', async () => {
+ const postFn = vi.fn().mockRejectedValue(makeHttpError(503))
+ const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+
+ await pushAndFlush(postFn)
+
+ expect(postFn).toHaveBeenCalledTimes(3)
+ expect(consoleSpy).toHaveBeenCalledWith('[evlog-pipeline] dropped 1 events', 'Request failed with status code 503')
+ })
+
+ it('does not retry a batch Loki rejects with a 4xx response', async () => {
+ const postFn = vi.fn().mockRejectedValue(makeHttpError(400))
+ const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+
+ await pushAndFlush(postFn)
+
+ expect(postFn).toHaveBeenCalledOnce()
+ expect(consoleSpy).toHaveBeenCalledOnce()
+ expect(consoleSpy.mock.calls[0][0]).toContain('[evlog-loki] dropped 1 events')
+ })
+})