🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): allow longer event propagation batches (#17391)
Max Deichmann25d39151492b
diff --git a/worker/src/__tests__/eventPropagationHealth.test.ts b/worker/src/__tests__/eventPropagationHealth.test.tsindex ef66698052..2d3df90328 100644--- a/worker/src/__tests__/eventPropagationHealth.test.ts+++ b/worker/src/__tests__/eventPropagationHealth.test.ts@@ -4,6 +4,7 @@ import {evaluateEventPropagationStuck,getEventPropagationHealth,} from "../features/health";+import { getLastRunStartedAt } from "../features/eventPropagation/handleEventPropagationJob";// Pin the write mode to events_only: the propagation worker registers for any// mode that writes to events_full (dual AND events_only), so the health gate@@ -32,6 +33,20 @@ describe("getEventPropagationHealth", () => {expect(health.enabled).toBe(true);expect(health.stuck).toBe(false); // no heartbeat yet -> not stuck});++ it("allows a long propagation run but still detects an expired heartbeat", async () => {+ vi.mocked(getLastRunStartedAt).mockResolvedValueOnce(+ Date.now() - 30 * 60_000,+ );++ expect((await getEventPropagationHealth()).stuck).toBe(false);++ vi.mocked(getLastRunStartedAt).mockResolvedValueOnce(+ Date.now() - 36 * 60_000,+ );++ expect((await getEventPropagationHealth()).stuck).toBe(true);+ });});describe("evaluateEventPropagationStuck", () => {diff --git a/worker/src/env.ts b/worker/src/env.tsindex a3212519fd..ef41a6834c 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -618,7 +618,7 @@ const EnvSchema = z.object({// slot is wedged. The heartbeat is refreshed at the top of every invocation and// per-chunk during the experiment backfill, so the threshold only needs to// exceed the longest un-heartbeated step — a single CH INSERT (request_timeout- // 10 min). 15 min leaves headroom.+ // 30 min). 35 min leaves headroom.//// Probes using this flag MUST set initialDelaySeconds >= 60s (one cron cycle):// the heartbeat is only refreshed when the minute-boundary cron next runs, so a@@ -628,7 +628,7 @@ const EnvSchema = z.object({.number().positive().int()- .default(15),+ .default(35),// Liveness threshold for the opt-in ?failIfQueueConsumptionStuck=true health// check: fail once this container's BullMQ workers have neither picked up nordiff --git a/worker/src/features/eventPropagation/handleEventPropagationJob.ts b/worker/src/features/eventPropagation/handleEventPropagationJob.tsindex d52fa67fd8..aa1f717a4a 100644--- a/worker/src/features/eventPropagation/handleEventPropagationJob.ts+++ b/worker/src/features/eventPropagation/handleEventPropagationJob.ts@@ -338,7 +338,7 @@ export const handleEventPropagationJob = async (${excludeProjectIdsInClause ? `AND obs.project_id ${excludeProjectIdsInClause}` : ""}`,clickhouseConfigs: {- request_timeout: 600000, // 10 minutes timeout+ request_timeout: 1_800_000, // 30 minutes timeout},clickhouseSettings: {parallel_view_processing: 1,