Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): allow longer event propagation batches (#17391)

Max Deichmann25d39151492b
diff --git a/worker/src/__tests__/eventPropagationHealth.test.ts b/worker/src/__tests__/eventPropagationHealth.test.ts
index ef66698052..2d3df90328 100644
--- a/worker/src/__tests__/eventPropagationHealth.test.ts
+++ b/worker/src/__tests__/eventPropagationHealth.test.ts
@@ -4,6 +4,7 @@ import {
evaluateEventPropagationStuck,
getEventPropagationHealth,
} from "../features/health";
+import { getLastRunStartedAt } from "../features/eventPropagation/handleEventPropagationJob";
// Pin the write mode to events_only: the propagation worker registers for any
// mode that writes to events_full (dual AND events_only), so the health gate
@@ -32,6 +33,20 @@ describe("getEventPropagationHealth", () => {
expect(health.enabled).toBe(true);
expect(health.stuck).toBe(false); // no heartbeat yet -> not stuck
});
+
+ it("allows a long propagation run but still detects an expired heartbeat", async () => {
+ vi.mocked(getLastRunStartedAt).mockResolvedValueOnce(
+ Date.now() - 30 * 60_000,
+ );
+
+ expect((await getEventPropagationHealth()).stuck).toBe(false);
+
+ vi.mocked(getLastRunStartedAt).mockResolvedValueOnce(
+ Date.now() - 36 * 60_000,
+ );
+
+ expect((await getEventPropagationHealth()).stuck).toBe(true);
+ });
});
describe("evaluateEventPropagationStuck", () => {
diff --git a/worker/src/env.ts b/worker/src/env.ts
index a3212519fd..ef41a6834c 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -618,7 +618,7 @@ const EnvSchema = z.object({
// slot is wedged. The heartbeat is refreshed at the top of every invocation and
// per-chunk during the experiment backfill, so the threshold only needs to
// exceed the longest un-heartbeated step — a single CH INSERT (request_timeout
- // 10 min). 15 min leaves headroom.
+ // 30 min). 35 min leaves headroom.
//
// Probes using this flag MUST set initialDelaySeconds >= 60s (one cron cycle):
// the heartbeat is only refreshed when the minute-boundary cron next runs, so a
@@ -628,7 +628,7 @@ const EnvSchema = z.object({
.number()
.positive()
.int()
- .default(15),
+ .default(35),
// Liveness threshold for the opt-in ?failIfQueueConsumptionStuck=true health
// check: fail once this container's BullMQ workers have neither picked up nor
diff --git a/worker/src/features/eventPropagation/handleEventPropagationJob.ts b/worker/src/features/eventPropagation/handleEventPropagationJob.ts
index d52fa67fd8..aa1f717a4a 100644
--- a/worker/src/features/eventPropagation/handleEventPropagationJob.ts
+++ b/worker/src/features/eventPropagation/handleEventPropagationJob.ts
@@ -338,7 +338,7 @@ export const handleEventPropagationJob = async (
${excludeProjectIdsInClause ? `AND obs.project_id ${excludeProjectIdsInClause}` : ""}
`,
clickhouseConfigs: {
- request_timeout: 600000, // 10 minutes timeout
+ request_timeout: 1_800_000, // 30 minutes timeout
},
clickhouseSettings: {
parallel_view_processing: 1,