🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { describe, expect, it, vi } from "vitest";
import {
evaluateEventPropagationStuck,
getEventPropagationHealth,
} from "../features/health";
import { getLastRunStartedAt } from "../features/eventPropagation/handleEventPropagationJob";
// Pin the write mode to events_only: the propagation worker registers for any
// mode that writes to events_full (dual AND events_only), so the health gate
// must stay active for both. See worker/src/app.ts.
vi.mock("../env", async (importOriginal) => {
const mod = await importOriginal<typeof import("../env")>();
return {
...mod,
env: {
...mod.env,
QUEUE_CONSUMER_EVENT_PROPAGATION_QUEUE_IS_ENABLED: "true",
LANGFUSE_MIGRATION_V4_WRITE_MODE: "events_only",
},
};
});
vi.mock("../features/eventPropagation/handleEventPropagationJob", () => ({
getLastRunStartedAt: vi.fn().mockResolvedValue(null),
getLastProcessedPartition: vi.fn().mockResolvedValue(null),
}));
describe("getEventPropagationHealth", () => {
it("keeps the liveness gate enabled in events_only write mode", async () => {
const health = await getEventPropagationHealth();
expect(health.enabled).toBe(true);
expect(health.stuck).toBe(false); // no heartbeat yet -> not stuck
});
it("allows a long propagation run but still detects an expired heartbeat", async () => {
vi.mocked(getLastRunStartedAt).mockResolvedValueOnce(
Date.now() - 30 * 60_000,
);
expect((await getEventPropagationHealth()).stuck).toBe(false);
vi.mocked(getLastRunStartedAt).mockResolvedValueOnce(
Date.now() - 36 * 60_000,
);
expect((await getEventPropagationHealth()).stuck).toBe(true);
});
});
describe("evaluateEventPropagationStuck", () => {
const nowMs = 1_700_000_000_000;
const thresholdSeconds = 15 * 60;
it("is not stuck when the last run started recently", () => {
const result = evaluateEventPropagationStuck({
enabled: true,
nowMs,
lastRunStartedAtMs: nowMs - 60_000, // 1 min ago
lastProcessedPartition: null,
thresholdSeconds,
});
expect(result.stuck).toBe(false);
expect(result.secondsSinceLastRun).toBe(60);
expect(result.lastRunStartedAt).toBe(
new Date(nowMs - 60_000).toISOString(),
);
});
it("is stuck when the last run started longer ago than the threshold", () => {
const result = evaluateEventPropagationStuck({
enabled: true,
nowMs,
lastRunStartedAtMs: nowMs - 20 * 60_000, // 20 min ago > 15 min
lastProcessedPartition: null,
thresholdSeconds,
});
expect(result.stuck).toBe(true);
expect(result.secondsSinceLastRun).toBe(1200);
});
it("is not stuck exactly at the threshold (strictly greater than)", () => {
const result = evaluateEventPropagationStuck({
enabled: true,
nowMs,
lastRunStartedAtMs: nowMs - thresholdSeconds * 1000,
lastProcessedPartition: null,
thresholdSeconds,
});
expect(result.secondsSinceLastRun).toBe(thresholdSeconds);
expect(result.stuck).toBe(false);
});
it("is not stuck when the job has never run yet (no heartbeat key)", () => {
const result = evaluateEventPropagationStuck({
enabled: true,
nowMs,
lastRunStartedAtMs: null,
lastProcessedPartition: null,
thresholdSeconds,
});
expect(result.stuck).toBe(false);
expect(result.secondsSinceLastRun).toBeNull();
expect(result.lastRunStartedAt).toBeNull();
});
it("is never stuck when event propagation is disabled, even if very stale", () => {
const result = evaluateEventPropagationStuck({
enabled: false,
nowMs,
lastRunStartedAtMs: nowMs - 60 * 60_000, // 1 hour ago
lastProcessedPartition: null,
thresholdSeconds,
});
expect(result.stuck).toBe(false);
expect(result.enabled).toBe(false);
});
it("reports the propagation delay but does NOT let it drive stuck", () => {
// A large propagation delay ("behind") combined with a recent run start
// ("running") must stay healthy — restart would not help a slow backlog.
const result = evaluateEventPropagationStuck({
enabled: true,
nowMs,
lastRunStartedAtMs: nowMs - 30_000, // ran 30s ago -> not stuck
lastProcessedPartition: new Date(nowMs - 45 * 60_000).toISOString(), // 45 min behind
thresholdSeconds,
});
expect(result.propagationDelaySeconds).toBe(45 * 60);
expect(result.stuck).toBe(false);
});
it("leaves the propagation delay null for an unparsable cursor value", () => {
const result = evaluateEventPropagationStuck({
enabled: true,
nowMs,
lastRunStartedAtMs: nowMs - 30_000,
lastProcessedPartition: "not-a-date",
thresholdSeconds,
});
expect(result.propagationDelaySeconds).toBeNull();
});
});