Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

9.8 KB
Raw Blame
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { context as otelContext, type Context } from "@opentelemetry/api";

const telemetry = vi.hoisted(() => ({
  contextWithLangfuseProps: vi.fn(),
  recordDistribution: vi.fn(),
  recordGauge: vi.fn(),
  recordIncrement: vi.fn(),
  span: { setAttribute: vi.fn() },
  traceException: vi.fn(),
}));

vi.mock("@langfuse/shared/src/server", () => ({
  contextWithLangfuseProps: telemetry.contextWithLangfuseProps,
  getCurrentSpan: vi.fn(() => telemetry.span),
  instrumentAsync: vi.fn(
    (
      _options: unknown,
      callback: (span: typeof telemetry.span) => Promise<unknown>,
    ) => callback(telemetry.span),
  ),
  logger: {
    debug: vi.fn(),
    error: vi.fn(),
    info: vi.fn(),
    warn: vi.fn(),
  },
  recordDistribution: telemetry.recordDistribution,
  recordGauge: telemetry.recordGauge,
  recordIncrement: telemetry.recordIncrement,
  redis: null,
  traceException: telemetry.traceException,
}));

import { PeriodicRunner } from "../utils/PeriodicRunner";
import { PeriodicExclusiveRunner } from "../utils/PeriodicExclusiveRunner";

// Tracks the context `otelContext.with` activates, so a test can tell whether
// `execute()` runs inside it.
const activeContexts: Context[] = [];
vi.spyOn(otelContext, "with").mockImplementation(
  (ctx, fn, thisArg, ...args) => {
    activeContexts.push(ctx);
    try {
      return fn.call(thisArg, ...args);
    } finally {
      activeContexts.pop();
    }
  },
);

/**
 * Helper to flush microtasks without advancing timers
 *
 * JavaScript has different "task queues" that execute code in a specific order:
 * 1. Synchronous code runs first
 * 2. Microtasks (like Promise callbacks) run next
 * 3. Macrotasks (like setTimeout) run after microtasks
 *
 * When testing async code with fake timers, we often need to let microtasks
 * complete WITHOUT advancing the timer (which would also trigger macrotasks).
 *
 * This function works by:
 * - Creating a Promise that will resolve in the microtask queue
 * - Using `queueMicrotask()` to schedule the Promise resolution
 * - Awaiting that Promise, which pauses execution until all microtasks drain
 *
 * In these tests, we use it after calling `runner.start()` to ensure the
 * initial async execution completes before we check the results or advance timers.
 */
const flushMicrotasks = () => new Promise((r) => queueMicrotask(r));

const expectCompleted = (
  outcome: "success" | "failed" | "skipped",
  tags: Record<string, string>,
) =>
  expect(telemetry.recordIncrement).toHaveBeenCalledWith(
    "langfuse.periodic_runner.completed",
    1,
    { outcome, ...tags },
  );

// Test subclass that tracks execution
class TestRunner extends PeriodicRunner {
  public callCount = 0;
  public shouldThrow = false;
  public returnInterval: number | undefined = undefined;
  public initialDelayMsForTest = 0;
  public executeContext: Context | undefined;

  constructor() {
    super("test_runner");
  }

  protected get name(): string {
    return "test-runner";
  }

  protected get defaultIntervalMs(): number {
    return 1000;
  }

  protected get initialDelayMs(): number {
    return this.initialDelayMsForTest;
  }

  protected async execute(): Promise<number | void> {
    this.callCount++;
    this.executeContext = activeContexts.at(-1);
    if (this.shouldThrow) {
      this.shouldThrow = false; // Only throw once
      throw new Error("Test error");
    }
    return this.returnInterval;
  }
}

class TestExclusiveRunner extends PeriodicExclusiveRunner {
  public callCount = 0;
  public failNext = false;
  public lockAcquired = true;
  public lockNotAcquiredCount = 0;
  public readonly operationError = new Error("Caught operation error");

  constructor(lockMode: "stub" | "unavailable" | "release_failure" = "stub") {
    super({
      name: "test-exclusive-runner",
      metricName: "test_exclusive_runner",
      metricScope: "test_scope",
      lockKey: "test-exclusive-runner-lock",
      lockTtlSeconds: 60,
      onUnavailable: "fail",
    });
    if (lockMode !== "unavailable") {
      this.lock.acquire = async () =>
        this.lockAcquired ? "acquired" : "held_by_other";
    }
    if (lockMode !== "release_failure") {
      this.lock.release = async () => true;
    }
  }

  protected get defaultIntervalMs(): number {
    return 1000;
  }

  protected async execute(): Promise<void> {
    await this.withLock(
      async () => {
        this.callCount++;
        if (this.failNext) {
          this.failNext = false;
          throw this.operationError;
        }
      },
      undefined,
      () => {
        this.lockNotAcquiredCount++;
      },
    );
  }
}

describe("PeriodicRunner", () => {
  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(new Date("2026-07-23T08:00:00.000Z"));
    vi.clearAllMocks();
  });

  afterEach(() => {
    vi.useRealTimers();
  });

  it("should call execute on start and record canonical metrics", async () => {
    const clickHouseContext = {} as Context;
    telemetry.contextWithLangfuseProps.mockReturnValue(clickHouseContext);
    const runner = new TestRunner();

    runner.start();
    await flushMicrotasks();

    expect(runner.callCount).toBe(1);
    expect(runner.executeContext).toBe(clickHouseContext);
    expect(telemetry.contextWithLangfuseProps).toHaveBeenCalledWith({
      clickhouse: {
        surface: "worker",
        route: "langfuse.periodic_runner.test_runner",
      },
    });
    expect(telemetry.recordIncrement).toHaveBeenCalledWith(
      "langfuse.periodic_runner.started",
      1,
      { runner: "test_runner" },
    );
    expectCompleted("success", { runner: "test_runner" });
    expect(telemetry.recordDistribution).toHaveBeenCalledWith(
      "langfuse.periodic_runner.duration_ms",
      0,
      {
        outcome: "success",
        runner: "test_runner",
        unit: "milliseconds",
      },
    );
    expect(telemetry.recordGauge).toHaveBeenCalledWith(
      "langfuse.periodic_runner.last_healthy_timestamp_seconds",
      Date.now() / 1000,
      { runner: "test_runner", unit: "seconds" },
    );
    runner.stop();
  });

  it("should delay the initial execution when configured", async () => {
    const runner = new TestRunner();
    runner.initialDelayMsForTest = 250;

    runner.start();
    await vi.advanceTimersByTimeAsync(249);
    expect(runner.callCount).toBe(0);

    await vi.advanceTimersByTimeAsync(1);
    expect(runner.callCount).toBe(1);
    runner.stop();
  });

  it("should call execute repeatedly at default interval", async () => {
    const runner = new TestRunner();

    runner.start();
    await flushMicrotasks(); // First execution

    expect(runner.callCount).toBe(1);

    await vi.advanceTimersByTimeAsync(1000); // Advance to trigger second execution

    expect(runner.callCount).toBe(2);
    runner.stop();
  });

  it("should use dynamic interval when execute returns a number", async () => {
    const runner = new TestRunner();
    runner.returnInterval = 500; // Return shorter interval

    runner.start();
    await flushMicrotasks(); // First execution

    expect(runner.callCount).toBe(1);

    // Should wait 500ms (not 1000ms) for next execution
    await vi.advanceTimersByTimeAsync(499);
    expect(runner.callCount).toBe(1); // Not yet

    await vi.advanceTimersByTimeAsync(1);
    expect(runner.callCount).toBe(2); // Now it fired

    runner.stop();
  });

  it("should stop scheduling when stop is called", async () => {
    const runner = new TestRunner();

    runner.start();
    await flushMicrotasks();
    expect(runner.callCount).toBe(1);

    runner.stop();

    await vi.advanceTimersByTimeAsync(5000);

    expect(runner.callCount).toBe(1); // No more executions
  });

  it("should continue scheduling after execute throws", async () => {
    const runner = new TestRunner();
    runner.shouldThrow = true;

    runner.start();
    await flushMicrotasks(); // First execution (throws)
    expect(runner.callCount).toBe(1);

    await vi.advanceTimersByTimeAsync(1000); // Second execution (succeeds)
    expect(runner.callCount).toBe(2);

    runner.stop();
  });

  it("should not start twice when start is called multiple times", async () => {
    const runner = new TestRunner();

    runner.start();
    runner.start(); // Second call should be ignored
    await flushMicrotasks();

    expect(runner.callCount).toBe(1);
    runner.stop();
  });

  it("marks a caught error failed and keeps the loop running", async () => {
    const runner = new TestExclusiveRunner();
    runner.failNext = true;

    runner.start();
    await vi.advanceTimersByTimeAsync(0);

    expect(telemetry.traceException).toHaveBeenCalledWith(
      runner.operationError,
    );
    expectCompleted("failed", {
      runner: "test_exclusive_runner",
      scope: "test_scope",
    });
    expect(telemetry.recordGauge).not.toHaveBeenCalled();

    await vi.advanceTimersByTimeAsync(1000);
    expect(runner.callCount).toBe(2);
    runner.stop();
  });

  it("records a lock miss as skipped", async () => {
    const runner = new TestExclusiveRunner();
    runner.lockAcquired = false;

    runner.start();
    await vi.advanceTimersByTimeAsync(0);

    expectCompleted("skipped", {
      runner: "test_exclusive_runner",
      scope: "test_scope",
    });
    expect(telemetry.traceException).not.toHaveBeenCalled();
    expect(telemetry.recordGauge).not.toHaveBeenCalled();
    expect(runner.lockNotAcquiredCount).toBe(1);
    runner.stop();
  });

  it.each(["unavailable", "release_failure"] as const)(
    "records a %s lock as failed",
    async (lockMode) => {
      const runner = new TestExclusiveRunner(lockMode);

      runner.start();
      await vi.advanceTimersByTimeAsync(0);

      expect(telemetry.traceException).toHaveBeenCalledWith(expect.any(Error));
      expectCompleted("failed", {
        runner: "test_exclusive_runner",
        scope: "test_scope",
      });
      expect(telemetry.recordGauge).not.toHaveBeenCalled();
      runner.stop();
    },
  );
});