Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

26.2 KB
Raw Blame
import { createHash, randomUUID } from "node:crypto";
import {
  getTracingReadinessSettings,
  logger,
  QueueJobs,
  redis,
  recordDistribution,
  recordGauge,
  recordIncrement,
  TraceBatchTraceSchema,
  TraceBatchQueue,
} from "@langfuse/shared/src/server";
import {
  isTopicsProcessingActive,
  isTopicsProjectEnabled,
} from "@langfuse/shared/topics/server";
import { env } from "../../env";
import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";
import {
  ACKNOWLEDGE_SCRIPT,
  CLEANUP_SIZE,
  evalReadiness,
  getTraceBatchFirstKeys,
  getTraceBatchFirstMember,
  getTraceBatchReadinessKeys,
  getTraceBatchShard,
  HYDRATE_SCRIPT,
  readinessScriptHashes,
  SNAPSHOT_SCRIPT,
  TRACK_SCRIPT,
} from "./traceBatchReadiness";
import {
  compareBatchesByOldestDue,
  compareNumbers,
  compareStrings,
  getTraceExclusionReason,
  type PendingTrace,
  prepareLocalityPartials,
  QUERY_BUFFER_MS,
  selectTraceBatches,
  toLimit,
  type TraceBatchWeightLimits,
  type TraceExclusionReason,
} from "./traceBatchSelection";
import {
  getDeterministicSamplingValue,
  shouldSampleEvaluation,
} from "../evaluation/deterministicSampling";

const CHUNK_SIZE = 1_000;
const REDIS_CHUNK_SIZE = 50;
const REDIS_PIPELINE_SIZE = 4;

function recordTrackingVolume(
  stage: "eligible" | "recorded",
  entries: Iterable<
    readonly [
      string,
      { eventUpdateCount: number; serializedEventBytes: number },
    ]
  >,
): void {
  let updates = 0;
  let bytes = 0;
  for (const [, state] of entries) {
    updates += state.eventUpdateCount;
    bytes += state.serializedEventBytes;
  }
  recordIncrement("langfuse.trace_batch.event_updates", updates, { stage });
  recordIncrement("langfuse.trace_batch.serialized_event_bytes", bytes, {
    stage,
  });
}

export async function trackTraceBatchActivity(
  projectId: string,
  events: {
    traceId: string;
    startTimeISO: string;
    serializedEventBytes: number;
  }[],
  idleMs?: number,
): Promise<void> {
  if (
    !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||
    env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED !== "true"
  )
    return;
  const samplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;
  // Topics summarizes every trace of an active project, so those bypass sampling.
  const topicsProject = await resolveTopicsProcessingActive(projectId);
  // Rate 0 admits only Topics projects; skip all tracking work for the rest.
  if (samplingRate === 0 && !topicsProject) return;

  const bounds = new Map<
    string,
    {
      minStart: number;
      maxStart: number;
      eventUpdateCount: number;
      serializedEventBytes: number;
    }
  >();
  for (const event of events) {
    const start = Date.parse(event.startTimeISO);
    if (!Number.isFinite(start)) continue;
    const previous = bounds.get(event.traceId);
    bounds.set(event.traceId, {
      minStart: Math.min(previous?.minStart ?? start, start),
      maxStart: Math.max(previous?.maxStart ?? start, start),
      eventUpdateCount: (previous?.eventUpdateCount ?? 0) + 1,
      serializedEventBytes:
        (previous?.serializedEventBytes ?? 0) + event.serializedEventBytes,
    });
  }
  if (bounds.size === 0) return;

  try {
    recordTrackingVolume("eligible", bounds);
    const admitAll = samplingRate === 1 || topicsProject;
    // Sample by trace ID so later observations and retries keep the same decision.
    // Dispatcher and consumer process admitted work without resampling.
    const entries = [...bounds].filter(
      ([traceId]) =>
        admitAll ||
        shouldSampleEvaluation({
          samplingValue: getDeterministicSamplingValue(traceId),
          samplingRate,
        }),
    );
    // Counts distinct trace IDs per ingestion batch, not globally unique traces.
    recordIncrement("langfuse.trace_batch.sampling_decisions", entries.length, {
      decision: "selected",
    });
    recordIncrement(
      "langfuse.trace_batch.sampling_decisions",
      bounds.size - entries.length,
      { decision: "excluded" },
    );
    if (entries.length === 0) return;
    const effectiveIdleMs = idleMs ?? (await resolveTraceIdleMs(projectId));
    if (!Number.isSafeInteger(effectiveIdleMs) || effectiveIdleMs <= 0)
      throw new Error("Trace batch idle time must be a positive integer");
    if (!redis) throw new Error("Trace batching requires Redis");
    const shards = new Map<number, typeof entries>();
    for (const entry of entries) {
      const shard = getTraceBatchShard(JSON.stringify([projectId, entry[0]]));
      const group = shards.get(shard) ?? [];
      group.push(entry);
      shards.set(shard, group);
    }
    const clock = await redis.time();
    const now = Number(clock[0]) * 1_000 + Math.floor(Number(clock[1]) / 1_000);
    // Each pipeline contains bounded scripts from one cluster slot.
    for (const [shard, group] of shards) {
      const { due, state: stateKey } = getTraceBatchReadinessKeys(shard);
      const first = getTraceBatchFirstKeys(shard, now);
      const keys = [due, stateKey, ...first.keys];
      for (
        let offset = 0;
        offset < group.length;
        offset += REDIS_CHUNK_SIZE * REDIS_PIPELINE_SIZE
      ) {
        const pipeline =
          group.length - offset > REDIS_CHUNK_SIZE
            ? redis.pipeline()
            : undefined;
        const commands: {
          chunk: typeof entries;
          args: (string | number | Buffer)[];
        }[] = [];
        for (
          let start = offset;
          start <
          Math.min(
            group.length,
            offset + REDIS_CHUNK_SIZE * REDIS_PIPELINE_SIZE,
          );
          start += REDIS_CHUNK_SIZE
        ) {
          const chunk = group.slice(start, start + REDIS_CHUNK_SIZE);
          const args = [
            effectiveIdleMs,
            env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,
            env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS,
            first.width,
            first.firstBucket,
            ...chunk.flatMap(([traceId, state]) => [
              JSON.stringify([projectId, traceId]),
              state.minStart,
              state.maxStart,
              // Unique even after dispatch deletes and a later arrival recreates state.
              randomUUID(),
              state.eventUpdateCount,
              state.serializedEventBytes,
              getTraceBatchFirstMember(JSON.stringify([projectId, traceId])),
            ]),
          ];
          pipeline?.evalsha(
            readinessScriptHashes.get(TRACK_SCRIPT)!,
            keys.length,
            ...keys,
            ...args,
          );
          commands.push({ chunk, args });
        }
        const results: [Error | null, unknown][] | null = pipeline
          ? await pipeline.exec()
          : [[null, await evalReadiness(TRACK_SCRIPT, keys, commands[0].args)]];
        if (!results)
          throw new Error("Trace batch tracking pipeline returned no results");
        let failure: unknown;
        // A command error does not undo successful siblings. Record every known result.
        for (const [index, { chunk, args }] of commands.entries()) {
          try {
            const [error, result] = results[index];
            let expired: number;
            if (error) {
              if (!error.message.startsWith("NOSCRIPT")) throw error;
              // Only this cache miss is safe to retry; successful siblings are not replayed.
              expired = Number(
                await redis.eval(TRACK_SCRIPT, keys.length, ...keys, ...args),
              );
            } else {
              expired = Number(result);
            }
            recordTrackingVolume("recorded", chunk);
            recordIncrement("langfuse.trace_batch.expired_traces", expired, {
              source: "ingestion",
            });
            recordIncrement(
              "langfuse.trace_batch.tracked_traces",
              chunk.length,
            );
          } catch (error) {
            if (failure === undefined) failure = error;
          }
        }
        if (failure !== undefined) throw failure;
      }
    }
  } catch (error) {
    // This optional load experiment must not fail normal event ingestion.
    recordIncrement("langfuse.trace_batch.tracking_errors", 1);
    logger.error("Failed to track trace batch activity", { error });
  }
}

async function resolveTopicsProcessingActive(
  projectId: string,
): Promise<boolean> {
  try {
    return await isTopicsProcessingActive(projectId);
  } catch (error) {
    // The allowlist keeps Topics traces during an outage; the consumer still
    // loads each project's models and skips projects without them.
    recordIncrement("langfuse.trace_batch.topics_gate_failures", 1);
    logger.warn("Failed to read Topics processing state, using allowlist", {
      projectId,
      error,
    });
    return isTopicsProjectEnabled(projectId);
  }
}

async function resolveTraceIdleMs(projectId: string): Promise<number> {
  let traceIdleMs: number | null = null;
  try {
    ({ traceIdleMs } = await getTracingReadinessSettings(projectId));
  } catch (error) {
    recordIncrement("langfuse.trace_batch.idle_time_read_failures", 1);
    logger.warn("Failed to read trace idle time, using the default", {
      projectId,
      error,
    });
  }
  if (traceIdleMs === null) return env.LANGFUSE_TRACE_BATCH_IDLE_MS;
  // A due time past the pending TTL would expire the trace before dispatch.
  const cap =
    env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS -
    env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS;
  return cap > 0 ? Math.min(traceIdleMs, cap) : traceIdleMs;
}

export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
  private stopping = false;
  private activeDispatch: Promise<void> | null = null;

  constructor() {
    super({
      name: "TraceBatchDispatcher",
      metricName: "trace_batch_dispatcher",
      lockKey: "{trace-batch}:dispatcher",
      lockTtlSeconds: 60,
      onUnavailable: "fail",
    });
  }

  protected get defaultIntervalMs(): number {
    return env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS;
  }

  public override stop(): void {
    this.stopping = true;
    super.stop();
  }

  public async drain(): Promise<void> {
    this.stop();
    if (!this.activeDispatch) return;
    let timeout: ReturnType<typeof setTimeout> | undefined;
    try {
      await Promise.race([
        this.activeDispatch,
        new Promise<void>((resolve) => {
          timeout = setTimeout(() => {
            logger.warn(
              "Trace batch dispatcher drain timed out after 5 seconds; continuing shutdown",
            );
            resolve();
          }, 5_000);
        }),
      ]);
    } finally {
      clearTimeout(timeout);
    }
  }

  protected async execute(): Promise<number | void> {
    if (
      !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||
      env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED !== "true" ||
      this.stopping ||
      this.activeDispatch
    )
      return;
    const startedAt = Date.now();
    const dispatch = this.withLock(() => this.dispatchDue()).then(() => {});
    this.activeDispatch = dispatch;
    try {
      await dispatch;
    } finally {
      this.activeDispatch = null;
    }
    // PeriodicRunner waits this delay after completion. Account for work time
    // to maintain the configured start cadence without overlapping runs.
    return Math.max(0, this.defaultIntervalMs - (Date.now() - startedAt));
  }

  private async dispatchDue(): Promise<void> {
    if (!redis) throw new Error("Trace batching requires Redis");
    const queue = TraceBatchQueue.getInstance();
    if (!queue) throw new Error("Trace batch queue is unavailable");
    const totals = { pending: 0, ready: 0, oldestAge: 0 };
    const members: {
      member: string;
      projectId: string;
      shard: number;
      cutoff: string;
      due: number;
    }[] = [];
    for (let shard = 0; shard < env.LANGFUSE_TRACE_BATCH_SHARD_COUNT; shard++) {
      if (this.stopping) return;
      const { due, state } = getTraceBatchReadinessKeys(shard);
      let cutoff = "";
      let now = 0;
      while (!this.stopping) {
        await this.extendLockOnProgress();
        const snapshot = (await evalReadiness(
          SNAPSHOT_SCRIPT,
          [due, state],
          [env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS, cutoff],
        )) as (string | number)[];
        const [clock, , pending, ready, oldest, expired] = snapshot.map(Number);
        now = clock;
        cutoff = String(snapshot[1]);
        recordIncrement("langfuse.trace_batch.expired_traces", expired, {
          source: "dispatcher",
        });
        if (expired < CLEANUP_SIZE) {
          totals.pending += pending;
          totals.ready += ready;
          totals.oldestAge = Math.max(
            totals.oldestAge,
            Math.max(0, now - oldest),
          );
          break;
        }
      }
      if (this.stopping) return;
      const dueMembers = await redis.zrange(
        due,
        `(${now - env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS}`,
        cutoff,
        "BYSCORE",
        "WITHSCORES",
      );
      recordDistribution(
        "langfuse.trace_batch.snapshot_size",
        dueMembers.length / 2,
      );
      for (let index = 0; index < dueMembers.length; index += 2) {
        const member = dueMembers[index];
        members.push({
          member,
          projectId: (JSON.parse(member) as [string, string])[0],
          shard,
          cutoff,
          due: Number(dueMembers[index + 1]),
        });
      }
    }
    // The lease moves between workers after every run, so per-shard series
    // would multiply custom metrics by the shard count on each dispatcher.
    recordGauge("langfuse.trace_batch.pending_traces", totals.pending);
    recordGauge("langfuse.trace_batch.ready_traces", totals.ready);
    recordGauge("langfuse.trace_batch.oldest_due_age_ms", totals.oldestAge);
    if (this.stopping) return;

    // Each range read captures one shard's due IDs. Pack the combined cohort so
    // Redis placement does not determine the ClickHouse query boundaries.
    const strategy = env.LANGFUSE_TRACE_BATCH_STRATEGY;
    // Project packing needs due order within each project, including across shards.
    members.sort(
      (a, b) =>
        compareStrings(a.projectId, b.projectId) ||
        compareNumbers(a.due, b.due) ||
        compareStrings(a.member, b.member),
    );
    const batchWeightLimits: TraceBatchWeightLimits = {
      maxEventUpdates: toLimit(
        env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES,
      ),
      maxSerializedEventBytes: toLimit(
        env.LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES,
      ),
    };
    const traceWeightLimits: TraceBatchWeightLimits = {
      maxEventUpdates: toLimit(
        env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES,
      ),
      maxSerializedEventBytes: toLimit(
        env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES,
      ),
    };

    const acknowledge = async (
      batch: readonly PendingTrace[],
    ): Promise<boolean[]> => {
      const removed = Array<boolean>(batch.length).fill(false);
      const groups = new Map<
        number,
        { entry: PendingTrace; index: number }[]
      >();
      batch.forEach((entry, index) => {
        const shard = getTraceBatchShard(entry.member);
        const group = groups.get(shard) ?? [];
        group.push({ entry, index });
        groups.set(shard, group);
      });
      for (const [shard, group] of groups) {
        const { due, state } = getTraceBatchReadinessKeys(shard);
        for (
          let offset = 0;
          offset < group.length;
          offset += REDIS_CHUNK_SIZE
        ) {
          const chunk = group.slice(offset, offset + REDIS_CHUNK_SIZE);
          const clock = await redis!.time();
          const first = getTraceBatchFirstKeys(
            shard,
            Number(clock[0]) * 1_000 + Math.floor(Number(clock[1]) / 1_000),
          );
          const flags = (await evalReadiness(
            ACKNOWLEDGE_SCRIPT,
            [due, state, ...first.keys],
            [
              env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS,
              first.width,
              first.firstBucket,
              ...chunk.flatMap(({ entry: { member, trace } }) => [
                member,
                trace.revision,
                getTraceBatchFirstMember(member),
              ]),
            ],
          )) as number[];
          chunk.forEach(({ index }, position) => {
            removed[index] = Number(flags[position]) === 1;
          });
        }
      }
      return removed;
    };

    const enqueued: PendingTrace[] = [];
    const flushEnqueued = async () => {
      if (enqueued.length === 0) return;
      const removed = await acknowledge(enqueued.splice(0));
      recordIncrement(
        "langfuse.trace_batch.reactivated_traces",
        removed.filter((wasRemoved) => !wasRemoved).length,
      );
    };

    // Excluded traces leave Redis like dispatched ones; activity after this
    // run's snapshot starts a fresh pending window that is evaluated again.
    // A trace whose revision changed before acknowledgement stays pending and
    // is reported only once a later run removes it.
    const exclude = async (
      excluded: readonly {
        entry: PendingTrace;
        reason: TraceExclusionReason;
      }[],
    ) => {
      if (excluded.length === 0) return;
      await this.extendLockOnProgress(true);
      if (this.stopping) return;
      const removed = await acknowledge(excluded.map(({ entry }) => entry));
      for (const [index, { entry, reason }] of excluded.entries()) {
        if (!removed[index]) continue;
        recordIncrement("langfuse.trace_batch.excluded_traces", 1, {
          reason,
          strategy,
        });
        logger.warn("Trace batch excluded oversized trace", {
          projectId: entry.trace.projectId,
          traceId: entry.trace.traceId,
          reason,
          estimatedEventUpdateCount: entry.estimates?.eventUpdateCount,
          estimatedSerializedEventBytes: entry.estimates?.serializedEventBytes,
          eventTimeEnvelopeMs: entry.trace.maxStart - entry.trace.minStart,
        });
      }
    };

    const enqueue = async (batch: PendingTrace[]) => {
      await this.extendLockOnProgress(true);
      if (this.stopping) return;
      const traces = batch.map(({ trace }) => trace);
      const id = createHash("sha256")
        .update(JSON.stringify(traces))
        .digest("hex");
      // Stable IDs reduce duplicate delivery if enqueue succeeds but ACK fails.
      // Reads remain retry-safe even if regrouping produces a different job ID.
      await queue.add(
        QueueJobs.TraceBatch,
        {
          id,
          timestamp: new Date(),
          name: QueueJobs.TraceBatch,
          payload: { traces },
        },
        { jobId: id },
      );
      enqueued.push(...batch);
      recordDistribution("langfuse.trace_batch.size", batch.length);
      recordDistribution(
        "langfuse.trace_batch.project_count",
        new Set(traces.map((trace) => trace.projectId)).size,
      );
      const fill = (() => {
        if (batch.length === 1) {
          return "singleton";
        }
        if (batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {
          return "full";
        }
        return "partial";
      })();
      recordIncrement("langfuse.trace_batch.dispatched_batches", 1, {
        strategy,
        fill,
      });
      recordDistribution(
        "langfuse.trace_batch.event_time_envelope_ms",
        Math.max(...traces.map(({ maxStart }) => maxStart)) -
          Math.min(...traces.map(({ minStart }) => minStart)) +
          2 * QUERY_BUFFER_MS,
        { strategy },
      );
      let estimatedEventUpdateCount = 0;
      let estimatedSerializedEventBytes = 0;
      let unavailableEstimates = 0;
      for (const entry of batch) {
        if (entry.estimates) {
          estimatedEventUpdateCount += entry.estimates.eventUpdateCount;
          estimatedSerializedEventBytes += entry.estimates.serializedEventBytes;
          recordDistribution(
            "langfuse.trace_batch.estimated_event_update_count",
            entry.estimates.eventUpdateCount,
            { scope: "trace", strategy },
          );
          recordDistribution(
            "langfuse.trace_batch.estimated_serialized_event_bytes",
            entry.estimates.serializedEventBytes,
            { scope: "trace", strategy },
          );
        } else {
          unavailableEstimates++;
        }
      }
      if (unavailableEstimates === 0) {
        recordDistribution(
          "langfuse.trace_batch.estimated_event_update_count",
          estimatedEventUpdateCount,
          { scope: "batch", strategy },
        );
        recordDistribution(
          "langfuse.trace_batch.estimated_serialized_event_bytes",
          estimatedSerializedEventBytes,
          { scope: "batch", strategy },
        );
      } else {
        recordIncrement(
          "langfuse.trace_batch.estimates_unavailable",
          unavailableEstimates,
          {
            scope: "trace",
            strategy,
          },
        );
        recordIncrement("langfuse.trace_batch.estimates_unavailable", 1, {
          scope: "batch",
          strategy,
        });
      }
    };

    let projectBatchTail: PendingTrace[] = [];
    let localityPartials: PendingTrace[][] = [];
    for (
      let offset = 0;
      offset < members.length && !this.stopping;
      offset += CHUNK_SIZE
    ) {
      await this.extendLockOnProgress();
      const candidates = members.slice(offset, offset + CHUNK_SIZE);
      const byShard = new Map<number, typeof candidates>();
      for (const candidate of candidates) {
        const group = byShard.get(candidate.shard) ?? [];
        group.push(candidate);
        byShard.set(candidate.shard, group);
      }
      const hydratedByMember = new Map<string, [string, string]>();
      for (const [shard, group] of byShard) {
        const { due, state } = getTraceBatchReadinessKeys(shard);
        for (let start = 0; start < group.length; start += REDIS_CHUNK_SIZE) {
          await this.extendLockOnProgress();
          const part = (await evalReadiness(
            HYDRATE_SCRIPT,
            [due, state],
            [
              group[0].cutoff,
              env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,
              ...group
                .slice(start, start + REDIS_CHUNK_SIZE)
                .map(({ member }) => member),
            ],
          )) as string[];
          for (let index = 0; index < part.length; index += 3) {
            hydratedByMember.set(part[index], [
              part[index + 1],
              part[index + 2],
            ]);
          }
        }
      }
      recordIncrement(
        "langfuse.trace_batch.skipped_traces",
        candidates.length - hydratedByMember.size,
      );
      const hydratedCandidates: PendingTrace[] = [];
      const excluded: {
        entry: PendingTrace;
        reason: TraceExclusionReason;
      }[] = [];
      for (const { member } of candidates) {
        if (this.stopping) return;
        const hydrated = hydratedByMember.get(member);
        if (!hydrated) continue;
        const [projectId, traceId] = JSON.parse(member) as [string, string];
        const state = JSON.parse(hydrated[0]);
        const entry: PendingTrace = {
          member,
          due: Number(hydrated[1]),
          estimates:
            Number.isSafeInteger(state.eventUpdateCount) &&
            state.eventUpdateCount >= 0 &&
            Number.isSafeInteger(state.serializedEventBytes) &&
            state.serializedEventBytes >= 0
              ? {
                  eventUpdateCount: state.eventUpdateCount,
                  serializedEventBytes: state.serializedEventBytes,
                }
              : undefined,
          trace: TraceBatchTraceSchema.parse({
            ...state,
            projectId,
            traceId,
          }),
        };
        const reason = getTraceExclusionReason(entry, traceWeightLimits);
        if (reason) excluded.push({ entry, reason });
        else hydratedCandidates.push(entry);
      }
      await exclude(excluded);
      if (this.stopping) return;

      const carriedTraceCount =
        strategy === "project"
          ? projectBatchTail.length
          : localityPartials.reduce((count, batch) => count + batch.length, 0);
      const selectorInput =
        strategy === "project"
          ? [...projectBatchTail, ...hydratedCandidates]
          : [...localityPartials.flat(), ...hydratedCandidates];
      recordDistribution(
        "langfuse.trace_batch.candidate_buffer_size",
        hydratedCandidates.length + carriedTraceCount,
        { strategy },
      );
      const selectorStartedAt = performance.now();
      const selected = selectTraceBatches(
        selectorInput,
        env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
        strategy,
        batchWeightLimits,
      );
      recordDistribution(
        "langfuse.trace_batch.selector_duration_ms",
        performance.now() - selectorStartedAt,
        { strategy },
      );

      try {
        if (strategy === "project") {
          projectBatchTail = [];
          if (selected.at(-1)?.length !== env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {
            projectBatchTail = selected.pop() ?? [];
          }
          for (const batch of selected) {
            if (this.stopping) return;
            await enqueue(batch);
          }
          continue;
        }

        const fullBatches = selected.filter(
          (batch) => batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
        );
        const partialBatches = selected.filter(
          (batch) => batch.length < env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
        );
        for (const batch of fullBatches) {
          if (this.stopping) return;
          await enqueue(batch);
        }
        await this.extendLockOnProgress();
        if (this.stopping) return;
        const prepared = prepareLocalityPartials(
          partialBatches,
          env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
          batchWeightLimits,
        );
        localityPartials = prepared.carry;
        for (const batch of prepared.dispatch) {
          if (this.stopping) return;
          await enqueue(batch);
        }
      } finally {
        await flushEnqueued();
      }
    }
    if (this.stopping) return;
    try {
      if (strategy === "project") {
        if (projectBatchTail.length > 0) await enqueue(projectBatchTail);
        return;
      }
      for (const batch of localityPartials.toSorted(
        compareBatchesByOldestDue,
      )) {
        if (this.stopping) return;
        await enqueue(batch);
      }
    } finally {
      await flushEnqueued();
    }
  }
}