🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(worker): move trace batch readiness scripts and selection into dedicated files (#19363)
Lysander Kiesel651918f06705
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.tsindex e6a21fc269..04aa2835ed 100644--- a/worker/src/__tests__/traceBatching.test.ts+++ b/worker/src/__tests__/traceBatching.test.ts@@ -35,12 +35,14 @@ import {getTraceBatchShard,} from "../features/traceBatching/traceBatchReadiness";import {- prepareLocalityPartials,- selectTraceBatches,- type PendingTrace,trackTraceBatchActivity,TraceBatchDispatcher,} from "../features/traceBatching/traceBatching";+import {+ prepareLocalityPartials,+ selectTraceBatches,+ type PendingTrace,+} from "../features/traceBatching/traceBatchSelection";import { otelIngestionQueueProcessorBuilder } from "../queues/otelIngestionQueue";import { traceBatchQueueProcessor } from "../queues/traceBatchQueue";import { ClickhouseWriter } from "../services/ClickhouseWriter";diff --git a/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts b/worker/src/features/traceBatching/TraceBatchMetricsRunner.tsindex 3ec7bdb888..ee363a6f54 100644--- a/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts+++ b/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts@@ -10,29 +10,11 @@ import { PeriodicRunner } from "../../utils/PeriodicRunner";import {getTraceBatchFirstKeys,getTraceBatchReadinessKeys,+ MEMORY_SCRIPT,} from "./traceBatchReadiness";const METRIC_PREFIX = "langfuse.trace_batch";const MEMORY_INTERVAL_MS = 60_000;-// EVAL applies the shared client's key prefix and routes keys to their-// cluster slot. Raw MEMORY commands do not identify keys in ioredis metadata.-const MEMORY_SCRIPT = `-local result = {- redis.call('MEMORY', 'USAGE', KEYS[1], 'SAMPLES', 5) or 0,- redis.call('ZCARD', KEYS[1]),- redis.call('MEMORY', 'USAGE', KEYS[2], 'SAMPLES', 5) or 0,- redis.call('HLEN', KEYS[2])-}-local bytes = 0-local entries = 0-for i = 3, #KEYS do- bytes = bytes + (redis.call('MEMORY', 'USAGE', KEYS[i], 'SAMPLES', 5) or 0)- entries = entries + redis.call('HLEN', KEYS[i])-end-table.insert(result, bytes)-table.insert(result, entries)-return result-`;export class TraceBatchMetricsRunner extends PeriodicRunner {private lastMemorySampleAt: number | null = null;diff --git a/worker/src/features/traceBatching/traceBatchReadiness.ts b/worker/src/features/traceBatching/traceBatchReadiness.tsindex 2ee136c0a2..77a9481551 100644--- a/worker/src/features/traceBatching/traceBatchReadiness.ts+++ b/worker/src/features/traceBatching/traceBatchReadiness.ts@@ -1,5 +1,5 @@import { createHash } from "node:crypto";-import { getShardIndex } from "@langfuse/shared/src/server";+import { getShardIndex, redis } from "@langfuse/shared/src/server";import { env } from "../../env";export function getTraceBatchReadinessKeys(shard: number) {@@ -33,3 +33,240 @@ export function getTraceBatchFirstKeys(shard: number, now: number) {export function getTraceBatchFirstMember(member: string): Buffer {return createHash("sha256").update(member).digest().subarray(0, 16);}++export const CLEANUP_SIZE = 100;++// Retain traces for a bounded time after readiness. Reuse the due index so+// cleanup needs no full hash scan. Whole-key expiry is an inactivity backstop.+const EXPIRE_PENDING_SCRIPT = `+ local function expirePending(now, retention, limit)+ local expired = redis.call('ZRANGE', KEYS[1], '-inf', now - retention,+ 'BYSCORE', 'LIMIT', 0, limit)+ for _, member in ipairs(expired) do+ redis.call('ZREM', KEYS[1], member)+ redis.call('HDEL', KEYS[2], member)+ end+ return #expired+ end+`;++export const TRACK_SCRIPT = `+ ${EXPIRE_PENDING_SCRIPT}+ local clock = redis.call('TIME')+ local now = clock[1] * 1000 + math.floor(clock[2] / 1000)+ local firstTtl = tonumber(ARGV[3])+ local width = tonumber(ARGV[4])+ local bucket = math.floor(now / width)+ local current = bucket - tonumber(ARGV[5]) + 3+ if current < 3 or current > #KEYS then+ return redis.error_reply('Trace readiness bucket clock changed; retry intake')+ end+ for i = 6, #ARGV, 7 do+ local member = ARGV[i]+ local minStart = tonumber(ARGV[i + 1])+ local maxStart = tonumber(ARGV[i + 2])+ local eventUpdateCount = tonumber(ARGV[i + 4])+ local serializedEventBytes = tonumber(ARGV[i + 5])+ local identity = ARGV[i + 6]+ local previous = redis.call('HGET', KEYS[2], member)+ local state = previous and cjson.decode(previous)+ local priorDue = tonumber(redis.call('ZSCORE', KEYS[1], member))+ if not priorDue or priorDue <= now - tonumber(ARGV[2]) then state = nil end+ if state and state.historyBucket then+ local old = state.historyBucket - tonumber(ARGV[5]) + 3+ if old >= 3 and old <= #KEYS and old ~= current then+ redis.call('HDEL', KEYS[old], identity)+ end+ else+ local retainedUpdates = 0+ local retainedBytes = 0+ local estimatesKnown = true+ for key = 3, #KEYS do+ local retained = redis.call('HGET', KEYS[key], identity)+ if retained then+ local bounds = cmsgpack.unpack(retained)+ if bounds[3] > now then+ minStart = math.min(minStart, bounds[1])+ maxStart = math.max(maxStart, bounds[2])+ if bounds[4] and bounds[5] then+ retainedUpdates = math.max(retainedUpdates, bounds[4])+ retainedBytes = math.max(retainedBytes, bounds[5])+ else+ estimatesKnown = false+ end+ end+ if key ~= current then redis.call('HDEL', KEYS[key], identity) end+ end+ end+ if not estimatesKnown then+ eventUpdateCount = nil+ serializedEventBytes = nil+ elseif not state then+ eventUpdateCount = eventUpdateCount + retainedUpdates+ serializedEventBytes = serializedEventBytes + retainedBytes+ end+ end+ if state then+ minStart = math.min(minStart, state.minStart)+ maxStart = math.max(maxStart, state.maxStart)+ if eventUpdateCount and serializedEventBytes and state.eventUpdateCount and state.serializedEventBytes then+ eventUpdateCount = eventUpdateCount + state.eventUpdateCount+ serializedEventBytes = serializedEventBytes + state.serializedEventBytes+ else+ -- Missing history cannot be reconstructed from a later update.+ eventUpdateCount = nil+ serializedEventBytes = nil+ end+ end+ redis.call('HSET', KEYS[2], member, cjson.encode({+ minStart = minStart, maxStart = maxStart, revision = ARGV[i + 3], historyBucket = bucket,+ eventUpdateCount = eventUpdateCount, serializedEventBytes = serializedEventBytes+ }))+ redis.call('ZADD', KEYS[1], now + tonumber(ARGV[1]), member)+ redis.call('HSET', KEYS[current], identity,+ cmsgpack.pack({minStart, maxStart, now + firstTtl, eventUpdateCount, serializedEventBytes}))+ end+ local firstExpiry = math.max((bucket + 1) * width + firstTtl,+ redis.call('PEXPIRETIME', KEYS[current]))+ -- Integer command arguments need decimal strings, including round timestamps.+ redis.call('PEXPIREAT', KEYS[current], string.format('%.0f', firstExpiry))+ local retention = tonumber(ARGV[2])+ local expired = expirePending(now, retention, ${CLEANUP_SIZE})+ -- The shared inactivity backstop covers the latest required member deadline.+ local expiresAt = math.max(now + tonumber(ARGV[1]) + retention,+ redis.call('PEXPIRETIME', KEYS[1]), redis.call('PEXPIRETIME', KEYS[2]))+ redis.call('PEXPIREAT', KEYS[1], string.format('%.0f', expiresAt))+ redis.call('PEXPIREAT', KEYS[2], string.format('%.0f', expiresAt))+ return expired+`;++// Cleanup stays bounded even when the dispatcher has accumulated a backlog.+// Keep the first Redis timestamp as the cutoff throughout this run.+export const SNAPSHOT_SCRIPT = `+ ${EXPIRE_PENDING_SCRIPT}+ local clock = redis.call('TIME')+ local now = clock[1] * 1000 + math.floor(clock[2] / 1000)+ local retention = tonumber(ARGV[1])+ local cutoff = tonumber(ARGV[2]) or now+ local expired = expirePending(now, retention, ${CLEANUP_SIZE})+ local pending = redis.call('ZCARD', KEYS[1])+ local ready = math.max(0, redis.call('ZCOUNT', KEYS[1], '-inf', cutoff)+ - redis.call('ZCOUNT', KEYS[1], '-inf', now - retention))+ local first = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')+ return {now, cutoff, pending, ready, first[2] or tostring(now), expired}+`;++// Fetch state only while it is still due for this run. Checking the score and+// reading the revision atomically prevents dispatching freshly reactivated work.+export const HYDRATE_SCRIPT = `+ local clock = redis.call('TIME')+ local now = clock[1] * 1000 + math.floor(clock[2] / 1000)+ local result = {}+ for i = 3, #ARGV do+ local member = ARGV[i]+ local due = tonumber(redis.call('ZSCORE', KEYS[1], member))+ if due and due <= tonumber(ARGV[1]) and due > now - tonumber(ARGV[2]) then+ local state = redis.call('HGET', KEYS[2], member)+ if state then+ table.insert(result, member)+ table.insert(result, state)+ table.insert(result, tostring(due))+ else+ redis.call('ZREM', KEYS[1], member)+ end+ end+ end+ return result+`;++// Returns one flag per member, in argument order: 1 when removed, 0 when a+// newer revision keeps the trace pending.+export const ACKNOWLEDGE_SCRIPT = `+ local clock = redis.call('TIME')+ local now = clock[1] * 1000 + math.floor(clock[2] / 1000)+ local width = tonumber(ARGV[2])+ local bucket = math.floor(now / width)+ local current = bucket - tonumber(ARGV[3]) + 3+ if current < 3 or current > #KEYS then+ return redis.error_reply('Trace readiness bucket clock changed; retry acknowledgement')+ end+ local removed = {}+ local retained = false+ for i = 4, #ARGV, 3 do+ local state = redis.call('HGET', KEYS[2], ARGV[i])+ local decoded = state and cjson.decode(state)+ if decoded and decoded.revision == ARGV[i + 1] then+ local identity = ARGV[i + 2]+ if decoded.historyBucket then+ local old = decoded.historyBucket - tonumber(ARGV[3]) + 3+ if old >= 3 and old <= #KEYS and old ~= current then redis.call('HDEL', KEYS[old], identity) end+ else+ for key = 3, #KEYS do+ if key ~= current then redis.call('HDEL', KEYS[key], identity) end+ end+ end+ redis.call('HSET', KEYS[current], identity,+ cmsgpack.pack({decoded.minStart, decoded.maxStart, now + tonumber(ARGV[1]), decoded.eventUpdateCount, decoded.serializedEventBytes}))+ retained = true+ redis.call('ZREM', KEYS[1], ARGV[i])+ redis.call('HDEL', KEYS[2], ARGV[i])+ table.insert(removed, 1)+ else+ table.insert(removed, 0)+ end+ end+ if retained then+ local expiresAt = math.max((bucket + 1) * width + tonumber(ARGV[1]),+ redis.call('PEXPIRETIME', KEYS[current]))+ redis.call('PEXPIREAT', KEYS[current], string.format('%.0f', expiresAt))+ end+ return removed+`;++export const readinessScriptHashes = new Map(+ [TRACK_SCRIPT, SNAPSHOT_SCRIPT, HYDRATE_SCRIPT, ACKNOWLEDGE_SCRIPT].map(+ (script) =>+ [script, createHash("sha1").update(script).digest("hex")] as const,+ ),+);++export async function evalReadiness(+ script: string,+ keys: string[],+ args: (string | number | Buffer)[],+): Promise<unknown> {+ if (!redis) throw new Error("Trace batching requires Redis");+ try {+ return await redis.evalsha(+ readinessScriptHashes.get(script)!,+ keys.length,+ ...keys,+ ...args,+ );+ } catch (error) {+ // SCRIPT FLUSH or a promoted primary can have an empty script cache.+ if (!(error instanceof Error) || !error.message.startsWith("NOSCRIPT"))+ throw error;+ return redis.eval(script, keys.length, ...keys, ...args);+ }+}++// EVAL applies the shared client's key prefix and routes keys to their+// cluster slot. Raw MEMORY commands do not identify keys in ioredis metadata.+export const MEMORY_SCRIPT = `+local result = {+ redis.call('MEMORY', 'USAGE', KEYS[1], 'SAMPLES', 5) or 0,+ redis.call('ZCARD', KEYS[1]),+ redis.call('MEMORY', 'USAGE', KEYS[2], 'SAMPLES', 5) or 0,+ redis.call('HLEN', KEYS[2])+}+local bytes = 0+local entries = 0+for i = 3, #KEYS do+ bytes = bytes + (redis.call('MEMORY', 'USAGE', KEYS[i], 'SAMPLES', 5) or 0)+ entries = entries + redis.call('HLEN', KEYS[i])+end+table.insert(result, bytes)+table.insert(result, entries)+return result+`;diff --git a/worker/src/features/traceBatching/traceBatchSelection.ts b/worker/src/features/traceBatching/traceBatchSelection.tsnew file mode 100644index 0000000000..8acabc39e3--- /dev/null+++ b/worker/src/features/traceBatching/traceBatchSelection.ts@@ -0,0 +1,540 @@+import { xxh32 } from "@node-rs/xxhash";+import {+ type QueueName,+ type TQueueJobTypes,+} from "@langfuse/shared/src/server";++export const QUERY_BUFFER_MS = 2 * 60_000;+const MINUTE_MS = 60_000;+const NARROW_BATCH_MAX_ENVELOPE_MS = 60 * MINUTE_MS;+const WIDE_MAX_EXPANSION_RATIO = 0.25;++export type TraceBatchStrategy = "project" | "locality";+export type PendingTrace = {+ member: string;+ due: number;+ estimates?: { eventUpdateCount: number; serializedEventBytes: number };+ trace: TQueueJobTypes[QueueName.TraceBatch]["payload"]["traces"][number];+};++/**+ * Upper bounds on the summed ingestion estimates of one dispatched batch.+ * Traces without estimates weigh nothing, and a single trace always forms a+ * batch even when it alone exceeds a budget.+ */+export type TraceBatchWeightLimits = {+ maxEventUpdates: number;+ maxSerializedEventBytes: number;+};++const UNBOUNDED_WEIGHT_LIMITS: TraceBatchWeightLimits = {+ maxEventUpdates: Number.POSITIVE_INFINITY,+ maxSerializedEventBytes: Number.POSITIVE_INFINITY,+};++export const toLimit = (value: number) =>+ value > 0 ? value : Number.POSITIVE_INFINITY;++const getEventUpdates = ({ estimates }: PendingTrace) =>+ estimates?.eventUpdateCount ?? 0;++const getSerializedEventBytes = ({ estimates }: PendingTrace) =>+ estimates?.serializedEventBytes ?? 0;++const fitsWeightLimits = (+ eventUpdates: number,+ serializedEventBytes: number,+ limits: TraceBatchWeightLimits,+) =>+ eventUpdates <= limits.maxEventUpdates &&+ serializedEventBytes <= limits.maxSerializedEventBytes;++const isWithinWeightLimits = (+ batch: readonly PendingTrace[],+ limits: TraceBatchWeightLimits,+) =>+ batch.length <= 1 ||+ fitsWeightLimits(+ batch.reduce((total, entry) => total + getEventUpdates(entry), 0),+ batch.reduce((total, entry) => total + getSerializedEventBytes(entry), 0),+ limits,+ );++// A heavy trace uses more than half of a batch budget.+const isHeavyTrace = (entry: PendingTrace, limits: TraceBatchWeightLimits) =>+ getEventUpdates(entry) * 2 > limits.maxEventUpdates ||+ getSerializedEventBytes(entry) * 2 > limits.maxSerializedEventBytes;++export type TraceExclusionReason = "event_updates" | "serialized_bytes";++export const getTraceExclusionReason = (+ { estimates }: PendingTrace,+ limits: TraceBatchWeightLimits,+): TraceExclusionReason | undefined => {+ if (!estimates) return undefined;+ if (estimates.eventUpdateCount > limits.maxEventUpdates)+ return "event_updates";+ if (estimates.serializedEventBytes > limits.maxSerializedEventBytes)+ return "serialized_bytes";+ return undefined;+};++export const compareNumbers = (left: number, right: number) => left - right;++export const compareStrings = (left: string, right: string) => {+ if (left < right) {+ return -1;+ }+ if (left > right) {+ return 1;+ }+ return 0;+};++const toMinuteBucket = (timestamp: number) => Math.floor(timestamp / MINUTE_MS);++const getSpanMs = ({ trace }: PendingTrace) => trace.maxStart - trace.minStart;++const allowedEnvelopeMs = (seed: PendingTrace) =>+ Math.max(+ NARROW_BATCH_MAX_ENVELOPE_MS,+ getSpanMs(seed) * (1 + WIDE_MAX_EXPANSION_RATIO),+ );++type LocalityCandidate = {+ entry: PendingTrace;+ minMinute: number;+ maxMinute: number;+ traceHash: number;+};++const toLocalityCandidate = (entry: PendingTrace): LocalityCandidate => ({+ entry,+ minMinute: toMinuteBucket(entry.trace.minStart),+ maxMinute: toMinuteBucket(entry.trace.maxStart),+ traceHash: xxh32(entry.trace.traceId),+});++type BatchSelectionCost = {+ crossProjectBoundaries: number;+ minuteHashCells: number;+ traceHashGap: number;+};++const compareByClickHouseLocality = (+ left: LocalityCandidate,+ right: LocalityCandidate,+) =>+ compareStrings(left.entry.trace.projectId, right.entry.trace.projectId) ||+ compareNumbers(left.minMinute, right.minMinute) ||+ compareNumbers(left.maxMinute, right.maxMinute) ||+ compareNumbers(left.traceHash, right.traceHash) ||+ compareStrings(left.entry.trace.traceId, right.entry.trace.traceId);++const canonicalizeLocalityBatch = (+ batch: readonly PendingTrace[],+): PendingTrace[] =>+ batch+ .map(toLocalityCandidate)+ .toSorted(compareByClickHouseLocality)+ .map(({ entry }) => entry);++const compareCanonicalBatches = (+ left: readonly PendingTrace[],+ right: readonly PendingTrace[],+) => {+ const leftCandidates = left.map(toLocalityCandidate);+ const rightCandidates = right.map(toLocalityCandidate);+ const sharedLength = Math.min(leftCandidates.length, rightCandidates.length);+ for (let index = 0; index < sharedLength; index++) {+ const comparison = compareByClickHouseLocality(+ leftCandidates[index],+ rightCandidates[index],+ );+ if (comparison !== 0) return comparison;+ }+ return compareNumbers(leftCandidates.length, rightCandidates.length);+};++const getBatchBounds = (batch: readonly PendingTrace[]) => ({+ minStart: Math.min(...batch.map(({ trace }) => trace.minStart)),+ maxStart: Math.max(...batch.map(({ trace }) => trace.maxStart)),+});++const isFeasibleLocalityBatch = (+ batch: readonly PendingTrace[],+ maxBatchSize: number,+ weightLimits: TraceBatchWeightLimits,+) => {+ if (batch.length === 0 || batch.length > maxBatchSize) return false;+ if (!isWithinWeightLimits(batch, weightLimits)) return false;+ const canonical = canonicalizeLocalityBatch(batch);+ const { minStart, maxStart } = getBatchBounds(canonical);+ return maxStart - minStart <= allowedEnvelopeMs(canonical[0]);+};++const compareBatchSelectionCost = (+ left: BatchSelectionCost,+ right: BatchSelectionCost,+) =>+ compareNumbers(left.crossProjectBoundaries, right.crossProjectBoundaries) ||+ compareNumbers(left.minuteHashCells, right.minuteHashCells) ||+ compareNumbers(left.traceHashGap, right.traceHashGap);++const addBatchSelectionCost = (+ left: BatchSelectionCost,+ right: BatchSelectionCost,+): BatchSelectionCost => ({+ crossProjectBoundaries:+ left.crossProjectBoundaries + right.crossProjectBoundaries,+ minuteHashCells: left.minuteHashCells + right.minuteHashCells,+ traceHashGap: left.traceHashGap + right.traceHashGap,+});++const ZERO_BATCH_SELECTION_COST: BatchSelectionCost = {+ crossProjectBoundaries: 0,+ minuteHashCells: 0,+ traceHashGap: 0,+};++const selectLocalityBatches = (+ candidates: readonly PendingTrace[],+ maxBatchSize: number,+ weightLimits: TraceBatchWeightLimits,+): PendingTrace[][] => {+ const sorted = candidates+ .map(toLocalityCandidate)+ .toSorted(compareByClickHouseLocality);+ const candidateCount = sorted.length;+ const stride = Math.min(maxBatchSize, candidateCount) + 1;+ const minuteHashCellCosts = new Float64Array(candidateCount * stride);+ const feasibleSlices = new Uint8Array(candidateCount * stride);+ const projectBoundaryPrefix = new Uint32Array(candidateCount + 1);+ const traceHashGapPrefix = new Float64Array(candidateCount + 1);++ for (let index = 1; index < candidateCount; index++) {+ const previous = sorted[index - 1];+ const current = sorted[index];+ const sameProject =+ previous.entry.trace.projectId === current.entry.trace.projectId;+ const sameTimeRange =+ previous.minMinute === current.minMinute &&+ previous.maxMinute === current.maxMinute;+ projectBoundaryPrefix[index + 1] =+ projectBoundaryPrefix[index] + (sameProject ? 0 : 1);+ traceHashGapPrefix[index + 1] =+ traceHashGapPrefix[index] ++ (sameProject && sameTimeRange+ ? current.traceHash - previous.traceHash+ : 0);+ }++ for (let start = 0; start < candidateCount; start++) {+ let minMinute = Number.POSITIVE_INFINITY;+ let maxMinute = Number.NEGATIVE_INFINITY;+ let minStart = Number.POSITIVE_INFINITY;+ let maxStart = Number.NEGATIVE_INFINITY;+ let eventUpdates = 0;+ let serializedEventBytes = 0;+ const seedEnvelopeMs = allowedEnvelopeMs(sorted[start].entry);+ const maxLength = Math.min(maxBatchSize, candidateCount - start);+ for (let length = 1; length <= maxLength; length++) {+ const candidate = sorted[start + length - 1];+ eventUpdates += getEventUpdates(candidate.entry);+ serializedEventBytes += getSerializedEventBytes(candidate.entry);+ // Weights are nonnegative, so every longer slice is also over budget.+ if (+ length > 1 &&+ !fitsWeightLimits(eventUpdates, serializedEventBytes, weightLimits)+ )+ break;+ minMinute = Math.min(minMinute, candidate.minMinute);+ maxMinute = Math.max(maxMinute, candidate.maxMinute);+ minStart = Math.min(minStart, candidate.entry.trace.minStart);+ maxStart = Math.max(maxStart, candidate.entry.trace.maxStart);+ const slot = start * stride + length;+ // The shared time predicate applies every selected trace hash across the+ // complete envelope, so minute × hash count approximates PK cells read.+ minuteHashCellCosts[slot] = (maxMinute - minMinute + 1) * length;+ // Narrow groups stay within one hour. Wide groups may not grow past+ // 125% of the first trace in locality order, the slice seed.+ feasibleSlices[slot] = maxStart - minStart <= seedEnvelopeMs ? 1 : 0;+ }+ }++ const minJobs = new Int32Array(candidateCount + 1).fill(-1);+ minJobs[0] = 0;+ for (let end = 1; end <= candidateCount; end++) {+ const minStart = Math.max(0, end - maxBatchSize);+ let best = Number.MAX_SAFE_INTEGER;+ for (let start = minStart; start < end; start++) {+ if (minJobs[start] < 0) continue;+ if (!feasibleSlices[start * stride + (end - start)]) continue;+ best = Math.min(best, minJobs[start] + 1);+ }+ if (best === Number.MAX_SAFE_INTEGER) {+ throw new Error("Unable to partition trace batch candidates");+ }+ minJobs[end] = best;+ }+ const batchCount = minJobs[candidateCount];++ const predecessors = Array.from({ length: batchCount + 1 }, () =>+ new Int32Array(candidateCount + 1).fill(-1),+ );+ let previousCosts: Array<BatchSelectionCost | null> = Array(+ candidateCount + 1,+ ).fill(null);+ previousCosts[0] = ZERO_BATCH_SELECTION_COST;++ // Find the cheapest path through all prefixes using exactly batchCount jobs.+ for (let batchIndex = 1; batchIndex <= batchCount; batchIndex++) {+ const currentCosts: Array<BatchSelectionCost | null> = Array(+ candidateCount + 1,+ ).fill(null);+ const remainingJobs = batchCount - batchIndex;+ // Leave enough traces for the remaining nonempty, size-bounded batches.+ const minEnd = Math.max(+ batchIndex,+ candidateCount - remainingJobs * maxBatchSize,+ );+ const maxEnd = Math.min(+ candidateCount - remainingJobs,+ batchIndex * maxBatchSize,+ );+ for (let end = minEnd; end <= maxEnd; end++) {+ if (minJobs[end] > batchIndex) continue;+ const minStart = Math.max(batchIndex - 1, end - maxBatchSize);+ for (let start = minStart; start < end; start++) {+ const previousCost = previousCosts[start];+ if (!previousCost) continue;+ const length = end - start;+ if (!feasibleSlices[start * stride + length]) continue;+ const batchCost: BatchSelectionCost = {+ crossProjectBoundaries:+ projectBoundaryPrefix[end] - projectBoundaryPrefix[start + 1],+ minuteHashCells: minuteHashCellCosts[start * stride + length],+ traceHashGap: traceHashGapPrefix[end] - traceHashGapPrefix[start + 1],+ };+ const candidateCost = addBatchSelectionCost(previousCost, batchCost);+ const currentCost = currentCosts[end];+ if (+ !currentCost ||+ compareBatchSelectionCost(candidateCost, currentCost) < 0+ ) {+ currentCosts[end] = candidateCost;+ predecessors[batchIndex][end] = start;+ }+ }+ }+ previousCosts = currentCosts;+ }++ const batches = Array<PendingTrace[]>(batchCount);+ let end = candidateCount;+ for (let batchIndex = batchCount; batchIndex > 0; batchIndex--) {+ const start = predecessors[batchIndex][end];+ if (start < 0)+ throw new Error("Unable to partition trace batch candidates");+ batches[batchIndex - 1] = sorted+ .slice(start, end)+ .map(({ entry }) => entry);+ end = start;+ }+ return batches;+};++/**+ * Deterministically assigns one bounded hydrated candidate window.+ *+ * Locality follows the events table's physical order: project, start-time+ * minute, and xxHash32(trace ID). A batch may not exceed one hour, or 125% of+ * its first trace's span, whichever is larger. Extra jobs are added only when+ * that envelope cap forbids a cheaper fill. Among feasible partitions with+ * that minimum job count, dynamic programming first avoids crossing projects,+ * then minimizes minute × trace-hash cells, then splits the largest hash gaps.+ * Both strategies also close a multi-trace batch before its estimated event+ * updates or serialized bytes would exceed `weightLimits`. Locality packs+ * traces above half of either budget in a separate lane.+ *+ * Selection uses O(n × k × maxBatchSize) time and+ * O(n × min(n, maxBatchSize) + n × k) memory, where k is the fewest feasible+ * jobs (at least ceil(n / maxBatchSize), up to n for disjoint envelopes).+ * Locality callers bound n to CHUNK_SIZE (1,000) hydrated candidates plus+ * at most maxBatchSize - 1 traces across all retained partials.+ * Project mode also accepts a tail of at most maxBatchSize - 1 traces and is O(n).+ */+export function selectTraceBatches(+ candidates: readonly PendingTrace[],+ maxBatchSize: number,+ strategy: TraceBatchStrategy,+ weightLimits: TraceBatchWeightLimits = UNBOUNDED_WEIGHT_LIMITS,+): PendingTrace[][] {+ if (candidates.length === 0) return [];+ if (strategy === "project") {+ const batches: PendingTrace[][] = [];+ let batch: PendingTrace[] = [];+ let eventUpdates = 0;+ let serializedEventBytes = 0;+ for (const candidate of candidates) {+ const candidateEventUpdates = getEventUpdates(candidate);+ const candidateBytes = getSerializedEventBytes(candidate);+ if (+ batch.length > 0 &&+ (batch.length === maxBatchSize ||+ !fitsWeightLimits(+ eventUpdates + candidateEventUpdates,+ serializedEventBytes + candidateBytes,+ weightLimits,+ ))+ ) {+ batches.push(batch);+ batch = [];+ eventUpdates = 0;+ serializedEventBytes = 0;+ }+ batch.push(candidate);+ eventUpdates += candidateEventUpdates;+ serializedEventBytes += candidateBytes;+ }+ batches.push(batch);+ return batches;+ }++ // Two heavy traces cannot share a batch on the dimension that makes them+ // heavy, so packing them separately keeps them from fragmenting the light+ // traces around them in locality order.+ const light: PendingTrace[] = [];+ const heavy: PendingTrace[] = [];+ for (const candidate of candidates) {+ if (isHeavyTrace(candidate, weightLimits)) heavy.push(candidate);+ else light.push(candidate);+ }+ return [+ ...selectLocalityBatches(light, maxBatchSize, weightLimits),+ ...selectLocalityBatches(heavy, maxBatchSize, weightLimits),+ ];+}++export const compareBatchesByOldestDue = (+ left: readonly PendingTrace[],+ right: readonly PendingTrace[],+) =>+ compareNumbers(+ Math.min(...left.map(({ due }) => due)),+ Math.min(...right.map(({ due }) => due)),+ ) || compareCanonicalBatches(left, right);++const canCoalesceLocalityPartials = (+ left: readonly PendingTrace[],+ right: readonly PendingTrace[],+ maxBatchSize: number,+ weightLimits: TraceBatchWeightLimits,+) => {+ if (left.length + right.length > maxBatchSize) return false;+ const projectId = left[0]?.trace.projectId;+ if (+ !projectId ||+ left.some(({ trace }) => trace.projectId !== projectId) ||+ right.some(({ trace }) => trace.projectId !== projectId)+ ) {+ return false;+ }+ const leftBounds = getBatchBounds(left);+ const rightBounds = getBatchBounds(right);+ const bufferedIntervalsOverlap =+ leftBounds.minStart - QUERY_BUFFER_MS <=+ rightBounds.maxStart + QUERY_BUFFER_MS &&+ rightBounds.minStart - QUERY_BUFFER_MS <=+ leftBounds.maxStart + QUERY_BUFFER_MS;+ return (+ bufferedIntervalsOverlap &&+ isFeasibleLocalityBatch([...left, ...right], maxBatchSize, weightLimits)+ );+};++/**+ * Coalesce compatible locality partials and retain a globally bounded carry.+ * The dispatcher calls this once per hydrated window; work never survives the+ * current dispatch run.+ */+export function prepareLocalityPartials(+ partials: readonly (readonly PendingTrace[])[],+ maxBatchSize: number,+ weightLimits: TraceBatchWeightLimits = UNBOUNDED_WEIGHT_LIMITS,+): { dispatch: PendingTrace[][]; carry: PendingTrace[][] } {+ const batches = partials+ .filter((batch) => batch.length > 0)+ .map(canonicalizeLocalityBatch)+ .toSorted(compareCanonicalBatches);++ while (true) {+ let best:+ | {+ leftIndex: number;+ rightIndex: number;+ envelopeIncrease: number;+ merged: PendingTrace[];+ }+ | undefined;+ for (let leftIndex = 0; leftIndex < batches.length; leftIndex++) {+ for (+ let rightIndex = leftIndex + 1;+ rightIndex < batches.length;+ rightIndex+++ ) {+ const left = batches[leftIndex];+ const right = batches[rightIndex];+ if (+ !canCoalesceLocalityPartials(left, right, maxBatchSize, weightLimits)+ )+ continue;+ const leftBounds = getBatchBounds(left);+ const rightBounds = getBatchBounds(right);+ const merged = canonicalizeLocalityBatch([...left, ...right]);+ const mergedBounds = getBatchBounds(merged);+ const envelopeIncrease =+ mergedBounds.maxStart -+ mergedBounds.minStart -+ Math.max(+ leftBounds.maxStart - leftBounds.minStart,+ rightBounds.maxStart - rightBounds.minStart,+ );+ if (+ !best ||+ envelopeIncrease < best.envelopeIncrease ||+ (envelopeIncrease === best.envelopeIncrease &&+ compareCanonicalBatches(merged, best.merged) < 0)+ ) {+ best = { leftIndex, rightIndex, envelopeIncrease, merged };+ }+ }+ }+ if (!best) break;+ batches[best.leftIndex] = best.merged;+ batches.splice(best.rightIndex, 1);+ batches.sort(compareCanonicalBatches);+ }++ const dispatch = batches+ .filter((batch) => batch.length === maxBatchSize)+ .toSorted(compareBatchesByOldestDue);+ const carry = batches+ .filter((batch) => batch.length < maxBatchSize)+ .toSorted(compareCanonicalBatches);+ const carryBudget = maxBatchSize - 1;+ let carriedTraceCount = carry.reduce(+ (count, batch) => count + batch.length,+ 0,+ );+ while (carriedTraceCount > carryBudget) {+ carry.sort(compareBatchesByOldestDue);+ const batch = carry.shift();+ if (!batch) break;+ dispatch.push(batch);+ carriedTraceCount -= batch.length;+ }+ dispatch.sort(compareBatchesByOldestDue);+ carry.sort(compareCanonicalBatches);+ return { dispatch, carry };+}diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.tsindex f2bd8a2ef5..da4434bd34 100644--- a/worker/src/features/traceBatching/traceBatching.ts+++ b/worker/src/features/traceBatching/traceBatching.ts@@ -1,6 +1,5 @@/* eslint-disable @typescript-eslint/no-use-before-define */import { createHash, randomUUID } from "node:crypto";-import { xxh32 } from "@node-rs/xxhash";import {getTracingReadinessSettings,logger,@@ -11,8 +10,6 @@ import {recordIncrement,TraceBatchTraceSchema,TraceBatchQueue,- type QueueName,- type TQueueJobTypes,} from "@langfuse/shared/src/server";import {isTopicsProcessingActive,@@ -21,11 +18,31 @@ import {import { env } from "../../env";import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";import {+ ACKNOWLEDGE_SCRIPT,+ CLEANUP_SIZE,+ evalReadiness,getTraceBatchFirstKeys,getTraceBatchFirstMember,getTraceBatchReadinessKeys,getTraceBatchShard,+ HYDRATE_SCRIPT,+ readinessScriptHashes,+ SNAPSHOT_SCRIPT,+ TRACK_SCRIPT,} from "./traceBatchReadiness";+import {+ compareBatchesByOldestDue,+ compareNumbers,+ compareStrings,+ getTraceExclusionReason,+ type PendingTrace,+ prepareLocalityPartials,+ QUERY_BUFFER_MS,+ selectTraceBatches,+ toLimit,+ type TraceBatchWeightLimits,+ type TraceExclusionReason,+} from "./traceBatchSelection";import {getDeterministicSamplingValue,shouldSampleEvaluation,@@ -34,756 +51,6 @@ import {const CHUNK_SIZE = 1_000;const REDIS_CHUNK_SIZE = 50;const REDIS_PIPELINE_SIZE = 4;-const CLEANUP_SIZE = 100;-const QUERY_BUFFER_MS = 2 * 60_000;-const MINUTE_MS = 60_000;-const NARROW_BATCH_MAX_ENVELOPE_MS = 60 * MINUTE_MS;-const WIDE_MAX_EXPANSION_RATIO = 0.25;--export type TraceBatchStrategy = "project" | "locality";-export type PendingTrace = {- member: string;- due: number;- estimates?: { eventUpdateCount: number; serializedEventBytes: number };- trace: TQueueJobTypes[QueueName.TraceBatch]["payload"]["traces"][number];-};--/**- * Upper bounds on the summed ingestion estimates of one dispatched batch.- * Traces without estimates weigh nothing, and a single trace always forms a- * batch even when it alone exceeds a budget.- */-export type TraceBatchWeightLimits = {- maxEventUpdates: number;- maxSerializedEventBytes: number;-};--const UNBOUNDED_WEIGHT_LIMITS: TraceBatchWeightLimits = {- maxEventUpdates: Number.POSITIVE_INFINITY,- maxSerializedEventBytes: Number.POSITIVE_INFINITY,-};--const toLimit = (value: number) =>- value > 0 ? value : Number.POSITIVE_INFINITY;--const getEventUpdates = ({ estimates }: PendingTrace) =>- estimates?.eventUpdateCount ?? 0;--const getSerializedEventBytes = ({ estimates }: PendingTrace) =>- estimates?.serializedEventBytes ?? 0;--const fitsWeightLimits = (- eventUpdates: number,- serializedEventBytes: number,- limits: TraceBatchWeightLimits,-) =>- eventUpdates <= limits.maxEventUpdates &&- serializedEventBytes <= limits.maxSerializedEventBytes;--const isWithinWeightLimits = (- batch: readonly PendingTrace[],- limits: TraceBatchWeightLimits,-) =>- batch.length <= 1 ||- fitsWeightLimits(- batch.reduce((total, entry) => total + getEventUpdates(entry), 0),- batch.reduce((total, entry) => total + getSerializedEventBytes(entry), 0),- limits,- );--// A heavy trace uses more than half of a batch budget.-const isHeavyTrace = (entry: PendingTrace, limits: TraceBatchWeightLimits) =>- getEventUpdates(entry) * 2 > limits.maxEventUpdates ||- getSerializedEventBytes(entry) * 2 > limits.maxSerializedEventBytes;--type TraceExclusionReason = "event_updates" | "serialized_bytes";--const getTraceExclusionReason = (- { estimates }: PendingTrace,- limits: TraceBatchWeightLimits,-): TraceExclusionReason | undefined => {- if (!estimates) return undefined;- if (estimates.eventUpdateCount > limits.maxEventUpdates)- return "event_updates";- if (estimates.serializedEventBytes > limits.maxSerializedEventBytes)- return "serialized_bytes";- return undefined;-};--const compareNumbers = (left: number, right: number) => left - right;--const compareStrings = (left: string, right: string) => {- if (left < right) {- return -1;- }- if (left > right) {- return 1;- }- return 0;-};--const toMinuteBucket = (timestamp: number) => Math.floor(timestamp / MINUTE_MS);--const getSpanMs = ({ trace }: PendingTrace) => trace.maxStart - trace.minStart;--const allowedEnvelopeMs = (seed: PendingTrace) =>- Math.max(- NARROW_BATCH_MAX_ENVELOPE_MS,- getSpanMs(seed) * (1 + WIDE_MAX_EXPANSION_RATIO),- );--type LocalityCandidate = {- entry: PendingTrace;- minMinute: number;- maxMinute: number;- traceHash: number;-};--const toLocalityCandidate = (entry: PendingTrace): LocalityCandidate => ({- entry,- minMinute: toMinuteBucket(entry.trace.minStart),- maxMinute: toMinuteBucket(entry.trace.maxStart),- traceHash: xxh32(entry.trace.traceId),-});--type BatchSelectionCost = {- crossProjectBoundaries: number;- minuteHashCells: number;- traceHashGap: number;-};--const compareByClickHouseLocality = (- left: LocalityCandidate,- right: LocalityCandidate,-) =>- compareStrings(left.entry.trace.projectId, right.entry.trace.projectId) ||- compareNumbers(left.minMinute, right.minMinute) ||- compareNumbers(left.maxMinute, right.maxMinute) ||- compareNumbers(left.traceHash, right.traceHash) ||- compareStrings(left.entry.trace.traceId, right.entry.trace.traceId);--const canonicalizeLocalityBatch = (- batch: readonly PendingTrace[],-): PendingTrace[] =>- batch- .map(toLocalityCandidate)- .toSorted(compareByClickHouseLocality)- .map(({ entry }) => entry);--const compareCanonicalBatches = (- left: readonly PendingTrace[],- right: readonly PendingTrace[],-) => {- const leftCandidates = left.map(toLocalityCandidate);- const rightCandidates = right.map(toLocalityCandidate);- const sharedLength = Math.min(leftCandidates.length, rightCandidates.length);- for (let index = 0; index < sharedLength; index++) {- const comparison = compareByClickHouseLocality(- leftCandidates[index],- rightCandidates[index],- );- if (comparison !== 0) return comparison;- }- return compareNumbers(leftCandidates.length, rightCandidates.length);-};--const getBatchBounds = (batch: readonly PendingTrace[]) => ({- minStart: Math.min(...batch.map(({ trace }) => trace.minStart)),- maxStart: Math.max(...batch.map(({ trace }) => trace.maxStart)),-});--const isFeasibleLocalityBatch = (- batch: readonly PendingTrace[],- maxBatchSize: number,- weightLimits: TraceBatchWeightLimits,-) => {- if (batch.length === 0 || batch.length > maxBatchSize) return false;- if (!isWithinWeightLimits(batch, weightLimits)) return false;- const canonical = canonicalizeLocalityBatch(batch);- const { minStart, maxStart } = getBatchBounds(canonical);- return maxStart - minStart <= allowedEnvelopeMs(canonical[0]);-};--const compareBatchSelectionCost = (- left: BatchSelectionCost,- right: BatchSelectionCost,-) =>- compareNumbers(left.crossProjectBoundaries, right.crossProjectBoundaries) ||- compareNumbers(left.minuteHashCells, right.minuteHashCells) ||- compareNumbers(left.traceHashGap, right.traceHashGap);--const addBatchSelectionCost = (- left: BatchSelectionCost,- right: BatchSelectionCost,-): BatchSelectionCost => ({- crossProjectBoundaries:- left.crossProjectBoundaries + right.crossProjectBoundaries,- minuteHashCells: left.minuteHashCells + right.minuteHashCells,- traceHashGap: left.traceHashGap + right.traceHashGap,-});--const ZERO_BATCH_SELECTION_COST: BatchSelectionCost = {- crossProjectBoundaries: 0,- minuteHashCells: 0,- traceHashGap: 0,-};--const selectLocalityBatches = (- candidates: readonly PendingTrace[],- maxBatchSize: number,- weightLimits: TraceBatchWeightLimits,-): PendingTrace[][] => {- const sorted = candidates- .map(toLocalityCandidate)- .toSorted(compareByClickHouseLocality);- const candidateCount = sorted.length;- const stride = Math.min(maxBatchSize, candidateCount) + 1;- const minuteHashCellCosts = new Float64Array(candidateCount * stride);- const feasibleSlices = new Uint8Array(candidateCount * stride);- const projectBoundaryPrefix = new Uint32Array(candidateCount + 1);- const traceHashGapPrefix = new Float64Array(candidateCount + 1);-- for (let index = 1; index < candidateCount; index++) {- const previous = sorted[index - 1];- const current = sorted[index];- const sameProject =- previous.entry.trace.projectId === current.entry.trace.projectId;- const sameTimeRange =- previous.minMinute === current.minMinute &&- previous.maxMinute === current.maxMinute;- projectBoundaryPrefix[index + 1] =- projectBoundaryPrefix[index] + (sameProject ? 0 : 1);- traceHashGapPrefix[index + 1] =- traceHashGapPrefix[index] +- (sameProject && sameTimeRange- ? current.traceHash - previous.traceHash- : 0);- }-- for (let start = 0; start < candidateCount; start++) {- let minMinute = Number.POSITIVE_INFINITY;- let maxMinute = Number.NEGATIVE_INFINITY;- let minStart = Number.POSITIVE_INFINITY;- let maxStart = Number.NEGATIVE_INFINITY;- let eventUpdates = 0;- let serializedEventBytes = 0;- const seedEnvelopeMs = allowedEnvelopeMs(sorted[start].entry);- const maxLength = Math.min(maxBatchSize, candidateCount - start);- for (let length = 1; length <= maxLength; length++) {- const candidate = sorted[start + length - 1];- eventUpdates += getEventUpdates(candidate.entry);- serializedEventBytes += getSerializedEventBytes(candidate.entry);- // Weights are nonnegative, so every longer slice is also over budget.- if (- length > 1 &&- !fitsWeightLimits(eventUpdates, serializedEventBytes, weightLimits)- )- break;- minMinute = Math.min(minMinute, candidate.minMinute);- maxMinute = Math.max(maxMinute, candidate.maxMinute);- minStart = Math.min(minStart, candidate.entry.trace.minStart);- maxStart = Math.max(maxStart, candidate.entry.trace.maxStart);- const slot = start * stride + length;- // The shared time predicate applies every selected trace hash across the- // complete envelope, so minute × hash count approximates PK cells read.- minuteHashCellCosts[slot] = (maxMinute - minMinute + 1) * length;- // Narrow groups stay within one hour. Wide groups may not grow past- // 125% of the first trace in locality order, the slice seed.- feasibleSlices[slot] = maxStart - minStart <= seedEnvelopeMs ? 1 : 0;- }- }-- const minJobs = new Int32Array(candidateCount + 1).fill(-1);- minJobs[0] = 0;- for (let end = 1; end <= candidateCount; end++) {- const minStart = Math.max(0, end - maxBatchSize);- let best = Number.MAX_SAFE_INTEGER;- for (let start = minStart; start < end; start++) {- if (minJobs[start] < 0) continue;- if (!feasibleSlices[start * stride + (end - start)]) continue;- best = Math.min(best, minJobs[start] + 1);- }- if (best === Number.MAX_SAFE_INTEGER) {- throw new Error("Unable to partition trace batch candidates");- }- minJobs[end] = best;- }- const batchCount = minJobs[candidateCount];-- const predecessors = Array.from({ length: batchCount + 1 }, () =>- new Int32Array(candidateCount + 1).fill(-1),- );- let previousCosts: Array<BatchSelectionCost | null> = Array(- candidateCount + 1,- ).fill(null);- previousCosts[0] = ZERO_BATCH_SELECTION_COST;-- // Find the cheapest path through all prefixes using exactly batchCount jobs.- for (let batchIndex = 1; batchIndex <= batchCount; batchIndex++) {- const currentCosts: Array<BatchSelectionCost | null> = Array(- candidateCount + 1,- ).fill(null);- const remainingJobs = batchCount - batchIndex;- // Leave enough traces for the remaining nonempty, size-bounded batches.- const minEnd = Math.max(- batchIndex,- candidateCount - remainingJobs * maxBatchSize,- );- const maxEnd = Math.min(- candidateCount - remainingJobs,- batchIndex * maxBatchSize,- );- for (let end = minEnd; end <= maxEnd; end++) {- if (minJobs[end] > batchIndex) continue;- const minStart = Math.max(batchIndex - 1, end - maxBatchSize);- for (let start = minStart; start < end; start++) {- const previousCost = previousCosts[start];- if (!previousCost) continue;- const length = end - start;- if (!feasibleSlices[start * stride + length]) continue;- const batchCost: BatchSelectionCost = {- crossProjectBoundaries:- projectBoundaryPrefix[end] - projectBoundaryPrefix[start + 1],- minuteHashCells: minuteHashCellCosts[start * stride + length],- traceHashGap: traceHashGapPrefix[end] - traceHashGapPrefix[start + 1],- };- const candidateCost = addBatchSelectionCost(previousCost, batchCost);- const currentCost = currentCosts[end];- if (- !currentCost ||- compareBatchSelectionCost(candidateCost, currentCost) < 0- ) {- currentCosts[end] = candidateCost;- predecessors[batchIndex][end] = start;- }- }- }- previousCosts = currentCosts;- }-- const batches = Array<PendingTrace[]>(batchCount);- let end = candidateCount;- for (let batchIndex = batchCount; batchIndex > 0; batchIndex--) {- const start = predecessors[batchIndex][end];- if (start < 0)- throw new Error("Unable to partition trace batch candidates");- batches[batchIndex - 1] = sorted- .slice(start, end)- .map(({ entry }) => entry);- end = start;- }- return batches;-};--/**- * Deterministically assigns one bounded hydrated candidate window.- *- * Locality follows the events table's physical order: project, start-time- * minute, and xxHash32(trace ID). A batch may not exceed one hour, or 125% of- * its first trace's span, whichever is larger. Extra jobs are added only when- * that envelope cap forbids a cheaper fill. Among feasible partitions with- * that minimum job count, dynamic programming first avoids crossing projects,- * then minimizes minute × trace-hash cells, then splits the largest hash gaps.- * Both strategies also close a multi-trace batch before its estimated event- * updates or serialized bytes would exceed `weightLimits`. Locality packs- * traces above half of either budget in a separate lane.- *- * Selection uses O(n × k × maxBatchSize) time and- * O(n × min(n, maxBatchSize) + n × k) memory, where k is the fewest feasible- * jobs (at least ceil(n / maxBatchSize), up to n for disjoint envelopes).- * Locality callers bound n to CHUNK_SIZE (1,000) hydrated candidates plus- * at most maxBatchSize - 1 traces across all retained partials.- * Project mode also accepts a tail of at most maxBatchSize - 1 traces and is O(n).- */-export function selectTraceBatches(- candidates: readonly PendingTrace[],- maxBatchSize: number,- strategy: TraceBatchStrategy,- weightLimits: TraceBatchWeightLimits = UNBOUNDED_WEIGHT_LIMITS,-): PendingTrace[][] {- if (candidates.length === 0) return [];- if (strategy === "project") {- const batches: PendingTrace[][] = [];- let batch: PendingTrace[] = [];- let eventUpdates = 0;- let serializedEventBytes = 0;- for (const candidate of candidates) {- const candidateEventUpdates = getEventUpdates(candidate);- const candidateBytes = getSerializedEventBytes(candidate);- if (- batch.length > 0 &&- (batch.length === maxBatchSize ||- !fitsWeightLimits(- eventUpdates + candidateEventUpdates,- serializedEventBytes + candidateBytes,- weightLimits,- ))- ) {- batches.push(batch);- batch = [];- eventUpdates = 0;- serializedEventBytes = 0;- }- batch.push(candidate);- eventUpdates += candidateEventUpdates;- serializedEventBytes += candidateBytes;- }- batches.push(batch);- return batches;- }-- // Two heavy traces cannot share a batch on the dimension that makes them- // heavy, so packing them separately keeps them from fragmenting the light- // traces around them in locality order.- const light: PendingTrace[] = [];- const heavy: PendingTrace[] = [];- for (const candidate of candidates) {- if (isHeavyTrace(candidate, weightLimits)) heavy.push(candidate);- else light.push(candidate);- }- return [- ...selectLocalityBatches(light, maxBatchSize, weightLimits),- ...selectLocalityBatches(heavy, maxBatchSize, weightLimits),- ];-}--const compareBatchesByOldestDue = (- left: readonly PendingTrace[],- right: readonly PendingTrace[],-) =>- compareNumbers(- Math.min(...left.map(({ due }) => due)),- Math.min(...right.map(({ due }) => due)),- ) || compareCanonicalBatches(left, right);--const canCoalesceLocalityPartials = (- left: readonly PendingTrace[],- right: readonly PendingTrace[],- maxBatchSize: number,- weightLimits: TraceBatchWeightLimits,-) => {- if (left.length + right.length > maxBatchSize) return false;- const projectId = left[0]?.trace.projectId;- if (- !projectId ||- left.some(({ trace }) => trace.projectId !== projectId) ||- right.some(({ trace }) => trace.projectId !== projectId)- ) {- return false;- }- const leftBounds = getBatchBounds(left);- const rightBounds = getBatchBounds(right);- const bufferedIntervalsOverlap =- leftBounds.minStart - QUERY_BUFFER_MS <=- rightBounds.maxStart + QUERY_BUFFER_MS &&- rightBounds.minStart - QUERY_BUFFER_MS <=- leftBounds.maxStart + QUERY_BUFFER_MS;- return (- bufferedIntervalsOverlap &&- isFeasibleLocalityBatch([...left, ...right], maxBatchSize, weightLimits)- );-};--/**- * Coalesce compatible locality partials and retain a globally bounded carry.- * The dispatcher calls this once per hydrated window; work never survives the- * current dispatch run.- */-export function prepareLocalityPartials(- partials: readonly (readonly PendingTrace[])[],- maxBatchSize: number,- weightLimits: TraceBatchWeightLimits = UNBOUNDED_WEIGHT_LIMITS,-): { dispatch: PendingTrace[][]; carry: PendingTrace[][] } {- const batches = partials- .filter((batch) => batch.length > 0)- .map(canonicalizeLocalityBatch)- .toSorted(compareCanonicalBatches);-- while (true) {- let best:- | {- leftIndex: number;- rightIndex: number;- envelopeIncrease: number;- merged: PendingTrace[];- }- | undefined;- for (let leftIndex = 0; leftIndex < batches.length; leftIndex++) {- for (- let rightIndex = leftIndex + 1;- rightIndex < batches.length;- rightIndex++- ) {- const left = batches[leftIndex];- const right = batches[rightIndex];- if (- !canCoalesceLocalityPartials(left, right, maxBatchSize, weightLimits)- )- continue;- const leftBounds = getBatchBounds(left);- const rightBounds = getBatchBounds(right);- const merged = canonicalizeLocalityBatch([...left, ...right]);- const mergedBounds = getBatchBounds(merged);- const envelopeIncrease =- mergedBounds.maxStart -- mergedBounds.minStart -- Math.max(- leftBounds.maxStart - leftBounds.minStart,- rightBounds.maxStart - rightBounds.minStart,- );- if (- !best ||- envelopeIncrease < best.envelopeIncrease ||- (envelopeIncrease === best.envelopeIncrease &&- compareCanonicalBatches(merged, best.merged) < 0)- ) {- best = { leftIndex, rightIndex, envelopeIncrease, merged };- }- }- }- if (!best) break;- batches[best.leftIndex] = best.merged;- batches.splice(best.rightIndex, 1);- batches.sort(compareCanonicalBatches);- }-- const dispatch = batches- .filter((batch) => batch.length === maxBatchSize)- .toSorted(compareBatchesByOldestDue);- const carry = batches- .filter((batch) => batch.length < maxBatchSize)- .toSorted(compareCanonicalBatches);- const carryBudget = maxBatchSize - 1;- let carriedTraceCount = carry.reduce(- (count, batch) => count + batch.length,- 0,- );- while (carriedTraceCount > carryBudget) {- carry.sort(compareBatchesByOldestDue);- const batch = carry.shift();- if (!batch) break;- dispatch.push(batch);- carriedTraceCount -= batch.length;- }- dispatch.sort(compareBatchesByOldestDue);- carry.sort(compareCanonicalBatches);- return { dispatch, carry };-}--// Retain traces for a bounded time after readiness. Reuse the due index so-// cleanup needs no full hash scan. Whole-key expiry is an inactivity backstop.-const EXPIRE_PENDING_SCRIPT = `- local function expirePending(now, retention, limit)- local expired = redis.call('ZRANGE', KEYS[1], '-inf', now - retention,- 'BYSCORE', 'LIMIT', 0, limit)- for _, member in ipairs(expired) do- redis.call('ZREM', KEYS[1], member)- redis.call('HDEL', KEYS[2], member)- end- return #expired- end-`;--const TRACK_SCRIPT = `- ${EXPIRE_PENDING_SCRIPT}- local clock = redis.call('TIME')- local now = clock[1] * 1000 + math.floor(clock[2] / 1000)- local firstTtl = tonumber(ARGV[3])- local width = tonumber(ARGV[4])- local bucket = math.floor(now / width)- local current = bucket - tonumber(ARGV[5]) + 3- if current < 3 or current > #KEYS then- return redis.error_reply('Trace readiness bucket clock changed; retry intake')- end- for i = 6, #ARGV, 7 do- local member = ARGV[i]- local minStart = tonumber(ARGV[i + 1])- local maxStart = tonumber(ARGV[i + 2])- local eventUpdateCount = tonumber(ARGV[i + 4])- local serializedEventBytes = tonumber(ARGV[i + 5])- local identity = ARGV[i + 6]- local previous = redis.call('HGET', KEYS[2], member)- local state = previous and cjson.decode(previous)- local priorDue = tonumber(redis.call('ZSCORE', KEYS[1], member))- if not priorDue or priorDue <= now - tonumber(ARGV[2]) then state = nil end- if state and state.historyBucket then- local old = state.historyBucket - tonumber(ARGV[5]) + 3- if old >= 3 and old <= #KEYS and old ~= current then- redis.call('HDEL', KEYS[old], identity)- end- else- local retainedUpdates = 0- local retainedBytes = 0- local estimatesKnown = true- for key = 3, #KEYS do- local retained = redis.call('HGET', KEYS[key], identity)- if retained then- local bounds = cmsgpack.unpack(retained)- if bounds[3] > now then- minStart = math.min(minStart, bounds[1])- maxStart = math.max(maxStart, bounds[2])- if bounds[4] and bounds[5] then- retainedUpdates = math.max(retainedUpdates, bounds[4])- retainedBytes = math.max(retainedBytes, bounds[5])- else- estimatesKnown = false- end- end- if key ~= current then redis.call('HDEL', KEYS[key], identity) end- end- end- if not estimatesKnown then- eventUpdateCount = nil- serializedEventBytes = nil- elseif not state then- eventUpdateCount = eventUpdateCount + retainedUpdates- serializedEventBytes = serializedEventBytes + retainedBytes- end- end- if state then- minStart = math.min(minStart, state.minStart)- maxStart = math.max(maxStart, state.maxStart)- if eventUpdateCount and serializedEventBytes and state.eventUpdateCount and state.serializedEventBytes then- eventUpdateCount = eventUpdateCount + state.eventUpdateCount- serializedEventBytes = serializedEventBytes + state.serializedEventBytes- else- -- Missing history cannot be reconstructed from a later update.- eventUpdateCount = nil- serializedEventBytes = nil- end- end- redis.call('HSET', KEYS[2], member, cjson.encode({- minStart = minStart, maxStart = maxStart, revision = ARGV[i + 3], historyBucket = bucket,- eventUpdateCount = eventUpdateCount, serializedEventBytes = serializedEventBytes- }))- redis.call('ZADD', KEYS[1], now + tonumber(ARGV[1]), member)- redis.call('HSET', KEYS[current], identity,- cmsgpack.pack({minStart, maxStart, now + firstTtl, eventUpdateCount, serializedEventBytes}))- end- local firstExpiry = math.max((bucket + 1) * width + firstTtl,- redis.call('PEXPIRETIME', KEYS[current]))- -- Integer command arguments need decimal strings, including round timestamps.- redis.call('PEXPIREAT', KEYS[current], string.format('%.0f', firstExpiry))- local retention = tonumber(ARGV[2])- local expired = expirePending(now, retention, ${CLEANUP_SIZE})- -- The shared inactivity backstop covers the latest required member deadline.- local expiresAt = math.max(now + tonumber(ARGV[1]) + retention,- redis.call('PEXPIRETIME', KEYS[1]), redis.call('PEXPIRETIME', KEYS[2]))- redis.call('PEXPIREAT', KEYS[1], string.format('%.0f', expiresAt))- redis.call('PEXPIREAT', KEYS[2], string.format('%.0f', expiresAt))- return expired-`;--// Cleanup stays bounded even when the dispatcher has accumulated a backlog.-// Keep the first Redis timestamp as the cutoff throughout this run.-const SNAPSHOT_SCRIPT = `- ${EXPIRE_PENDING_SCRIPT}- local clock = redis.call('TIME')- local now = clock[1] * 1000 + math.floor(clock[2] / 1000)- local retention = tonumber(ARGV[1])- local cutoff = tonumber(ARGV[2]) or now- local expired = expirePending(now, retention, ${CLEANUP_SIZE})- local pending = redis.call('ZCARD', KEYS[1])- local ready = math.max(0, redis.call('ZCOUNT', KEYS[1], '-inf', cutoff)- - redis.call('ZCOUNT', KEYS[1], '-inf', now - retention))- local first = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')- return {now, cutoff, pending, ready, first[2] or tostring(now), expired}-`;--// Fetch state only while it is still due for this run. Checking the score and-// reading the revision atomically prevents dispatching freshly reactivated work.-const HYDRATE_SCRIPT = `- local clock = redis.call('TIME')- local now = clock[1] * 1000 + math.floor(clock[2] / 1000)- local result = {}- for i = 3, #ARGV do- local member = ARGV[i]- local due = tonumber(redis.call('ZSCORE', KEYS[1], member))- if due and due <= tonumber(ARGV[1]) and due > now - tonumber(ARGV[2]) then- local state = redis.call('HGET', KEYS[2], member)- if state then- table.insert(result, member)- table.insert(result, state)- table.insert(result, tostring(due))- else- redis.call('ZREM', KEYS[1], member)- end- end- end- return result-`;--// Returns one flag per member, in argument order: 1 when removed, 0 when a-// newer revision keeps the trace pending.-const ACKNOWLEDGE_SCRIPT = `- local clock = redis.call('TIME')- local now = clock[1] * 1000 + math.floor(clock[2] / 1000)- local width = tonumber(ARGV[2])- local bucket = math.floor(now / width)- local current = bucket - tonumber(ARGV[3]) + 3- if current < 3 or current > #KEYS then- return redis.error_reply('Trace readiness bucket clock changed; retry acknowledgement')- end- local removed = {}- local retained = false- for i = 4, #ARGV, 3 do- local state = redis.call('HGET', KEYS[2], ARGV[i])- local decoded = state and cjson.decode(state)- if decoded and decoded.revision == ARGV[i + 1] then- local identity = ARGV[i + 2]- if decoded.historyBucket then- local old = decoded.historyBucket - tonumber(ARGV[3]) + 3- if old >= 3 and old <= #KEYS and old ~= current then redis.call('HDEL', KEYS[old], identity) end- else- for key = 3, #KEYS do- if key ~= current then redis.call('HDEL', KEYS[key], identity) end- end- end- redis.call('HSET', KEYS[current], identity,- cmsgpack.pack({decoded.minStart, decoded.maxStart, now + tonumber(ARGV[1]), decoded.eventUpdateCount, decoded.serializedEventBytes}))- retained = true- redis.call('ZREM', KEYS[1], ARGV[i])- redis.call('HDEL', KEYS[2], ARGV[i])- table.insert(removed, 1)- else- table.insert(removed, 0)- end- end- if retained then- local expiresAt = math.max((bucket + 1) * width + tonumber(ARGV[1]),- redis.call('PEXPIRETIME', KEYS[current]))- redis.call('PEXPIREAT', KEYS[current], string.format('%.0f', expiresAt))- end- return removed-`;--const readinessScriptHashes = new Map(- [TRACK_SCRIPT, SNAPSHOT_SCRIPT, HYDRATE_SCRIPT, ACKNOWLEDGE_SCRIPT].map(- (script) =>- [script, createHash("sha1").update(script).digest("hex")] as const,- ),-);--async function evalReadiness(- script: string,- keys: string[],- args: (string | number | Buffer)[],-): Promise<unknown> {- if (!redis) throw new Error("Trace batching requires Redis");- try {- return await redis.evalsha(- readinessScriptHashes.get(script)!,- keys.length,- ...keys,- ...args,- );- } catch (error) {- // SCRIPT FLUSH or a promoted primary can have an empty script cache.- if (!(error instanceof Error) || !error.message.startsWith("NOSCRIPT"))- throw error;- return redis.eval(script, keys.length, ...keys, ...args);- }-}function recordTrackingVolume(stage: "eligible" | "recorded",